Merge master r12-8312.devel/c++-coroutines

	* Merge master r12-8312-gb85e79dce149.

35 files changed, 3139 insertions, 387 deletions

diff --git a/gcc/config/aarch64/aarch64-builtins.cc b/gcc/config/aarch64/aarch64-builtins.cc
index 6ebeee7..c21476d 100644
--- a/gcc/config/aarch64/aarch64-builtins.cc
+++ b/gcc/config/aarch64/aarch64-builtins.cc
@@ -1664,6 +1664,14 @@ aarch64_init_ls64_builtins (void)
       = aarch64_general_add_builtin (data[i].name, data[i].type, data[i].code);
 }
 
+/* Implement #pragma GCC aarch64 "arm_acle.h".  */
+void
+handle_arm_acle_h (void)
+{
+  if (TARGET_LS64)
+    aarch64_init_ls64_builtins ();
+}
+
 /* Initialize fpsr fpcr getters and setters.  */
 
 static void
@@ -1755,9 +1763,6 @@ aarch64_general_init_builtins (void)
 
   if (TARGET_MEMTAG)
     aarch64_init_memtag_builtins ();
-
-  if (TARGET_LS64)
-    aarch64_init_ls64_builtins ();
 }
 
 /* Implement TARGET_BUILTIN_DECL for the AARCH64_BUILTIN_GENERAL group.  */
diff --git a/gcc/config/aarch64/aarch64-c.cc b/gcc/config/aarch64/aarch64-c.cc
index caf8e33..767ee0c 100644
--- a/gcc/config/aarch64/aarch64-c.cc
+++ b/gcc/config/aarch64/aarch64-c.cc
@@ -302,6 +302,8 @@ aarch64_pragma_aarch64 (cpp_reader *)
     aarch64_sve::handle_arm_sve_h ();
   else if (strcmp (name, "arm_neon.h") == 0)
     handle_arm_neon_h ();
+  else if (strcmp (name, "arm_acle.h") == 0)
+    handle_arm_acle_h ();
   else
     error ("unknown %<#pragma GCC aarch64%> option %qs", name);
 }
diff --git a/gcc/config/aarch64/aarch64-protos.h b/gcc/config/aarch64/aarch64-protos.h
index c6f13ee..2ac781d 100644
--- a/gcc/config/aarch64/aarch64-protos.h
+++ b/gcc/config/aarch64/aarch64-protos.h
@@ -995,6 +995,7 @@ rtx aarch64_general_expand_builtin (unsigned int, tree, rtx, int);
 tree aarch64_general_builtin_decl (unsigned, bool);
 tree aarch64_general_builtin_rsqrt (unsigned int);
 tree aarch64_builtin_vectorized_function (unsigned int, tree, tree);
+void handle_arm_acle_h (void);
 void handle_arm_neon_h (void);
 
 namespace aarch64_sve {
diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md
index 1873342..a00e1c6 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -3385,20 +3385,6 @@
 
 ;; 'across lanes' add.
 
-(define_expand "reduc_plus_scal_<mode>"
-  [(match_operand:<VEL> 0 "register_operand")
-   (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand")]
-	       UNSPEC_ADDV)]
-  "TARGET_SIMD"
-  {
-    rtx elt = aarch64_endian_lane_rtx (<MODE>mode, 0);
-    rtx scratch = gen_reg_rtx (<MODE>mode);
-    emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
-    emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
-    DONE;
-  }
-)
-
 (define_insn "aarch64_faddp<mode>"
  [(set (match_operand:VHSDF 0 "register_operand" "=w")
        (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")
@@ -3409,31 +3395,22 @@
   [(set_attr "type" "neon_fp_reduc_add_<stype><q>")]
 )
 
-(define_insn "aarch64_reduc_plus_internal<mode>"
- [(set (match_operand:VDQV 0 "register_operand" "=w")
-       (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
+(define_insn "reduc_plus_scal_<mode>"
+ [(set (match_operand:<VEL> 0 "register_operand" "=w")
+       (unspec:<VEL> [(match_operand:VDQV 1 "register_operand" "w")]
 		    UNSPEC_ADDV))]
  "TARGET_SIMD"
  "add<VDQV:vp>\\t%<Vetype>0, %1.<Vtype>"
   [(set_attr "type" "neon_reduc_add<q>")]
 )
 
-(define_insn "aarch64_<su>addlv<mode>"
- [(set (match_operand:<VWIDE_S> 0 "register_operand" "=w")
-       (unspec:<VWIDE_S> [(match_operand:VDQV_L 1 "register_operand" "w")]
-		    USADDLV))]
- "TARGET_SIMD"
- "<su>addl<vp>\\t%<Vwstype>0<Vwsuf>, %1.<Vtype>"
-  [(set_attr "type" "neon_reduc_add<q>")]
-)
-
-(define_insn "aarch64_<su>addlp<mode>"
- [(set (match_operand:<VDBLW> 0 "register_operand" "=w")
-       (unspec:<VDBLW> [(match_operand:VDQV_L 1 "register_operand" "w")]
-		    USADDLP))]
+(define_insn "reduc_plus_scal_v2si"
+ [(set (match_operand:SI 0 "register_operand" "=w")
+       (unspec:SI [(match_operand:V2SI 1 "register_operand" "w")]
+		    UNSPEC_ADDV))]
  "TARGET_SIMD"
- "<su>addlp\\t%0.<Vwhalf>, %1.<Vtype>"
-  [(set_attr "type" "neon_reduc_add<q>")]
+ "addp\\t%0.2s, %1.2s, %1.2s"
+  [(set_attr "type" "neon_reduc_add")]
 )
 
 ;; ADDV with result zero-extended to SI/DImode (for popcount).
@@ -3447,15 +3424,6 @@
   [(set_attr "type" "neon_reduc_add<VDQV_E:q>")]
 )
 
-(define_insn "aarch64_reduc_plus_internalv2si"
- [(set (match_operand:V2SI 0 "register_operand" "=w")
-       (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
-		    UNSPEC_ADDV))]
- "TARGET_SIMD"
- "addp\\t%0.2s, %1.2s, %1.2s"
-  [(set_attr "type" "neon_reduc_add")]
-)
-
 (define_insn "reduc_plus_scal_<mode>"
  [(set (match_operand:<VEL> 0 "register_operand" "=w")
        (unspec:<VEL> [(match_operand:V2F 1 "register_operand" "w")]
@@ -3467,7 +3435,7 @@
 
 (define_expand "reduc_plus_scal_v4sf"
  [(set (match_operand:SF 0 "register_operand")
-       (unspec:V4SF [(match_operand:V4SF 1 "register_operand")]
+       (unspec:SF [(match_operand:V4SF 1 "register_operand")]
 		    UNSPEC_FADDV))]
  "TARGET_SIMD"
 {
@@ -3479,6 +3447,24 @@
   DONE;
 })
 
+(define_insn "aarch64_<su>addlv<mode>"
+ [(set (match_operand:<VWIDE_S> 0 "register_operand" "=w")
+       (unspec:<VWIDE_S> [(match_operand:VDQV_L 1 "register_operand" "w")]
+		    USADDLV))]
+ "TARGET_SIMD"
+ "<su>addl<vp>\\t%<Vwstype>0<Vwsuf>, %1.<Vtype>"
+  [(set_attr "type" "neon_reduc_add<q>")]
+)
+
+(define_insn "aarch64_<su>addlp<mode>"
+ [(set (match_operand:<VDBLW> 0 "register_operand" "=w")
+       (unspec:<VDBLW> [(match_operand:VDQV_L 1 "register_operand" "w")]
+		    USADDLP))]
+ "TARGET_SIMD"
+ "<su>addlp\\t%0.<Vwhalf>, %1.<Vtype>"
+  [(set_attr "type" "neon_reduc_add<q>")]
+)
+
 (define_insn "clrsb<mode>2"
   [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
         (clrsb:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index 3e2a6fb..f650abb 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -15637,7 +15637,7 @@ private:
   unsigned int adjust_body_cost (loop_vec_info, const aarch64_vector_costs *,
 				 unsigned int);
   bool prefer_unrolled_loop () const;
-  unsigned int determine_suggested_unroll_factor ();
+  unsigned int determine_suggested_unroll_factor (loop_vec_info);
 
   /* True if we have performed one-time initialization based on the
      vec_info.  */
@@ -16746,7 +16746,8 @@ adjust_body_cost_sve (const aarch64_vec_op_count *ops,
 }
 
 unsigned int
-aarch64_vector_costs::determine_suggested_unroll_factor ()
+aarch64_vector_costs::
+determine_suggested_unroll_factor (loop_vec_info loop_vinfo)
 {
   bool sve = m_vec_flags & VEC_ANY_SVE;
   /* If we are trying to unroll an Advanced SIMD main loop that contains
@@ -16760,6 +16761,7 @@ aarch64_vector_costs::determine_suggested_unroll_factor ()
     return 1;
 
   unsigned int max_unroll_factor = 1;
+  auto vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
   for (auto vec_ops : m_ops)
     {
       aarch64_simd_vec_issue_info const *vec_issue
@@ -16768,7 +16770,8 @@ aarch64_vector_costs::determine_suggested_unroll_factor ()
 	return 1;
       /* Limit unroll factor to a value adjustable by the user, the default
 	 value is 4. */
-      unsigned int unroll_factor = aarch64_vect_unroll_limit;
+      unsigned int unroll_factor = MIN (aarch64_vect_unroll_limit,
+					(int) known_alignment (vf));
       unsigned int factor
        = vec_ops.reduction_latency > 1 ? vec_ops.reduction_latency : 1;
       unsigned int temp;
@@ -16946,7 +16949,8 @@ aarch64_vector_costs::finish_cost (const vector_costs *uncast_scalar_costs)
     {
       m_costs[vect_body] = adjust_body_cost (loop_vinfo, scalar_costs,
 					     m_costs[vect_body]);
-      m_suggested_unroll_factor = determine_suggested_unroll_factor ();
+      m_suggested_unroll_factor
+	= determine_suggested_unroll_factor (loop_vinfo);
     }
 
   /* Apply the heuristic described above m_stp_sequence_cost.  Prefer
@@ -18053,6 +18057,9 @@ aarch64_validate_mtune (const char *str, const struct processor **res)
   return false;
 }
 
+static_assert (TARGET_CPU_generic < TARGET_CPU_MASK,
+	       "TARGET_CPU_NBITS is big enough");
+
 /* Return the CPU corresponding to the enum CPU.
    If it doesn't specify a cpu, return the default.  */
 
@@ -18062,12 +18069,12 @@ aarch64_get_tune_cpu (enum aarch64_processor cpu)
   if (cpu != aarch64_none)
     return &all_cores[cpu];
 
-  /* The & 0x3f is to extract the bottom 6 bits that encode the
-     default cpu as selected by the --with-cpu GCC configure option
+  /* The & TARGET_CPU_MASK is to extract the bottom TARGET_CPU_NBITS bits that
+     encode the default cpu as selected by the --with-cpu GCC configure option
      in config.gcc.
      ???: The whole TARGET_CPU_DEFAULT and AARCH64_CPU_DEFAULT_FLAGS
      flags mechanism should be reworked to make it more sane.  */
-  return &all_cores[TARGET_CPU_DEFAULT & 0x3f];
+  return &all_cores[TARGET_CPU_DEFAULT & TARGET_CPU_MASK];
 }
 
 /* Return the architecture corresponding to the enum ARCH.
@@ -18079,7 +18086,8 @@ aarch64_get_arch (enum aarch64_arch arch)
   if (arch != aarch64_no_arch)
     return &all_architectures[arch];
 
-  const struct processor *cpu = &all_cores[TARGET_CPU_DEFAULT & 0x3f];
+  const struct processor *cpu
+    = &all_cores[TARGET_CPU_DEFAULT & TARGET_CPU_MASK];
 
   return &all_architectures[cpu->arch];
 }
@@ -18166,7 +18174,7 @@ aarch64_override_options (void)
 	{
 	  /* Get default configure-time CPU.  */
 	  selected_cpu = aarch64_get_tune_cpu (aarch64_none);
-	  aarch64_isa_flags = TARGET_CPU_DEFAULT >> 6;
+	  aarch64_isa_flags = TARGET_CPU_DEFAULT >> TARGET_CPU_NBITS;
 	}
 
       if (selected_tune)
diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
index efa46ac..359b6e85 100644
--- a/gcc/config/aarch64/aarch64.h
+++ b/gcc/config/aarch64/aarch64.h
@@ -813,10 +813,16 @@ enum target_cpus
   TARGET_CPU_generic
 };
 
+/* Define how many bits are used to represent the CPU in TARGET_CPU_DEFAULT.
+   This needs to be big enough to fit the value of TARGET_CPU_generic.
+   All bits after this are used to represent the AARCH64_CPU_DEFAULT_FLAGS.  */
+#define TARGET_CPU_NBITS 8
+#define TARGET_CPU_MASK ((1 << TARGET_CPU_NBITS) - 1)
+
 /* If there is no CPU defined at configure, use generic as default.  */
 #ifndef TARGET_CPU_DEFAULT
 #define TARGET_CPU_DEFAULT \
-  (TARGET_CPU_generic | (AARCH64_CPU_DEFAULT_FLAGS << 6))
+  (TARGET_CPU_generic | (AARCH64_CPU_DEFAULT_FLAGS << TARGET_CPU_NBITS))
 #endif
 
 /* If inserting NOP before a mult-accumulate insn remember to adjust the
diff --git a/gcc/config/aarch64/arm_acle.h b/gcc/config/aarch64/arm_acle.h
index ecd852f..9775a48 100644
--- a/gcc/config/aarch64/arm_acle.h
+++ b/gcc/config/aarch64/arm_acle.h
@@ -29,6 +29,8 @@
 
 #include <stdint.h>
 
+#pragma GCC aarch64 "arm_acle.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
diff --git a/gcc/config/arm/t-aprofile b/gcc/config/arm/t-aprofile
index 574951c..fe2ec88 100644
--- a/gcc/config/arm/t-aprofile
+++ b/gcc/config/arm/t-aprofile
@@ -26,8 +26,8 @@
 
 # Arch and FPU variants to build libraries with
 
-MULTI_ARCH_OPTS_A       = march=armv7-a/march=armv7-a+fp/march=armv7-a+simd/march=armv7ve+simd/march=armv8-a/march=armv8-a+simd/march=armv9-a/march=armv9-a+simd
-MULTI_ARCH_DIRS_A       = v7-a v7-a+fp v7-a+simd v7ve+simd v8-a v8-a+simd v9-a v9-a+simd
+MULTI_ARCH_OPTS_A       = march=armv7-a/march=armv7-a+fp/march=armv7-a+simd/march=armv7ve+simd/march=armv8-a/march=armv8-a+simd
+MULTI_ARCH_DIRS_A       = v7-a v7-a+fp v7-a+simd v7ve+simd v8-a v8-a+simd
 
 # ARMv7-A - build nofp, fp-d16 and SIMD variants
 
@@ -46,11 +46,6 @@ MULTILIB_REQUIRED	+= mthumb/march=armv8-a/mfloat-abi=soft
 MULTILIB_REQUIRED	+= mthumb/march=armv8-a+simd/mfloat-abi=hard
 MULTILIB_REQUIRED	+= mthumb/march=armv8-a+simd/mfloat-abi=softfp
 
-# Armv9-A - build nofp and SIMD variants.
-MULTILIB_REQUIRED	+= mthumb/march=armv9-a/mfloat-abi=soft
-MULTILIB_REQUIRED	+= mthumb/march=armv9-a+simd/mfloat-abi=hard
-MULTILIB_REQUIRED	+= mthumb/march=armv9-a+simd/mfloat-abi=softfp
-
 # Matches
 
 # Arch Matches
@@ -135,14 +130,12 @@ MULTILIB_MATCHES	+= $(foreach ARCH, $(v8_6_a_simd_variants), \
 			     march?armv8-a+simd=march?armv8.6-a$(ARCH))
 
 # Armv9 without SIMD: map down to base architecture
-MULTILIB_MATCHES	+= $(foreach ARCH, $(v9_a_nosimd_variants), \
-			     march?armv9-a=march?armv9-a$(ARCH))
+MULTILIB_MATCHES    += march?armv8-a=march?armv9-a
+# No variants without SIMD.
 
 # Armv9 with SIMD: map down to base arch + simd
-MULTILIB_MATCHES	+= march?armv9-a+simd=march?armv9-a+crc+simd \
-			   $(foreach ARCH, $(filter-out +simd, $(v9_a_simd_variants)), \
-			     march?armv9-a+simd=march?armv9-a$(ARCH) \
-			     march?armv9-a+simd=march?armv9-a+crc$(ARCH))
+MULTILIB_MATCHES    += $(foreach ARCH, $(v9_a_simd_variants), \
+			     march?armv8-a+simd=march?armv9-a$(ARCH))
 
 # Use Thumb libraries for everything.
 
@@ -150,13 +143,11 @@ MULTILIB_REUSE		+= mthumb/march.armv7-a/mfloat-abi.soft=marm/march.armv7-a/mfloa
 
 MULTILIB_REUSE		+= mthumb/march.armv8-a/mfloat-abi.soft=marm/march.armv8-a/mfloat-abi.soft
 
-MULTILIB_REUSE		+= mthumb/march.armv9-a/mfloat-abi.soft=marm/march.armv9-a/mfloat-abi.soft
-
 MULTILIB_REUSE		+= $(foreach ABI, hard softfp, \
-			     $(foreach ARCH, armv7-a+fp armv7-a+simd armv7ve+simd armv8-a+simd armv9-a+simd, \
+			     $(foreach ARCH, armv7-a+fp armv7-a+simd armv7ve+simd armv8-a+simd, \
 			       mthumb/march.$(ARCH)/mfloat-abi.$(ABI)=marm/march.$(ARCH)/mfloat-abi.$(ABI)))
 
 # Softfp but no FP, use the soft-float libraries.
 MULTILIB_REUSE		+= $(foreach MODE, arm thumb, \
-			     $(foreach ARCH, armv7-a armv8-a armv9-a, \
+			     $(foreach ARCH, armv7-a armv8-a, \
 			       mthumb/march.$(ARCH)/mfloat-abi.soft=m$(MODE)/march.$(ARCH)/mfloat-abi.softfp))
diff --git a/gcc/config/arm/t-multilib b/gcc/config/arm/t-multilib
index ea258b1..6bb58d3 100644
--- a/gcc/config/arm/t-multilib
+++ b/gcc/config/arm/t-multilib
@@ -78,7 +78,6 @@ v8_4_a_simd_variants	:= $(call all_feat_combs, simd fp16 crypto i8mm bf16)
 v8_5_a_simd_variants	:= $(call all_feat_combs, simd fp16 crypto i8mm bf16)
 v8_6_a_simd_variants	:= $(call all_feat_combs, simd fp16 crypto i8mm bf16)
 v8_r_nosimd_variants	:= +crc
-v9_a_nosimd_variants	:= +crc
 v9_a_simd_variants	:= $(call all_feat_combs, simd fp16 crypto i8mm bf16)
 
 ifneq (,$(HAS_APROFILE))
@@ -206,14 +205,10 @@ MULTILIB_MATCHES	+= $(foreach ARCH, $(v8_6_a_simd_variants), \
 
 # Armv9
 MULTILIB_MATCHES	+= march?armv7=march?armv9-a
-MULTILIB_MATCHES	+= $(foreach ARCH, $(v9_a_nosimd_variants), \
-			     march?armv7=march?armv9-a$(ARCH))
 
 # Armv9 with SIMD
-MULTILIB_MATCHES	+= march?armv7+fp=march?armv9-a+crc+simd \
-			   $(foreach ARCH, $(v9_a_simd_variants), \
-			     march?armv7+fp=march?armv9-a$(ARCH) \
-			     march?armv7+fp=march?armv9-a+crc$(ARCH))
+MULTILIB_MATCHES	+= $(foreach ARCH, $(v9_a_simd_variants), \
+			     march?armv7+fp=march?armv9-a$(ARCH))
 endif		# Not APROFILE.
 
 # Use Thumb libraries for everything.
diff --git a/gcc/config/bfin/bfin.md b/gcc/config/bfin/bfin.md
index 0e44653..56b2472 100644
--- a/gcc/config/bfin/bfin.md
+++ b/gcc/config/bfin/bfin.md
@@ -1741,7 +1741,7 @@
 	(ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "d") (const_int 1))
 		(zero_extend:SI (reg:BI REG_CC))))
    (set (reg:BI REG_CC)
-	(zero_extract:BI (match_dup 1) (const_int 31) (const_int 0)))]
+	(zero_extract:BI (match_dup 1) (const_int 1) (const_int 31)))]
   ""
   "%0 = ROT %1 BY 1%!"
   [(set_attr "type" "dsp32shiftimm")])
diff --git a/gcc/config/freebsd.h b/gcc/config/freebsd.h
index 28ebcad..d89ee7d 100644
--- a/gcc/config/freebsd.h
+++ b/gcc/config/freebsd.h
@@ -55,7 +55,7 @@ along with GCC; see the file COPYING3.  If not see
 #endif
 
 #undef TARGET_LIBC_HAS_FUNCTION
-#define TARGET_LIBC_HAS_FUNCTION no_c99_libc_has_function
+#define TARGET_LIBC_HAS_FUNCTION bsd_libc_has_function
 
 /* Use --as-needed -lgcc_s for eh support.  */
 #ifdef HAVE_LD_AS_NEEDED
diff --git a/gcc/config/gcn/gcn.cc b/gcc/config/gcn/gcn.cc
index 402f025..e2e9335 100644
--- a/gcc/config/gcn/gcn.cc
+++ b/gcc/config/gcn/gcn.cc
@@ -5588,8 +5588,9 @@ gcn_print_lds_decl (FILE *f, tree var)
       fprintf (f, "%u", gang_private_hwm);
       gang_private_hwm += size;
       if (gang_private_hwm > gang_private_size_opt)
-	error ("gang-private data-share memory exhausted (increase with "
-	       "%<-mgang-private-size=<number>%>)");
+	error ("%d bytes of gang-private data-share memory exhausted"
+	       " (increase with %<-mgang-private-size=%d%>, for example)",
+	       gang_private_size_opt, gang_private_hwm);
     }
 }
 
diff --git a/gcc/config/i386/avx512fintrin.h b/gcc/config/i386/avx512fintrin.h
index 29511fd..77d6249 100644
--- a/gcc/config/i386/avx512fintrin.h
+++ b/gcc/config/i386/avx512fintrin.h
@@ -3286,31 +3286,67 @@ _mm_maskz_scalef_round_ss (__mmask8 __U, __m128 __A, __m128 __B, const int __R)
 						      (__mmask8) __U, __R);
 }
 #else
-#define _mm512_scalef_round_pd(A, B, C)            \
-    (__m512d)__builtin_ia32_scalefpd512_mask(A, B, (__v8df)_mm512_undefined_pd(), -1, C)
-
-#define _mm512_mask_scalef_round_pd(W, U, A, B, C) \
-    (__m512d)__builtin_ia32_scalefpd512_mask(A, B, W, U, C)
-
-#define _mm512_maskz_scalef_round_pd(U, A, B, C)   \
-    (__m512d)__builtin_ia32_scalefpd512_mask(A, B, (__v8df)_mm512_setzero_pd(), U, C)
+#define _mm512_scalef_round_pd(A, B, C)					\
+  ((__m512d)								\
+   __builtin_ia32_scalefpd512_mask((A), (B),				\
+				   (__v8df) _mm512_undefined_pd(),	\
+				   -1, (C)))
+
+#define _mm512_mask_scalef_round_pd(W, U, A, B, C)			\
+  ((__m512d) __builtin_ia32_scalefpd512_mask((A), (B), (W), (U), (C)))
+
+#define _mm512_maskz_scalef_round_pd(U, A, B, C)			\
+  ((__m512d)								\
+   __builtin_ia32_scalefpd512_mask((A), (B),				\
+				   (__v8df) _mm512_setzero_pd(),	\
+				   (U), (C)))
+
+#define _mm512_scalef_round_ps(A, B, C)					\
+  ((__m512)								\
+   __builtin_ia32_scalefps512_mask((A), (B),				\
+				   (__v16sf) _mm512_undefined_ps(),	\
+				   -1, (C)))
+
+#define _mm512_mask_scalef_round_ps(W, U, A, B, C)			\
+  ((__m512) __builtin_ia32_scalefps512_mask((A), (B), (W), (U), (C)))
+
+#define _mm512_maskz_scalef_round_ps(U, A, B, C)			\
+  ((__m512)								\
+   __builtin_ia32_scalefps512_mask((A), (B),				\
+				   (__v16sf) _mm512_setzero_ps(),	\
+				   (U), (C)))
+
+#define _mm_scalef_round_sd(A, B, C)					\
+  ((__m128d)								\
+   __builtin_ia32_scalefsd_mask_round ((A), (B),			\
+				       (__v2df) _mm_undefined_pd (),	\
+				       -1, (C)))
 
-#define _mm512_scalef_round_ps(A, B, C)            \
-    (__m512)__builtin_ia32_scalefps512_mask(A, B, (__v16sf)_mm512_undefined_ps(), -1, C)
+#define _mm_scalef_round_ss(A, B, C)					\
+  ((__m128)								\
+   __builtin_ia32_scalefss_mask_round ((A), (B),			\
+				       (__v4sf) _mm_undefined_ps (),	\
+				       -1, (C)))
 
-#define _mm512_mask_scalef_round_ps(W, U, A, B, C) \
-    (__m512)__builtin_ia32_scalefps512_mask(A, B, W, U, C)
+#define _mm_mask_scalef_round_sd(W, U, A, B, C)				\
+  ((__m128d)								\
+   __builtin_ia32_scalefsd_mask_round ((A), (B), (W), (U), (C)))
 
-#define _mm512_maskz_scalef_round_ps(U, A, B, C)   \
-    (__m512)__builtin_ia32_scalefps512_mask(A, B, (__v16sf)_mm512_setzero_ps(), U, C)
+#define _mm_mask_scalef_round_ss(W, U, A, B, C)				\
+  ((__m128)								\
+   __builtin_ia32_scalefss_mask_round ((A), (B), (W), (U), (C)))
 
-#define _mm_scalef_round_sd(A, B, C)            \
-    (__m128d)__builtin_ia32_scalefsd_mask_round (A, B, \
-	(__v2df)_mm_setzero_pd (), -1, C)
+#define _mm_maskz_scalef_round_sd(U, A, B, C)				\
+  ((__m128d)								\
+   __builtin_ia32_scalefsd_mask_round ((A), (B),			\
+				       (__v2df) _mm_setzero_pd (),	\
+				       (U), (C)))
 
-#define _mm_scalef_round_ss(A, B, C)            \
-    (__m128)__builtin_ia32_scalefss_mask_round (A, B, \
-	(__v4sf)_mm_setzero_ps (), -1, C)
+#define _mm_maskz_scalef_round_ss(U, A, B, C)				\
+  ((__m128)								\
+   __builtin_ia32_scalefss_mask_round ((A), (B),			\
+				       (__v4sf) _mm_setzero_ps (),	\
+				       (U), (C)))
 #endif
 
 #define _mm_mask_scalef_sd(W, U, A, B) \
diff --git a/gcc/config/i386/i386-expand.cc b/gcc/config/i386/i386-expand.cc
index 794315ee..31780b6 100644
--- a/gcc/config/i386/i386-expand.cc
+++ b/gcc/config/i386/i386-expand.cc
@@ -3136,6 +3136,8 @@ ix86_expand_int_movcc (rtx operands[])
   bool sign_bit_compare_p = false;
   rtx op0 = XEXP (operands[1], 0);
   rtx op1 = XEXP (operands[1], 1);
+  rtx op2 = operands[2];
+  rtx op3 = operands[3];
 
   if (GET_MODE (op0) == TImode
       || (GET_MODE (op0) == DImode
@@ -3153,17 +3155,29 @@ ix86_expand_int_movcc (rtx operands[])
       || (op1 == constm1_rtx && (code == GT || code == LE)))
     sign_bit_compare_p = true;
 
+  /* op0 == op1 ? op0 : op3 is equivalent to op0 == op1 ? op1 : op3,
+     but if op1 is a constant, the latter form allows more optimizations,
+     either through the last 2 ops being constant handling, or the one
+     constant and one variable cases.  On the other side, for cmov the
+     former might be better as we don't need to load the constant into
+     another register.  */
+  if (code == EQ && CONST_INT_P (op1) && rtx_equal_p (op0, op2))
+    op2 = op1;
+  /* Similarly for op0 != op1 ? op2 : op0 and op0 != op1 ? op2 : op1.  */
+  else if (code == NE && CONST_INT_P (op1) && rtx_equal_p (op0, op3))
+    op3 = op1;
+
   /* Don't attempt mode expansion here -- if we had to expand 5 or 6
      HImode insns, we'd be swallowed in word prefix ops.  */
 
   if ((mode != HImode || TARGET_FAST_PREFIX)
       && (mode != (TARGET_64BIT ? TImode : DImode))
-      && CONST_INT_P (operands[2])
-      && CONST_INT_P (operands[3]))
+      && CONST_INT_P (op2)
+      && CONST_INT_P (op3))
     {
       rtx out = operands[0];
-      HOST_WIDE_INT ct = INTVAL (operands[2]);
-      HOST_WIDE_INT cf = INTVAL (operands[3]);
+      HOST_WIDE_INT ct = INTVAL (op2);
+      HOST_WIDE_INT cf = INTVAL (op3);
       HOST_WIDE_INT diff;
 
       diff = ct - cf;
@@ -3559,6 +3573,9 @@ ix86_expand_int_movcc (rtx operands[])
       if (BRANCH_COST (optimize_insn_for_speed_p (), false) <= 2)
 	return false;
 
+      operands[2] = op2;
+      operands[3] = op3;
+
       /* If one of the two operands is an interesting constant, load a
 	 constant with the above and mask it in with a logical operation.  */
 
@@ -17036,7 +17053,8 @@ ix86_emit_fp_unordered_jump (rtx label)
 
 /* Output code to perform an sinh XFmode calculation.  */
 
-void ix86_emit_i387_sinh (rtx op0, rtx op1)
+void
+ix86_emit_i387_sinh (rtx op0, rtx op1)
 {
   rtx e1 = gen_reg_rtx (XFmode);
   rtx e2 = gen_reg_rtx (XFmode);
@@ -17084,7 +17102,8 @@ void ix86_emit_i387_sinh (rtx op0, rtx op1)
 
 /* Output code to perform an cosh XFmode calculation.  */
 
-void ix86_emit_i387_cosh (rtx op0, rtx op1)
+void
+ix86_emit_i387_cosh (rtx op0, rtx op1)
 {
   rtx e1 = gen_reg_rtx (XFmode);
   rtx e2 = gen_reg_rtx (XFmode);
@@ -17106,7 +17125,8 @@ void ix86_emit_i387_cosh (rtx op0, rtx op1)
 
 /* Output code to perform an tanh XFmode calculation.  */
 
-void ix86_emit_i387_tanh (rtx op0, rtx op1)
+void
+ix86_emit_i387_tanh (rtx op0, rtx op1)
 {
   rtx e1 = gen_reg_rtx (XFmode);
   rtx e2 = gen_reg_rtx (XFmode);
@@ -17152,7 +17172,8 @@ void ix86_emit_i387_tanh (rtx op0, rtx op1)
 
 /* Output code to perform an asinh XFmode calculation.  */
 
-void ix86_emit_i387_asinh (rtx op0, rtx op1)
+void
+ix86_emit_i387_asinh (rtx op0, rtx op1)
 {
   rtx e1 = gen_reg_rtx (XFmode);
   rtx e2 = gen_reg_rtx (XFmode);
@@ -17204,7 +17225,8 @@ void ix86_emit_i387_asinh (rtx op0, rtx op1)
 
 /* Output code to perform an acosh XFmode calculation.  */
 
-void ix86_emit_i387_acosh (rtx op0, rtx op1)
+void
+ix86_emit_i387_acosh (rtx op0, rtx op1)
 {
   rtx e1 = gen_reg_rtx (XFmode);
   rtx e2 = gen_reg_rtx (XFmode);
@@ -17230,7 +17252,8 @@ void ix86_emit_i387_acosh (rtx op0, rtx op1)
 
 /* Output code to perform an atanh XFmode calculation.  */
 
-void ix86_emit_i387_atanh (rtx op0, rtx op1)
+void
+ix86_emit_i387_atanh (rtx op0, rtx op1)
 {
   rtx e1 = gen_reg_rtx (XFmode);
   rtx e2 = gen_reg_rtx (XFmode);
@@ -17281,7 +17304,8 @@ void ix86_emit_i387_atanh (rtx op0, rtx op1)
 
 /* Output code to perform a log1p XFmode calculation.  */
 
-void ix86_emit_i387_log1p (rtx op0, rtx op1)
+void
+ix86_emit_i387_log1p (rtx op0, rtx op1)
 {
   rtx_code_label *label1 = gen_label_rtx ();
   rtx_code_label *label2 = gen_label_rtx ();
@@ -17291,6 +17315,11 @@ void ix86_emit_i387_log1p (rtx op0, rtx op1)
   rtx cst, cstln2, cst1;
   rtx_insn *insn;
 
+  /* The emit_jump call emits pending stack adjust, make sure it is emitted
+     before the conditional jump, otherwise the stack adjustment will be
+     only conditional.  */
+  do_pending_stack_adjust ();
+
   cst = const_double_from_real_value
     (REAL_VALUE_ATOF ("0.29289321881345247561810596348408353", XFmode), XFmode);
   cstln2 = force_reg (XFmode, standard_80387_constant_rtx (4)); /* fldln2 */
@@ -17320,7 +17349,8 @@ void ix86_emit_i387_log1p (rtx op0, rtx op1)
 }
 
 /* Emit code for round calculation.  */
-void ix86_emit_i387_round (rtx op0, rtx op1)
+void
+ix86_emit_i387_round (rtx op0, rtx op1)
 {
   machine_mode inmode = GET_MODE (op1);
   machine_mode outmode = GET_MODE (op0);
@@ -17434,7 +17464,8 @@ void ix86_emit_i387_round (rtx op0, rtx op1)
 /* Output code to perform a Newton-Rhapson approximation of a single precision
    floating point divide [http://en.wikipedia.org/wiki/N-th_root_algorithm].  */
 
-void ix86_emit_swdivsf (rtx res, rtx a, rtx b, machine_mode mode)
+void
+ix86_emit_swdivsf (rtx res, rtx a, rtx b, machine_mode mode)
 {
   rtx x0, x1, e0, e1;
 
@@ -17485,7 +17516,8 @@ void ix86_emit_swdivsf (rtx res, rtx a, rtx b, machine_mode mode)
 /* Output code to perform a Newton-Rhapson approximation of a
    single precision floating point [reciprocal] square root.  */
 
-void ix86_emit_swsqrtsf (rtx res, rtx a, machine_mode mode, bool recip)
+void
+ix86_emit_swsqrtsf (rtx res, rtx a, machine_mode mode, bool recip)
 {
   rtx x0, e0, e1, e2, e3, mthree, mhalf;
   REAL_VALUE_TYPE r;
@@ -23240,9 +23272,10 @@ ix86_expand_divmod_libfunc (rtx libfunc, machine_mode mode,
   *rem_p = rem;
 }
 
-void ix86_expand_atomic_fetch_op_loop (rtx target, rtx mem, rtx val,
-				       enum rtx_code code, bool after,
-				       bool doubleword)
+void
+ix86_expand_atomic_fetch_op_loop (rtx target, rtx mem, rtx val,
+				  enum rtx_code code, bool after,
+				  bool doubleword)
 {
   rtx old_reg, new_reg, old_mem, success;
   machine_mode mode = GET_MODE (target);
@@ -23286,10 +23319,11 @@ void ix86_expand_atomic_fetch_op_loop (rtx target, rtx mem, rtx val,
    it will be relaxed to an atomic load + compare, and skip
    cmpxchg instruction if mem != exp_input.  */
 
-void ix86_expand_cmpxchg_loop (rtx *ptarget_bool, rtx target_val,
-			       rtx mem, rtx exp_input, rtx new_input,
-			       rtx mem_model, bool doubleword,
-			       rtx_code_label *loop_label)
+void
+ix86_expand_cmpxchg_loop (rtx *ptarget_bool, rtx target_val,
+			  rtx mem, rtx exp_input, rtx new_input,
+			  rtx mem_model, bool doubleword,
+			  rtx_code_label *loop_label)
 {
   rtx_code_label *cmp_label = NULL;
   rtx_code_label *done_label = NULL;
@@ -23388,6 +23422,7 @@ void ix86_expand_cmpxchg_loop (rtx *ptarget_bool, rtx target_val,
 
     /* If mem is not expected, pause and loop back.  */
     emit_label (cmp_label);
+    emit_move_insn (target_val, new_mem);
     emit_insn (gen_pause ());
     emit_jump_insn (gen_jump (loop_label));
     emit_barrier ();
diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
index c959b71..b16df5b 100644
--- a/gcc/config/i386/i386.cc
+++ b/gcc/config/i386/i386.cc
@@ -4891,6 +4891,7 @@ ix86_gimplify_va_arg (tree valist, tree type, gimple_seq *pre_p,
 	{
 	  int i, prev_size = 0;
 	  tree temp = create_tmp_var (type, "va_arg_tmp");
+	  TREE_ADDRESSABLE (temp) = 1;
 
 	  /* addr = &temp; */
 	  t = build1 (ADDR_EXPR, build_pointer_type (type), temp);
@@ -6524,7 +6525,8 @@ ix86_initial_elimination_offset (int from, int to)
 }
 
 /* Emits a warning for unsupported msabi to sysv pro/epilogues.  */
-void warn_once_call_ms2sysv_xlogues (const char *feature)
+void
+warn_once_call_ms2sysv_xlogues (const char *feature)
 {
   static bool warned_once = false;
   if (!warned_once)
@@ -18806,7 +18808,8 @@ ix86_veclibabi_svml (combined_fn fn, tree type_out, tree type_in)
       return NULL_TREE;
     }
 
-  tree fndecl = mathfn_built_in (TREE_TYPE (type_in), fn);
+  tree fndecl = mathfn_built_in (el_mode == DFmode
+				 ? double_type_node : float_type_node, fn);
   bname = IDENTIFIER_POINTER (DECL_NAME (fndecl));
 
   if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_LOGF)
@@ -18898,7 +18901,8 @@ ix86_veclibabi_acml (combined_fn fn, tree type_out, tree type_in)
       return NULL_TREE;
     }
 
-  tree fndecl = mathfn_built_in (TREE_TYPE (type_in), fn);
+  tree fndecl = mathfn_built_in (el_mode == DFmode
+				 ? double_type_node : float_type_node, fn);
   bname = IDENTIFIER_POINTER (DECL_NAME (fndecl));
   sprintf (name + 7, "%s", bname+10);
 
diff --git a/gcc/config/i386/smmintrin.h b/gcc/config/i386/smmintrin.h
index b42b212..eb6a451 100644
--- a/gcc/config/i386/smmintrin.h
+++ b/gcc/config/i386/smmintrin.h
@@ -810,17 +810,11 @@ _mm_cmpgt_epi64 (__m128i __X, __m128i __Y)
 
 #include <popcntintrin.h>
 
-#ifndef __SSE4_1__
+#ifndef __CRC32__
 #pragma GCC push_options
-#pragma GCC target("sse4.1")
-#define __DISABLE_SSE4_1__
-#endif /* __SSE4_1__ */
-
-#ifndef __SSE4_2__
-#pragma GCC push_options
-#pragma GCC target("sse4.2")
-#define __DISABLE_SSE4_2__
-#endif /* __SSE4_1__ */
+#pragma GCC target("crc32")
+#define __DISABLE_CRC32__
+#endif /* __CRC32__ */
 
 /* Accumulate CRC32 (polynomial 0x11EDC6F41) value.  */
 extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
@@ -849,14 +843,9 @@ _mm_crc32_u64 (unsigned long long __C, unsigned long long __V)
 }
 #endif
 
-#ifdef __DISABLE_SSE4_2__
-#undef __DISABLE_SSE4_2__
+#ifdef __DISABLE_CRC32__
+#undef __DISABLE_CRC32__
 #pragma GCC pop_options
-#endif /* __DISABLE_SSE4_2__ */
-
-#ifdef __DISABLE_SSE4_1__
-#undef __DISABLE_SSE4_1__
-#pragma GCC pop_options
-#endif /* __DISABLE_SSE4_1__ */
+#endif /* __DISABLE_CRC32__ */
 
 #endif /* _SMMINTRIN_H_INCLUDED */
diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
index a852c16..5e93aa2 100644
--- a/gcc/config/i386/sse.md
+++ b/gcc/config/i386/sse.md
@@ -327,9 +327,7 @@
 
 ;; 128-, 256- and 512-bit float vector modes for bitwise operations
 (define_mode_iterator VFB
-  [(V32HF "TARGET_AVX512FP16")
-   (V16HF "TARGET_AVX512FP16")
-   (V8HF "TARGET_AVX512FP16")
+  [(V32HF "TARGET_AVX512F") (V16HF "TARGET_AVX") (V8HF "TARGET_SSE2")
    (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
    (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
 
@@ -340,8 +338,7 @@
 
 ;; 128- and 256-bit float vector modes for bitwise operations
 (define_mode_iterator VFB_128_256
-  [(V16HF "TARGET_AVX512FP16")
-   (V8HF "TARGET_AVX512FP16")
+  [(V16HF "TARGET_AVX") (V8HF "TARGET_SSE2")
    (V8SF "TARGET_AVX") V4SF
    (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
 
@@ -399,7 +396,7 @@
 
 ;; All 512bit vector float modes for bitwise operations
 (define_mode_iterator VFB_512
-  [(V32HF "TARGET_AVX512FP16") V16SF V8DF])
+  [V32HF V16SF V8DF])
 
 (define_mode_iterator VI48_AVX512VL
   [V16SI (V8SI  "TARGET_AVX512VL") (V4SI  "TARGET_AVX512VL")
@@ -4581,7 +4578,8 @@
 	  (not:VFB_128_256
 	    (match_operand:VFB_128_256 1 "register_operand" "0,x,v,v"))
 	  (match_operand:VFB_128_256 2 "vector_operand" "xBm,xm,vm,vm")))]
-  "TARGET_SSE && <mask_avx512vl_condition>"
+  "TARGET_SSE && <mask_avx512vl_condition>
+   && (!<mask_applied> || <ssescalarmode>mode != HFmode)"
 {
   char buf[128];
   const char *ops;
@@ -4648,7 +4646,7 @@
 	  (not:VFB_512
 	    (match_operand:VFB_512 1 "register_operand" "v"))
 	  (match_operand:VFB_512 2 "nonimmediate_operand" "vm")))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && (!<mask_applied> || <ssescalarmode>mode != HFmode)"
 {
   char buf[128];
   const char *ops;
@@ -4683,7 +4681,8 @@
        (any_logic:VFB_128_256
          (match_operand:VFB_128_256 1 "vector_operand")
          (match_operand:VFB_128_256 2 "vector_operand")))]
-  "TARGET_SSE && <mask_avx512vl_condition>"
+  "TARGET_SSE && <mask_avx512vl_condition>
+   && (!<mask_applied> || <ssescalarmode>mode != HFmode)"
   "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
 
 (define_expand "<code><mode>3<mask_name>"
@@ -4691,7 +4690,7 @@
        (any_logic:VFB_512
          (match_operand:VFB_512 1 "nonimmediate_operand")
          (match_operand:VFB_512 2 "nonimmediate_operand")))]
-  "TARGET_AVX512F"
+  "TARGET_AVX512F && (!<mask_applied> || <ssescalarmode>mode != HFmode)"
   "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
 
 (define_insn "*<code><mode>3<mask_name>"
@@ -4700,6 +4699,7 @@
 	  (match_operand:VFB_128_256 1 "vector_operand" "%0,x,v,v")
 	  (match_operand:VFB_128_256 2 "vector_operand" "xBm,xm,vm,vm")))]
   "TARGET_SSE && <mask_avx512vl_condition>
+   && (!<mask_applied> || <ssescalarmode>mode != HFmode)
    && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
 {
   char buf[128];
@@ -4766,7 +4766,8 @@
 	(any_logic:VFB_512
 	  (match_operand:VFB_512 1 "nonimmediate_operand" "%v")
 	  (match_operand:VFB_512 2 "nonimmediate_operand" "vm")))]
-  "TARGET_AVX512F && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
+  "TARGET_AVX512F && !(MEM_P (operands[1]) && MEM_P (operands[2]))
+   && (!<mask_applied> || <ssescalarmode>mode != HFmode)"
 {
   char buf[128];
   const char *ops;
@@ -16741,17 +16742,6 @@
 	  (match_operand:<avx512fmaskmode> 4 "register_operand")))]
   "TARGET_AVX512F")
 
-(define_expand "<sse2_avx2>_andnot<mode>3_mask"
-  [(set (match_operand:VI12_AVX512VL 0 "register_operand")
-	(vec_merge:VI12_AVX512VL
-	  (and:VI12_AVX512VL
-	    (not:VI12_AVX512VL
-	      (match_operand:VI12_AVX512VL 1 "register_operand"))
-	    (match_operand:VI12_AVX512VL 2 "nonimmediate_operand"))
-	  (match_operand:VI12_AVX512VL 3 "nonimm_or_0_operand")
-	  (match_operand:<avx512fmaskmode> 4 "register_operand")))]
-  "TARGET_AVX512BW")
-
 (define_insn "*andnot<mode>3"
   [(set (match_operand:VI 0 "register_operand" "=x,x,v")
 	(and:VI
diff --git a/gcc/config/loongarch/loongarch.cc b/gcc/config/loongarch/loongarch.cc
index 6e24111..80046b6 100644
--- a/gcc/config/loongarch/loongarch.cc
+++ b/gcc/config/loongarch/loongarch.cc
@@ -329,6 +329,9 @@ loongarch_flatten_aggregate_field (const_tree type,
 	    if (!TYPE_P (TREE_TYPE (f)))
 	      return -1;
 
+	    if (DECL_SIZE (f) && integer_zerop (DECL_SIZE (f)))
+	      continue;
+
 	    HOST_WIDE_INT pos = offset + int_byte_position (f);
 	    n = loongarch_flatten_aggregate_field (TREE_TYPE (f), fields, n,
 						   pos);
@@ -473,13 +476,14 @@ loongarch_pass_aggregate_in_fpr_and_gpr_p (const_tree type,
 
 static rtx
 loongarch_pass_fpr_single (machine_mode type_mode, unsigned regno,
-			   machine_mode value_mode)
+			   machine_mode value_mode,
+			   HOST_WIDE_INT offset)
 {
   rtx x = gen_rtx_REG (value_mode, regno);
 
   if (type_mode != value_mode)
     {
-      x = gen_rtx_EXPR_LIST (VOIDmode, x, const0_rtx);
+      x = gen_rtx_EXPR_LIST (VOIDmode, x, GEN_INT (offset));
       x = gen_rtx_PARALLEL (type_mode, gen_rtvec (1, x));
     }
   return x;
@@ -539,7 +543,8 @@ loongarch_get_arg_info (struct loongarch_arg_info *info,
 	  {
 	  case 1:
 	    return loongarch_pass_fpr_single (mode, fregno,
-					      TYPE_MODE (fields[0].type));
+					      TYPE_MODE (fields[0].type),
+					      fields[0].offset);
 
 	  case 2:
 	    return loongarch_pass_fpr_pair (mode, fregno,
diff --git a/gcc/config/loongarch/loongarch.md b/gcc/config/loongarch/loongarch.md
index 6c57c8b..d3c809e 100644
--- a/gcc/config/loongarch/loongarch.md
+++ b/gcc/config/loongarch/loongarch.md
@@ -713,6 +713,12 @@
 ;;
 
 ;; Float division and modulus.
+(define_expand "div<mode>3"
+  [(set (match_operand:ANYF 0 "register_operand")
+	(div:ANYF (match_operand:ANYF 1 "reg_or_1_operand")
+		  (match_operand:ANYF 2 "register_operand")))]
+  "")
+
 (define_insn "*div<mode>3"
   [(set (match_operand:ANYF 0 "register_operand" "=f")
 	(div:ANYF (match_operand:ANYF 1 "register_operand" "f")
@@ -2047,13 +2053,17 @@
 
 (define_insn "loongarch_ibar"
   [(unspec_volatile:SI
-      [(match_operand 0 "const_uimm15_operand")] UNSPECV_IBAR)]
+      [(match_operand 0 "const_uimm15_operand")]
+       UNSPECV_IBAR)
+   (clobber (mem:BLK (scratch)))]
   ""
   "ibar\t%0")
 
 (define_insn "loongarch_dbar"
   [(unspec_volatile:SI
-      [(match_operand 0 "const_uimm15_operand")] UNSPECV_DBAR)]
+      [(match_operand 0 "const_uimm15_operand")]
+       UNSPECV_DBAR)
+   (clobber (mem:BLK (scratch)))]
   ""
   "dbar\t%0")
 
@@ -2072,13 +2082,17 @@
 
 (define_insn "loongarch_syscall"
   [(unspec_volatile:SI
-      [(match_operand 0 "const_uimm15_operand")] UNSPECV_SYSCALL)]
+      [(match_operand 0 "const_uimm15_operand")]
+       UNSPECV_SYSCALL)
+   (clobber (mem:BLK (scratch)))]
   ""
   "syscall\t%0")
 
 (define_insn "loongarch_break"
   [(unspec_volatile:SI
-      [(match_operand 0 "const_uimm15_operand")] UNSPECV_BREAK)]
+      [(match_operand 0 "const_uimm15_operand")]
+       UNSPECV_BREAK)
+   (clobber (mem:BLK (scratch)))]
   ""
   "break\t%0")
 
@@ -2103,7 +2117,8 @@
 (define_insn "loongarch_csrrd_<d>"
   [(set (match_operand:GPR 0 "register_operand" "=r")
 	(unspec_volatile:GPR [(match_operand  1 "const_uimm14_operand")]
-			      UNSPECV_CSRRD))]
+			      UNSPECV_CSRRD))
+   (clobber (mem:BLK (scratch)))]
   ""
   "csrrd\t%0,%1"
   [(set_attr "type" "load")
@@ -2114,7 +2129,8 @@
 	  (unspec_volatile:GPR
 	    [(match_operand:GPR 1 "register_operand" "0")
 	     (match_operand 2 "const_uimm14_operand")]
-	     UNSPECV_CSRWR))]
+	     UNSPECV_CSRWR))
+   (clobber (mem:BLK (scratch)))]
   ""
   "csrwr\t%0,%2"
   [(set_attr "type" "store")
@@ -2126,7 +2142,8 @@
 	    [(match_operand:GPR 1 "register_operand" "0")
 	     (match_operand:GPR 2 "register_operand" "q")
 	     (match_operand 3 "const_uimm14_operand")]
-	     UNSPECV_CSRXCHG))]
+	     UNSPECV_CSRXCHG))
+   (clobber (mem:BLK (scratch)))]
   ""
   "csrxchg\t%0,%2,%3"
   [(set_attr "type" "load")
@@ -2135,7 +2152,8 @@
 (define_insn "loongarch_iocsrrd_<size>"
   [(set (match_operand:QHWD 0 "register_operand" "=r")
 	(unspec_volatile:QHWD [(match_operand:SI 1 "register_operand" "r")]
-			      UNSPECV_IOCSRRD))]
+			      UNSPECV_IOCSRRD))
+   (clobber (mem:BLK (scratch)))]
   ""
   "iocsrrd.<size>\t%0,%1"
   [(set_attr "type" "load")
@@ -2144,7 +2162,8 @@
 (define_insn "loongarch_iocsrwr_<size>"
   [(unspec_volatile:QHWD [(match_operand:QHWD 0 "register_operand" "r")
 			  (match_operand:SI 1 "register_operand" "r")]
-			  UNSPECV_IOCSRWR)]
+			  UNSPECV_IOCSRWR)
+   (clobber (mem:BLK (scratch)))]
   ""
   "iocsrwr.<size>\t%0,%1"
   [(set_attr "type" "load")
@@ -2154,7 +2173,8 @@
   [(unspec_volatile:X [(match_operand 0 "const_uimm5_operand")
 		       (match_operand:X 1 "register_operand" "r")
 		       (match_operand 2 "const_imm12_operand")]
-		       UNSPECV_CACOP)]
+		       UNSPECV_CACOP)
+   (clobber (mem:BLK (scratch)))]
   ""
   "cacop\t%0,%1,%2"
   [(set_attr "type" "load")
@@ -2164,7 +2184,8 @@
   [(unspec_volatile:X [(match_operand:X 0 "register_operand" "r")
 		       (match_operand:X 1 "register_operand" "r")
 		       (match_operand 2 "const_uimm5_operand")]
-		       UNSPECV_LDDIR)]
+		       UNSPECV_LDDIR)
+   (clobber (mem:BLK (scratch)))]
   ""
   "lddir\t%0,%1,%2"
   [(set_attr "type" "load")
@@ -2173,7 +2194,8 @@
 (define_insn "loongarch_ldpte_<d>"
   [(unspec_volatile:X [(match_operand:X 0 "register_operand" "r")
 		       (match_operand 1 "const_uimm5_operand")]
-		       UNSPECV_LDPTE)]
+		       UNSPECV_LDPTE)
+   (clobber (mem:BLK (scratch)))]
   ""
   "ldpte\t%0,%1"
   [(set_attr "type" "load")
diff --git a/gcc/config/nvptx/nvptx.h b/gcc/config/nvptx/nvptx.h
index 3b06f33..ed72c25 100644
--- a/gcc/config/nvptx/nvptx.h
+++ b/gcc/config/nvptx/nvptx.h
@@ -29,25 +29,6 @@
 
 #define STARTFILE_SPEC "%{mmainkernel:crt0.o}"
 
-/* Newer versions of CUDA no longer support sm_30, and nvptx-tools as
-   currently doesn't handle that gracefully when verifying
-   ( https://github.com/MentorEmbedded/nvptx-tools/issues/30 ).  Work around
-   this by verifying with sm_35 when having misa=sm_30 (either implicitly
-   or explicitly).  */
-#define ASM_SPEC				\
-  "%{"						\
-  /* Explict misa=sm_30.  */			\
-  "misa=sm_30:-m sm_35"				\
-  /* Separator.	 */				\
-  "; "						\
-  /* Catch-all.	 */				\
-  "misa=*:-m %*"				\
-  /* Separator.	 */				\
-  "; "						\
-  /* Implicit misa=sm_30.  */			\
-  ":-m sm_35"					\
-  "}"
-
 #define TARGET_CPU_CPP_BUILTINS() nvptx_cpu_cpp_builtins ()
 
 /* Avoid the default in ../../gcc.cc, which adds "-pthread", which is not
diff --git a/gcc/config/nvptx/nvptx.opt b/gcc/config/nvptx/nvptx.opt
index 55a1057..c5a5668 100644
--- a/gcc/config/nvptx/nvptx.opt
+++ b/gcc/config/nvptx/nvptx.opt
@@ -52,7 +52,6 @@ mgomp
 Target Mask(GOMP)
 Generate code for OpenMP offloading: enables -msoft-stack and -muniform-simt.
 
-; Default needs to be in sync with default in ASM_SPEC in nvptx.h.
 misa=
 Target RejectNegative ToLower Joined Enum(ptx_isa) Var(ptx_isa_option) Init(PTX_ISA_SM30)
 Specify the PTX ISA target architecture to use.
diff --git a/gcc/config/riscv/arch-canonicalize b/gcc/config/riscv/arch-canonicalize
index 49a6204..f36a2ca 100755
--- a/gcc/config/riscv/arch-canonicalize
+++ b/gcc/config/riscv/arch-canonicalize
@@ -20,14 +20,18 @@
 # along with GCC; see the file COPYING3.  If not see
 # <http://www.gnu.org/licenses/>.
 
+# TODO: Extract riscv_subset_t from riscv-common.cc and make it can be compiled
+#       standalone to replace this script, that also prevents us implementing
+#       that twice and keep sync again and again.
 
 from __future__ import print_function
 import sys
+import argparse
 import collections
 import itertools
 from functools import reduce
 
-
+SUPPORTED_ISA_SPEC = ["2.2", "20190608", "20191213"]
 CANONICAL_ORDER = "imafdgqlcbjktpvn"
 LONG_EXT_PREFIXES = ['z', 's', 'h', 'x']
 
@@ -35,29 +39,42 @@ LONG_EXT_PREFIXES = ['z', 's', 'h', 'x']
 # IMPLIED_EXT(ext) -> implied extension list.
 #
 IMPLIED_EXT = {
-  "d" : ["f"],
-  "zk" : ["zkn"],
-  "zk" : ["zkr"],
-  "zk" : ["zkt"],
-  "zkn" : ["zbkb"],
-  "zkn" : ["zbkc"],
-  "zkn" : ["zbkx"],
-  "zkn" : ["zkne"],
-  "zkn" : ["zknd"],
-  "zkn" : ["zknh"],
-  "zks" : ["zbkb"],
-  "zks" : ["zbkc"],
-  "zks" : ["zbkx"],
-  "zks" : ["zksed"],
-  "zks" : ["zksh"],
+  "d" : ["f", "zicsr"],
+  "f" : ["zicsr"],
+  "zk" : ["zkn", "zkr", "zkt"],
+  "zkn" : ["zbkb", "zbkc", "zbkx", "zkne", "zknd", "zknh"],
+  "zks" : ["zbkb", "zbkc", "zbkx", "zksed", "zksh"],
+
+  "v" : ["zvl128b", "zve64d"],
+  "zve32x" : ["zvl32b"],
+  "zve64x" : ["zve32x", "zvl64b"],
+  "zve32f" : ["f", "zve32x"],
+  "zve64f" : ["f", "zve32f", "zve64x"],
+  "zve64d" : ["d", "zve64f"],
+
+  "zvl64b" : ["zvl32b"],
+  "zvl128b" : ["zvl64b"],
+  "zvl256b" : ["zvl128b"],
+  "zvl512b" : ["zvl256b"],
+  "zvl1024b" : ["zvl512b"],
+  "zvl2048b" : ["zvl1024b"],
+  "zvl4096b" : ["zvl2048b"],
+  "zvl8192b" : ["zvl4096b"],
+  "zvl16384b" : ["zvl8192b"],
+  "zvl32768b" : ["zvl16384b"],
+  "zvl65536b" : ["zvl32768b"],
 }
 
-def arch_canonicalize(arch):
+def arch_canonicalize(arch, isa_spec):
   # TODO: Support extension version.
+  is_isa_spec_2p2 = isa_spec == '2.2'
   new_arch = ""
+  extra_long_ext = []
   if arch[:5] in ['rv32e', 'rv32i', 'rv32g', 'rv64i', 'rv64g']:
-    # TODO: We should expand g to imad_zifencei once we support newer spec.
     new_arch = arch[:5].replace("g", "imafd")
+    if arch[:5] in ['rv32g', 'rv64g']:
+      if not is_isa_spec_2p2:
+        extra_long_ext = ['zicsr', 'zifencei']
   else:
     raise Exception("Unexpected arch: `%s`" % arch[:5])
 
@@ -74,15 +91,24 @@ def arch_canonicalize(arch):
     long_exts = []
     std_exts = list(arch[5:])
 
+  long_exts += extra_long_ext
+
   #
   # Handle implied extensions.
   #
-  for ext in std_exts + long_exts:
-    if ext in IMPLIED_EXT:
-      implied_exts = IMPLIED_EXT[ext]
-      for implied_ext in implied_exts:
-        if implied_ext not in std_exts + long_exts:
-          long_exts.append(implied_ext)
+  any_change = True
+  while any_change:
+    any_change = False
+    for ext in std_exts + long_exts:
+      if ext in IMPLIED_EXT:
+        implied_exts = IMPLIED_EXT[ext]
+        for implied_ext in implied_exts:
+          if implied_ext == 'zicsr' and is_isa_spec_2p2:
+              continue
+
+          if implied_ext not in std_exts + long_exts:
+            long_exts.append(implied_ext)
+            any_change = True
 
   # Single letter extension might appear in the long_exts list,
   # becasue we just append extensions list to the arch string.
@@ -99,6 +125,9 @@ def arch_canonicalize(arch):
     return (exts.startswith("x"), exts.startswith("zxm"),
             LONG_EXT_PREFIXES.index(exts[0]), canonical_sort, exts[1:])
 
+  # Removing duplicates.
+  long_exts = list(set(long_exts))
+
   # Multi-letter extension must be in lexicographic order.
   long_exts = list(sorted(filter(lambda x:len(x) != 1, long_exts),
                           key=longext_sort))
@@ -118,11 +147,20 @@ def arch_canonicalize(arch):
   # Concat rest of the multi-char extensions.
   if long_exts:
     new_arch += "_" + "_".join(long_exts)
+
   return new_arch
 
 if len(sys.argv) < 2:
   print ("Usage: %s <arch_str> [<arch_str>*]" % sys.argv)
   sys.exit(1)
 
-for arg in sys.argv[1:]:
-  print (arch_canonicalize(arg))
+parser = argparse.ArgumentParser()
+parser.add_argument('-misa-spec', type=str,
+                    default='20191213',
+                    choices=SUPPORTED_ISA_SPEC)
+parser.add_argument('arch_strs', nargs=argparse.REMAINDER)
+
+args = parser.parse_args()
+
+for arch in args.arch_strs:
+  print (arch_canonicalize(arch, args.misa_spec))
diff --git a/gcc/config/riscv/multilib-generator b/gcc/config/riscv/multilib-generator
index 1ea2fb2..36698d4 100755
--- a/gcc/config/riscv/multilib-generator
+++ b/gcc/config/riscv/multilib-generator
@@ -46,16 +46,18 @@ import argparse
 # TODO: Add test for this script.
 #
 
+SUPPORTED_ISA_SPEC = ["2.2", "20190608", "20191213"]
 arches = collections.OrderedDict()
 abis = collections.OrderedDict()
 required = []
 reuse = []
 
-def arch_canonicalize(arch):
+def arch_canonicalize(arch, isa_spec):
   this_file = os.path.abspath(os.path.join( __file__))
   arch_can_script = \
     os.path.join(os.path.dirname(this_file), "arch-canonicalize")
-  proc = subprocess.Popen([sys.executable, arch_can_script, arch],
+  proc = subprocess.Popen([sys.executable, arch_can_script,
+                          '-misa-spec=%s' % isa_spec, arch],
                           stdout=subprocess.PIPE)
   out, err = proc.communicate()
   return out.decode().strip()
@@ -133,6 +135,9 @@ options = filter(lambda x:x.startswith("--"), sys.argv[1:])
 
 parser = argparse.ArgumentParser()
 parser.add_argument("--cmodel", type=str)
+parser.add_argument('-misa-spec', type=str,
+                    default='20191213',
+                    choices=SUPPORTED_ISA_SPEC)
 parser.add_argument("cfgs", type=str, nargs='*')
 args = parser.parse_args()
 
@@ -158,13 +163,14 @@ for cmodel in cmodels:
     if cmodel == "compact" and arch.startswith("rv32"):
       continue
 
-    arch = arch_canonicalize (arch)
+    arch = arch_canonicalize (arch, args.misa_spec)
     arches[arch] = 1
     abis[abi] = 1
     extra = list(filter(None, extra.split(',')))
     ext_combs = expand_combination(ext)
     alts = sum([[x] + [x + y for y in ext_combs] for x in [arch] + extra], [])
-    alts = list(map(arch_canonicalize, alts))
+    alts = filter(lambda x: len(x) != 0, alts)
+    alts = list(map(lambda a : arch_canonicalize(a, args.misa_spec), alts))
 
     # Drop duplicated entry.
     alts = unique(alts)
diff --git a/gcc/config/rs6000/rs6000-builtins.def b/gcc/config/rs6000/rs6000-builtins.def
index 0f527c5..f4a9f24 100644
--- a/gcc/config/rs6000/rs6000-builtins.def
+++ b/gcc/config/rs6000/rs6000-builtins.def
@@ -1190,9 +1190,6 @@
   const vd __builtin_altivec_neg_v2df (vd);
     NEG_V2DF negv2df2 {}
 
-  const vsll __builtin_altivec_neg_v2di (vsll);
-    NEG_V2DI negv2di2 {}
-
   void __builtin_altivec_stvx_v2df (vd, signed long, void *);
     STVX_V2DF altivec_stvx_v2df {stvec}
 
@@ -2136,6 +2133,9 @@
   const vus __builtin_altivec_nand_v8hi_uns (vus, vus);
     NAND_V8HI_UNS nandv8hi3 {}
 
+  const vsll __builtin_altivec_neg_v2di (vsll);
+    NEG_V2DI negv2di2 {}
+
   const vsc __builtin_altivec_orc_v16qi (vsc, vsc);
     ORC_V16QI orcv16qi3 {}
 
diff --git a/gcc/config/rs6000/rs6000.cc b/gcc/config/rs6000/rs6000.cc
index ceaddaf..bc61959 100644
--- a/gcc/config/rs6000/rs6000.cc
+++ b/gcc/config/rs6000/rs6000.cc
@@ -25678,11 +25678,20 @@ rs6000_sibcall_aix (rtx value, rtx func_desc, rtx tlsarg, rtx cookie)
   rtx r12 = NULL_RTX;
   rtx func_addr = func_desc;
 
-  gcc_assert (INTVAL (cookie) == 0);
-
   if (global_tlsarg)
     tlsarg = global_tlsarg;
 
+  /* Handle longcall attributes.  */
+  if (INTVAL (cookie) & CALL_LONG && SYMBOL_REF_P (func_desc))
+    {
+      /* PCREL can do a sibling call to a longcall function
+	 because we don't need to restore the TOC register.  */
+      gcc_assert (rs6000_pcrel_p ());
+      func_desc = rs6000_longcall_ref (func_desc, tlsarg);
+    }
+  else
+    gcc_assert (INTVAL (cookie) == 0);
+
   /* For ELFv2, r12 and CTR need to hold the function address
      for an indirect call.  */
   if (GET_CODE (func_desc) != SYMBOL_REF && DEFAULT_ABI == ABI_ELFv2)
diff --git a/gcc/config/rs6000/rs6000.md b/gcc/config/rs6000/rs6000.md
index fdfbc65..64049a6 100644
--- a/gcc/config/rs6000/rs6000.md
+++ b/gcc/config/rs6000/rs6000.md
@@ -835,8 +835,8 @@
 ;; complex forms.  Basic data transfer is done later.
 
 (define_insn "zero_extendqi<mode>2"
-  [(set (match_operand:EXTQI 0 "gpc_reg_operand" "=r,r,^wa,^v")
-	(zero_extend:EXTQI (match_operand:QI 1 "reg_or_mem_operand" "m,r,Z,v")))]
+  [(set (match_operand:EXTQI 0 "gpc_reg_operand" "=r,r,wa,^v")
+	(zero_extend:EXTQI (match_operand:QI 1 "reg_or_mem_operand" "m,r,?Z,v")))]
   ""
   "@
    lbz%U1%X1 %0,%1
@@ -889,8 +889,8 @@
 
 
 (define_insn "zero_extendhi<mode>2"
-  [(set (match_operand:EXTHI 0 "gpc_reg_operand" "=r,r,^wa,^v")
-	(zero_extend:EXTHI (match_operand:HI 1 "reg_or_mem_operand" "m,r,Z,v")))]
+  [(set (match_operand:EXTHI 0 "gpc_reg_operand" "=r,r,wa,^v")
+	(zero_extend:EXTHI (match_operand:HI 1 "reg_or_mem_operand" "m,r,?Z,v")))]
   ""
   "@
    lhz%U1%X1 %0,%1
@@ -944,7 +944,7 @@
 
 (define_insn "zero_extendsi<mode>2"
   [(set (match_operand:EXTSI 0 "gpc_reg_operand" "=r,r,d,wa,wa,r,wa")
-	(zero_extend:EXTSI (match_operand:SI 1 "reg_or_mem_operand" "m,r,Z,Z,r,wa,wa")))]
+	(zero_extend:EXTSI (match_operand:SI 1 "reg_or_mem_operand" "m,r,?Z,?Z,r,wa,wa")))]
   ""
   "@
    lwz%U1%X1 %0,%1
@@ -7496,7 +7496,7 @@
   [(set (match_operand:SI 0 "nonimmediate_operand"
 	  "=r,         r,
 	   r,          d,          v,
-	   m,          Z,          Z,
+	   m,          ?Z,         ?Z,
 	   r,          r,          r,          r,
 	   wa,         wa,         wa,         v,
 	   wa,         v,          v,
@@ -7504,7 +7504,7 @@
 	   r,          *h,         *h")
 	(match_operand:SI 1 "input_operand"
 	  "r,          U,
-	   m,          Z,          Z,
+	   m,          ?Z,         ?Z,
 	   r,          d,          v,
 	   I,          L,          eI,         n,
 	   wa,         O,          wM,         wB,
@@ -7785,11 +7785,11 @@
 ;;		MTVSRWZ     MF%1       MT%1       NOP
 (define_insn "*mov<mode>_internal"
   [(set (match_operand:QHI 0 "nonimmediate_operand"
-		"=r,        r,         wa,        m,         Z,         r,
+		"=r,        r,         wa,        m,         ?Z,        r,
 		 wa,        wa,        wa,        v,         ?v,        r,
 		 wa,        r,         *c*l,      *h")
 	(match_operand:QHI 1 "input_operand"
-		"r,         m,         Z,         r,         wa,        i,
+		"r,         m,         ?Z,        r,         wa,        i,
 		 wa,        O,         wM,        wB,        wS,        wa,
 		 r,         *h,        r,         0"))]
   "gpc_reg_operand (operands[0], <MODE>mode)
@@ -7973,10 +7973,10 @@
 ;;	FMR          MR         MT%0       MF%1       NOP
 (define_insn "movsd_hardfloat"
   [(set (match_operand:SD 0 "nonimmediate_operand"
-	 "=!r,       d,         m,         Z,         ?d,        ?r,
+	 "=!r,       d,         m,         ?Z,        ?d,        ?r,
 	  f,         !r,        *c*l,      !r,        *h")
 	(match_operand:SD 1 "input_operand"
-	 "m,         Z,         r,         wx,        r,         d,
+	 "m,         ?Z,        r,         wx,        r,         d,
 	  f,         r,         r,         *h,        0"))]
   "(register_operand (operands[0], SDmode)
    || register_operand (operands[1], SDmode))
@@ -14580,10 +14580,10 @@
   [(set_attr "type" "fp,fpstore,mtvsr,mfvsr,store")])
 
 (define_insn_and_split "unpack<mode>_nodm"
-  [(set (match_operand:<FP128_64> 0 "nonimmediate_operand" "=d,m")
+  [(set (match_operand:<FP128_64> 0 "nonimmediate_operand" "=d,m,m")
 	(unspec:<FP128_64>
-	 [(match_operand:FMOVE128 1 "register_operand" "d,d")
-	  (match_operand:QI 2 "const_0_to_1_operand" "i,i")]
+	 [(match_operand:FMOVE128 1 "register_operand" "d,d,r")
+	  (match_operand:QI 2 "const_0_to_1_operand" "i,i,i")]
 	 UNSPEC_UNPACK_128BIT))]
   "(!TARGET_POWERPC64 || !TARGET_DIRECT_MOVE) && FLOAT128_2REG_P (<MODE>mode)"
   "#"
@@ -14600,15 +14600,28 @@
 
   operands[3] = gen_rtx_REG (<FP128_64>mode, fp_regno);
 }
-  [(set_attr "type" "fp,fpstore")])
+  [(set_attr "type" "fp,fpstore,store")])
 
-(define_insn_and_split "pack<mode>"
+(define_expand "pack<mode>"
+  [(use (match_operand:FMOVE128 0 "register_operand"))
+   (use (match_operand:<FP128_64> 1 "register_operand"))
+   (use (match_operand:<FP128_64> 2 "register_operand"))]
+  "FLOAT128_2REG_P (<MODE>mode)"
+{
+  if (TARGET_HARD_FLOAT)
+    emit_insn (gen_pack<mode>_hard (operands[0], operands[1], operands[2]));
+  else
+    emit_insn (gen_pack<mode>_soft (operands[0], operands[1], operands[2]));
+  DONE;
+})
+
+(define_insn_and_split "pack<mode>_hard"
   [(set (match_operand:FMOVE128 0 "register_operand" "=&d")
 	(unspec:FMOVE128
 	 [(match_operand:<FP128_64> 1 "register_operand" "d")
 	  (match_operand:<FP128_64> 2 "register_operand" "d")]
 	 UNSPEC_PACK_128BIT))]
-  "FLOAT128_2REG_P (<MODE>mode)"
+  "FLOAT128_2REG_P (<MODE>mode) && TARGET_HARD_FLOAT"
   "#"
   "&& reload_completed"
   [(set (match_dup 3) (match_dup 1))
@@ -14626,6 +14639,34 @@
   [(set_attr "type" "fp")
    (set_attr "length" "8")])
 
+(define_insn_and_split "pack<mode>_soft"
+  [(set (match_operand:FMOVE128 0 "register_operand" "=&r")
+	(unspec:FMOVE128
+	 [(match_operand:<FP128_64> 1 "register_operand" "r")
+	  (match_operand:<FP128_64> 2 "register_operand" "r")]
+	 UNSPEC_PACK_128BIT))]
+  "FLOAT128_2REG_P (<MODE>mode) && TARGET_SOFT_FLOAT"
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 3) (match_dup 1))
+   (set (match_dup 4) (match_dup 2))]
+{
+  unsigned dest_hi = REGNO (operands[0]);
+  unsigned dest_lo = dest_hi + (TARGET_POWERPC64 ? 1 : 2);
+
+  gcc_assert (!IN_RANGE (REGNO (operands[1]), dest_hi, dest_lo));
+  gcc_assert (!IN_RANGE (REGNO (operands[2]), dest_hi, dest_lo));
+
+  operands[3] = gen_rtx_REG (<FP128_64>mode, dest_hi);
+  operands[4] = gen_rtx_REG (<FP128_64>mode, dest_lo);
+}
+  [(set_attr "type" "integer")
+   (set (attr "length")
+	(if_then_else
+	 (match_test "TARGET_POWERPC64")
+	 (const_string "8")
+	 (const_string "16")))])
+
 (define_insn "unpack<mode>"
   [(set (match_operand:DI 0 "register_operand" "=wa,wa")
 	(unspec:DI [(match_operand:FMOVE128_VSX 1 "register_operand" "0,wa")
diff --git a/gcc/config/s390/3931.md b/gcc/config/s390/3931.md
new file mode 100644
index 0000000..bc97bc5
--- /dev/null
+++ b/gcc/config/s390/3931.md
@@ -0,0 +1,2562 @@
+;; Scheduling description for z16.
+;;   Copyright (C) 2022 Free Software Foundation, Inc.
+;;   Contributed by Robin Dapp (rdapp@linux.ibm.com)
+
+;; This file is part of GCC.
+
+;; GCC is free software; you can redistribute it and/or modify it under
+;; the terms of the GNU General Public License as published by the Free
+;; Software Foundation; either version 3, or (at your option) any later
+;; version.
+
+;; GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+;; WARRANTY; without even the implied warranty of MERCHANTABILITY or
+;; FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+;; for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GCC; see the file COPYING3.  If not see
+;; <http://www.gnu.org/licenses/>.
+
+(define_attr "z16_unit_fpd" ""
+(cond [(eq_attr "mnemonic"
+"ddb,
+ddbr,
+deb,
+debr,
+dxbr,
+sqdb,
+sqdbr,
+sqeb,
+sqebr,
+sqxbr,
+vfddb,
+vfdsb,
+vfsqdb,
+vfsqsb,
+wfddb,
+wfdsb,
+wfdxb,
+wfsqdb,
+wfsqxb"
+)
+ (const_int 1)] (const_int 0)))
+
+(define_attr "z16_unit_fxa" ""
+(cond [(eq_attr "mnemonic"
+"a,
+afi,
+ag,
+agf,
+agfi,
+agfr,
+agh,
+aghi,
+aghik,
+agr,
+agrk,
+ah,
+ahi,
+ahik,
+ahy,
+al,
+alc,
+alcg,
+alcgr,
+alcr,
+alfi,
+alg,
+algf,
+algfi,
+algfr,
+alghsik,
+algr,
+algrk,
+alhsik,
+alr,
+alrk,
+aly,
+ar,
+ark,
+ay,
+bras,
+brasl,
+etnd,
+exrl,
+flogr,
+ic,
+icm,
+icmh,
+icmy,
+icy,
+iihf,
+iilf,
+ipm,
+la,
+larl,
+lay,
+lb,
+lbr,
+lcgr,
+lcr,
+lgb,
+lgbr,
+lgf,
+lgfi,
+lgfr,
+lgfrl,
+lgh,
+lghi,
+lghr,
+lghrl,
+lgr,
+lh,
+lhi,
+lhr,
+lhrl,
+lhy,
+llcr,
+llgcr,
+llgfr,
+llghr,
+llgtr,
+llhr,
+llihf,
+llihh,
+llihl,
+llilf,
+llilh,
+llill,
+lngr,
+lnr,
+loc,
+locg,
+locghi,
+locgr,
+lochi,
+locr,
+lpgr,
+lpr,
+lr,
+lrv,
+lrvg,
+lrvgr,
+lrvh,
+lrvr,
+lt,
+ltg,
+ltgf,
+ltgfr,
+ltgr,
+ltr,
+m,
+mfy,
+mg,
+mgh,
+mghi,
+mgrk,
+mh,
+mhi,
+mhy,
+ml,
+mlg,
+mlgr,
+mlr,
+mr,
+ms,
+msc,
+msfi,
+msg,
+msgc,
+msgf,
+msgfi,
+msgfr,
+msgr,
+msgrkc,
+msr,
+msrkc,
+msy,
+n,
+ncgrk,
+ncrk,
+ng,
+ngr,
+ngrk,
+nihf,
+nihh,
+nihl,
+nilf,
+nilh,
+nill,
+nngrk,
+nnrk,
+nogrk,
+nork,
+nr,
+nrk,
+nxgrk,
+nxrk,
+ny,
+o,
+ocgrk,
+ocrk,
+og,
+ogr,
+ogrk,
+oihf,
+oihh,
+oihl,
+oilf,
+oilh,
+oill,
+or,
+ork,
+oy,
+pfpo,
+popcnt,
+risbg,
+risbgn,
+rll,
+rllg,
+s,
+selgr,
+selr,
+sg,
+sgf,
+sgfr,
+sgh,
+sgr,
+sgrk,
+sh,
+shy,
+sl,
+slb,
+slbg,
+slbgr,
+slbr,
+slfi,
+slg,
+slgf,
+slgfi,
+slgfr,
+slgr,
+slgrk,
+sll,
+sllg,
+sllk,
+slr,
+slrk,
+sly,
+sr,
+sra,
+srag,
+srak,
+srk,
+srl,
+srlg,
+srlk,
+sy,
+x,
+xg,
+xgr,
+xgrk,
+xihf,
+xilf,
+xr,
+xrk,
+xy"
+)
+ (const_int 1)] (const_int 0)))
+
+(define_attr "z16_unit_fxb" ""
+(cond [(eq_attr "mnemonic"
+"agsi,
+algsi,
+alsi,
+asi,
+b,
+bc,
+bcr,
+bi,
+br,
+brcl,
+c,
+cfi,
+cg,
+cgf,
+cgfi,
+cgfr,
+cgfrl,
+cgh,
+cghi,
+cghrl,
+cghsi,
+cgit,
+cgr,
+cgrl,
+cgrt,
+ch,
+chi,
+chrl,
+chsi,
+chy,
+cit,
+cl,
+clfhsi,
+clfi,
+clfit,
+clg,
+clgf,
+clgfi,
+clgfr,
+clgfrl,
+clghrl,
+clghsi,
+clgit,
+clgr,
+clgrl,
+clgrt,
+clgt,
+clhhsi,
+clhrl,
+cli,
+cliy,
+clm,
+clmy,
+clr,
+clrl,
+clrt,
+clt,
+cly,
+cr,
+crl,
+crt,
+cy,
+j,
+jg,
+laa,
+laag,
+lan,
+lang,
+lao,
+laog,
+lat,
+lax,
+laxg,
+lcdfr,
+ldgr,
+ldr,
+lgat,
+lgdr,
+lndfr,
+lpdfr,
+lzdr,
+lzer,
+mvghi,
+mvhhi,
+mvhi,
+mvi,
+mviy,
+ni,
+niy,
+nop,
+nopr,
+ntstg,
+oi,
+oiy,
+ppa,
+st,
+stc,
+stcy,
+std,
+stdy,
+ste,
+stey,
+stg,
+stgrl,
+sth,
+sthrl,
+sthy,
+stoc,
+stocg,
+strl,
+strv,
+strvg,
+strvh,
+sty,
+tend,
+tm,
+tmh,
+tmhh,
+tmhl,
+tml,
+tmlh,
+tmll,
+tmy,
+vlgvb,
+vlgvf,
+vlgvg,
+vlgvh,
+vlr,
+vlvgb,
+vlvgf,
+vlvgg,
+vlvgh,
+vlvgp,
+vst,
+vstbr,
+vstbrf,
+vstbrg,
+vstbrh,
+vstbrq,
+vstebrf,
+vstebrg,
+vstef,
+vsteg,
+vsterf,
+vsterg,
+vsterh,
+vstl,
+vstrl,
+vstrlr,
+xi,
+xiy"
+)
+ (const_int 1)] (const_int 0)))
+
+(define_attr "z16_unit_fxd" ""
+(cond [(eq_attr "mnemonic"
+"dlgr,
+dlr,
+dr,
+dsgfr,
+dsgr"
+)
+ (const_int 1)] (const_int 0)))
+
+(define_attr "z16_unit_lsu" ""
+(cond [(eq_attr "mnemonic"
+"a,
+adb,
+aeb,
+ag,
+agf,
+agh,
+agsi,
+ah,
+ahy,
+al,
+alc,
+alcg,
+alg,
+algf,
+algsi,
+alsi,
+aly,
+asi,
+ay,
+c,
+cdb,
+ceb,
+cg,
+cgf,
+cgfrl,
+cgh,
+cghrl,
+cghsi,
+cgrl,
+ch,
+chrl,
+chsi,
+chy,
+cl,
+clc,
+clfhsi,
+clg,
+clgf,
+clgfrl,
+clghrl,
+clghsi,
+clgrl,
+clgt,
+clhhsi,
+clhrl,
+cli,
+cliy,
+clm,
+clmy,
+clrl,
+clt,
+cly,
+crl,
+cy,
+ddb,
+deb,
+ear,
+ic,
+icm,
+icmh,
+icmy,
+icy,
+kdb,
+keb,
+l,
+laa,
+laag,
+lan,
+lang,
+lao,
+laog,
+lat,
+lax,
+laxg,
+lb,
+lcbb,
+ld,
+lde,
+ldeb,
+ldy,
+le,
+ley,
+lg,
+lgat,
+lgb,
+lgf,
+lgfrl,
+lgh,
+lghrl,
+lgrl,
+lh,
+lhrl,
+lhy,
+llc,
+llgc,
+llgf,
+llgfrl,
+llgh,
+llghrl,
+llgt,
+llh,
+llhrl,
+loc,
+locg,
+lrl,
+lrv,
+lrvg,
+lrvh,
+lt,
+ltg,
+ltgf,
+ly,
+m,
+madb,
+maeb,
+mdb,
+meeb,
+mfy,
+mg,
+mgh,
+mh,
+mhy,
+ml,
+mlg,
+ms,
+msc,
+msdb,
+mseb,
+msg,
+msgc,
+msgf,
+msy,
+mvghi,
+mvhhi,
+mvhi,
+mvi,
+mviy,
+n,
+ng,
+ni,
+niy,
+ntstg,
+ny,
+o,
+og,
+oi,
+oiy,
+oy,
+s,
+sar,
+sdb,
+seb,
+sfpc,
+sg,
+sgf,
+sgh,
+sh,
+shy,
+sl,
+slb,
+slbg,
+slg,
+slgf,
+sly,
+sqdb,
+sqeb,
+st,
+stc,
+stcy,
+std,
+stdy,
+ste,
+stey,
+stg,
+stgrl,
+sth,
+sthrl,
+sthy,
+stoc,
+stocg,
+strl,
+strv,
+strvg,
+strvh,
+sty,
+sy,
+tabort,
+tm,
+tmy,
+vl,
+vlbb,
+vlbr,
+vlbrf,
+vlbrg,
+vlbrh,
+vlbrq,
+vlbrrepf,
+vlbrrepg,
+vlbrreph,
+vleb,
+vlebrf,
+vlebrg,
+vlebrh,
+vlef,
+vleg,
+vleh,
+vlerf,
+vlerg,
+vlerh,
+vll,
+vllebrzf,
+vllebrzg,
+vllebrzh,
+vllezb,
+vllezf,
+vllezg,
+vllezh,
+vllezlf,
+vlrepb,
+vlrepf,
+vlrepg,
+vlreph,
+vlrl,
+vlrlr,
+vst,
+vstbr,
+vstbrf,
+vstbrg,
+vstbrh,
+vstbrq,
+vstebrf,
+vstebrg,
+vstef,
+vsteg,
+vsterf,
+vsterg,
+vsterh,
+vstl,
+vstrl,
+vstrlr,
+x,
+xg,
+xi,
+xiy,
+xy"
+)
+ (const_int 1)] (const_int 0)))
+
+(define_attr "z16_unit_vfu" ""
+(cond [(eq_attr "mnemonic"
+"adb,
+adbr,
+adtr,
+aeb,
+aebr,
+axbr,
+axtr,
+cdb,
+cdbr,
+cdtr,
+ceb,
+cebr,
+cpsdr,
+cxbr,
+cxtr,
+ddtr,
+dxtr,
+fidbr,
+fidbra,
+fidtr,
+fiebr,
+fiebra,
+fixbr,
+fixbra,
+fixtr,
+kdb,
+kdbr,
+kdtr,
+keb,
+kebr,
+kxbr,
+kxtr,
+lcdbr,
+lcebr,
+lcxbr,
+ldeb,
+ldebr,
+ldetr,
+le,
+ledbr,
+ledtr,
+ler,
+ley,
+lndbr,
+lnebr,
+lnxbr,
+lpdbr,
+lpebr,
+lpxbr,
+ltdbr,
+ltdtr,
+ltebr,
+ltxbr,
+ltxtr,
+lxdb,
+lxdbr,
+lxdtr,
+lxeb,
+lxebr,
+madb,
+madbr,
+maeb,
+maebr,
+mdb,
+mdbr,
+mdtr,
+meeb,
+meebr,
+msdb,
+msdbr,
+mseb,
+msebr,
+mxbr,
+mxtr,
+sdb,
+sdbr,
+sdtr,
+seb,
+sebr,
+sxbr,
+sxtr,
+tcdb,
+tceb,
+tcxb,
+tdcdt,
+tdcet,
+tdcxt,
+vab,
+vaccb,
+vacccq,
+vaccf,
+vaccg,
+vacch,
+vaccq,
+vacq,
+vaf,
+vag,
+vah,
+vaq,
+vavgb,
+vavgf,
+vavgg,
+vavgh,
+vavglb,
+vavglf,
+vavglg,
+vavglh,
+vbperm,
+vcdgb,
+vcdlgb,
+vcefb,
+vcelfb,
+vceqb,
+vceqbs,
+vceqf,
+vceqfs,
+vceqg,
+vceqgs,
+vceqh,
+vceqhs,
+vcfeb,
+vcfn,
+vcgdb,
+vchb,
+vchbs,
+vchf,
+vchfs,
+vchg,
+vchgs,
+vchh,
+vchhs,
+vchlb,
+vchlbs,
+vchlf,
+vchlfs,
+vchlg,
+vchlgs,
+vchlh,
+vchlhs,
+vcksm,
+vclfeb,
+vclfnh,
+vclfnl,
+vclgdb,
+vclzb,
+vclzf,
+vclzg,
+vclzh,
+vcnf,
+vcrnf,
+vctzb,
+vctzf,
+vctzg,
+vctzh,
+verimb,
+verimf,
+verimg,
+verimh,
+verllb,
+verllf,
+verllg,
+verllh,
+verllvb,
+verllvf,
+verllvg,
+verllvh,
+veslb,
+veslf,
+veslg,
+veslh,
+veslvb,
+veslvf,
+veslvg,
+veslvh,
+vesrab,
+vesraf,
+vesrag,
+vesrah,
+vesravb,
+vesravf,
+vesravg,
+vesravh,
+vesrlb,
+vesrlf,
+vesrlg,
+vesrlh,
+vesrlvb,
+vesrlvf,
+vesrlvg,
+vesrlvh,
+vfadb,
+vfasb,
+vfcedb,
+vfcedbs,
+vfcesb,
+vfcesbs,
+vfchdb,
+vfchdbs,
+vfchedb,
+vfchedbs,
+vfchesb,
+vfchesbs,
+vfchsb,
+vfchsbs,
+vfeeb,
+vfeef,
+vfeeh,
+vfeezbs,
+vfeezfs,
+vfeezhs,
+vfeneb,
+vfenef,
+vfeneh,
+vfenezb,
+vfenezf,
+vfenezh,
+vfidb,
+vfisb,
+vfkedb,
+vfkesb,
+vfkhdb,
+vfkhedb,
+vfkhesb,
+vfkhsb,
+vflcdb,
+vflcsb,
+vflndb,
+vflnsb,
+vflpdb,
+vflpsb,
+vfmadb,
+vfmasb,
+vfmaxdb,
+vfmaxsb,
+vfmdb,
+vfmindb,
+vfminsb,
+vfmsb,
+vfmsdb,
+vfmssb,
+vfnmadb,
+vfnmasb,
+vfnmsdb,
+vfnmssb,
+vfsdb,
+vfssb,
+vftcidb,
+vftcisb,
+vgbm,
+vgfmab,
+vgfmaf,
+vgfmag,
+vgfmah,
+vgfmb,
+vgfmf,
+vgfmg,
+vgfmh,
+vgm,
+vgmb,
+vgmf,
+vgmg,
+vgmh,
+vistrb,
+vistrbs,
+vistrf,
+vistrfs,
+vistrh,
+vistrhs,
+vlcb,
+vlcf,
+vlcg,
+vlch,
+vldeb,
+vleb,
+vlebrf,
+vlebrg,
+vlebrh,
+vledb,
+vlef,
+vleg,
+vleh,
+vleib,
+vleif,
+vleig,
+vleih,
+vlpb,
+vlpf,
+vlpg,
+vlph,
+vmaeb,
+vmaef,
+vmaeh,
+vmahb,
+vmahf,
+vmahh,
+vmalb,
+vmaleb,
+vmalef,
+vmaleh,
+vmalf,
+vmalhb,
+vmalhf,
+vmalhh,
+vmalhw,
+vmalob,
+vmalof,
+vmaloh,
+vmaob,
+vmaof,
+vmaoh,
+vmeb,
+vmef,
+vmeh,
+vmhb,
+vmhf,
+vmhh,
+vmlb,
+vmleb,
+vmlef,
+vmleh,
+vmlf,
+vmlhb,
+vmlhf,
+vmlhh,
+vmlhw,
+vmlob,
+vmlof,
+vmloh,
+vmnb,
+vmnf,
+vmng,
+vmnh,
+vmnlb,
+vmnlf,
+vmnlg,
+vmnlh,
+vmob,
+vmof,
+vmoh,
+vmrhb,
+vmrhf,
+vmrhg,
+vmrhh,
+vmrlb,
+vmrlf,
+vmrlg,
+vmrlh,
+vmslg,
+vmxb,
+vmxf,
+vmxg,
+vmxh,
+vmxlb,
+vmxlf,
+vmxlg,
+vmxlh,
+vn,
+vnc,
+vnn,
+vno,
+vnot,
+vnx,
+vo,
+voc,
+vone,
+vpdi,
+vperm,
+vpkf,
+vpkg,
+vpkh,
+vpklsf,
+vpklsfs,
+vpklsg,
+vpklsgs,
+vpklsh,
+vpklshs,
+vpksf,
+vpksfs,
+vpksg,
+vpksgs,
+vpksh,
+vpkshs,
+vpopct,
+vpopctb,
+vpopctf,
+vpopctg,
+vpopcth,
+vrepb,
+vrepf,
+vrepg,
+vreph,
+vrepi,
+vrepib,
+vrepif,
+vrepig,
+vrepih,
+vsb,
+vsbcbiq,
+vsbiq,
+vscbib,
+vscbif,
+vscbig,
+vscbih,
+vscbiq,
+vsegb,
+vsegf,
+vsegh,
+vsel,
+vsf,
+vsg,
+vsh,
+vsl,
+vslb,
+vsld,
+vsldb,
+vsq,
+vsra,
+vsrab,
+vsrd,
+vsrl,
+vsrlb,
+vsumb,
+vsumgf,
+vsumgh,
+vsumh,
+vsumqf,
+vsumqg,
+vtm,
+vuphb,
+vuphf,
+vuphh,
+vuplb,
+vuplf,
+vuplhb,
+vuplhf,
+vuplhh,
+vuplhw,
+vupllb,
+vupllf,
+vupllh,
+vx,
+vzero,
+wcdgb,
+wcdlgb,
+wcefb,
+wcelfb,
+wcfeb,
+wcgdb,
+wclfeb,
+wclgdb,
+wfadb,
+wfasb,
+wfaxb,
+wfcdb,
+wfcedb,
+wfcesb,
+wfcexb,
+wfcexbs,
+wfchdb,
+wfchedb,
+wfchesb,
+wfchexb,
+wfchexbs,
+wfchsb,
+wfchxb,
+wfchxbs,
+wfcsb,
+wfcxb,
+wfidb,
+wfisb,
+wfixb,
+wfkdb,
+wfkedb,
+wfkesb,
+wfkexb,
+wfkhdb,
+wfkhedb,
+wfkhesb,
+wfkhexb,
+wfkhsb,
+wfkhxb,
+wfksb,
+wfkxb,
+wflcdb,
+wflcsb,
+wflcxb,
+wflld,
+wflndb,
+wflnsb,
+wflnxb,
+wflpdb,
+wflpsb,
+wflpxb,
+wflrx,
+wfmadb,
+wfmasb,
+wfmaxb,
+wfmaxxb,
+wfmdb,
+wfminxb,
+wfmsb,
+wfmsdb,
+wfmssb,
+wfmsxb,
+wfmxb,
+wfnmaxb,
+wfnmsxb,
+wfsdb,
+wfssb,
+wfsxb,
+wftcixb,
+wldeb,
+wledb"
+)
+ (const_int 1)] (const_int 0)))
+
+(define_attr "z16_cracked" ""
+(cond [(eq_attr "mnemonic"
+"bas,
+basr,
+cdfbr,
+cdftr,
+cdgbr,
+cdgtr,
+cdlfbr,
+cdlftr,
+cdlgbr,
+cdlgtr,
+cefbr,
+cegbr,
+celfbr,
+celgbr,
+cfdbr,
+cfebr,
+cfxbr,
+cgdbr,
+cgdtr,
+cgebr,
+cgxbr,
+cgxtr,
+chhsi,
+clfdbr,
+clfdtr,
+clfebr,
+clfxbr,
+clfxtr,
+clgdbr,
+clgdtr,
+clgebr,
+clgxbr,
+clgxtr,
+cs,
+csg,
+csy,
+d,
+efpc,
+ex,
+lcgfr,
+lngfr,
+lpgfr,
+lpq,
+lxr,
+lzxr,
+rxsbg,
+stpq,
+vgef,
+vgeg,
+vscef,
+vsceg,
+vsteb,
+vstebrh,
+vsteh"
+)
+ (const_int 1)] (const_int 0)))
+
+(define_attr "z16_expanded" ""
+(cond [(eq_attr "mnemonic"
+"cds,
+cdsg,
+cdsy,
+cxfbr,
+cxftr,
+cxgbr,
+cxgtr,
+cxlfbr,
+cxlftr,
+cxlgbr,
+cxlgtr,
+dl,
+dlg,
+dsg,
+dsgf,
+lam,
+lm,
+lmg,
+lmy,
+sldl,
+srda,
+srdl,
+stam,
+stm,
+stmg,
+stmy,
+tbegin,
+tbeginc"
+)
+ (const_int 1)] (const_int 0)))
+
+(define_attr "z16_groupalone" ""
+(cond [(eq_attr "mnemonic"
+"alc,
+alcg,
+alcgr,
+alcr,
+axbr,
+axtr,
+clc,
+cxbr,
+cxtr,
+dlgr,
+dlr,
+dr,
+dsgfr,
+dsgr,
+dxbr,
+dxtr,
+fixbr,
+fixbra,
+fixtr,
+flogr,
+kxbr,
+kxtr,
+lcxbr,
+lnxbr,
+lpxbr,
+ltxbr,
+ltxtr,
+lxdb,
+lxdbr,
+lxdtr,
+lxeb,
+lxebr,
+m,
+madb,
+maeb,
+maebr,
+mfy,
+mg,
+mgrk,
+ml,
+mlg,
+mlgr,
+mlr,
+mr,
+msdb,
+mseb,
+msebr,
+mvc,
+mxbr,
+mxtr,
+nc,
+oc,
+ppa,
+sfpc,
+slb,
+slbg,
+slbgr,
+slbr,
+sqxbr,
+sxbr,
+sxtr,
+tabort,
+tcxb,
+tdcxt,
+tend,
+xc"
+)
+ (const_int 1)] (const_int 0)))
+
+(define_attr "z16_endgroup" ""
+(cond [(eq_attr "mnemonic"
+"bras,
+brasl,
+exrl,
+ipm"
+)
+ (const_int 1)] (const_int 0)))
+
+(define_attr "z16_groupoftwo" ""
+(cond [(eq_attr "mnemonic"
+"vacccq,
+vacq,
+vfmadb,
+vfmasb,
+vfmsdb,
+vfmssb,
+vfnmadb,
+vfnmasb,
+vfnmsdb,
+vfnmssb,
+vgfmab,
+vgfmaf,
+vgfmag,
+vgfmah,
+vmaeb,
+vmaef,
+vmaeh,
+vmahb,
+vmahf,
+vmahh,
+vmalb,
+vmaleb,
+vmalef,
+vmaleh,
+vmalf,
+vmalhb,
+vmalhf,
+vmalhh,
+vmalhw,
+vmalob,
+vmalof,
+vmaloh,
+vmaob,
+vmaof,
+vmaoh,
+vmslg,
+vperm,
+vsbcbiq,
+vsbiq,
+vsel,
+wfmadb,
+wfmasb,
+wfmaxb,
+wfmsdb,
+wfmssb,
+wfmsxb,
+wfnmaxb,
+wfnmsxb"
+)
+ (const_int 1)] (const_int 0)))
+
+(define_insn_reservation "z16_0" 0
+  (and (eq_attr "cpu" "z16")
+(eq_attr "mnemonic"
+"a,
+afi,
+ag,
+agfi,
+aghi,
+aghik,
+agr,
+agrk,
+ahi,
+ahik,
+al,
+alfi,
+alg,
+algf,
+algfi,
+algfr,
+alghsik,
+algr,
+algrk,
+alhsik,
+alr,
+alrk,
+aly,
+ar,
+ark,
+ay,
+b,
+bc,
+bcr,
+bi,
+br,
+bras,
+brasl,
+brcl,
+c,
+cfi,
+cg,
+cgfi,
+cghi,
+cghsi,
+cgit,
+cgr,
+cgrl,
+cgrt,
+chi,
+chsi,
+cit,
+cl,
+clfhsi,
+clfi,
+clfit,
+clg,
+clgf,
+clgfi,
+clgfr,
+clgfrl,
+clghrl,
+clghsi,
+clgit,
+clgr,
+clgrl,
+clgrt,
+clgt,
+clhhsi,
+clhrl,
+cli,
+cliy,
+clr,
+clrl,
+clrt,
+clt,
+cly,
+cr,
+crl,
+crt,
+cy,
+etnd,
+exrl,
+ic,
+icm,
+icmh,
+icmy,
+icy,
+iihf,
+iilf,
+j,
+jg,
+la,
+larl,
+lat,
+lay,
+lb,
+lbr,
+lcdfr,
+lcgr,
+lcr,
+ldgr,
+ldr,
+lgat,
+lgb,
+lgbr,
+lgf,
+lgfi,
+lgfr,
+lgfrl,
+lgh,
+lghi,
+lghr,
+lghrl,
+lgr,
+lh,
+lhi,
+lhr,
+lhrl,
+lhy,
+llcr,
+llgcr,
+llgfr,
+llghr,
+llgtr,
+llhr,
+llihf,
+llihh,
+llihl,
+llilf,
+llilh,
+llill,
+lndfr,
+lngr,
+lnr,
+lpdfr,
+lpgr,
+lpr,
+lr,
+lrv,
+lrvg,
+lrvgr,
+lrvh,
+lrvr,
+lt,
+ltg,
+ltgf,
+ltgfr,
+ltgr,
+ltr,
+lzdr,
+lzer,
+n,
+ncgrk,
+ncrk,
+ng,
+ngr,
+ngrk,
+nihf,
+nihh,
+nihl,
+nilf,
+nilh,
+nill,
+nngrk,
+nnrk,
+nogrk,
+nop,
+nopr,
+nork,
+nr,
+nrk,
+nxgrk,
+nxrk,
+ny,
+o,
+ocgrk,
+ocrk,
+og,
+ogr,
+ogrk,
+oihf,
+oihh,
+oihl,
+oilf,
+oilh,
+oill,
+or,
+ork,
+oy,
+pfpo,
+risbg,
+risbgn,
+rll,
+rllg,
+rnsbg,
+rosbg,
+s,
+sg,
+sgr,
+sgrk,
+sl,
+sldl,
+slfi,
+slg,
+slgf,
+slgfi,
+slgfr,
+slgr,
+slgrk,
+sll,
+sllg,
+sllk,
+slr,
+slrk,
+sly,
+sr,
+sra,
+srag,
+srak,
+srda,
+srdl,
+srk,
+srl,
+srlg,
+srlk,
+sy,
+tm,
+tmh,
+tmhh,
+tmhl,
+tml,
+tmlh,
+tmll,
+tmy,
+vlr,
+vlvgb,
+vlvgf,
+vlvgg,
+vlvgh,
+x,
+xg,
+xgr,
+xgrk,
+xihf,
+xilf,
+xr,
+xrk,
+xy"
+)) "nothing")
+
+(define_insn_reservation "z16_1" 1
+  (and (eq_attr "cpu" "z16")
+(eq_attr "mnemonic"
+"agf,
+agfr,
+agh,
+agsi,
+ah,
+ahy,
+algsi,
+alsi,
+asi,
+cgf,
+cgfr,
+cgfrl,
+cgh,
+cghrl,
+ch,
+chrl,
+chy,
+clm,
+clmy,
+cpsdr,
+laa,
+laag,
+lan,
+lang,
+lao,
+laog,
+lax,
+laxg,
+le,
+ler,
+ley,
+loc,
+locg,
+locghi,
+locgr,
+lochi,
+locr,
+mvghi,
+mvhhi,
+mvhi,
+mvi,
+mviy,
+ni,
+niy,
+ntstg,
+oi,
+oiy,
+selgr,
+selr,
+sgf,
+sgfr,
+sgh,
+sh,
+shy,
+st,
+stc,
+stcy,
+stg,
+stgrl,
+sth,
+sthrl,
+sthy,
+stoc,
+stocg,
+strl,
+strv,
+strvg,
+strvh,
+sty,
+vab,
+vaccb,
+vacccq,
+vaccf,
+vaccg,
+vacch,
+vaccq,
+vacq,
+vaf,
+vag,
+vah,
+vaq,
+vavgb,
+vavgf,
+vavgg,
+vavgh,
+vavglb,
+vavglf,
+vavglg,
+vavglh,
+vbperm,
+vceqb,
+vceqf,
+vceqg,
+vceqh,
+vcfn,
+vchb,
+vchf,
+vchg,
+vchh,
+vchlb,
+vchlf,
+vchlg,
+vchlh,
+vclfnh,
+vclfnl,
+vclzb,
+vclzf,
+vclzg,
+vclzh,
+vcnf,
+vcrnf,
+vctzb,
+vctzf,
+vctzg,
+vctzh,
+verimb,
+verimf,
+verimg,
+verimh,
+verllb,
+verllf,
+verllg,
+verllh,
+verllvb,
+verllvf,
+verllvg,
+verllvh,
+veslb,
+veslf,
+veslg,
+veslh,
+veslvb,
+veslvf,
+veslvg,
+veslvh,
+vesrab,
+vesraf,
+vesrag,
+vesrah,
+vesravb,
+vesravf,
+vesravg,
+vesravh,
+vesrlb,
+vesrlf,
+vesrlg,
+vesrlh,
+vesrlvb,
+vesrlvf,
+vesrlvg,
+vesrlvh,
+vfcedb,
+vfcesb,
+vfchdb,
+vfchedb,
+vfchesb,
+vfchsb,
+vfkedb,
+vfkesb,
+vfkhdb,
+vfkhedb,
+vfkhesb,
+vfkhsb,
+vflcdb,
+vflcsb,
+vflndb,
+vflnsb,
+vflpdb,
+vflpsb,
+vfmaxdb,
+vfmaxsb,
+vfmindb,
+vfminsb,
+vgbm,
+vgm,
+vgmb,
+vgmf,
+vgmg,
+vgmh,
+vlcb,
+vlcf,
+vlcg,
+vlch,
+vleb,
+vlebrf,
+vlebrg,
+vlebrh,
+vlef,
+vleg,
+vleh,
+vleib,
+vleif,
+vleig,
+vleih,
+vlpb,
+vlpf,
+vlpg,
+vlph,
+vmnb,
+vmnf,
+vmng,
+vmnh,
+vmnlb,
+vmnlf,
+vmnlg,
+vmnlh,
+vmrhb,
+vmrhf,
+vmrhg,
+vmrhh,
+vmrlb,
+vmrlf,
+vmrlg,
+vmrlh,
+vmxb,
+vmxf,
+vmxg,
+vmxh,
+vmxlb,
+vmxlf,
+vmxlg,
+vmxlh,
+vn,
+vnc,
+vnn,
+vno,
+vnot,
+vnx,
+vo,
+voc,
+vone,
+vpdi,
+vperm,
+vpkf,
+vpkg,
+vpkh,
+vpklsf,
+vpklsg,
+vpklsh,
+vpksf,
+vpksg,
+vpksh,
+vpopct,
+vpopctb,
+vpopctf,
+vpopctg,
+vpopcth,
+vrepb,
+vrepf,
+vrepg,
+vreph,
+vrepi,
+vrepib,
+vrepif,
+vrepig,
+vrepih,
+vsb,
+vsbcbiq,
+vsbiq,
+vscbib,
+vscbif,
+vscbig,
+vscbih,
+vscbiq,
+vsegb,
+vsegf,
+vsegh,
+vsel,
+vsf,
+vsg,
+vsh,
+vsl,
+vslb,
+vsld,
+vsldb,
+vsq,
+vsra,
+vsrab,
+vsrd,
+vsrl,
+vsrlb,
+vuphb,
+vuphf,
+vuphh,
+vuplb,
+vuplf,
+vuplhb,
+vuplhf,
+vuplhh,
+vuplhw,
+vupllb,
+vupllf,
+vupllh,
+vx,
+vzero,
+wfcedb,
+wfcesb,
+wfcexb,
+wfchdb,
+wfchedb,
+wfchesb,
+wfchexb,
+wfchsb,
+wfchxb,
+wfkedb,
+wfkesb,
+wfkexb,
+wfkhdb,
+wfkhedb,
+wfkhesb,
+wfkhexb,
+wfkhsb,
+wfkhxb,
+wflcdb,
+wflcsb,
+wflcxb,
+wflndb,
+wflnsb,
+wflnxb,
+wflpdb,
+wflpsb,
+wflpxb,
+wfmaxxb,
+wfminxb,
+xi,
+xiy"
+)) "nothing")
+
+(define_insn_reservation "z16_2" 2
+  (and (eq_attr "cpu" "z16")
+(eq_attr "mnemonic"
+"cdb,
+cdbr,
+ceb,
+cebr,
+ear,
+ipm,
+kdb,
+kdbr,
+keb,
+kebr,
+l,
+lcbb,
+lcdbr,
+lcebr,
+ld,
+lde,
+ldy,
+lg,
+lgdr,
+lgrl,
+llc,
+llgc,
+llgf,
+llgfrl,
+llgh,
+llghrl,
+llgt,
+llh,
+llhrl,
+lm,
+lmg,
+lmy,
+lndbr,
+lnebr,
+lpdbr,
+lpebr,
+lrl,
+ltdbr,
+ltebr,
+ly,
+popcnt,
+sar,
+tcdb,
+tceb,
+vceqbs,
+vceqfs,
+vceqgs,
+vceqhs,
+vchbs,
+vchfs,
+vchgs,
+vchhs,
+vchlbs,
+vchlfs,
+vchlgs,
+vchlhs,
+vfcedbs,
+vfcesbs,
+vfchdbs,
+vfchedbs,
+vfchesbs,
+vfchsbs,
+vfeeb,
+vfeef,
+vfeeh,
+vfeneb,
+vfenef,
+vfeneh,
+vfenezb,
+vfenezf,
+vfenezh,
+vftcidb,
+vftcisb,
+vistrb,
+vistrf,
+vistrh,
+vlbrrepf,
+vlbrrepg,
+vlbrreph,
+vlgvb,
+vlgvf,
+vlgvg,
+vlgvh,
+vllebrzf,
+vllebrzg,
+vllebrzh,
+vllezb,
+vllezf,
+vllezg,
+vllezh,
+vllezlf,
+vlrepb,
+vlrepf,
+vlrepg,
+vlreph,
+vlrl,
+vlvgp,
+vpklsfs,
+vpklsgs,
+vpklshs,
+vpksfs,
+vpksgs,
+vpkshs,
+wfcdb,
+wfcexbs,
+wfchexbs,
+wfchxbs,
+wfcsb,
+wfcxb,
+wfkdb,
+wfksb,
+wfkxb,
+wftcixb"
+)) "nothing")
+
+(define_insn_reservation "z16_3" 3
+  (and (eq_attr "cpu" "z16")
+(eq_attr "mnemonic"
+"cds,
+cdsy,
+mgh,
+mghi,
+mh,
+mhi,
+mhy,
+std,
+stdy,
+ste,
+stey,
+vcksm,
+vfeezbs,
+vfeezfs,
+vfeezhs,
+vgfmab,
+vgfmaf,
+vgfmag,
+vgfmah,
+vgfmb,
+vgfmf,
+vgfmg,
+vgfmh,
+vistrbs,
+vistrfs,
+vistrhs,
+vl,
+vlbb,
+vlbr,
+vlbrf,
+vlbrg,
+vlbrh,
+vlbrq,
+vlerf,
+vlerg,
+vlerh,
+vll,
+vlrlr,
+vmaeb,
+vmaef,
+vmaeh,
+vmahb,
+vmahf,
+vmahh,
+vmalb,
+vmaleb,
+vmalef,
+vmaleh,
+vmalf,
+vmalhb,
+vmalhf,
+vmalhh,
+vmalhw,
+vmalob,
+vmalof,
+vmaloh,
+vmaob,
+vmaof,
+vmaoh,
+vmeb,
+vmef,
+vmeh,
+vmhb,
+vmhf,
+vmhh,
+vmlb,
+vmleb,
+vmlef,
+vmleh,
+vmlf,
+vmlhb,
+vmlhf,
+vmlhh,
+vmlhw,
+vmlob,
+vmlof,
+vmloh,
+vmob,
+vmof,
+vmoh,
+vsumb,
+vsumgf,
+vsumgh,
+vsumh,
+vsumqf,
+vsumqg,
+vtm"
+)) "nothing")
+
+(define_insn_reservation "z16_4" 4
+  (and (eq_attr "cpu" "z16")
+(eq_attr "mnemonic"
+"bas,
+basr,
+chhsi,
+clc,
+ex,
+lam,
+lcgfr,
+lngfr,
+lpgfr,
+lxr,
+lzxr,
+ms,
+msfi,
+msgf,
+msgfi,
+msgfr,
+msr,
+msy,
+mvc,
+nc,
+oc,
+ppa,
+rxsbg,
+tabort,
+tbegin,
+tbeginc,
+tend,
+vst,
+vstbr,
+vstbrf,
+vstbrg,
+vstbrh,
+vstbrq,
+vstebrf,
+vstebrg,
+vstef,
+vsteg,
+vsterf,
+vsterg,
+vsterh,
+vstl,
+vstrl,
+vstrlr,
+xc"
+)) "nothing")
+
+(define_insn_reservation "z16_5" 5
+  (and (eq_attr "cpu" "z16")
+(eq_attr "mnemonic"
+"adb,
+adbr,
+aeb,
+aebr,
+alc,
+alcg,
+alcgr,
+alcr,
+cs,
+csg,
+csy,
+fidbr,
+fidbra,
+fiebr,
+fiebra,
+ldeb,
+ldebr,
+ledbr,
+madbr,
+mdb,
+mdbr,
+meeb,
+meebr,
+msc,
+msdbr,
+msrkc,
+sdb,
+sdbr,
+seb,
+sebr,
+slb,
+slbg,
+slbgr,
+slbr,
+stm,
+stmg,
+stmy,
+vcdgb,
+vcdlgb,
+vcefb,
+vcelfb,
+vcfeb,
+vcgdb,
+vclfeb,
+vclgdb,
+vfadb,
+vfasb,
+vfidb,
+vfisb,
+vfmadb,
+vfmasb,
+vfmdb,
+vfmsb,
+vfmsdb,
+vfmssb,
+vfnmadb,
+vfnmasb,
+vfnmsdb,
+vfnmssb,
+vfsdb,
+vfssb,
+vldeb,
+vledb,
+vmslg,
+wcdgb,
+wcdlgb,
+wcefb,
+wcelfb,
+wcfeb,
+wcgdb,
+wclfeb,
+wclgdb,
+wfadb,
+wfasb,
+wfidb,
+wfisb,
+wflld,
+wfmadb,
+wfmasb,
+wfmdb,
+wfmsb,
+wfmsdb,
+wfmssb,
+wfsdb,
+wfssb,
+wldeb,
+wledb"
+)) "nothing")
+
+(define_insn_reservation "z16_6" 6
+  (and (eq_attr "cpu" "z16")
+(eq_attr "mnemonic"
+"msg,
+msgr,
+sfpc"
+)) "nothing")
+
+(define_insn_reservation "z16_7" 7
+  (and (eq_attr "cpu" "z16")
+(eq_attr "mnemonic"
+"adtr,
+cdtr,
+fidtr,
+kdtr,
+ldetr,
+ltdtr,
+msgc,
+msgrkc,
+sdtr,
+tdcdt,
+tdcet,
+vgef,
+vgeg"
+)) "nothing")
+
+(define_insn_reservation "z16_8" 8
+  (and (eq_attr "cpu" "z16")
+(eq_attr "mnemonic"
+"cdsg,
+flogr,
+lpq,
+stpq,
+vsteb,
+vstebrh,
+vsteh"
+)) "nothing")
+
+(define_insn_reservation "z16_9" 9
+  (and (eq_attr "cpu" "z16")
+(eq_attr "mnemonic"
+"cdfbr,
+cdgbr,
+cdlfbr,
+cdlgbr,
+cefbr,
+cegbr,
+celfbr,
+celgbr,
+cxfbr,
+cxgbr,
+cxlfbr,
+cxlgbr,
+m,
+madb,
+maeb,
+maebr,
+mfy,
+ml,
+mlr,
+mr,
+msdb,
+mseb,
+msebr,
+stam,
+wfaxb,
+wfixb,
+wflrx,
+wfsxb"
+)) "nothing")
+
+(define_insn_reservation "z16_10" 10
+  (and (eq_attr "cpu" "z16")
+(eq_attr "mnemonic"
+"lxdb,
+lxdbr,
+lxeb,
+lxebr,
+vscef,
+vsceg"
+)) "nothing")
+
+(define_insn_reservation "z16_11" 11
+  (and (eq_attr "cpu" "z16")
+(eq_attr "mnemonic"
+"cfdbr,
+cfebr,
+cgdbr,
+cgebr,
+clfdbr,
+clfebr,
+clgdbr,
+clgebr,
+mg,
+mgrk,
+mlg,
+mlgr"
+)) "nothing")
+
+(define_insn_reservation "z16_12" 12
+  (and (eq_attr "cpu" "z16")
+(eq_attr "mnemonic"
+"cxbr,
+cxftr,
+cxlftr,
+cxtr,
+kxbr,
+kxtr,
+tcxb,
+tdcxt"
+)) "nothing")
+
+(define_insn_reservation "z16_13" 13
+  (and (eq_attr "cpu" "z16")
+(eq_attr "mnemonic"
+"axbr,
+axtr,
+fixbr,
+fixbra,
+fixtr,
+lcxbr,
+lnxbr,
+lpxbr,
+ltxbr,
+ltxtr,
+lxdtr,
+sxbr,
+sxtr"
+)) "nothing")
+
+(define_insn_reservation "z16_14" 14
+  (and (eq_attr "cpu" "z16")
+(eq_attr "mnemonic"
+"cfxbr,
+cgxbr,
+clfxbr,
+clgxbr,
+ledtr"
+)) "nothing")
+
+(define_insn_reservation "z16_16" 16
+  (and (eq_attr "cpu" "z16")
+(eq_attr "mnemonic"
+"cdftr,
+cdlftr"
+)) "nothing")
+
+(define_insn_reservation "z16_20" 20
+  (and (eq_attr "cpu" "z16")
+(eq_attr "mnemonic"
+"cdgtr,
+cdlgtr,
+cgdtr,
+cgxtr,
+clfdtr,
+clfxtr,
+clgdtr,
+clgxtr,
+cxgtr,
+cxlgtr,
+d,
+ddb,
+ddbr,
+ddtr,
+deb,
+debr,
+dl,
+dlg,
+dlgr,
+dlr,
+dr,
+dsg,
+dsgf,
+dsgfr,
+dsgr,
+dxbr,
+dxtr,
+efpc,
+mdtr,
+mxbr,
+mxtr,
+sqdb,
+sqdbr,
+sqeb,
+sqebr,
+sqxbr,
+vfddb,
+vfdsb,
+vfsqdb,
+vfsqsb,
+wfddb,
+wfdsb,
+wfdxb,
+wfmaxb,
+wfmsxb,
+wfmxb,
+wfnmaxb,
+wfnmsxb,
+wfsqdb,
+wfsqxb"
+)) "nothing")
+
diff --git a/gcc/config/s390/driver-native.cc b/gcc/config/s390/driver-native.cc
index 48524c4..b5eb222 100644
--- a/gcc/config/s390/driver-native.cc
+++ b/gcc/config/s390/driver-native.cc
@@ -123,8 +123,12 @@ s390_host_detect_local_cpu (int argc, const char **argv)
 	    case 0x8562:
 	      cpu = "z15";
 	      break;
+	    case 0x3931:
+	    case 0x3932:
+	      cpu = "z16";
+	      break;
 	    default:
-	      cpu = "arch14";
+	      cpu = "z16";
 	      break;
 	    }
 	}
diff --git a/gcc/config/s390/s390-opts.h b/gcc/config/s390/s390-opts.h
index 1ec8463..4ef82ac 100644
--- a/gcc/config/s390/s390-opts.h
+++ b/gcc/config/s390/s390-opts.h
@@ -38,7 +38,7 @@ enum processor_type
   PROCESSOR_2964_Z13,
   PROCESSOR_3906_Z14,
   PROCESSOR_8561_Z15,
-  PROCESSOR_ARCH14,
+  PROCESSOR_3931_Z16,
   PROCESSOR_NATIVE,
   PROCESSOR_max
 };
diff --git a/gcc/config/s390/s390-protos.h b/gcc/config/s390/s390-protos.h
index e625159..fd4acaa 100644
--- a/gcc/config/s390/s390-protos.h
+++ b/gcc/config/s390/s390-protos.h
@@ -49,7 +49,6 @@ extern void s390_function_profiler (FILE *, int);
 extern void s390_set_has_landing_pad_p (bool);
 extern bool s390_hard_regno_rename_ok (unsigned int, unsigned int);
 extern int s390_class_max_nregs (enum reg_class, machine_mode);
-extern bool s390_function_arg_vector (machine_mode, const_tree);
 extern bool s390_return_addr_from_memory(void);
 extern bool s390_fma_allowed_p (machine_mode);
 #if S390_USE_TARGET_ATTRIBUTE
diff --git a/gcc/config/s390/s390.cc b/gcc/config/s390/s390.cc
index d2af6d8..7c3bd6c 100644
--- a/gcc/config/s390/s390.cc
+++ b/gcc/config/s390/s390.cc
@@ -337,7 +337,7 @@ const struct s390_processor processor_table[] =
   { "z13",    "z13",    PROCESSOR_2964_Z13,    &zEC12_cost,  11 },
   { "z14",    "arch12", PROCESSOR_3906_Z14,    &zEC12_cost,  12 },
   { "z15",    "arch13", PROCESSOR_8561_Z15,    &zEC12_cost,  13 },
-  { "arch14", "arch14", PROCESSOR_ARCH14,      &zEC12_cost,  14 },
+  { "z16",    "arch14", PROCESSOR_3931_Z16,    &zEC12_cost,  14 },
   { "native", "",       PROCESSOR_NATIVE,      NULL,         0  }
 };
 
@@ -853,12 +853,6 @@ s390_expand_builtin (tree exp, rtx target, rtx subtarget ATTRIBUTE_UNUSED,
 	  error ("Builtin %qF requires z15 or higher", fndecl);
 	  return const0_rtx;
 	}
-
-      if ((bflags & B_NNPA) && !TARGET_NNPA)
-	{
-	  error ("Builtin %qF requires arch14 or higher.", fndecl);
-	  return const0_rtx;
-	}
     }
   if (fcode >= S390_OVERLOADED_BUILTIN_VAR_OFFSET
       && fcode < S390_ALL_BUILTIN_MAX)
@@ -8525,7 +8519,7 @@ s390_issue_rate (void)
     case PROCESSOR_2827_ZEC12:
     case PROCESSOR_2964_Z13:
     case PROCESSOR_3906_Z14:
-    case PROCESSOR_ARCH14:
+    case PROCESSOR_3931_Z16:
     default:
       return 1;
     }
@@ -12148,29 +12142,26 @@ s390_function_arg_size (machine_mode mode, const_tree type)
   gcc_unreachable ();
 }
 
-/* Return true if a function argument of type TYPE and mode MODE
-   is to be passed in a vector register, if available.  */
+/* Return true if a variable of TYPE should be passed as single value
+   with type CODE. If STRICT_SIZE_CHECK_P is true the sizes of the
+   record type and the field type must match.
 
-bool
-s390_function_arg_vector (machine_mode mode, const_tree type)
+   The ABI says that record types with a single member are treated
+   just like that member would be.  This function is a helper to
+   detect such cases.  The function also produces the proper
+   diagnostics for cases where the outcome might be different
+   depending on the GCC version.  */
+static bool
+s390_single_field_struct_p (enum tree_code code, const_tree type,
+			    bool strict_size_check_p)
 {
-  if (!TARGET_VX_ABI)
-    return false;
-
-  if (s390_function_arg_size (mode, type) > 16)
-    return false;
-
-  /* No type info available for some library calls ...  */
-  if (!type)
-    return VECTOR_MODE_P (mode);
-
-  /* The ABI says that record types with a single member are treated
-     just like that member would be.  */
   int empty_base_seen = 0;
+  bool zero_width_bf_skipped_p = false;
   const_tree orig_type = type;
+
   while (TREE_CODE (type) == RECORD_TYPE)
     {
-      tree field, single = NULL_TREE;
+      tree field, single_type = NULL_TREE;
 
       for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
 	{
@@ -12187,48 +12178,108 @@ s390_function_arg_vector (machine_mode mode, const_tree type)
 	      continue;
 	    }
 
-	  if (single == NULL_TREE)
-	    single = TREE_TYPE (field);
+	  if (DECL_FIELD_CXX_ZERO_WIDTH_BIT_FIELD (field))
+	    {
+	      zero_width_bf_skipped_p = true;
+	      continue;
+	    }
+
+	  if (single_type == NULL_TREE)
+	    single_type = TREE_TYPE (field);
 	  else
 	    return false;
 	}
 
-      if (single == NULL_TREE)
+      if (single_type == NULL_TREE)
 	return false;
-      else
-	{
-	  /* If the field declaration adds extra byte due to
-	     e.g. padding this is not accepted as vector type.  */
-	  if (int_size_in_bytes (single) <= 0
-	      || int_size_in_bytes (single) != int_size_in_bytes (type))
-	    return false;
-	  type = single;
-	}
+
+      /* Reaching this point we have a struct with a single member and
+	 zero or more zero-sized bit-fields which have been skipped in the
+	 past.  */
+
+      /* If ZERO_WIDTH_BF_SKIPPED_P then the struct will not be accepted.  In case
+	 we are not supposed to emit a warning exit early.  */
+      if (zero_width_bf_skipped_p && !warn_psabi)
+	return false;
+
+      /* If the field declaration adds extra bytes due to padding this
+	 is not accepted with STRICT_SIZE_CHECK_P.  */
+      if (strict_size_check_p
+	  && (int_size_in_bytes (single_type) <= 0
+	      || int_size_in_bytes (single_type) != int_size_in_bytes (type)))
+	return false;
+
+      type = single_type;
     }
 
-  if (!VECTOR_TYPE_P (type))
+  if (TREE_CODE (type) != code)
     return false;
 
-  if (warn_psabi && empty_base_seen)
+  if (warn_psabi)
     {
-      static unsigned last_reported_type_uid;
       unsigned uid = TYPE_UID (TYPE_MAIN_VARIANT (orig_type));
-      if (uid != last_reported_type_uid)
-	{
-	  const char *url = CHANGES_ROOT_URL "gcc-10/changes.html#empty_base";
-	  last_reported_type_uid = uid;
-	  if (empty_base_seen & 1)
-	    inform (input_location,
-		    "parameter passing for argument of type %qT when C++17 "
-		    "is enabled changed to match C++14 %{in GCC 10.1%}",
-		    orig_type, url);
-	  else
-	    inform (input_location,
-		    "parameter passing for argument of type %qT with "
-		    "%<[[no_unique_address]]%> members changed "
-		    "%{in GCC 10.1%}", orig_type, url);
+
+      if (empty_base_seen)
+	{
+	  static unsigned last_reported_type_uid_empty_base;
+	  if (uid != last_reported_type_uid_empty_base)
+	    {
+	      last_reported_type_uid_empty_base = uid;
+	      const char *url = CHANGES_ROOT_URL "gcc-10/changes.html#empty_base";
+	      if (empty_base_seen & 1)
+		inform (input_location,
+			"parameter passing for argument of type %qT when C++17 "
+			"is enabled changed to match C++14 %{in GCC 10.1%}",
+			orig_type, url);
+	      else
+		inform (input_location,
+			"parameter passing for argument of type %qT with "
+			"%<[[no_unique_address]]%> members changed "
+			"%{in GCC 10.1%}", orig_type, url);
+	    }
+	}
+
+      /* For C++ older GCCs ignored zero width bitfields and therefore
+	 passed structs more often as single values than GCC 12 does.
+	 So diagnostics are only required in cases where we do NOT
+	 accept the struct to be passed as single value.  */
+      if (zero_width_bf_skipped_p)
+	{
+	  static unsigned last_reported_type_uid_zero_width;
+	  if (uid != last_reported_type_uid_zero_width)
+	    {
+	      last_reported_type_uid_zero_width = uid;
+	      inform (input_location,
+		      "parameter passing for argument of type %qT with "
+		      "zero-width bit fields members changed in GCC 12",
+		      orig_type);
+	    }
 	}
     }
+
+  return !zero_width_bf_skipped_p;
+}
+
+
+/* Return true if a function argument of type TYPE and mode MODE
+   is to be passed in a vector register, if available.  */
+
+static bool
+s390_function_arg_vector (machine_mode mode, const_tree type)
+{
+  if (!TARGET_VX_ABI)
+    return false;
+
+  if (s390_function_arg_size (mode, type) > 16)
+    return false;
+
+  /* No type info available for some library calls ...  */
+  if (!type)
+    return VECTOR_MODE_P (mode);
+
+  if (!s390_single_field_struct_p (VECTOR_TYPE, type, true))
+    return false;
+
   return true;
 }
 
@@ -12249,64 +12300,9 @@ s390_function_arg_float (machine_mode mode, const_tree type)
   if (!type)
     return mode == SFmode || mode == DFmode || mode == SDmode || mode == DDmode;
 
-  /* The ABI says that record types with a single member are treated
-     just like that member would be.  */
-  int empty_base_seen = 0;
-  const_tree orig_type = type;
-  while (TREE_CODE (type) == RECORD_TYPE)
-    {
-      tree field, single = NULL_TREE;
-
-      for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
-	{
-	  if (TREE_CODE (field) != FIELD_DECL)
-	    continue;
-	  if (DECL_FIELD_ABI_IGNORED (field))
-	    {
-	      if (lookup_attribute ("no_unique_address",
-				    DECL_ATTRIBUTES (field)))
-		empty_base_seen |= 2;
-	      else
-		empty_base_seen |= 1;
-	      continue;
-	    }
-
-	  if (single == NULL_TREE)
-	    single = TREE_TYPE (field);
-	  else
-	    return false;
-	}
-
-      if (single == NULL_TREE)
-	return false;
-      else
-	type = single;
-    }
-
-  if (TREE_CODE (type) != REAL_TYPE)
+  if (!s390_single_field_struct_p (REAL_TYPE, type, false))
     return false;
 
-  if (warn_psabi && empty_base_seen)
-    {
-      static unsigned last_reported_type_uid;
-      unsigned uid = TYPE_UID (TYPE_MAIN_VARIANT (orig_type));
-      if (uid != last_reported_type_uid)
-	{
-	  const char *url = CHANGES_ROOT_URL "gcc-10/changes.html#empty_base";
-	  last_reported_type_uid = uid;
-	  if (empty_base_seen & 1)
-	    inform (input_location,
-		    "parameter passing for argument of type %qT when C++17 "
-		    "is enabled changed to match C++14 %{in GCC 10.1%}",
-		    orig_type, url);
-	  else
-	    inform (input_location,
-		    "parameter passing for argument of type %qT with "
-		    "%<[[no_unique_address]]%> members changed "
-		    "%{in GCC 10.1%}", orig_type, url);
-	}
-    }
-
   return true;
 }
 
@@ -14879,7 +14875,6 @@ s390_get_sched_attrmask (rtx_insn *insn)
 	mask |= S390_SCHED_ATTR_MASK_GROUPOFTWO;
       break;
     case PROCESSOR_8561_Z15:
-    case PROCESSOR_ARCH14:
       if (get_attr_z15_cracked (insn))
 	mask |= S390_SCHED_ATTR_MASK_CRACKED;
       if (get_attr_z15_expanded (insn))
@@ -14891,6 +14886,18 @@ s390_get_sched_attrmask (rtx_insn *insn)
       if (get_attr_z15_groupoftwo (insn))
 	mask |= S390_SCHED_ATTR_MASK_GROUPOFTWO;
       break;
+    case PROCESSOR_3931_Z16:
+      if (get_attr_z16_cracked (insn))
+	mask |= S390_SCHED_ATTR_MASK_CRACKED;
+      if (get_attr_z16_expanded (insn))
+	mask |= S390_SCHED_ATTR_MASK_EXPANDED;
+      if (get_attr_z16_endgroup (insn))
+	mask |= S390_SCHED_ATTR_MASK_ENDGROUP;
+      if (get_attr_z16_groupalone (insn))
+	mask |= S390_SCHED_ATTR_MASK_GROUPALONE;
+      if (get_attr_z16_groupoftwo (insn))
+	mask |= S390_SCHED_ATTR_MASK_GROUPOFTWO;
+      break;
     default:
       gcc_unreachable ();
     }
@@ -14927,7 +14934,6 @@ s390_get_unit_mask (rtx_insn *insn, int *units)
 	mask |= 1 << 3;
       break;
     case PROCESSOR_8561_Z15:
-    case PROCESSOR_ARCH14:
       *units = 4;
       if (get_attr_z15_unit_lsu (insn))
 	mask |= 1 << 0;
@@ -14938,6 +14944,17 @@ s390_get_unit_mask (rtx_insn *insn, int *units)
       if (get_attr_z15_unit_vfu (insn))
 	mask |= 1 << 3;
       break;
+    case PROCESSOR_3931_Z16:
+      *units = 4;
+      if (get_attr_z16_unit_lsu (insn))
+	mask |= 1 << 0;
+      if (get_attr_z16_unit_fxa (insn))
+	mask |= 1 << 1;
+      if (get_attr_z16_unit_fxb (insn))
+	mask |= 1 << 2;
+      if (get_attr_z16_unit_vfu (insn))
+	mask |= 1 << 3;
+      break;
     default:
       gcc_unreachable ();
     }
@@ -14951,7 +14968,7 @@ s390_is_fpd (rtx_insn *insn)
     return false;
 
   return get_attr_z13_unit_fpd (insn) || get_attr_z14_unit_fpd (insn)
-    || get_attr_z15_unit_fpd (insn);
+    || get_attr_z15_unit_fpd (insn) || get_attr_z16_unit_fpd (insn);
 }
 
 static bool
@@ -14961,7 +14978,7 @@ s390_is_fxd (rtx_insn *insn)
     return false;
 
   return get_attr_z13_unit_fxd (insn) || get_attr_z14_unit_fxd (insn)
-    || get_attr_z15_unit_fxd (insn);
+    || get_attr_z15_unit_fxd (insn) || get_attr_z16_unit_fxd (insn);
 }
 
 /* Returns TRUE if INSN is a long-running instruction.  */
diff --git a/gcc/config/s390/s390.h b/gcc/config/s390/s390.h
index 5a64048..2e1bc71 100644
--- a/gcc/config/s390/s390.h
+++ b/gcc/config/s390/s390.h
@@ -43,12 +43,12 @@ enum processor_flags
   PF_VXE2 = 8192,
   PF_Z15 = 16384,
   PF_NNPA = 32768,
-  PF_ARCH14 = 65536
+  PF_Z16 = 65536
 };
 
 /* This is necessary to avoid a warning about comparing different enum
    types.  */
-#define s390_tune_attr ((enum attr_cpu)(s390_tune > PROCESSOR_8561_Z15 ? PROCESSOR_8561_Z15 : s390_tune ))
+#define s390_tune_attr ((enum attr_cpu)(s390_tune > PROCESSOR_3931_Z16 ? PROCESSOR_3931_Z16 : s390_tune ))
 
 /* These flags indicate that the generated code should run on a cpu
    providing the respective hardware facility regardless of the
@@ -110,10 +110,10 @@ enum processor_flags
 	(s390_arch_flags & PF_VXE2)
 #define TARGET_CPU_VXE2_P(opts) \
 	(opts->x_s390_arch_flags & PF_VXE2)
-#define TARGET_CPU_ARCH14 \
-	(s390_arch_flags & PF_ARCH14)
-#define TARGET_CPU_ARCH14_P(opts) \
-	(opts->x_s390_arch_flags & PF_ARCH14)
+#define TARGET_CPU_Z16 \
+	(s390_arch_flags & PF_Z16)
+#define TARGET_CPU_Z16_P(opts) \
+	(opts->x_s390_arch_flags & PF_Z16)
 #define TARGET_CPU_NNPA \
 	(s390_arch_flags & PF_NNPA)
 #define TARGET_CPU_NNPA_P(opts) \
@@ -177,9 +177,9 @@ enum processor_flags
 	(TARGET_VX && TARGET_CPU_VXE2)
 #define TARGET_VXE2_P(opts)						\
 	(TARGET_VX_P (opts) && TARGET_CPU_VXE2_P (opts))
-#define TARGET_ARCH14 (TARGET_ZARCH && TARGET_CPU_ARCH14)
-#define TARGET_ARCH14_P(opts)						\
-	(TARGET_ZARCH_P (opts->x_target_flags) && TARGET_CPU_ARCH14_P (opts))
+#define TARGET_Z16 (TARGET_ZARCH && TARGET_CPU_Z16)
+#define TARGET_Z16_P(opts)						\
+	(TARGET_ZARCH_P (opts->x_target_flags) && TARGET_CPU_Z16_P (opts))
 #define TARGET_NNPA					\
 	(TARGET_ZARCH && TARGET_CPU_NNPA)
 #define TARGET_NNPA_P(opts)						\
diff --git a/gcc/config/s390/s390.md b/gcc/config/s390/s390.md
index d0f233e..55c0064 100644
--- a/gcc/config/s390/s390.md
+++ b/gcc/config/s390/s390.md
@@ -518,11 +518,11 @@
 ;; Processor type.  This attribute must exactly match the processor_type
 ;; enumeration in s390.h.
 
-(define_attr "cpu" "z900,z990,z9_109,z9_ec,z10,z196,zEC12,z13,z14,z15"
+(define_attr "cpu" "z900,z990,z9_109,z9_ec,z10,z196,zEC12,z13,z14,z15,z16"
   (const (symbol_ref "s390_tune_attr")))
 
 (define_attr "cpu_facility"
-  "standard,ieee,zarch,cpu_zarch,longdisp,extimm,dfp,z10,z196,zEC12,vx,z13,z14,vxe,z15,vxe2,arch14,nnpa"
+  "standard,ieee,zarch,cpu_zarch,longdisp,extimm,dfp,z10,z196,zEC12,vx,z13,z14,vxe,z15,vxe2,z16,nnpa"
   (const_string "standard"))
 
 (define_attr "enabled" ""
@@ -588,8 +588,8 @@
 	      (match_test "TARGET_VXE2"))
 	 (const_int 1)
 
-	 (and (eq_attr "cpu_facility" "arch14")
-	      (match_test "TARGET_ARCH14"))
+	 (and (eq_attr "cpu_facility" "z16")
+	      (match_test "TARGET_Z16"))
 	 (const_int 1)
 
          (and (eq_attr "cpu_facility" "nnpa")
@@ -629,6 +629,9 @@
 ;; Pipeline description for z15
 (include "8561.md")
 
+;; Pipeline description for z16
+(include "3931.md")
+
 ;; Predicates
 (include "predicates.md")
 
diff --git a/gcc/config/s390/s390.opt b/gcc/config/s390/s390.opt
index 5068486..9e8d3bf 100644
--- a/gcc/config/s390/s390.opt
+++ b/gcc/config/s390/s390.opt
@@ -116,7 +116,10 @@ EnumValue
 Enum(processor_type) String(arch13) Value(PROCESSOR_8561_Z15)
 
 EnumValue
-Enum(processor_type) String(arch14) Value(PROCESSOR_ARCH14)
+Enum(processor_type) String(arch14) Value(PROCESSOR_3931_Z16)
+
+EnumValue
+Enum(processor_type) String(z16) Value(PROCESSOR_3931_Z16)
 
 EnumValue
 Enum(processor_type) String(native) Value(PROCESSOR_NATIVE) DriverOnly
diff --git a/gcc/config/sparc/sparc.cc b/gcc/config/sparc/sparc.cc
index bb4ce88..467a9f1 100644
--- a/gcc/config/sparc/sparc.cc
+++ b/gcc/config/sparc/sparc.cc
@@ -8884,8 +8884,20 @@ epilogue_renumber (rtx *where, int test)
       if (REGNO (*where) >= 8 && REGNO (*where) < 24)      /* oX or lX */
 	return 1;
       if (! test && REGNO (*where) >= 24 && REGNO (*where) < 32)
-	*where = gen_rtx_REG (GET_MODE (*where), OUTGOING_REGNO (REGNO(*where)));
-      /* fallthrough */
+	{
+	  if (ORIGINAL_REGNO (*where))
+	    {
+	      rtx n = gen_raw_REG (GET_MODE (*where),
+				   OUTGOING_REGNO (REGNO (*where)));
+	      ORIGINAL_REGNO (n) = ORIGINAL_REGNO (*where);
+	      *where = n;
+	    }
+	  else
+	    *where = gen_rtx_REG (GET_MODE (*where),
+				  OUTGOING_REGNO (REGNO (*where)));
+	}
+      return 0;
+
     case SCRATCH:
     case PC:
     case CONST_INT: