diff options
Diffstat (limited to 'gcc')
62 files changed, 2709 insertions, 1932 deletions
diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 19c28f1..3df799b 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,254 @@ +2025-10-15 Andrew MacLeod <amacleod@redhat.com> + + PR tree-optimization/121468 + PR tree-optimization/121206 + PR tree-optimization/122200 + * value-range.cc (irange_bitmask::range_from_mask): New. + (irange::snap): Add explicit overflow flag. + (irange::snap_subranges): Use overflow flag. + (irange::set_range_from_bitmask): Use range_from_mask. + (test_irange_snap_bounds): Adjust for improved ranges. + * value-range.h (irange::range_from_mask): Add prototype. + (irange::snap): Adjust prototype. + +2025-10-15 Tobias Burnus <tburnus@baylibre.com> + + * config/gcn/gcn-devices.def (gfx942, gfx950): Set generic name + to GFX9_4_GENERIC. + * config/gcn/t-omp-device: Include generic names for OpenMP's + ISA trait. + +2025-10-15 Andrew Pinski <andrew.pinski@oss.qualcomm.com> + + * print-tree.cc (print_node): Print out clique/base + for MEM_REF and TARGET_MEM_REF. + +2025-10-15 Richard Earnshaw <rearnsha@arm.com> + + PR target/118460 + * config/arm/arm.cc (arm_canonicalize_comparison): For floating- + point comparisons, swap the operand order if that will be more + likely to produce a comparison that can be used with VSEL. + (arm_validize_comparison): Make sure that HFmode comparisons + are compatible with VSEL. + +2025-10-15 Andrew Pinski <andrew.pinski@oss.qualcomm.com> + + PR tree-optimization/122037 + * tree-ssa-dce.cc (eliminate_unnecessary_stmts): Remove + __builtin_stack_save when the lhs is unused. + +2025-10-15 Alice Carlotti <alice.carlotti@arm.com> + + * config/aarch64/aarch64-sys-regs.def: Copy from Binutils. + * config/aarch64/aarch64.cc (F_ARCHEXT): Delete flag. + * config/aarch64/aarch64.h + (AARCH64_FL_AMU): Delete unused macro. + (AARCH64_FL_SCXTNUM): Ditto. + (AARCH64_FL_ID_PFR2): Ditto. + (AARCH64_FL_AIE): Ditto. + (AARCH64_FL_DEBUGv8p9): Ditto. + (AARCH64_FL_FGT2): Ditto. + (AARCH64_FL_PFAR): Ditto. + (AARCH64_FL_PMUv3_ICNTR): Ditto. + (AARCH64_FL_PMUv3_SS): Ditto. + (AARCH64_FL_PMUv3p9): Ditto. + (AARCH64_FL_S1PIE): Ditto. + (AARCH64_FL_S1POE): Ditto. + (AARCH64_FL_S2PIE): Ditto. + (AARCH64_FL_S2POE): Ditto. + (AARCH64_FL_SCTLR2): Ditto. + (AARCH64_FL_SEBEP): Ditto. + (AARCH64_FL_SPE_FDS): Ditto. + (AARCH64_FL_TCR2): Ditto. + +2025-10-15 Sebastian Pop <spop@nvidia.com> + + * doc/invoke.texi (ftree-parallelize-loops): Update. + * common.opt (ftree-parallelize-loops): Add alias that maps to + special value INT_MAX for runtime thread detection. + * tree-parloops.cc (create_parallel_loop): Use INT_MAX for runtime + detection. Call gimple_build_omp_parallel without building a + OMP_CLAUSE_NUM_THREADS clause. + (gen_parallel_loop): For auto-detection, use a conservative + estimate of 2 threads. + (parallelize_loops): Same. + +2025-10-15 Christophe Lyon <christophe.lyon@linaro.org> + + PR target/122189 + * config/arm/iterators.md (VxCIQ_carry, VxCIQ_M_carry, VxCQ_carry) + (VxCQ_M_carry): New iterators. + * config/arm/mve.md (get_fpscr_nzcvqc, set_fpscr_nzcvqc): Use + unspec instead of unspec_volatile. + (vadciq, vadciq_m, vadcq, vadcq_m): Use vfpcc in operation. Use a + different unspec code for carry calcultation. + * config/arm/unspecs.md (VADCQ_U_carry, VADCQ_M_U_carry) + (VADCQ_S_carry, VADCQ_M_S_carry, VSBCIQ_U_carry ,VSBCIQ_S_carry + ,VSBCIQ_M_U_carry ,VSBCIQ_M_S_carry ,VSBCQ_U_carry ,VSBCQ_S_carry + ,VSBCQ_M_U_carry ,VSBCQ_M_S_carry ,VADCIQ_U_carry + ,VADCIQ_M_U_carry ,VADCIQ_S_carry ,VADCIQ_M_S_carry): New unspec + codes. + +2025-10-15 Roger Sayle <roger@nextmovesoftware.com> + + PR rtl-optimization/122266 + * combine.cc (struct reg_stat_type): Change types of sign_bit_copies + and last_set_sign_bit_copies to unsigned short, to avoid overflows + on TImode (and wider) values. + +2025-10-15 Jan Hubicka <hubicka@ucw.cz> + + * auto-profile.cc (scale_bb_profile): Use + profile_count::max_prefer_initialized. + (afdo_adjust_guessed_profile): Likewise. + * bb-reorder.cc (edge_order): Do not use max. + * cfghooks.cc (merge_blocks): Likewise. + * ipa-fnsummary.cc (param_change_prob): Likewise. + * ipa-inline-transform.cc (inline_transform): Likewise. + * predict.cc (update_max_bb_count): Likewise. + (estimate_bb_frequencies): Likewise. + (rebuild_frequencies): Likewise. + * tree-ssa-loop-unswitch.cc (struct unswitch_predicate): Likewise. + * profile-count.h (profile_count::max): Rename to + (profile_count::max_prefer_initialized): this; update handling + of qualities. + +2025-10-15 Haochen Jiang <haochen.jiang@intel.com> + + * common/config/i386/cpuinfo.h + (get_intel_cpu): Handle Wildcat Lake. + * common/config/i386/i386-common.cc (processor_name): + Add Wildcat Lake. + * doc/invoke.texi: Ditto. + +2025-10-15 Haochen Jiang <haochen.jiang@intel.com> + + * config/i386/i386.h + (PTA_PANTHERLAKE): Remove PREFETCHI. + (PTA_DIAMONDRAPIDS): Remove USER_MSR. + * doc/invoke.texi: Correct documentation. + +2025-10-15 Pan Li <pan2.li@intel.com> + + * config/riscv/autovec-opt.md: Take concrete op instead + of any_widen_binop for vwaddu/vwsubu wx combine. + +2025-10-14 Richard Biener <rguenther@suse.de> + + * tree-vectorizer.h (REDUC_GROUP_FIRST_ELEMENT, + REDUC_GROUP_NEXT_ELEMENT, REDUC_GROUP_SIZE): Remove. + * tree-vect-slp.cc (REDUC_GROUP_FIRST_ELEMENT): Re-instantiate + here. + +2025-10-14 Richard Biener <rguenther@suse.de> + + * tree-vect-slp.cc (vect_analyze_slp_reduction): Move + reduction chain discovery ... + (vect_analyze_slp_reduc_chain): ... here. + +2025-10-14 Richard Biener <rguenther@suse.de> + + * tree-vect-loop.cc (vect_create_epilog_for_reduction): Move + bitsize compute down to where it is used and consistently + use vectype1 for element extraction. + +2025-10-14 Tamar Christina <tamar.christina@arm.com> + + PR tree-optimization/121949 + * tree-vect-patterns.cc (vect_recog_vector_vector_shift_pattern): Remove + restriction on internal_def. + +2025-10-14 Robin Dapp <rdapp@ventanamicro.com> + + * tree-vect-stmts.cc (get_load_store_type): Add load-permutation + checks and setting of slp_perm. + (vectorizable_store): Remove perm_ok argument. + (vectorizable_load): Ditto and replace slp_perm by ls.slp_perm. + * tree-vectorizer.h (struct vect_load_store_data): Add slp_perm. + +2025-10-14 Richard Biener <rguenther@suse.de> + + * tree-vectorizer.h (vect_reduc_info_s::is_reduc_chain): New. + (_loop_vec_info::reduction_chains): Remove. + (LOOP_VINFO_REDUCTION_CHAINS): Likewise. + * tree-vect-patterns.cc (vect_reassociating_reduction_p): + Do not special-case reduction group stmts. + * tree-vect-loop.cc (vect_is_simple_reduction): Remove + reduction chain handling. + (vect_analyze_scalar_cycles_1): Remove slp parameter and adjust. + (vect_analyze_scalar_cycles): Likewise. + (vect_fixup_reduc_chain): Remove. + (vect_fixup_scalar_cycles_with_patterns): Likewise. + (vect_analyze_loop_2): Adjust. + (vect_create_epilog_for_reduction): Check the reduction info + for whether this is a reduction chain. + (vect_transform_cycle_phi): Likewise. + (vectorizable_reduction): Likewise. Simplify code for all-SLP. + * tree-vect-slp.cc (vect_analyze_slp_reduc_chain): Simplify. + (vect_analyze_slp_reduction): New function, perform reduction + chain discovery here. + (vect_analyze_slp): Remove reduction chain handling. + Use vect_analyze_slp_reduction for possible reduction chain + processing. + +2025-10-14 Haochen Jiang <haochen.jiang@intel.com> + + * common/config/i386/cpuinfo.h + (get_available_features): Remove AMX-TRANSPOSE. + * common/config/i386/i386-common.cc + (OPTION_MASK_ISA2_AMX_TRANSPOSE_SET): Removed. + (OPTION_MASK_ISA2_AMX_TRANSPOSE_UNSET): Ditto. + (ix86_handle_option): Remove amx-transpose handle. + * common/config/i386/i386-cpuinfo.h + (enum processor_features): Remove FEATURE_AMX_TRANSPOSE. + Set FEATURE_AMX_MOVRS value. + * common/config/i386/i386-isas.h: Remove AMX-TRANSPOSE. + * config.gcc: Do not include amxtransposeintrin.h. + * config/i386/amxmovrsintrin.h: Remove AMX-TRANSPOSE intrins. + * config/i386/amxtransposeintrin.h: Ditto. + * config/i386/cpuid.h (bit_AMX_TRANSPOSE): Removed. + * config/i386/i386.h (PTA_DIAMONDRAPIDS): Remove AMX-TRANSPOSE. + * config/i386/i386-c.cc (ix86_target_macros_internal): Remove + AMX_TRANSPOSE. + * config/i386/i386-isa.def (AMX_TRANSPOSE): Removed. + * config/i386/i386-options.cc + (ix86_valid_target_attribute_inner_p): Remove AMX-TRANSPOSE. + * config/i386/i386.opt: Ditto. + * config/i386/i386.opt.urls: Ditto. + * config/i386/immintrin.h: Remove amxtransposeintrin.h. + * doc/extend.texi: Remove amx-transpose. + * doc/invoke.texi: Ditto. + * doc/sourcebuild.texi: Ditto. + +2025-10-14 Andrew Pinski <andrew.pinski@oss.qualcomm.com> + + * tree-ssa-phiopt.cc (pass_phiopt::execute): Disable + cselim-limited and factor out operations for -Og. + +2025-10-14 Andrew Pinski <andrew.pinski@oss.qualcomm.com> + + PR tree-optimization/122178 + * tree-ssa-phiopt.cc (cond_if_else_store_replacement_1): Handle + clobber statements. + +2025-10-14 Andrew Pinski <andrew.pinski@oss.qualcomm.com> + + PR tree-optimization/122182 + * tree-ssa-dom.cc (cprop_operand): Don't check may_propagate_copy_into_asm. + * tree-ssa-propagate.cc (substitute_and_fold_engine::replace_uses_in): Don't + check may_propagate_copy_into_asm. + (may_propagate_copy_into_asm): Remove. + * tree-ssa-propagate.h (may_propagate_copy_into_asm): Remove. + +2025-10-14 Zhongyao Chen <chenzhongyao.hit@gmail.com> + + * common/config/riscv/riscv-common.cc (riscv_subset_list::get_profile_name): + New function. + * config/riscv/riscv-c.cc (riscv_cpu_cpp_builtins): Define + profile macro if a profile is detected. + * config/riscv/riscv-subset.h (riscv_subset_list::get_profile_name): Declare. + 2025-10-13 Shreya Munnangi <smunnangi1@ventanamicro.com> PR target/120811 diff --git a/gcc/DATESTAMP b/gcc/DATESTAMP index 75df97b..8ca925d 100644 --- a/gcc/DATESTAMP +++ b/gcc/DATESTAMP @@ -1 +1 @@ -20251014 +20251016 diff --git a/gcc/auto-profile.cc b/gcc/auto-profile.cc index 6971204..cf7a219 100644 --- a/gcc/auto-profile.cc +++ b/gcc/auto-profile.cc @@ -3600,11 +3600,12 @@ scale_bb_profile () { profile_count cnt = bb->count; bbs.safe_push (bb); - max_count = max_count.max (bb->count); + max_count = profile_count::max_prefer_initialized (max_count, cnt); if (afdo_set_bb_count (bb, zero_bbs)) { std::swap (cnt, bb->count); - max_count_in_fn = max_count_in_fn.max (cnt); + max_count_in_fn + = profile_count::max_prefer_initialized (max_count_in_fn, cnt); add_scale (&scales, cnt, bb->count); } } @@ -3646,7 +3647,8 @@ afdo_adjust_guessed_profile (bb_set *annotated_bb) if (is_bb_annotated (seed_bb, *annotated_bb)) { component[seed_bb->index] = 1; - max_count_in_fn = max_count_in_fn.max (seed_bb->count); + max_count_in_fn + = profile_count::max_prefer_initialized (max_count_in_fn, seed_bb->count); } else component[seed_bb->index] = 0; @@ -3669,7 +3671,7 @@ afdo_adjust_guessed_profile (bb_set *annotated_bb) basic_block b = stack.pop (); bbs.quick_push (b); - max_count = max_count.max (b->count); + max_count = profile_count::max_prefer_initialized (max_count, b->count); for (edge e: b->preds) if (!component[e->src->index]) diff --git a/gcc/bb-reorder.cc b/gcc/bb-reorder.cc index 641b492..e4efdee 100644 --- a/gcc/bb-reorder.cc +++ b/gcc/bb-reorder.cc @@ -2389,8 +2389,10 @@ edge_order (const void *ve1, const void *ve2) /* Since profile_count::operator< does not establish a strict weak order in presence of uninitialized counts, use 'max': this makes them appear as if having execution frequency less than any initialized count. */ - profile_count m = c1.max (c2); - return (m == c2) - (m == c1); + gcov_type gc1 = c1.initialized_p () ? c1.to_gcov_type () : 0; + gcov_type gc2 = c2.initialized_p () ? c2.to_gcov_type () : 0; + gcov_type m = MAX (gc1, gc2); + return (m == gc1) - (m == gc2); } /* Reorder basic blocks using the "simple" algorithm. This tries to diff --git a/gcc/c-family/ChangeLog b/gcc/c-family/ChangeLog index 07ea6aa..16cdcf3 100644 --- a/gcc/c-family/ChangeLog +++ b/gcc/c-family/ChangeLog @@ -1,3 +1,19 @@ +2025-10-14 Jakub Jelinek <jakub@redhat.com> + + * c.opt (Wflex-array-member-not-at-end, Wignored-qualifiers, + Wopenacc-parallelism, Wstrict-flex-arrays, Wsync-nand, + fstrict-flex-arrays, fstrict-flex-arrays=): Enable also for ObjC and + ObjC++ next to C and C++. + (Wmisleading-indentation, Wopenmp-simd): Likewise. Also change + LangEnabledBy from just C C++ to C ObjC C++ ObjC++. + (Wplacement-new, Wplacement-new=, fcontract-assumption-mode=, + fcontract-build-level=, fcontract-strict-declarations=, + fcontract-mode=, fcontract-continuation-mode=, fcontract-role=, + fcontract-semantic=, fcoroutines, flang-info-include-translate, + flang-info-include-translate-not, flang-info-include-translate=, + flang-info-module-cmi, flang-info-module-cmi=): Enable also + for ObjC++ next to C++. + 2025-10-13 Iain Sandoe <iain@sandoe.co.uk> * c.opt: Enable Wignored-attributes for Objective-C and diff --git a/gcc/c-family/c.opt b/gcc/c-family/c.opt index a7fd14a..b7ce67a 100644 --- a/gcc/c-family/c.opt +++ b/gcc/c-family/c.opt @@ -774,7 +774,7 @@ C++ ObjC++ Var(warn_extra_semi) Init(-1) Warning Warn about semicolon after in-class function definition. Wflex-array-member-not-at-end -C C++ Var(warn_flex_array_member_not_at_end) Warning +C ObjC C++ ObjC++ Var(warn_flex_array_member_not_at_end) Warning Warn when a structure containing a C99 flexible array member as the last field is not at the end of another structure. @@ -866,7 +866,7 @@ C ObjC C++ ObjC++ Var(warn_if_not_aligned) Init(1) Warning Warn when the field in a struct is not aligned. Wignored-qualifiers -C C++ Var(warn_ignored_qualifiers) Warning EnabledBy(Wextra) +C ObjC C++ ObjC++ Var(warn_ignored_qualifiers) Warning EnabledBy(Wextra) Warn whenever type qualifiers are ignored. Wignored-attributes @@ -1013,7 +1013,7 @@ C ObjC C++ ObjC++ Var(warn_memset_transposed_args) Warning LangEnabledBy(C ObjC Warn about suspicious calls to memset where the third argument is constant literal zero and the second is not. Wmisleading-indentation -C C++ Common Var(warn_misleading_indentation) Warning LangEnabledBy(C C++,Wall) +C ObjC C++ ObjC++ Common Var(warn_misleading_indentation) Warning LangEnabledBy(C ObjC C++ ObjC++,Wall) Warn when the indentation of the code does not reflect the block structure. Wmismatched-dealloc @@ -1187,7 +1187,7 @@ C ObjC Var(warn_old_style_definition) Init(-1) Warning Warn if an old-style parameter definition is used. Wopenacc-parallelism -C C++ Var(warn_openacc_parallelism) Warning +C ObjC C++ ObjC++ Var(warn_openacc_parallelism) Warning Warn about potentially suboptimal choices related to OpenACC parallelism. Wopenmp @@ -1195,7 +1195,7 @@ C ObjC C++ ObjC++ Warning Var(warn_openmp) Init(1) Warn about suspicious OpenMP code. Wopenmp-simd -C C++ Var(warn_openmp_simd) Warning LangEnabledBy(C C++,Wall) +C ObjC C++ ObjC++ Var(warn_openmp_simd) Warning LangEnabledBy(C ObjC C++ ObjC++,Wall) Warn if a simd directive is overridden by the vectorizer cost model. Woverlength-strings @@ -1243,11 +1243,11 @@ C++ ObjC++ Var(warn_pessimizing_move) Warning LangEnabledBy(C++ ObjC++, Wall) Warn about calling std::move on a local object in a return statement preventing copy elision. Wplacement-new -C++ Warning Alias(Wplacement-new=, 1, 0) +C++ ObjC++ Warning Alias(Wplacement-new=, 1, 0) Warn for placement new expressions with undefined behavior. Wplacement-new= -C++ Joined RejectNegative UInteger Var(warn_placement_new) Init(-1) Warning IntegerRange(0, 2) +C++ ObjC++ Joined RejectNegative UInteger Var(warn_placement_new) Init(-1) Warning IntegerRange(0, 2) Warn for placement new expressions with undefined behavior. Wpmf-conversions @@ -1417,7 +1417,7 @@ C ObjC C++ ObjC++ LangEnabledBy(C ObjC C++ ObjC++,Wall, 3, 0) IntegerRange(0, 3) ; Wstrict-flex-arrays -C C++ Var(warn_strict_flex_arrays) Warning +C ObjC C++ ObjC++ Var(warn_strict_flex_arrays) Warning Warn about improper usages of flexible array members according to the level of -fstrict-flex-arrays. @@ -1495,7 +1495,7 @@ C ObjC C++ ObjC++ Var(warn_switch_outside_range) Warning Init(1) Warn about switch values that are outside of the switch's type range. Wsync-nand -C C++ Var(warn_sync_nand) Init(1) Warning +C ObjC C++ ObjC++ Var(warn_sync_nand) Init(1) Warning Warn when __sync_fetch_and_nand and __sync_nand_and_fetch built-in functions are used. Wsynth @@ -1900,35 +1900,35 @@ EnumValue Enum(on_off) String(on) Value(1) fcontract-assumption-mode= -C++ Joined RejectNegative +C++ ObjC++ Joined RejectNegative -fcontract-assumption-mode=[on|off] Enable or disable treating axiom level contracts as assumptions (default on). fcontract-build-level= -C++ Joined RejectNegative +C++ ObjC++ Joined RejectNegative -fcontract-build-level=[off|default|audit] Specify max contract level to generate runtime checks for. fcontract-strict-declarations= -C++ Var(flag_contract_strict_declarations) Enum(on_off) Joined Init(0) RejectNegative +C++ ObjC++ Var(flag_contract_strict_declarations) Enum(on_off) Joined Init(0) RejectNegative -fcontract-strict-declarations=[on|off] Enable or disable warnings on generalized redeclaration of functions with contracts (default off). fcontract-mode= -C++ Var(flag_contract_mode) Enum(on_off) Joined Init(1) RejectNegative +C++ ObjC++ Var(flag_contract_mode) Enum(on_off) Joined Init(1) RejectNegative -fcontract-mode=[on|off] Enable or disable all contract facilities (default on). fcontract-continuation-mode= -C++ Joined RejectNegative +C++ ObjC++ Joined RejectNegative -fcontract-continuation-mode=[on|off] Enable or disable contract continuation mode (default off). fcontract-role= -C++ Joined RejectNegative +C++ ObjC++ Joined RejectNegative -fcontract-role=<name>:<semantics> Specify the semantics for all levels in a role (default, review), or a custom contract role with given semantics (ex: opt:assume,assume,assume). fcontract-semantic= -C++ Joined RejectNegative +C++ ObjC++ Joined RejectNegative -fcontract-semantic=<level>:<semantic> Specify the concrete semantics for level. fcoroutines -C++ LTO Var(flag_coroutines) +C++ ObjC++ LTO Var(flag_coroutines) Enable C++ coroutines (experimental). fdebug-cpp @@ -2130,23 +2130,23 @@ C ObjC Var(warn_compare_distinct_pointer_types) Warning Init(1) Warn if pointers of distinct types are compared without a cast. flang-info-include-translate -C++ Var(note_include_translate_yes) +C++ ObjC++ Var(note_include_translate_yes) Note #include directives translated to import declarations. flang-info-include-translate-not -C++ Var(note_include_translate_no) +C++ ObjC++ Var(note_include_translate_no) Note #include directives not translated to import declarations, and not known to be textual. flang-info-include-translate= -C++ Joined RejectNegative MissingArgError(missing header name) +C++ ObjC++ Joined RejectNegative MissingArgError(missing header name) Note a #include translation of a specific header. flang-info-module-cmi -C++ Var(note_module_cmi_yes) +C++ ObjC++ Var(note_module_cmi_yes) Note Compiled Module Interface pathnames. flang-info-module-cmi= -C++ Joined RejectNegative MissingArgError(missing module name) +C++ ObjC++ Joined RejectNegative MissingArgError(missing module name) Note Compiled Module Interface pathname of a specific module or header-unit. fmax-include-depth= @@ -2357,10 +2357,10 @@ C++ ObjC++ Var(flag_sized_deallocation) Init(-1) Enable C++14 sized deallocation support. fstrict-flex-arrays -C C++ Common Alias(fstrict-flex-arrays=,3,0) +C ObjC C++ ObjC++ Common Alias(fstrict-flex-arrays=,3,0) fstrict-flex-arrays= -C C++ Common Joined RejectNegative UInteger Var(flag_strict_flex_arrays) Init(0) IntegerRange(0,3) +C ObjC C++ ObjC++ Common Joined RejectNegative UInteger Var(flag_strict_flex_arrays) Init(0) IntegerRange(0,3) -fstrict-flex-arrays=<level> Control when to treat the trailing array of a structure as a flexible array member for the purposes of accessing the elements of such an array. The default is treating all trailing arrays of structures as flexible array members. fsquangle diff --git a/gcc/cfghooks.cc b/gcc/cfghooks.cc index 25bc5d4..01169e2 100644 --- a/gcc/cfghooks.cc +++ b/gcc/cfghooks.cc @@ -824,7 +824,7 @@ merge_blocks (basic_block a, basic_block b) else if (a->count.quality () < b->count.quality ()) a->count = b->count; else if (a->count.quality () == b->count.quality ()) - a->count = a->count.max (b->count); + a->count = profile_count::max_prefer_initialized (a->count, b->count); cfg_hooks->merge_blocks (a, b); diff --git a/gcc/combine.cc b/gcc/combine.cc index 4dbc1f6..6696322 100644 --- a/gcc/combine.cc +++ b/gcc/combine.cc @@ -195,7 +195,7 @@ struct reg_stat_type { sign bits copies it was known to have when it was last set. */ unsigned HOST_WIDE_INT last_set_nonzero_bits; - char last_set_sign_bit_copies; + unsigned short last_set_sign_bit_copies; ENUM_BITFIELD(machine_mode) last_set_mode : MACHINE_MODE_BITSIZE; /* Set to true if references to register n in expressions should not be @@ -216,7 +216,7 @@ struct reg_stat_type { If an entry is zero, it means that we don't know anything special. */ - unsigned char sign_bit_copies; + unsigned short sign_bit_copies; unsigned HOST_WIDE_INT nonzero_bits; diff --git a/gcc/common.opt b/gcc/common.opt index 6c993a8..9b8fbf6 100644 --- a/gcc/common.opt +++ b/gcc/common.opt @@ -3303,6 +3303,10 @@ ftree-parallelize-loops= Common Joined RejectNegative UInteger Var(flag_tree_parallelize_loops) Init(1) Optimization -ftree-parallelize-loops=<number> Enable automatic parallelization of loops. +ftree-parallelize-loops +Common Alias(ftree-parallelize-loops=,2147483647,1) +Enable automatic parallelization of loops. + ftree-phiprop Common Var(flag_tree_phiprop) Init(1) Optimization Enable hoisting loads from conditional pointers. diff --git a/gcc/common/config/i386/cpuinfo.h b/gcc/common/config/i386/cpuinfo.h index 4efa2c0..9c18c04 100644 --- a/gcc/common/config/i386/cpuinfo.h +++ b/gcc/common/config/i386/cpuinfo.h @@ -628,6 +628,8 @@ get_intel_cpu (struct __processor_model *cpu_model, break; case 0xcc: /* Panther Lake. */ + case 0xd5: + /* Wildcat Lake. */ cpu = "pantherlake"; CHECK___builtin_cpu_is ("corei7"); CHECK___builtin_cpu_is ("pantherlake"); diff --git a/gcc/common/config/i386/i386-common.cc b/gcc/common/config/i386/i386-common.cc index d3509e1..c71f2c1 100644 --- a/gcc/common/config/i386/i386-common.cc +++ b/gcc/common/config/i386/i386-common.cc @@ -2270,6 +2270,8 @@ const pta processor_alias_table[] = M_CPU_SUBTYPE (INTEL_COREI7_PANTHERLAKE), P_PROC_AVX2}, {"diamondrapids", PROCESSOR_DIAMONDRAPIDS, CPU_HASWELL, PTA_DIAMONDRAPIDS, M_CPU_SUBTYPE (INTEL_COREI7_DIAMONDRAPIDS), P_PROC_AVX10_1}, + {"wildcatlake", PROCESSOR_PANTHERLAKE, CPU_HASWELL, PTA_PANTHERLAKE, + M_CPU_SUBTYPE (INTEL_COREI7_PANTHERLAKE), P_PROC_AVX2}, {"bonnell", PROCESSOR_BONNELL, CPU_ATOM, PTA_BONNELL, M_CPU_TYPE (INTEL_BONNELL), P_PROC_SSSE3}, {"atom", PROCESSOR_BONNELL, CPU_ATOM, PTA_BONNELL, diff --git a/gcc/config/aarch64/aarch64-sys-regs.def b/gcc/config/aarch64/aarch64-sys-regs.def index d7ef6da..51aab23 100644 --- a/gcc/config/aarch64/aarch64-sys-regs.def +++ b/gcc/config/aarch64/aarch64-sys-regs.def @@ -36,1127 +36,1254 @@ The FEATURES field maps onto ISA flags and specifies the architectural feature requirements of the system register. */ - SYSREG ("accdata_el1", CPENC (3,0,13,0,5), 0, AARCH64_NO_FEATURES) - SYSREG ("actlr_el1", CPENC (3,0,1,0,1), 0, AARCH64_NO_FEATURES) - SYSREG ("actlr_el2", CPENC (3,4,1,0,1), 0, AARCH64_NO_FEATURES) - SYSREG ("actlr_el3", CPENC (3,6,1,0,1), 0, AARCH64_NO_FEATURES) - SYSREG ("afsr0_el1", CPENC (3,0,5,1,0), 0, AARCH64_NO_FEATURES) - SYSREG ("afsr0_el12", CPENC (3,5,5,1,0), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("afsr0_el2", CPENC (3,4,5,1,0), 0, AARCH64_NO_FEATURES) - SYSREG ("afsr0_el3", CPENC (3,6,5,1,0), 0, AARCH64_NO_FEATURES) - SYSREG ("afsr1_el1", CPENC (3,0,5,1,1), 0, AARCH64_NO_FEATURES) - SYSREG ("afsr1_el12", CPENC (3,5,5,1,1), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("afsr1_el2", CPENC (3,4,5,1,1), 0, AARCH64_NO_FEATURES) - SYSREG ("afsr1_el3", CPENC (3,6,5,1,1), 0, AARCH64_NO_FEATURES) - SYSREG ("aidr_el1", CPENC (3,1,0,0,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("allint", CPENC (3,0,4,3,0), F_ARCHEXT, AARCH64_FEATURE (V8_8A)) - SYSREG ("amair_el1", CPENC (3,0,10,3,0), 0, AARCH64_NO_FEATURES) - SYSREG ("amair_el12", CPENC (3,5,10,3,0), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("amair_el2", CPENC (3,4,10,3,0), 0, AARCH64_NO_FEATURES) - SYSREG ("amair_el3", CPENC (3,6,10,3,0), 0, AARCH64_NO_FEATURES) - SYSREG ("amair2_el1", CPENC (3,0,10,3,1), F_ARCHEXT, AARCH64_FEATURE (AIE)) - SYSREG ("amair2_el12", CPENC (3,5,10,3,1), F_ARCHEXT, AARCH64_FEATURE (AIE)) - SYSREG ("amair2_el2", CPENC (3,4,10,3,1), F_ARCHEXT, AARCH64_FEATURE (AIE)) - SYSREG ("amair2_el3", CPENC (3,6,10,3,1), F_ARCHEXT, AARCH64_FEATURE (AIE)) - SYSREG ("amcfgr_el0", CPENC (3,3,13,2,1), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amcg1idr_el0", CPENC (3,3,13,2,6), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amcgcr_el0", CPENC (3,3,13,2,2), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amcntenclr0_el0", CPENC (3,3,13,2,4), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amcntenclr1_el0", CPENC (3,3,13,3,0), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amcntenset0_el0", CPENC (3,3,13,2,5), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amcntenset1_el0", CPENC (3,3,13,3,1), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amcr_el0", CPENC (3,3,13,2,0), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr00_el0", CPENC (3,3,13,4,0), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr01_el0", CPENC (3,3,13,4,1), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr02_el0", CPENC (3,3,13,4,2), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr03_el0", CPENC (3,3,13,4,3), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr10_el0", CPENC (3,3,13,12,0), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr110_el0", CPENC (3,3,13,13,2), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr111_el0", CPENC (3,3,13,13,3), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr112_el0", CPENC (3,3,13,13,4), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr113_el0", CPENC (3,3,13,13,5), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr114_el0", CPENC (3,3,13,13,6), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr115_el0", CPENC (3,3,13,13,7), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr11_el0", CPENC (3,3,13,12,1), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr12_el0", CPENC (3,3,13,12,2), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr13_el0", CPENC (3,3,13,12,3), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr14_el0", CPENC (3,3,13,12,4), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr15_el0", CPENC (3,3,13,12,5), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr16_el0", CPENC (3,3,13,12,6), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr17_el0", CPENC (3,3,13,12,7), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr18_el0", CPENC (3,3,13,13,0), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntr19_el0", CPENC (3,3,13,13,1), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevcntvoff00_el2", CPENC (3,4,13,8,0), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff010_el2", CPENC (3,4,13,9,2), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff011_el2", CPENC (3,4,13,9,3), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff012_el2", CPENC (3,4,13,9,4), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff013_el2", CPENC (3,4,13,9,5), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff014_el2", CPENC (3,4,13,9,6), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff015_el2", CPENC (3,4,13,9,7), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff01_el2", CPENC (3,4,13,8,1), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff02_el2", CPENC (3,4,13,8,2), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff03_el2", CPENC (3,4,13,8,3), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff04_el2", CPENC (3,4,13,8,4), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff05_el2", CPENC (3,4,13,8,5), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff06_el2", CPENC (3,4,13,8,6), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff07_el2", CPENC (3,4,13,8,7), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff08_el2", CPENC (3,4,13,9,0), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff09_el2", CPENC (3,4,13,9,1), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff10_el2", CPENC (3,4,13,10,0), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff110_el2", CPENC (3,4,13,11,2), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff111_el2", CPENC (3,4,13,11,3), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff112_el2", CPENC (3,4,13,11,4), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff113_el2", CPENC (3,4,13,11,5), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff114_el2", CPENC (3,4,13,11,6), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff115_el2", CPENC (3,4,13,11,7), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff11_el2", CPENC (3,4,13,10,1), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff12_el2", CPENC (3,4,13,10,2), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff13_el2", CPENC (3,4,13,10,3), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff14_el2", CPENC (3,4,13,10,4), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff15_el2", CPENC (3,4,13,10,5), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff16_el2", CPENC (3,4,13,10,6), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff17_el2", CPENC (3,4,13,10,7), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff18_el2", CPENC (3,4,13,11,0), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevcntvoff19_el2", CPENC (3,4,13,11,1), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("amevtyper00_el0", CPENC (3,3,13,6,0), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper01_el0", CPENC (3,3,13,6,1), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper02_el0", CPENC (3,3,13,6,2), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper03_el0", CPENC (3,3,13,6,3), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper10_el0", CPENC (3,3,13,14,0), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper110_el0", CPENC (3,3,13,15,2), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper111_el0", CPENC (3,3,13,15,3), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper112_el0", CPENC (3,3,13,15,4), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper113_el0", CPENC (3,3,13,15,5), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper114_el0", CPENC (3,3,13,15,6), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper115_el0", CPENC (3,3,13,15,7), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper11_el0", CPENC (3,3,13,14,1), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper12_el0", CPENC (3,3,13,14,2), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper13_el0", CPENC (3,3,13,14,3), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper14_el0", CPENC (3,3,13,14,4), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper15_el0", CPENC (3,3,13,14,5), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper16_el0", CPENC (3,3,13,14,6), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper17_el0", CPENC (3,3,13,14,7), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper18_el0", CPENC (3,3,13,15,0), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amevtyper19_el0", CPENC (3,3,13,15,1), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("amuserenr_el0", CPENC (3,3,13,2,3), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("apdakeyhi_el1", CPENC (3,0,2,2,1), F_ARCHEXT, AARCH64_FEATURE (V8_3A)) - SYSREG ("apdakeylo_el1", CPENC (3,0,2,2,0), F_ARCHEXT, AARCH64_FEATURE (V8_3A)) - SYSREG ("apdbkeyhi_el1", CPENC (3,0,2,2,3), F_ARCHEXT, AARCH64_FEATURE (V8_3A)) - SYSREG ("apdbkeylo_el1", CPENC (3,0,2,2,2), F_ARCHEXT, AARCH64_FEATURE (V8_3A)) - SYSREG ("apgakeyhi_el1", CPENC (3,0,2,3,1), F_ARCHEXT, AARCH64_FEATURE (V8_3A)) - SYSREG ("apgakeylo_el1", CPENC (3,0,2,3,0), F_ARCHEXT, AARCH64_FEATURE (V8_3A)) - SYSREG ("apiakeyhi_el1", CPENC (3,0,2,1,1), F_ARCHEXT, AARCH64_FEATURE (V8_3A)) - SYSREG ("apiakeylo_el1", CPENC (3,0,2,1,0), F_ARCHEXT, AARCH64_FEATURE (V8_3A)) - SYSREG ("apibkeyhi_el1", CPENC (3,0,2,1,3), F_ARCHEXT, AARCH64_FEATURE (V8_3A)) - SYSREG ("apibkeylo_el1", CPENC (3,0,2,1,2), F_ARCHEXT, AARCH64_FEATURE (V8_3A)) - SYSREG ("brbcr_el1", CPENC (2,1,9,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("brbcr_el12", CPENC (2,5,9,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("brbcr_el2", CPENC (2,4,9,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("brbfcr_el1", CPENC (2,1,9,0,1), 0, AARCH64_NO_FEATURES) - SYSREG ("brbidr0_el1", CPENC (2,1,9,2,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf0_el1", CPENC (2,1,8,0,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf10_el1", CPENC (2,1,8,10,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf11_el1", CPENC (2,1,8,11,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf12_el1", CPENC (2,1,8,12,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf13_el1", CPENC (2,1,8,13,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf14_el1", CPENC (2,1,8,14,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf15_el1", CPENC (2,1,8,15,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf16_el1", CPENC (2,1,8,0,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf17_el1", CPENC (2,1,8,1,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf18_el1", CPENC (2,1,8,2,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf19_el1", CPENC (2,1,8,3,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf1_el1", CPENC (2,1,8,1,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf20_el1", CPENC (2,1,8,4,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf21_el1", CPENC (2,1,8,5,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf22_el1", CPENC (2,1,8,6,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf23_el1", CPENC (2,1,8,7,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf24_el1", CPENC (2,1,8,8,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf25_el1", CPENC (2,1,8,9,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf26_el1", CPENC (2,1,8,10,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf27_el1", CPENC (2,1,8,11,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf28_el1", CPENC (2,1,8,12,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf29_el1", CPENC (2,1,8,13,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf2_el1", CPENC (2,1,8,2,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf30_el1", CPENC (2,1,8,14,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf31_el1", CPENC (2,1,8,15,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf3_el1", CPENC (2,1,8,3,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf4_el1", CPENC (2,1,8,4,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf5_el1", CPENC (2,1,8,5,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf6_el1", CPENC (2,1,8,6,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf7_el1", CPENC (2,1,8,7,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf8_el1", CPENC (2,1,8,8,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinf9_el1", CPENC (2,1,8,9,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbinfinj_el1", CPENC (2,1,9,1,0), 0, AARCH64_NO_FEATURES) - SYSREG ("brbsrc0_el1", CPENC (2,1,8,0,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc10_el1", CPENC (2,1,8,10,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc11_el1", CPENC (2,1,8,11,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc12_el1", CPENC (2,1,8,12,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc13_el1", CPENC (2,1,8,13,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc14_el1", CPENC (2,1,8,14,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc15_el1", CPENC (2,1,8,15,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc16_el1", CPENC (2,1,8,0,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc17_el1", CPENC (2,1,8,1,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc18_el1", CPENC (2,1,8,2,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc19_el1", CPENC (2,1,8,3,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc1_el1", CPENC (2,1,8,1,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc20_el1", CPENC (2,1,8,4,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc21_el1", CPENC (2,1,8,5,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc22_el1", CPENC (2,1,8,6,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc23_el1", CPENC (2,1,8,7,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc24_el1", CPENC (2,1,8,8,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc25_el1", CPENC (2,1,8,9,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc26_el1", CPENC (2,1,8,10,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc27_el1", CPENC (2,1,8,11,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc28_el1", CPENC (2,1,8,12,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc29_el1", CPENC (2,1,8,13,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc2_el1", CPENC (2,1,8,2,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc30_el1", CPENC (2,1,8,14,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc31_el1", CPENC (2,1,8,15,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc3_el1", CPENC (2,1,8,3,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc4_el1", CPENC (2,1,8,4,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc5_el1", CPENC (2,1,8,5,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc6_el1", CPENC (2,1,8,6,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc7_el1", CPENC (2,1,8,7,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc8_el1", CPENC (2,1,8,8,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrc9_el1", CPENC (2,1,8,9,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbsrcinj_el1", CPENC (2,1,9,1,1), 0, AARCH64_NO_FEATURES) - SYSREG ("brbtgt0_el1", CPENC (2,1,8,0,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt10_el1", CPENC (2,1,8,10,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt11_el1", CPENC (2,1,8,11,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt12_el1", CPENC (2,1,8,12,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt13_el1", CPENC (2,1,8,13,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt14_el1", CPENC (2,1,8,14,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt15_el1", CPENC (2,1,8,15,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt16_el1", CPENC (2,1,8,0,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt17_el1", CPENC (2,1,8,1,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt18_el1", CPENC (2,1,8,2,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt19_el1", CPENC (2,1,8,3,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt1_el1", CPENC (2,1,8,1,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt20_el1", CPENC (2,1,8,4,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt21_el1", CPENC (2,1,8,5,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt22_el1", CPENC (2,1,8,6,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt23_el1", CPENC (2,1,8,7,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt24_el1", CPENC (2,1,8,8,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt25_el1", CPENC (2,1,8,9,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt26_el1", CPENC (2,1,8,10,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt27_el1", CPENC (2,1,8,11,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt28_el1", CPENC (2,1,8,12,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt29_el1", CPENC (2,1,8,13,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt2_el1", CPENC (2,1,8,2,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt30_el1", CPENC (2,1,8,14,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt31_el1", CPENC (2,1,8,15,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt3_el1", CPENC (2,1,8,3,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt4_el1", CPENC (2,1,8,4,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt5_el1", CPENC (2,1,8,5,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt6_el1", CPENC (2,1,8,6,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt7_el1", CPENC (2,1,8,7,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt8_el1", CPENC (2,1,8,8,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgt9_el1", CPENC (2,1,8,9,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("brbtgtinj_el1", CPENC (2,1,9,1,2), 0, AARCH64_NO_FEATURES) - SYSREG ("brbts_el1", CPENC (2,1,9,0,2), 0, AARCH64_NO_FEATURES) - SYSREG ("ccsidr2_el1", CPENC (3,1,0,0,2), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (V8_3A)) - SYSREG ("ccsidr_el1", CPENC (3,1,0,0,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("clidr_el1", CPENC (3,1,0,0,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("cntfrq_el0", CPENC (3,3,14,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("cnthctl_el2", CPENC (3,4,14,1,0), 0, AARCH64_NO_FEATURES) - SYSREG ("cnthp_ctl_el2", CPENC (3,4,14,2,1), 0, AARCH64_NO_FEATURES) - SYSREG ("cnthp_cval_el2", CPENC (3,4,14,2,2), 0, AARCH64_NO_FEATURES) - SYSREG ("cnthp_tval_el2", CPENC (3,4,14,2,0), 0, AARCH64_NO_FEATURES) - SYSREG ("cnthps_ctl_el2", CPENC (3,4,14,5,1), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("cnthps_cval_el2", CPENC (3,4,14,5,2), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("cnthps_tval_el2", CPENC (3,4,14,5,0), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("cnthv_ctl_el2", CPENC (3,4,14,3,1), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("cnthv_cval_el2", CPENC (3,4,14,3,2), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("cnthv_tval_el2", CPENC (3,4,14,3,0), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("cnthvs_ctl_el2", CPENC (3,4,14,4,1), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("cnthvs_cval_el2", CPENC (3,4,14,4,2), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("cnthvs_tval_el2", CPENC (3,4,14,4,0), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("cntkctl_el1", CPENC (3,0,14,1,0), 0, AARCH64_NO_FEATURES) - SYSREG ("cntkctl_el12", CPENC (3,5,14,1,0), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("cntp_ctl_el0", CPENC (3,3,14,2,1), 0, AARCH64_NO_FEATURES) - SYSREG ("cntp_ctl_el02", CPENC (3,5,14,2,1), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("cntp_cval_el0", CPENC (3,3,14,2,2), 0, AARCH64_NO_FEATURES) - SYSREG ("cntp_cval_el02", CPENC (3,5,14,2,2), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("cntp_tval_el0", CPENC (3,3,14,2,0), 0, AARCH64_NO_FEATURES) - SYSREG ("cntp_tval_el02", CPENC (3,5,14,2,0), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("cntpct_el0", CPENC (3,3,14,0,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("cntpctss_el0", CPENC (3,3,14,0,5), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("cntpoff_el2", CPENC (3,4,14,0,6), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("cntps_ctl_el1", CPENC (3,7,14,2,1), 0, AARCH64_NO_FEATURES) - SYSREG ("cntps_cval_el1", CPENC (3,7,14,2,2), 0, AARCH64_NO_FEATURES) - SYSREG ("cntps_tval_el1", CPENC (3,7,14,2,0), 0, AARCH64_NO_FEATURES) - SYSREG ("cntv_ctl_el0", CPENC (3,3,14,3,1), 0, AARCH64_NO_FEATURES) - SYSREG ("cntv_ctl_el02", CPENC (3,5,14,3,1), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("cntv_cval_el0", CPENC (3,3,14,3,2), 0, AARCH64_NO_FEATURES) - SYSREG ("cntv_cval_el02", CPENC (3,5,14,3,2), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("cntv_tval_el0", CPENC (3,3,14,3,0), 0, AARCH64_NO_FEATURES) - SYSREG ("cntv_tval_el02", CPENC (3,5,14,3,0), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("cntvct_el0", CPENC (3,3,14,0,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("cntvctss_el0", CPENC (3,3,14,0,6), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("cntvoff_el2", CPENC (3,4,14,0,3), 0, AARCH64_NO_FEATURES) - SYSREG ("contextidr_el1", CPENC (3,0,13,0,1), 0, AARCH64_NO_FEATURES) - SYSREG ("contextidr_el12", CPENC (3,5,13,0,1), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("contextidr_el2", CPENC (3,4,13,0,1), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("cpacr_el1", CPENC (3,0,1,0,2), 0, AARCH64_NO_FEATURES) - SYSREG ("cpacr_el12", CPENC (3,5,1,0,2), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("cptr_el2", CPENC (3,4,1,1,2), 0, AARCH64_NO_FEATURES) - SYSREG ("cptr_el3", CPENC (3,6,1,1,2), 0, AARCH64_NO_FEATURES) - SYSREG ("csrcr_el0", CPENC (2,3,8,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("csrcr_el1", CPENC (2,0,8,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("csrcr_el12", CPENC (2,5,8,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("csrcr_el2", CPENC (2,4,8,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("csridr_el0", CPENC (2,3,8,0,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("csrptr_el0", CPENC (2,3,8,0,1), 0, AARCH64_NO_FEATURES) - SYSREG ("csrptr_el1", CPENC (2,0,8,0,1), 0, AARCH64_NO_FEATURES) - SYSREG ("csrptr_el12", CPENC (2,5,8,0,1), 0, AARCH64_NO_FEATURES) - SYSREG ("csrptr_el2", CPENC (2,4,8,0,1), 0, AARCH64_NO_FEATURES) - SYSREG ("csrptridx_el0", CPENC (2,3,8,0,3), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("csrptridx_el1", CPENC (2,0,8,0,3), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("csrptridx_el2", CPENC (2,4,8,0,3), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("csselr_el1", CPENC (3,2,0,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("ctr_el0", CPENC (3,3,0,0,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("currentel", CPENC (3,0,4,2,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("dacr32_el2", CPENC (3,4,3,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("daif", CPENC (3,3,4,2,1), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgauthstatus_el1", CPENC (2,0,7,14,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("dbgbcr0_el1", CPENC (2,0,0,0,5), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbcr10_el1", CPENC (2,0,0,10,5), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbcr11_el1", CPENC (2,0,0,11,5), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbcr12_el1", CPENC (2,0,0,12,5), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbcr13_el1", CPENC (2,0,0,13,5), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbcr14_el1", CPENC (2,0,0,14,5), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbcr15_el1", CPENC (2,0,0,15,5), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbcr1_el1", CPENC (2,0,0,1,5), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbcr2_el1", CPENC (2,0,0,2,5), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbcr3_el1", CPENC (2,0,0,3,5), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbcr4_el1", CPENC (2,0,0,4,5), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbcr5_el1", CPENC (2,0,0,5,5), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbcr6_el1", CPENC (2,0,0,6,5), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbcr7_el1", CPENC (2,0,0,7,5), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbcr8_el1", CPENC (2,0,0,8,5), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbcr9_el1", CPENC (2,0,0,9,5), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbvr0_el1", CPENC (2,0,0,0,4), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbvr10_el1", CPENC (2,0,0,10,4), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbvr11_el1", CPENC (2,0,0,11,4), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbvr12_el1", CPENC (2,0,0,12,4), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbvr13_el1", CPENC (2,0,0,13,4), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbvr14_el1", CPENC (2,0,0,14,4), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbvr15_el1", CPENC (2,0,0,15,4), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbvr1_el1", CPENC (2,0,0,1,4), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbvr2_el1", CPENC (2,0,0,2,4), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbvr3_el1", CPENC (2,0,0,3,4), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbvr4_el1", CPENC (2,0,0,4,4), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbvr5_el1", CPENC (2,0,0,5,4), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbvr6_el1", CPENC (2,0,0,6,4), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbvr7_el1", CPENC (2,0,0,7,4), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbvr8_el1", CPENC (2,0,0,8,4), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgbvr9_el1", CPENC (2,0,0,9,4), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgclaimclr_el1", CPENC (2,0,7,9,6), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgclaimset_el1", CPENC (2,0,7,8,6), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgdtr_el0", CPENC (2,3,0,4,0), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgdtrrx_el0", CPENC (2,3,0,5,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("dbgdtrtx_el0", CPENC (2,3,0,5,0), F_REG_WRITE, AARCH64_NO_FEATURES) - SYSREG ("dbgprcr_el1", CPENC (2,0,1,4,4), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgvcr32_el2", CPENC (2,4,0,7,0), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwcr0_el1", CPENC (2,0,0,0,7), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwcr10_el1", CPENC (2,0,0,10,7), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwcr11_el1", CPENC (2,0,0,11,7), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwcr12_el1", CPENC (2,0,0,12,7), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwcr13_el1", CPENC (2,0,0,13,7), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwcr14_el1", CPENC (2,0,0,14,7), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwcr15_el1", CPENC (2,0,0,15,7), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwcr1_el1", CPENC (2,0,0,1,7), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwcr2_el1", CPENC (2,0,0,2,7), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwcr3_el1", CPENC (2,0,0,3,7), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwcr4_el1", CPENC (2,0,0,4,7), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwcr5_el1", CPENC (2,0,0,5,7), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwcr6_el1", CPENC (2,0,0,6,7), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwcr7_el1", CPENC (2,0,0,7,7), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwcr8_el1", CPENC (2,0,0,8,7), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwcr9_el1", CPENC (2,0,0,9,7), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwvr0_el1", CPENC (2,0,0,0,6), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwvr10_el1", CPENC (2,0,0,10,6), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwvr11_el1", CPENC (2,0,0,11,6), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwvr12_el1", CPENC (2,0,0,12,6), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwvr13_el1", CPENC (2,0,0,13,6), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwvr14_el1", CPENC (2,0,0,14,6), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwvr15_el1", CPENC (2,0,0,15,6), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwvr1_el1", CPENC (2,0,0,1,6), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwvr2_el1", CPENC (2,0,0,2,6), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwvr3_el1", CPENC (2,0,0,3,6), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwvr4_el1", CPENC (2,0,0,4,6), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwvr5_el1", CPENC (2,0,0,5,6), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwvr6_el1", CPENC (2,0,0,6,6), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwvr7_el1", CPENC (2,0,0,7,6), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwvr8_el1", CPENC (2,0,0,8,6), 0, AARCH64_NO_FEATURES) - SYSREG ("dbgwvr9_el1", CPENC (2,0,0,9,6), 0, AARCH64_NO_FEATURES) - SYSREG ("dczid_el0", CPENC (3,3,0,0,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("disr_el1", CPENC (3,0,12,1,1), F_ARCHEXT, AARCH64_FEATURE (RAS)) - SYSREG ("dit", CPENC (3,3,4,2,5), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("dlr_el0", CPENC (3,3,4,5,1), 0, AARCH64_NO_FEATURES) - SYSREG ("dspsr_el0", CPENC (3,3,4,5,0), 0, AARCH64_NO_FEATURES) - SYSREG ("elr_el1", CPENC (3,0,4,0,1), 0, AARCH64_NO_FEATURES) - SYSREG ("elr_el12", CPENC (3,5,4,0,1), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("elr_el2", CPENC (3,4,4,0,1), 0, AARCH64_NO_FEATURES) - SYSREG ("elr_el3", CPENC (3,6,4,0,1), 0, AARCH64_NO_FEATURES) - SYSREG ("erridr_el1", CPENC (3,0,5,3,0), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (RAS)) - SYSREG ("errselr_el1", CPENC (3,0,5,3,1), F_ARCHEXT, AARCH64_FEATURE (RAS)) - SYSREG ("erxaddr_el1", CPENC (3,0,5,4,3), F_ARCHEXT, AARCH64_FEATURE (RAS)) - SYSREG ("erxctlr_el1", CPENC (3,0,5,4,1), F_ARCHEXT, AARCH64_FEATURE (RAS)) - SYSREG ("erxfr_el1", CPENC (3,0,5,4,0), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (RAS)) - SYSREG ("erxgsr_el1", CPENC (3,0,5,3,2), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (RASv2)) - SYSREG ("erxmisc0_el1", CPENC (3,0,5,5,0), F_ARCHEXT, AARCH64_FEATURE (RAS)) - SYSREG ("erxmisc1_el1", CPENC (3,0,5,5,1), F_ARCHEXT, AARCH64_FEATURE (RAS)) - SYSREG ("erxmisc2_el1", CPENC (3,0,5,5,2), F_ARCHEXT, AARCH64_FEATURE (RAS)) - SYSREG ("erxmisc3_el1", CPENC (3,0,5,5,3), F_ARCHEXT, AARCH64_FEATURE (RAS)) - SYSREG ("erxpfgcdn_el1", CPENC (3,0,5,4,6), F_ARCHEXT, AARCH64_FEATURE (RAS)) - SYSREG ("erxpfgctl_el1", CPENC (3,0,5,4,5), F_ARCHEXT, AARCH64_FEATURE (RAS)) - SYSREG ("erxpfgf_el1", CPENC (3,0,5,4,4), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (RAS)) - SYSREG ("erxstatus_el1", CPENC (3,0,5,4,2), F_ARCHEXT, AARCH64_FEATURE (RAS)) - SYSREG ("esr_el1", CPENC (3,0,5,2,0), 0, AARCH64_NO_FEATURES) - SYSREG ("esr_el12", CPENC (3,5,5,2,0), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("esr_el2", CPENC (3,4,5,2,0), 0, AARCH64_NO_FEATURES) - SYSREG ("esr_el3", CPENC (3,6,5,2,0), 0, AARCH64_NO_FEATURES) - SYSREG ("far_el1", CPENC (3,0,6,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("far_el12", CPENC (3,5,6,0,0), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("far_el2", CPENC (3,4,6,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("far_el3", CPENC (3,6,6,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("fpcr", CPENC (3,3,4,4,0), 0, AARCH64_NO_FEATURES) - SYSREG ("fpexc32_el2", CPENC (3,4,5,3,0), 0, AARCH64_NO_FEATURES) - SYSREG ("fpsr", CPENC (3,3,4,4,1), 0, AARCH64_NO_FEATURES) - SYSREG ("gcspr_el0", CPENC (3,3,2,5,1), F_ARCHEXT, AARCH64_FEATURE (GCS)) - SYSREG ("gcspr_el1", CPENC (3,0,2,5,1), F_ARCHEXT, AARCH64_FEATURE (GCS)) - SYSREG ("gcspr_el2", CPENC (3,4,2,5,1), F_ARCHEXT, AARCH64_FEATURE (GCS)) - SYSREG ("gcspr_el12", CPENC (3,5,2,5,1), F_ARCHEXT, AARCH64_FEATURE (GCS)) - SYSREG ("gcspr_el3", CPENC (3,6,2,5,1), F_ARCHEXT, AARCH64_FEATURE (GCS)) - SYSREG ("gcscre0_el1", CPENC (3,0,2,5,2), F_ARCHEXT, AARCH64_FEATURE (GCS)) - SYSREG ("gcscr_el1", CPENC (3,0,2,5,0), F_ARCHEXT, AARCH64_FEATURE (GCS)) - SYSREG ("gcscr_el2", CPENC (3,4,2,5,0), F_ARCHEXT, AARCH64_FEATURE (GCS)) - SYSREG ("gcscr_el12", CPENC (3,5,2,5,0), F_ARCHEXT, AARCH64_FEATURE (GCS)) - SYSREG ("gcscr_el3", CPENC (3,6,2,5,0), F_ARCHEXT, AARCH64_FEATURE (GCS)) - SYSREG ("gcr_el1", CPENC (3,0,1,0,6), F_ARCHEXT, AARCH64_FEATURE (MEMTAG)) - SYSREG ("gmid_el1", CPENC (3,1,0,0,4), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (MEMTAG)) - SYSREG ("gpccr_el3", CPENC (3,6,2,1,6), 0, AARCH64_NO_FEATURES) - SYSREG ("gptbr_el3", CPENC (3,6,2,1,4), 0, AARCH64_NO_FEATURES) - SYSREG ("hacr_el2", CPENC (3,4,1,1,7), 0, AARCH64_NO_FEATURES) - SYSREG ("hafgrtr_el2", CPENC (3,4,3,1,6), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("hcr_el2", CPENC (3,4,1,1,0), 0, AARCH64_NO_FEATURES) - SYSREG ("hcrx_el2", CPENC (3,4,1,2,2), F_ARCHEXT, AARCH64_FEATURE (V8_7A)) - SYSREG ("hdfgrtr_el2", CPENC (3,4,3,1,4), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("hdfgrtr2_el2", CPENC (3,4,3,1,0), F_ARCHEXT, AARCH64_FEATURE (FGT2)) - SYSREG ("hdfgwtr_el2", CPENC (3,4,3,1,5), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("hdfgwtr2_el2", CPENC (3,4,3,1,1), F_ARCHEXT, AARCH64_FEATURE (FGT2)) - SYSREG ("hfgitr_el2", CPENC (3,4,1,1,6), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("hfgrtr_el2", CPENC (3,4,1,1,4), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("hfgrtr2_el2", CPENC (3,4,3,1,2), F_ARCHEXT, AARCH64_FEATURE (FGT2)) - SYSREG ("hfgwtr_el2", CPENC (3,4,1,1,5), F_ARCHEXT, AARCH64_FEATURE (V8_6A)) - SYSREG ("hfgwtr2_el2", CPENC (3,4,3,1,3), F_ARCHEXT, AARCH64_FEATURE (FGT2)) - SYSREG ("hpfar_el2", CPENC (3,4,6,0,4), 0, AARCH64_NO_FEATURES) - SYSREG ("hstr_el2", CPENC (3,4,1,1,3), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_ap0r0_el1", CPENC (3,0,12,8,4), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_ap0r1_el1", CPENC (3,0,12,8,5), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_ap0r2_el1", CPENC (3,0,12,8,6), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_ap0r3_el1", CPENC (3,0,12,8,7), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_ap1r0_el1", CPENC (3,0,12,9,0), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_ap1r1_el1", CPENC (3,0,12,9,1), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_ap1r2_el1", CPENC (3,0,12,9,2), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_ap1r3_el1", CPENC (3,0,12,9,3), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_asgi1r_el1", CPENC (3,0,12,11,6), F_REG_WRITE, AARCH64_NO_FEATURES) - SYSREG ("icc_bpr0_el1", CPENC (3,0,12,8,3), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_bpr1_el1", CPENC (3,0,12,12,3), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_ctlr_el1", CPENC (3,0,12,12,4), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_ctlr_el3", CPENC (3,6,12,12,4), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_dir_el1", CPENC (3,0,12,11,1), F_REG_WRITE, AARCH64_NO_FEATURES) - SYSREG ("icc_eoir0_el1", CPENC (3,0,12,8,1), F_REG_WRITE, AARCH64_NO_FEATURES) - SYSREG ("icc_eoir1_el1", CPENC (3,0,12,12,1), F_REG_WRITE, AARCH64_NO_FEATURES) - SYSREG ("icc_hppir0_el1", CPENC (3,0,12,8,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("icc_hppir1_el1", CPENC (3,0,12,12,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("icc_iar0_el1", CPENC (3,0,12,8,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("icc_iar1_el1", CPENC (3,0,12,12,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("icc_igrpen0_el1", CPENC (3,0,12,12,6), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_igrpen1_el1", CPENC (3,0,12,12,7), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_igrpen1_el3", CPENC (3,6,12,12,7), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_nmiar1_el1", CPENC (3,0,12,9,5), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (V8_8A)) - SYSREG ("icc_pmr_el1", CPENC (3,0,4,6,0), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_rpr_el1", CPENC (3,0,12,11,3), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("icc_sgi0r_el1", CPENC (3,0,12,11,7), F_REG_WRITE, AARCH64_NO_FEATURES) - SYSREG ("icc_sgi1r_el1", CPENC (3,0,12,11,5), F_REG_WRITE, AARCH64_NO_FEATURES) - SYSREG ("icc_sre_el1", CPENC (3,0,12,12,5), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_sre_el2", CPENC (3,4,12,9,5), 0, AARCH64_NO_FEATURES) - SYSREG ("icc_sre_el3", CPENC (3,6,12,12,5), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_ap0r0_el2", CPENC (3,4,12,8,0), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_ap0r1_el2", CPENC (3,4,12,8,1), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_ap0r2_el2", CPENC (3,4,12,8,2), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_ap0r3_el2", CPENC (3,4,12,8,3), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_ap1r0_el2", CPENC (3,4,12,9,0), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_ap1r1_el2", CPENC (3,4,12,9,1), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_ap1r2_el2", CPENC (3,4,12,9,2), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_ap1r3_el2", CPENC (3,4,12,9,3), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_eisr_el2", CPENC (3,4,12,11,3), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("ich_elrsr_el2", CPENC (3,4,12,11,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("ich_hcr_el2", CPENC (3,4,12,11,0), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_lr0_el2", CPENC (3,4,12,12,0), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_lr10_el2", CPENC (3,4,12,13,2), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_lr11_el2", CPENC (3,4,12,13,3), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_lr12_el2", CPENC (3,4,12,13,4), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_lr13_el2", CPENC (3,4,12,13,5), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_lr14_el2", CPENC (3,4,12,13,6), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_lr15_el2", CPENC (3,4,12,13,7), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_lr1_el2", CPENC (3,4,12,12,1), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_lr2_el2", CPENC (3,4,12,12,2), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_lr3_el2", CPENC (3,4,12,12,3), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_lr4_el2", CPENC (3,4,12,12,4), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_lr5_el2", CPENC (3,4,12,12,5), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_lr6_el2", CPENC (3,4,12,12,6), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_lr7_el2", CPENC (3,4,12,12,7), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_lr8_el2", CPENC (3,4,12,13,0), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_lr9_el2", CPENC (3,4,12,13,1), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_misr_el2", CPENC (3,4,12,11,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("ich_vmcr_el2", CPENC (3,4,12,11,7), 0, AARCH64_NO_FEATURES) - SYSREG ("ich_vtr_el2", CPENC (3,4,12,11,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_aa64afr0_el1", CPENC (3,0,0,5,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_aa64afr1_el1", CPENC (3,0,0,5,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_aa64dfr0_el1", CPENC (3,0,0,5,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_aa64dfr1_el1", CPENC (3,0,0,5,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_aa64isar0_el1", CPENC (3,0,0,6,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_aa64isar1_el1", CPENC (3,0,0,6,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_aa64isar2_el1", CPENC (3,0,0,6,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_aa64isar3_el1", CPENC (3,0,0,6,3), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_aa64mmfr0_el1", CPENC (3,0,0,7,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_aa64mmfr1_el1", CPENC (3,0,0,7,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_aa64mmfr2_el1", CPENC (3,0,0,7,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_aa64mmfr3_el1", CPENC (3,0,0,7,3), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_aa64mmfr4_el1", CPENC (3,0,0,7,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_aa64pfr0_el1", CPENC (3,0,0,4,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_aa64pfr1_el1", CPENC (3,0,0,4,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_aa64smfr0_el1", CPENC (3,0,0,4,5), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (SME)) - SYSREG ("id_aa64zfr0_el1", CPENC (3,0,0,4,4), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (SVE)) - SYSREG ("id_afr0_el1", CPENC (3,0,0,1,3), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_dfr0_el1", CPENC (3,0,0,1,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_dfr1_el1", CPENC (3,0,0,3,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_isar0_el1", CPENC (3,0,0,2,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_isar1_el1", CPENC (3,0,0,2,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_isar2_el1", CPENC (3,0,0,2,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_isar3_el1", CPENC (3,0,0,2,3), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_isar4_el1", CPENC (3,0,0,2,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_isar5_el1", CPENC (3,0,0,2,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_isar6_el1", CPENC (3,0,0,2,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_mmfr0_el1", CPENC (3,0,0,1,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_mmfr1_el1", CPENC (3,0,0,1,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_mmfr2_el1", CPENC (3,0,0,1,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_mmfr3_el1", CPENC (3,0,0,1,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_mmfr4_el1", CPENC (3,0,0,2,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_mmfr5_el1", CPENC (3,0,0,3,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_pfr0_el1", CPENC (3,0,0,1,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_pfr1_el1", CPENC (3,0,0,1,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("id_pfr2_el1", CPENC (3,0,0,3,4), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (ID_PFR2)) - SYSREG ("ifsr32_el2", CPENC (3,4,5,0,1), 0, AARCH64_NO_FEATURES) - SYSREG ("isr_el1", CPENC (3,0,12,1,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("lorc_el1", CPENC (3,0,10,4,3), F_ARCHEXT, AARCH64_FEATURE (LOR)) - SYSREG ("lorea_el1", CPENC (3,0,10,4,1), F_ARCHEXT, AARCH64_FEATURE (LOR)) - SYSREG ("lorid_el1", CPENC (3,0,10,4,7), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (LOR)) - SYSREG ("lorn_el1", CPENC (3,0,10,4,2), F_ARCHEXT, AARCH64_FEATURE (LOR)) - SYSREG ("lorsa_el1", CPENC (3,0,10,4,0), F_ARCHEXT, AARCH64_FEATURE (LOR)) - SYSREG ("mair_el1", CPENC (3,0,10,2,0), 0, AARCH64_NO_FEATURES) - SYSREG ("mair_el12", CPENC (3,5,10,2,0), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("mair_el2", CPENC (3,4,10,2,0), 0, AARCH64_NO_FEATURES) - SYSREG ("mair_el3", CPENC (3,6,10,2,0), 0, AARCH64_NO_FEATURES) - SYSREG ("mair2_el1", CPENC (3,0,10,2,1), F_ARCHEXT, AARCH64_FEATURE (AIE)) - SYSREG ("mair2_el12", CPENC (3,5,10,2,1), F_ARCHEXT, AARCH64_FEATURE (AIE)) - SYSREG ("mair2_el2", CPENC (3,4,10,1,1), F_ARCHEXT, AARCH64_FEATURE (AIE)) - SYSREG ("mair2_el3", CPENC (3,6,10,1,1), F_ARCHEXT, AARCH64_FEATURE (AIE)) - SYSREG ("mdccint_el1", CPENC (2,0,0,2,0), 0, AARCH64_NO_FEATURES) - SYSREG ("mdccsr_el0", CPENC (2,3,0,1,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("mdcr_el2", CPENC (3,4,1,1,1), 0, AARCH64_NO_FEATURES) - SYSREG ("mdcr_el3", CPENC (3,6,1,3,1), 0, AARCH64_NO_FEATURES) - SYSREG ("mdrar_el1", CPENC (2,0,1,0,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("mdscr_el1", CPENC (2,0,0,2,2), 0, AARCH64_NO_FEATURES) - SYSREG ("mdselr_el1", CPENC (2,0,0,4,2), F_ARCHEXT, AARCH64_FEATURE (DEBUGv8p9)) - SYSREG ("mecid_a0_el2", CPENC (3,4,10,8,1), F_ARCHEXT, AARCH64_FEATURE (V8_7A)) - SYSREG ("mecid_a1_el2", CPENC (3,4,10,8,3), F_ARCHEXT, AARCH64_FEATURE (V8_7A)) - SYSREG ("mecid_p0_el2", CPENC (3,4,10,8,0), F_ARCHEXT, AARCH64_FEATURE (V8_7A)) - SYSREG ("mecid_p1_el2", CPENC (3,4,10,8,2), F_ARCHEXT, AARCH64_FEATURE (V8_7A)) - SYSREG ("mecid_rl_a_el3", CPENC (3,6,10,10,1), F_ARCHEXT, AARCH64_FEATURE (V8_7A)) - SYSREG ("mecidr_el2", CPENC (3,4,10,8,7), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (V8_7A)) - SYSREG ("mfar_el3", CPENC (3,6,6,0,5), 0, AARCH64_NO_FEATURES) - SYSREG ("midr_el1", CPENC (3,0,0,0,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("mpam0_el1", CPENC (3,0,10,5,1), 0, AARCH64_NO_FEATURES) - SYSREG ("mpam1_el1", CPENC (3,0,10,5,0), 0, AARCH64_NO_FEATURES) - SYSREG ("mpam1_el12", CPENC (3,5,10,5,0), 0, AARCH64_NO_FEATURES) - SYSREG ("mpam2_el2", CPENC (3,4,10,5,0), 0, AARCH64_NO_FEATURES) - SYSREG ("mpam3_el3", CPENC (3,6,10,5,0), 0, AARCH64_NO_FEATURES) - SYSREG ("mpamhcr_el2", CPENC (3,4,10,4,0), 0, AARCH64_NO_FEATURES) - SYSREG ("mpamidr_el1", CPENC (3,0,10,4,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("mpamsm_el1", CPENC (3,0,10,5,3), F_ARCHEXT, AARCH64_FEATURE (SME)) - SYSREG ("mpamvpm0_el2", CPENC (3,4,10,6,0), 0, AARCH64_NO_FEATURES) - SYSREG ("mpamvpm1_el2", CPENC (3,4,10,6,1), 0, AARCH64_NO_FEATURES) - SYSREG ("mpamvpm2_el2", CPENC (3,4,10,6,2), 0, AARCH64_NO_FEATURES) - SYSREG ("mpamvpm3_el2", CPENC (3,4,10,6,3), 0, AARCH64_NO_FEATURES) - SYSREG ("mpamvpm4_el2", CPENC (3,4,10,6,4), 0, AARCH64_NO_FEATURES) - SYSREG ("mpamvpm5_el2", CPENC (3,4,10,6,5), 0, AARCH64_NO_FEATURES) - SYSREG ("mpamvpm6_el2", CPENC (3,4,10,6,6), 0, AARCH64_NO_FEATURES) - SYSREG ("mpamvpm7_el2", CPENC (3,4,10,6,7), 0, AARCH64_NO_FEATURES) - SYSREG ("mpamvpmv_el2", CPENC (3,4,10,4,1), 0, AARCH64_NO_FEATURES) - SYSREG ("mpidr_el1", CPENC (3,0,0,0,5), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("mpuir_el1", CPENC (3,0,0,0,4), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("mpuir_el2", CPENC (3,4,0,0,4), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("mvfr0_el1", CPENC (3,0,0,3,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("mvfr1_el1", CPENC (3,0,0,3,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("mvfr2_el1", CPENC (3,0,0,3,2), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("nzcv", CPENC (3,3,4,2,0), 0, AARCH64_NO_FEATURES) - SYSREG ("osdlr_el1", CPENC (2,0,1,3,4), 0, AARCH64_NO_FEATURES) - SYSREG ("osdtrrx_el1", CPENC (2,0,0,0,2), 0, AARCH64_NO_FEATURES) - SYSREG ("osdtrtx_el1", CPENC (2,0,0,3,2), 0, AARCH64_NO_FEATURES) - SYSREG ("oseccr_el1", CPENC (2,0,0,6,2), 0, AARCH64_NO_FEATURES) - SYSREG ("oslar_el1", CPENC (2,0,1,0,4), F_REG_WRITE, AARCH64_NO_FEATURES) - SYSREG ("oslsr_el1", CPENC (2,0,1,1,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("pir_el1", CPENC (3,0,10,2,3), F_ARCHEXT, AARCH64_FEATURE (S1PIE)) - SYSREG ("pir_el12", CPENC (3,5,10,2,3), F_ARCHEXT, AARCH64_FEATURE (S1PIE)) - SYSREG ("pir_el2", CPENC (3,4,10,2,3), F_ARCHEXT, AARCH64_FEATURE (S1PIE)) - SYSREG ("pir_el3", CPENC (3,6,10,2,3), F_ARCHEXT, AARCH64_FEATURE (S1PIE)) - SYSREG ("pire0_el1", CPENC (3,0,10,2,2), F_ARCHEXT, AARCH64_FEATURE (S1PIE)) - SYSREG ("pire0_el12", CPENC (3,5,10,2,2), F_ARCHEXT, AARCH64_FEATURE (S1PIE)) - SYSREG ("pire0_el2", CPENC (3,4,10,2,2), F_ARCHEXT, AARCH64_FEATURE (S1PIE)) - SYSREG ("pan", CPENC (3,0,4,2,3), F_ARCHEXT, AARCH64_FEATURE (PAN)) - SYSREG ("par_el1", CPENC (3,0,7,4,0), F_REG_128, AARCH64_NO_FEATURES) - SYSREG ("pfar_el1", CPENC (3,0,6,0,5), F_ARCHEXT, AARCH64_FEATURE (PFAR)) - SYSREG ("pfar_el12", CPENC (3,5,6,0,5), F_ARCHEXT, AARCH64_FEATURE (PFAR)) - SYSREG ("pfar_el2", CPENC (3,4,6,0,5), F_ARCHEXT, AARCH64_FEATURE (PFAR)) - SYSREG ("pmbidr_el1", CPENC (3,0,9,10,7), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PROFILE)) - SYSREG ("pmblimitr_el1", CPENC (3,0,9,10,0), F_ARCHEXT, AARCH64_FEATURE (PROFILE)) - SYSREG ("pmbptr_el1", CPENC (3,0,9,10,1), F_ARCHEXT, AARCH64_FEATURE (PROFILE)) - SYSREG ("pmbsr_el1", CPENC (3,0,9,10,3), F_ARCHEXT, AARCH64_FEATURE (PROFILE)) - SYSREG ("pmccfiltr_el0", CPENC (3,3,14,15,7), 0, AARCH64_NO_FEATURES) - SYSREG ("pmccntr_el0", CPENC (3,3,9,13,0), 0, AARCH64_NO_FEATURES) - SYSREG ("pmccntsvr_el1", CPENC (2,0,14,11,7), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmceid0_el0", CPENC (3,3,9,12,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("pmceid1_el0", CPENC (3,3,9,12,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("pmcntenclr_el0", CPENC (3,3,9,12,2), 0, AARCH64_NO_FEATURES) - SYSREG ("pmcntenset_el0", CPENC (3,3,9,12,1), 0, AARCH64_NO_FEATURES) - SYSREG ("pmcr_el0", CPENC (3,3,9,12,0), 0, AARCH64_NO_FEATURES) - SYSREG ("pmecr_el1", CPENC (3,0,9,14,5), F_ARCHEXT, AARCH64_FEATURE (SEBEP)) - SYSREG ("pmevcntr0_el0", CPENC (3,3,14,8,0), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr10_el0", CPENC (3,3,14,9,2), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr11_el0", CPENC (3,3,14,9,3), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr12_el0", CPENC (3,3,14,9,4), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr13_el0", CPENC (3,3,14,9,5), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr14_el0", CPENC (3,3,14,9,6), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr15_el0", CPENC (3,3,14,9,7), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr16_el0", CPENC (3,3,14,10,0), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr17_el0", CPENC (3,3,14,10,1), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr18_el0", CPENC (3,3,14,10,2), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr19_el0", CPENC (3,3,14,10,3), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr1_el0", CPENC (3,3,14,8,1), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr20_el0", CPENC (3,3,14,10,4), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr21_el0", CPENC (3,3,14,10,5), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr22_el0", CPENC (3,3,14,10,6), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr23_el0", CPENC (3,3,14,10,7), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr24_el0", CPENC (3,3,14,11,0), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr25_el0", CPENC (3,3,14,11,1), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr26_el0", CPENC (3,3,14,11,2), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr27_el0", CPENC (3,3,14,11,3), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr28_el0", CPENC (3,3,14,11,4), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr29_el0", CPENC (3,3,14,11,5), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr2_el0", CPENC (3,3,14,8,2), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr30_el0", CPENC (3,3,14,11,6), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr3_el0", CPENC (3,3,14,8,3), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr4_el0", CPENC (3,3,14,8,4), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr5_el0", CPENC (3,3,14,8,5), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr6_el0", CPENC (3,3,14,8,6), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr7_el0", CPENC (3,3,14,8,7), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr8_el0", CPENC (3,3,14,9,0), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntr9_el0", CPENC (3,3,14,9,1), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevcntsvr0_el1", CPENC (2,0,14,8,0), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr10_el1", CPENC (2,0,14,9,2), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr11_el1", CPENC (2,0,14,9,3), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr12_el1", CPENC (2,0,14,9,4), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr13_el1", CPENC (2,0,14,9,5), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr14_el1", CPENC (2,0,14,9,6), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr15_el1", CPENC (2,0,14,9,7), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr16_el1", CPENC (2,0,14,10,0), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr17_el1", CPENC (2,0,14,10,1), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr18_el1", CPENC (2,0,14,10,2), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr19_el1", CPENC (2,0,14,10,3), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr1_el1", CPENC (2,0,14,8,1), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr20_el1", CPENC (2,0,14,10,4), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr21_el1", CPENC (2,0,14,10,5), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr22_el1", CPENC (2,0,14,10,6), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr23_el1", CPENC (2,0,14,10,7), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr24_el1", CPENC (2,0,14,11,0), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr25_el1", CPENC (2,0,14,11,1), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr26_el1", CPENC (2,0,14,11,2), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr27_el1", CPENC (2,0,14,11,3), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr28_el1", CPENC (2,0,14,11,4), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr29_el1", CPENC (2,0,14,11,5), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr2_el1", CPENC (2,0,14,8,2), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr30_el1", CPENC (2,0,14,11,6), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr3_el1", CPENC (2,0,14,8,3), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr4_el1", CPENC (2,0,14,8,4), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr5_el1", CPENC (2,0,14,8,5), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr6_el1", CPENC (2,0,14,8,6), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr7_el1", CPENC (2,0,14,8,7), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr8_el1", CPENC (2,0,14,9,0), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevcntsvr9_el1", CPENC (2,0,14,9,1), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmevtyper0_el0", CPENC (3,3,14,12,0), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper10_el0", CPENC (3,3,14,13,2), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper11_el0", CPENC (3,3,14,13,3), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper12_el0", CPENC (3,3,14,13,4), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper13_el0", CPENC (3,3,14,13,5), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper14_el0", CPENC (3,3,14,13,6), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper15_el0", CPENC (3,3,14,13,7), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper16_el0", CPENC (3,3,14,14,0), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper17_el0", CPENC (3,3,14,14,1), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper18_el0", CPENC (3,3,14,14,2), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper19_el0", CPENC (3,3,14,14,3), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper1_el0", CPENC (3,3,14,12,1), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper20_el0", CPENC (3,3,14,14,4), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper21_el0", CPENC (3,3,14,14,5), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper22_el0", CPENC (3,3,14,14,6), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper23_el0", CPENC (3,3,14,14,7), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper24_el0", CPENC (3,3,14,15,0), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper25_el0", CPENC (3,3,14,15,1), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper26_el0", CPENC (3,3,14,15,2), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper27_el0", CPENC (3,3,14,15,3), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper28_el0", CPENC (3,3,14,15,4), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper29_el0", CPENC (3,3,14,15,5), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper2_el0", CPENC (3,3,14,12,2), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper30_el0", CPENC (3,3,14,15,6), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper3_el0", CPENC (3,3,14,12,3), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper4_el0", CPENC (3,3,14,12,4), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper5_el0", CPENC (3,3,14,12,5), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper6_el0", CPENC (3,3,14,12,6), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper7_el0", CPENC (3,3,14,12,7), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper8_el0", CPENC (3,3,14,13,0), 0, AARCH64_NO_FEATURES) - SYSREG ("pmevtyper9_el0", CPENC (3,3,14,13,1), 0, AARCH64_NO_FEATURES) - SYSREG ("pmiar_el1", CPENC (3,0,9,14,7), F_ARCHEXT, AARCH64_FEATURE (SEBEP)) - SYSREG ("pmicfiltr_el0", CPENC (3,3,9,6,0), F_ARCHEXT, AARCH64_FEATURE (PMUv3_ICNTR)) - SYSREG ("pmicntr_el0", CPENC (3,3,9,4,0), F_ARCHEXT, AARCH64_FEATURE (PMUv3_ICNTR)) - SYSREG ("pmicntsvr_el1", CPENC (2,0,14,12,0), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmintenclr_el1", CPENC (3,0,9,14,2), 0, AARCH64_NO_FEATURES) - SYSREG ("pmintenset_el1", CPENC (3,0,9,14,1), 0, AARCH64_NO_FEATURES) - SYSREG ("pmmir_el1", CPENC (3,0,9,14,6), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("pmovsclr_el0", CPENC (3,3,9,12,3), 0, AARCH64_NO_FEATURES) - SYSREG ("pmovsset_el0", CPENC (3,3,9,14,3), 0, AARCH64_NO_FEATURES) - SYSREG ("pmscr_el1", CPENC (3,0,9,9,0), F_ARCHEXT, AARCH64_FEATURE (PROFILE)) - SYSREG ("pmscr_el12", CPENC (3,5,9,9,0), F_ARCHEXT, AARCH64_FEATURE (PROFILE)) - SYSREG ("pmscr_el2", CPENC (3,4,9,9,0), F_ARCHEXT, AARCH64_FEATURE (PROFILE)) - SYSREG ("pmsdsfr_el1", CPENC (3,4,9,10,4), F_ARCHEXT, AARCH64_FEATURE (SPE_FDS)) - SYSREG ("pmselr_el0", CPENC (3,3,9,12,5), 0, AARCH64_NO_FEATURES) - SYSREG ("pmsevfr_el1", CPENC (3,0,9,9,5), F_ARCHEXT, AARCH64_FEATURE (PROFILE)) - SYSREG ("pmsfcr_el1", CPENC (3,0,9,9,4), F_ARCHEXT, AARCH64_FEATURE (PROFILE)) - SYSREG ("pmsicr_el1", CPENC (3,0,9,9,2), F_ARCHEXT, AARCH64_FEATURE (PROFILE)) - SYSREG ("pmsidr_el1", CPENC (3,0,9,9,7), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (PROFILE)) - SYSREG ("pmsirr_el1", CPENC (3,0,9,9,3), F_ARCHEXT, AARCH64_FEATURE (PROFILE)) - SYSREG ("pmslatfr_el1", CPENC (3,0,9,9,6), F_ARCHEXT, AARCH64_FEATURE (PROFILE)) - SYSREG ("pmsnevfr_el1", CPENC (3,0,9,9,1), F_ARCHEXT, AARCH64_FEATURE (V8_7A)) - SYSREG ("pmsscr_el1", CPENC (3,0,9,13,3), F_ARCHEXT, AARCH64_FEATURE (PMUv3_SS)) - SYSREG ("pmswinc_el0", CPENC (3,3,9,12,4), F_REG_WRITE, AARCH64_NO_FEATURES) - SYSREG ("pmuacr_el1", CPENC (3,0,9,14,4), F_ARCHEXT, AARCH64_FEATURE (PMUv3p9)) - SYSREG ("pmuserenr_el0", CPENC (3,3,9,14,0), 0, AARCH64_NO_FEATURES) - SYSREG ("pmxevcntr_el0", CPENC (3,3,9,13,2), 0, AARCH64_NO_FEATURES) - SYSREG ("pmxevtyper_el0", CPENC (3,3,9,13,1), 0, AARCH64_NO_FEATURES) - SYSREG ("pmzr_el0", CPENC (3,3,9,13,4), F_REG_WRITE|F_ARCHEXT, AARCH64_FEATURE (PMUv3_ICNTR)) - SYSREG ("por_el0", CPENC (3,3,10,2,4), F_ARCHEXT, AARCH64_FEATURE (S1POE)) - SYSREG ("por_el1", CPENC (3,0,10,2,4), F_ARCHEXT, AARCH64_FEATURE (S1POE)) - SYSREG ("por_el12", CPENC (3,5,10,2,4), F_ARCHEXT, AARCH64_FEATURE (S1POE)) - SYSREG ("por_el2", CPENC (3,4,10,2,4), F_ARCHEXT, AARCH64_FEATURE (S1POE)) - SYSREG ("por_el3", CPENC (3,6,10,2,4), F_ARCHEXT, AARCH64_FEATURE (S1POE)) - SYSREG ("prbar10_el1", CPENC (3,0,6,13,0), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar10_el2", CPENC (3,4,6,13,0), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar11_el1", CPENC (3,0,6,13,4), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar11_el2", CPENC (3,4,6,13,4), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar12_el1", CPENC (3,0,6,14,0), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar12_el2", CPENC (3,4,6,14,0), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar13_el1", CPENC (3,0,6,14,4), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar13_el2", CPENC (3,4,6,14,4), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar14_el1", CPENC (3,0,6,15,0), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar14_el2", CPENC (3,4,6,15,0), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar15_el1", CPENC (3,0,6,15,4), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar15_el2", CPENC (3,4,6,15,4), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar1_el1", CPENC (3,0,6,8,4), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar1_el2", CPENC (3,4,6,8,4), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar2_el1", CPENC (3,0,6,9,0), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar2_el2", CPENC (3,4,6,9,0), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar3_el1", CPENC (3,0,6,9,4), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar3_el2", CPENC (3,4,6,9,4), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar4_el1", CPENC (3,0,6,10,0), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar4_el2", CPENC (3,4,6,10,0), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar5_el1", CPENC (3,0,6,10,4), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar5_el2", CPENC (3,4,6,10,4), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar6_el1", CPENC (3,0,6,11,0), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar6_el2", CPENC (3,4,6,11,0), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar7_el1", CPENC (3,0,6,11,4), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar7_el2", CPENC (3,4,6,11,4), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar8_el1", CPENC (3,0,6,12,0), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar8_el2", CPENC (3,4,6,12,0), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar9_el1", CPENC (3,0,6,12,4), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar9_el2", CPENC (3,4,6,12,4), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar_el1", CPENC (3,0,6,8,0), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prbar_el2", CPENC (3,4,6,8,0), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prenr_el1", CPENC (3,0,6,1,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prenr_el2", CPENC (3,4,6,1,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar10_el1", CPENC (3,0,6,13,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar10_el2", CPENC (3,4,6,13,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar11_el1", CPENC (3,0,6,13,5), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar11_el2", CPENC (3,4,6,13,5), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar12_el1", CPENC (3,0,6,14,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar12_el2", CPENC (3,4,6,14,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar13_el1", CPENC (3,0,6,14,5), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar13_el2", CPENC (3,4,6,14,5), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar14_el1", CPENC (3,0,6,15,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar14_el2", CPENC (3,4,6,15,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar15_el1", CPENC (3,0,6,15,5), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar15_el2", CPENC (3,4,6,15,5), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar1_el1", CPENC (3,0,6,8,5), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar1_el2", CPENC (3,4,6,8,5), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar2_el1", CPENC (3,0,6,9,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar2_el2", CPENC (3,4,6,9,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar3_el1", CPENC (3,0,6,9,5), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar3_el2", CPENC (3,4,6,9,5), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar4_el1", CPENC (3,0,6,10,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar4_el2", CPENC (3,4,6,10,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar5_el1", CPENC (3,0,6,10,5), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar5_el2", CPENC (3,4,6,10,5), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar6_el1", CPENC (3,0,6,11,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar6_el2", CPENC (3,4,6,11,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar7_el1", CPENC (3,0,6,11,5), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar7_el2", CPENC (3,4,6,11,5), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar8_el1", CPENC (3,0,6,12,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar8_el2", CPENC (3,4,6,12,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar9_el1", CPENC (3,0,6,12,5), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar9_el2", CPENC (3,4,6,12,5), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar_el1", CPENC (3,0,6,8,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prlar_el2", CPENC (3,4,6,8,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prselr_el1", CPENC (3,0,6,2,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("prselr_el2", CPENC (3,4,6,2,1), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("rcwmask_el1", CPENC (3,0,13,0,6), F_ARCHEXT|F_REG_128, AARCH64_FEATURE (THE)) - SYSREG ("rcwsmask_el1", CPENC (3,0,13,0,3), F_ARCHEXT|F_REG_128, AARCH64_FEATURE (THE)) - SYSREG ("revidr_el1", CPENC (3,0,0,0,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("rgsr_el1", CPENC (3,0,1,0,5), F_ARCHEXT, AARCH64_FEATURE (MEMTAG)) - SYSREG ("rmr_el1", CPENC (3,0,12,0,2), 0, AARCH64_NO_FEATURES) - SYSREG ("rmr_el2", CPENC (3,4,12,0,2), 0, AARCH64_NO_FEATURES) - SYSREG ("rmr_el3", CPENC (3,6,12,0,2), 0, AARCH64_NO_FEATURES) - SYSREG ("rndr", CPENC (3,3,2,4,0), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (RNG)) - SYSREG ("rndrrs", CPENC (3,3,2,4,1), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (RNG)) - SYSREG ("rvbar_el1", CPENC (3,0,12,0,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("rvbar_el2", CPENC (3,4,12,0,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("rvbar_el3", CPENC (3,6,12,0,1), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("scr_el3", CPENC (3,6,1,1,0), 0, AARCH64_NO_FEATURES) - SYSREG ("sctlr_el1", CPENC (3,0,1,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("sctlr_el12", CPENC (3,5,1,0,0), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("sctlr_el2", CPENC (3,4,1,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("sctlr_el3", CPENC (3,6,1,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("sctlr2_el1", CPENC (3,0,1,0,3), F_ARCHEXT, AARCH64_FEATURE (SCTLR2)) - SYSREG ("sctlr2_el12", CPENC (3,5,1,0,3), F_ARCHEXT, AARCH64_FEATURE (SCTLR2)) - SYSREG ("sctlr2_el2", CPENC (3,4,1,0,3), F_ARCHEXT, AARCH64_FEATURE (SCTLR2)) - SYSREG ("sctlr2_el3", CPENC (3,6,1,0,3), F_ARCHEXT, AARCH64_FEATURE (SCTLR2)) - SYSREG ("scxtnum_el0", CPENC (3,3,13,0,7), F_ARCHEXT, AARCH64_FEATURE (SCXTNUM)) - SYSREG ("scxtnum_el1", CPENC (3,0,13,0,7), F_ARCHEXT, AARCH64_FEATURE (SCXTNUM)) - SYSREG ("scxtnum_el12", CPENC (3,5,13,0,7), F_ARCHEXT, AARCH64_FEATURE (SCXTNUM)) - SYSREG ("scxtnum_el2", CPENC (3,4,13,0,7), F_ARCHEXT, AARCH64_FEATURE (SCXTNUM)) - SYSREG ("scxtnum_el3", CPENC (3,6,13,0,7), F_ARCHEXT, AARCH64_FEATURE (SCXTNUM)) - SYSREG ("sder32_el2", CPENC (3,4,1,3,1), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("sder32_el3", CPENC (3,6,1,1,1), 0, AARCH64_NO_FEATURES) - SYSREG ("smcr_el1", CPENC (3,0,1,2,6), F_ARCHEXT, AARCH64_FEATURE (SME)) - SYSREG ("smcr_el12", CPENC (3,5,1,2,6), F_ARCHEXT, AARCH64_FEATURE (SME)) - SYSREG ("smcr_el2", CPENC (3,4,1,2,6), F_ARCHEXT, AARCH64_FEATURE (SME)) - SYSREG ("smcr_el3", CPENC (3,6,1,2,6), F_ARCHEXT, AARCH64_FEATURE (SME)) - SYSREG ("smidr_el1", CPENC (3,1,0,0,6), F_REG_READ|F_ARCHEXT, AARCH64_FEATURE (SME)) - SYSREG ("smpri_el1", CPENC (3,0,1,2,4), F_ARCHEXT, AARCH64_FEATURE (SME)) - SYSREG ("smprimap_el2", CPENC (3,4,1,2,5), F_ARCHEXT, AARCH64_FEATURE (SME)) - SYSREG ("sp_el0", CPENC (3,0,4,1,0), 0, AARCH64_NO_FEATURES) - SYSREG ("sp_el1", CPENC (3,4,4,1,0), 0, AARCH64_NO_FEATURES) - SYSREG ("sp_el2", CPENC (3,6,4,1,0), 0, AARCH64_NO_FEATURES) - SYSREG ("spsel", CPENC (3,0,4,2,0), 0, AARCH64_NO_FEATURES) - SYSREG ("spsr_abt", CPENC (3,4,4,3,1), 0, AARCH64_NO_FEATURES) - SYSREG ("spsr_el1", CPENC (3,0,4,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("spsr_el12", CPENC (3,5,4,0,0), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("spsr_el2", CPENC (3,4,4,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("spsr_el3", CPENC (3,6,4,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("spsr_fiq", CPENC (3,4,4,3,3), 0, AARCH64_NO_FEATURES) - SYSREG ("spsr_hyp", CPENC (3,4,4,0,0), F_DEPRECATED, AARCH64_NO_FEATURES) - SYSREG ("spsr_irq", CPENC (3,4,4,3,0), 0, AARCH64_NO_FEATURES) - SYSREG ("spsr_svc", CPENC (3,0,4,0,0), F_DEPRECATED, AARCH64_NO_FEATURES) - SYSREG ("spsr_und", CPENC (3,4,4,3,2), 0, AARCH64_NO_FEATURES) - SYSREG ("ssbs", CPENC (3,3,4,2,6), F_ARCHEXT, AARCH64_FEATURE (SSBS)) - SYSREG ("svcr", CPENC (3,3,4,2,2), F_ARCHEXT, AARCH64_FEATURE (SME)) - SYSREG ("s2pir_el2", CPENC (3,4,10,2,5), F_ARCHEXT, AARCH64_FEATURE (S2PIE)) - SYSREG ("s2por_el1", CPENC (3,0,10,2,5), F_ARCHEXT, AARCH64_FEATURE (S2POE)) - SYSREG ("tco", CPENC (3,3,4,2,7), F_ARCHEXT, AARCH64_FEATURE (MEMTAG)) - SYSREG ("tcr_el1", CPENC (3,0,2,0,2), 0, AARCH64_NO_FEATURES) - SYSREG ("tcr_el12", CPENC (3,5,2,0,2), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("tcr_el2", CPENC (3,4,2,0,2), 0, AARCH64_NO_FEATURES) - SYSREG ("tcr_el3", CPENC (3,6,2,0,2), 0, AARCH64_NO_FEATURES) - SYSREG ("tcr2_el1", CPENC (3,0,2,0,3), F_ARCHEXT, AARCH64_FEATURE (TCR2)) - SYSREG ("tcr2_el12", CPENC (3,5,2,0,3), F_ARCHEXT, AARCH64_FEATURE (TCR2)) - SYSREG ("tcr2_el2", CPENC (3,4,2,0,3), F_ARCHEXT, AARCH64_FEATURE (TCR2)) - SYSREG ("teecr32_el1", CPENC (2,2,0,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("teehbr32_el1", CPENC (2,2,1,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("tfsr_el1", CPENC (3,0,5,6,0), F_ARCHEXT, AARCH64_FEATURE (MEMTAG)) - SYSREG ("tfsr_el12", CPENC (3,5,5,6,0), F_ARCHEXT, AARCH64_FEATURE (MEMTAG)) - SYSREG ("tfsr_el2", CPENC (3,4,5,6,0), F_ARCHEXT, AARCH64_FEATURE (MEMTAG)) - SYSREG ("tfsr_el3", CPENC (3,6,5,6,0), F_ARCHEXT, AARCH64_FEATURE (MEMTAG)) - SYSREG ("tfsre0_el1", CPENC (3,0,5,6,1), F_ARCHEXT, AARCH64_FEATURE (MEMTAG)) - SYSREG ("tpidr2_el0", CPENC (3,3,13,0,5), F_ARCHEXT, AARCH64_FEATURE (SME)) - SYSREG ("tpidr_el0", CPENC (3,3,13,0,2), 0, AARCH64_NO_FEATURES) - SYSREG ("tpidr_el1", CPENC (3,0,13,0,4), 0, AARCH64_NO_FEATURES) - SYSREG ("tpidr_el2", CPENC (3,4,13,0,2), 0, AARCH64_NO_FEATURES) - SYSREG ("tpidr_el3", CPENC (3,6,13,0,2), 0, AARCH64_NO_FEATURES) - SYSREG ("tpidrro_el0", CPENC (3,3,13,0,3), 0, AARCH64_NO_FEATURES) - SYSREG ("trbbaser_el1", CPENC (3,0,9,11,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trbidr_el1", CPENC (3,0,9,11,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trblimitr_el1", CPENC (3,0,9,11,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trbmar_el1", CPENC (3,0,9,11,4), 0, AARCH64_NO_FEATURES) - SYSREG ("trbptr_el1", CPENC (3,0,9,11,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trbsr_el1", CPENC (3,0,9,11,3), 0, AARCH64_NO_FEATURES) - SYSREG ("trbtrg_el1", CPENC (3,0,9,11,6), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacatr0", CPENC (2,1,2,0,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacatr1", CPENC (2,1,2,2,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacatr10", CPENC (2,1,2,4,3), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacatr11", CPENC (2,1,2,6,3), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacatr12", CPENC (2,1,2,8,3), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacatr13", CPENC (2,1,2,10,3), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacatr14", CPENC (2,1,2,12,3), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacatr15", CPENC (2,1,2,14,3), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacatr2", CPENC (2,1,2,4,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacatr3", CPENC (2,1,2,6,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacatr4", CPENC (2,1,2,8,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacatr5", CPENC (2,1,2,10,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacatr6", CPENC (2,1,2,12,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacatr7", CPENC (2,1,2,14,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacatr8", CPENC (2,1,2,0,3), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacatr9", CPENC (2,1,2,2,3), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacvr0", CPENC (2,1,2,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacvr1", CPENC (2,1,2,2,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacvr10", CPENC (2,1,2,4,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacvr11", CPENC (2,1,2,6,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacvr12", CPENC (2,1,2,8,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacvr13", CPENC (2,1,2,10,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacvr14", CPENC (2,1,2,12,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacvr15", CPENC (2,1,2,14,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacvr2", CPENC (2,1,2,4,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacvr3", CPENC (2,1,2,6,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacvr4", CPENC (2,1,2,8,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacvr5", CPENC (2,1,2,10,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacvr6", CPENC (2,1,2,12,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacvr7", CPENC (2,1,2,14,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacvr8", CPENC (2,1,2,0,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcacvr9", CPENC (2,1,2,2,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcauthstatus", CPENC (2,1,7,14,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcauxctlr", CPENC (2,1,0,6,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcbbctlr", CPENC (2,1,0,15,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcccctlr", CPENC (2,1,0,14,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trccidcctlr0", CPENC (2,1,3,0,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trccidcctlr1", CPENC (2,1,3,1,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trccidcvr0", CPENC (2,1,3,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trccidcvr1", CPENC (2,1,3,2,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trccidcvr2", CPENC (2,1,3,4,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trccidcvr3", CPENC (2,1,3,6,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trccidcvr4", CPENC (2,1,3,8,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trccidcvr5", CPENC (2,1,3,10,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trccidcvr6", CPENC (2,1,3,12,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trccidcvr7", CPENC (2,1,3,14,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trccidr0", CPENC (2,1,7,12,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trccidr1", CPENC (2,1,7,13,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trccidr2", CPENC (2,1,7,14,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trccidr3", CPENC (2,1,7,15,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcclaimclr", CPENC (2,1,7,9,6), 0, AARCH64_NO_FEATURES) - SYSREG ("trcclaimset", CPENC (2,1,7,8,6), 0, AARCH64_NO_FEATURES) - SYSREG ("trccntctlr0", CPENC (2,1,0,4,5), 0, AARCH64_NO_FEATURES) - SYSREG ("trccntctlr1", CPENC (2,1,0,5,5), 0, AARCH64_NO_FEATURES) - SYSREG ("trccntctlr2", CPENC (2,1,0,6,5), 0, AARCH64_NO_FEATURES) - SYSREG ("trccntctlr3", CPENC (2,1,0,7,5), 0, AARCH64_NO_FEATURES) - SYSREG ("trccntrldvr0", CPENC (2,1,0,0,5), 0, AARCH64_NO_FEATURES) - SYSREG ("trccntrldvr1", CPENC (2,1,0,1,5), 0, AARCH64_NO_FEATURES) - SYSREG ("trccntrldvr2", CPENC (2,1,0,2,5), 0, AARCH64_NO_FEATURES) - SYSREG ("trccntrldvr3", CPENC (2,1,0,3,5), 0, AARCH64_NO_FEATURES) - SYSREG ("trccntvr0", CPENC (2,1,0,8,5), 0, AARCH64_NO_FEATURES) - SYSREG ("trccntvr1", CPENC (2,1,0,9,5), 0, AARCH64_NO_FEATURES) - SYSREG ("trccntvr2", CPENC (2,1,0,10,5), 0, AARCH64_NO_FEATURES) - SYSREG ("trccntvr3", CPENC (2,1,0,11,5), 0, AARCH64_NO_FEATURES) - SYSREG ("trcconfigr", CPENC (2,1,0,4,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcdevaff0", CPENC (2,1,7,10,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcdevaff1", CPENC (2,1,7,11,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcdevarch", CPENC (2,1,7,15,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcdevid", CPENC (2,1,7,2,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcdevtype", CPENC (2,1,7,3,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcdvcmr0", CPENC (2,1,2,0,6), 0, AARCH64_NO_FEATURES) - SYSREG ("trcdvcmr1", CPENC (2,1,2,4,6), 0, AARCH64_NO_FEATURES) - SYSREG ("trcdvcmr2", CPENC (2,1,2,8,6), 0, AARCH64_NO_FEATURES) - SYSREG ("trcdvcmr3", CPENC (2,1,2,12,6), 0, AARCH64_NO_FEATURES) - SYSREG ("trcdvcmr4", CPENC (2,1,2,0,7), 0, AARCH64_NO_FEATURES) - SYSREG ("trcdvcmr5", CPENC (2,1,2,4,7), 0, AARCH64_NO_FEATURES) - SYSREG ("trcdvcmr6", CPENC (2,1,2,8,7), 0, AARCH64_NO_FEATURES) - SYSREG ("trcdvcmr7", CPENC (2,1,2,12,7), 0, AARCH64_NO_FEATURES) - SYSREG ("trcdvcvr0", CPENC (2,1,2,0,4), 0, AARCH64_NO_FEATURES) - SYSREG ("trcdvcvr1", CPENC (2,1,2,4,4), 0, AARCH64_NO_FEATURES) - SYSREG ("trcdvcvr2", CPENC (2,1,2,8,4), 0, AARCH64_NO_FEATURES) - SYSREG ("trcdvcvr3", CPENC (2,1,2,12,4), 0, AARCH64_NO_FEATURES) - SYSREG ("trcdvcvr4", CPENC (2,1,2,0,5), 0, AARCH64_NO_FEATURES) - SYSREG ("trcdvcvr5", CPENC (2,1,2,4,5), 0, AARCH64_NO_FEATURES) - SYSREG ("trcdvcvr6", CPENC (2,1,2,8,5), 0, AARCH64_NO_FEATURES) - SYSREG ("trcdvcvr7", CPENC (2,1,2,12,5), 0, AARCH64_NO_FEATURES) - SYSREG ("trceventctl0r", CPENC (2,1,0,8,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trceventctl1r", CPENC (2,1,0,9,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcextinselr", CPENC (2,1,0,8,4), F_REG_ALIAS, AARCH64_NO_FEATURES) - SYSREG ("trcextinselr0", CPENC (2,1,0,8,4), 0, AARCH64_NO_FEATURES) - SYSREG ("trcextinselr1", CPENC (2,1,0,9,4), 0, AARCH64_NO_FEATURES) - SYSREG ("trcextinselr2", CPENC (2,1,0,10,4), 0, AARCH64_NO_FEATURES) - SYSREG ("trcextinselr3", CPENC (2,1,0,11,4), 0, AARCH64_NO_FEATURES) - SYSREG ("trcidr0", CPENC (2,1,0,8,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcidr1", CPENC (2,1,0,9,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcidr10", CPENC (2,1,0,2,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcidr11", CPENC (2,1,0,3,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcidr12", CPENC (2,1,0,4,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcidr13", CPENC (2,1,0,5,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcidr2", CPENC (2,1,0,10,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcidr3", CPENC (2,1,0,11,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcidr4", CPENC (2,1,0,12,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcidr5", CPENC (2,1,0,13,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcidr6", CPENC (2,1,0,14,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcidr7", CPENC (2,1,0,15,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcidr8", CPENC (2,1,0,0,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcidr9", CPENC (2,1,0,1,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcimspec0", CPENC (2,1,0,0,7), 0, AARCH64_NO_FEATURES) - SYSREG ("trcimspec1", CPENC (2,1,0,1,7), 0, AARCH64_NO_FEATURES) - SYSREG ("trcimspec2", CPENC (2,1,0,2,7), 0, AARCH64_NO_FEATURES) - SYSREG ("trcimspec3", CPENC (2,1,0,3,7), 0, AARCH64_NO_FEATURES) - SYSREG ("trcimspec4", CPENC (2,1,0,4,7), 0, AARCH64_NO_FEATURES) - SYSREG ("trcimspec5", CPENC (2,1,0,5,7), 0, AARCH64_NO_FEATURES) - SYSREG ("trcimspec6", CPENC (2,1,0,6,7), 0, AARCH64_NO_FEATURES) - SYSREG ("trcimspec7", CPENC (2,1,0,7,7), 0, AARCH64_NO_FEATURES) - SYSREG ("trcitctrl", CPENC (2,1,7,0,4), 0, AARCH64_NO_FEATURES) - SYSREG ("trcitecr_el1", CPENC (3,0,1,2,3), F_ARCHEXT, AARCH64_FEATURE (ITE)) - SYSREG ("trcitecr_el12", CPENC (3,5,1,2,3), F_ARCHEXT, AARCH64_FEATURE (ITE)) - SYSREG ("trcitecr_el2", CPENC (3,4,1,2,3), F_ARCHEXT, AARCH64_FEATURE (ITE)) - SYSREG ("trciteedcr", CPENC (2,1,0,2,1), F_ARCHEXT, AARCH64_FEATURE (ITE)) - SYSREG ("trclar", CPENC (2,1,7,12,6), F_REG_WRITE, AARCH64_NO_FEATURES) - SYSREG ("trclsr", CPENC (2,1,7,13,6), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcoslar", CPENC (2,1,1,0,4), F_REG_WRITE, AARCH64_NO_FEATURES) - SYSREG ("trcoslsr", CPENC (2,1,1,1,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcpdcr", CPENC (2,1,1,4,4), 0, AARCH64_NO_FEATURES) - SYSREG ("trcpdsr", CPENC (2,1,1,5,4), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcpidr0", CPENC (2,1,7,8,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcpidr1", CPENC (2,1,7,9,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcpidr2", CPENC (2,1,7,10,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcpidr3", CPENC (2,1,7,11,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcpidr4", CPENC (2,1,7,4,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcpidr5", CPENC (2,1,7,5,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcpidr6", CPENC (2,1,7,6,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcpidr7", CPENC (2,1,7,7,7), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcprgctlr", CPENC (2,1,0,1,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcprocselr", CPENC (2,1,0,2,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcqctlr", CPENC (2,1,0,1,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr10", CPENC (2,1,1,10,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr11", CPENC (2,1,1,11,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr12", CPENC (2,1,1,12,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr13", CPENC (2,1,1,13,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr14", CPENC (2,1,1,14,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr15", CPENC (2,1,1,15,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr16", CPENC (2,1,1,0,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr17", CPENC (2,1,1,1,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr18", CPENC (2,1,1,2,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr19", CPENC (2,1,1,3,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr2", CPENC (2,1,1,2,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr20", CPENC (2,1,1,4,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr21", CPENC (2,1,1,5,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr22", CPENC (2,1,1,6,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr23", CPENC (2,1,1,7,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr24", CPENC (2,1,1,8,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr25", CPENC (2,1,1,9,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr26", CPENC (2,1,1,10,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr27", CPENC (2,1,1,11,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr28", CPENC (2,1,1,12,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr29", CPENC (2,1,1,13,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr3", CPENC (2,1,1,3,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr30", CPENC (2,1,1,14,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr31", CPENC (2,1,1,15,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr4", CPENC (2,1,1,4,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr5", CPENC (2,1,1,5,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr6", CPENC (2,1,1,6,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr7", CPENC (2,1,1,7,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr8", CPENC (2,1,1,8,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsctlr9", CPENC (2,1,1,9,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcrsr", CPENC (2,1,0,10,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcseqevr0", CPENC (2,1,0,0,4), 0, AARCH64_NO_FEATURES) - SYSREG ("trcseqevr1", CPENC (2,1,0,1,4), 0, AARCH64_NO_FEATURES) - SYSREG ("trcseqevr2", CPENC (2,1,0,2,4), 0, AARCH64_NO_FEATURES) - SYSREG ("trcseqrstevr", CPENC (2,1,0,6,4), 0, AARCH64_NO_FEATURES) - SYSREG ("trcseqstr", CPENC (2,1,0,7,4), 0, AARCH64_NO_FEATURES) - SYSREG ("trcssccr0", CPENC (2,1,1,0,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcssccr1", CPENC (2,1,1,1,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcssccr2", CPENC (2,1,1,2,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcssccr3", CPENC (2,1,1,3,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcssccr4", CPENC (2,1,1,4,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcssccr5", CPENC (2,1,1,5,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcssccr6", CPENC (2,1,1,6,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcssccr7", CPENC (2,1,1,7,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcsscsr0", CPENC (2,1,1,8,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcsscsr1", CPENC (2,1,1,9,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcsscsr2", CPENC (2,1,1,10,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcsscsr3", CPENC (2,1,1,11,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcsscsr4", CPENC (2,1,1,12,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcsscsr5", CPENC (2,1,1,13,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcsscsr6", CPENC (2,1,1,14,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcsscsr7", CPENC (2,1,1,15,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcsspcicr0", CPENC (2,1,1,0,3), 0, AARCH64_NO_FEATURES) - SYSREG ("trcsspcicr1", CPENC (2,1,1,1,3), 0, AARCH64_NO_FEATURES) - SYSREG ("trcsspcicr2", CPENC (2,1,1,2,3), 0, AARCH64_NO_FEATURES) - SYSREG ("trcsspcicr3", CPENC (2,1,1,3,3), 0, AARCH64_NO_FEATURES) - SYSREG ("trcsspcicr4", CPENC (2,1,1,4,3), 0, AARCH64_NO_FEATURES) - SYSREG ("trcsspcicr5", CPENC (2,1,1,5,3), 0, AARCH64_NO_FEATURES) - SYSREG ("trcsspcicr6", CPENC (2,1,1,6,3), 0, AARCH64_NO_FEATURES) - SYSREG ("trcsspcicr7", CPENC (2,1,1,7,3), 0, AARCH64_NO_FEATURES) - SYSREG ("trcstallctlr", CPENC (2,1,0,11,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcstatr", CPENC (2,1,0,3,0), F_REG_READ, AARCH64_NO_FEATURES) - SYSREG ("trcsyncpr", CPENC (2,1,0,13,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trctraceidr", CPENC (2,1,0,0,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trctsctlr", CPENC (2,1,0,12,0), 0, AARCH64_NO_FEATURES) - SYSREG ("trcvdarcctlr", CPENC (2,1,0,10,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcvdctlr", CPENC (2,1,0,8,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcvdsacctlr", CPENC (2,1,0,9,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcvictlr", CPENC (2,1,0,0,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcviiectlr", CPENC (2,1,0,1,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcvipcssctlr", CPENC (2,1,0,3,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcvissctlr", CPENC (2,1,0,2,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcvmidcctlr0", CPENC (2,1,3,2,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcvmidcctlr1", CPENC (2,1,3,3,2), 0, AARCH64_NO_FEATURES) - SYSREG ("trcvmidcvr0", CPENC (2,1,3,0,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcvmidcvr1", CPENC (2,1,3,2,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcvmidcvr2", CPENC (2,1,3,4,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcvmidcvr3", CPENC (2,1,3,6,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcvmidcvr4", CPENC (2,1,3,8,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcvmidcvr5", CPENC (2,1,3,10,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcvmidcvr6", CPENC (2,1,3,12,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trcvmidcvr7", CPENC (2,1,3,14,1), 0, AARCH64_NO_FEATURES) - SYSREG ("trfcr_el1", CPENC (3,0,1,2,1), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("trfcr_el12", CPENC (3,5,1,2,1), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("trfcr_el2", CPENC (3,4,1,2,1), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("ttbr0_el1", CPENC (3,0,2,0,0), F_REG_128, AARCH64_NO_FEATURES) - SYSREG ("ttbr0_el12", CPENC (3,5,2,0,0), F_ARCHEXT|F_REG_128, AARCH64_FEATURE (V8_1A)) - SYSREG ("ttbr0_el2", CPENC (3,4,2,0,0), F_ARCHEXT|F_REG_128, AARCH64_FEATURE (V8A)) - SYSREG ("ttbr0_el3", CPENC (3,6,2,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("ttbr1_el1", CPENC (3,0,2,0,1), F_REG_128, AARCH64_NO_FEATURES) - SYSREG ("ttbr1_el12", CPENC (3,5,2,0,1), F_ARCHEXT|F_REG_128, AARCH64_FEATURE (V8_1A)) - SYSREG ("ttbr1_el2", CPENC (3,4,2,0,1), F_ARCHEXT|F_REG_128, AARCH64_FEATURES (2, V8A, V8_1A)) - SYSREG ("uao", CPENC (3,0,4,2,4), F_ARCHEXT, AARCH64_FEATURE (V8_2A)) - SYSREG ("vbar_el1", CPENC (3,0,12,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("vbar_el12", CPENC (3,5,12,0,0), F_ARCHEXT, AARCH64_FEATURE (V8_1A)) - SYSREG ("vbar_el2", CPENC (3,4,12,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("vbar_el3", CPENC (3,6,12,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("vdisr_el2", CPENC (3,4,12,1,1), F_ARCHEXT, AARCH64_FEATURE (RAS)) - SYSREG ("vmecid_a_el2", CPENC (3,4,10,9,1), F_ARCHEXT, AARCH64_FEATURE (V8_7A)) - SYSREG ("vmecid_p_el2", CPENC (3,4,10,9,0), F_ARCHEXT, AARCH64_FEATURE (V8_7A)) - SYSREG ("vmpidr_el2", CPENC (3,4,0,0,5), 0, AARCH64_NO_FEATURES) - SYSREG ("vncr_el2", CPENC (3,4,2,2,0), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("vpidr_el2", CPENC (3,4,0,0,0), 0, AARCH64_NO_FEATURES) - SYSREG ("vsctlr_el2", CPENC (3,4,2,0,0), F_ARCHEXT, AARCH64_FEATURE (V8R)) - SYSREG ("vsesr_el2", CPENC (3,4,5,2,3), F_ARCHEXT, AARCH64_FEATURE (RAS)) - SYSREG ("vstcr_el2", CPENC (3,4,2,6,2), F_ARCHEXT, AARCH64_FEATURE (V8_4A)) - SYSREG ("vsttbr_el2", CPENC (3,4,2,6,0), F_ARCHEXT, AARCH64_FEATURES (2, V8A, V8_4A)) - SYSREG ("vtcr_el2", CPENC (3,4,2,1,2), 0, AARCH64_NO_FEATURES) - SYSREG ("vttbr_el2", CPENC (3,4,2,1,0), F_ARCHEXT|F_REG_128, AARCH64_FEATURE (V8A)) - SYSREG ("zcr_el1", CPENC (3,0,1,2,0), F_ARCHEXT, AARCH64_FEATURE (SVE)) - SYSREG ("zcr_el12", CPENC (3,5,1,2,0), F_ARCHEXT, AARCH64_FEATURE (SVE)) - SYSREG ("zcr_el2", CPENC (3,4,1,2,0), F_ARCHEXT, AARCH64_FEATURE (SVE)) - SYSREG ("zcr_el3", CPENC (3,6,1,2,0), F_ARCHEXT, AARCH64_FEATURE (SVE)) + SYSREG ("accdata_el1", CPENC (3,0,13,0,5), 0, AARCH64_FEATURE (LS64)) /* LS64_ACCDATA */ + SYSREG ("actlr_el1", CPENC (3,0,1,0,1), 0, AARCH64_NO_FEATURES) + SYSREG ("actlr_el12", CPENC (3,5,1,0,1), 0, AARCH64_NO_FEATURES) + SYSREG ("actlr_el2", CPENC (3,4,1,0,1), 0, AARCH64_NO_FEATURES) + SYSREG ("actlr_el3", CPENC (3,6,1,0,1), 0, AARCH64_NO_FEATURES) + SYSREG ("actlralias_el1", CPENC (3,0,1,4,5), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("actlrmask_el1", CPENC (3,0,1,4,1), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("actlrmask_el12", CPENC (3,5,1,4,1), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("actlrmask_el2", CPENC (3,4,1,4,1), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("afsr0_el1", CPENC (3,0,5,1,0), 0, AARCH64_NO_FEATURES) + SYSREG ("afsr0_el12", CPENC (3,5,5,1,0), 0, AARCH64_NO_FEATURES) + SYSREG ("afsr0_el2", CPENC (3,4,5,1,0), 0, AARCH64_NO_FEATURES) + SYSREG ("afsr0_el3", CPENC (3,6,5,1,0), 0, AARCH64_NO_FEATURES) + SYSREG ("afsr1_el1", CPENC (3,0,5,1,1), 0, AARCH64_NO_FEATURES) + SYSREG ("afsr1_el12", CPENC (3,5,5,1,1), 0, AARCH64_NO_FEATURES) + SYSREG ("afsr1_el2", CPENC (3,4,5,1,1), 0, AARCH64_NO_FEATURES) + SYSREG ("afsr1_el3", CPENC (3,6,5,1,1), 0, AARCH64_NO_FEATURES) + SYSREG ("aidr_el1", CPENC (3,1,0,0,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("allint", CPENC (3,0,4,3,0), 0, AARCH64_FEATURE (V8_7A)) /* NMI */ + SYSREG ("amair_el1", CPENC (3,0,10,3,0), 0, AARCH64_NO_FEATURES) + SYSREG ("amair_el12", CPENC (3,5,10,3,0), 0, AARCH64_NO_FEATURES) + SYSREG ("amair_el2", CPENC (3,4,10,3,0), 0, AARCH64_NO_FEATURES) + SYSREG ("amair_el3", CPENC (3,6,10,3,0), 0, AARCH64_NO_FEATURES) + SYSREG ("amair2_el1", CPENC (3,0,10,3,1), 0, AARCH64_FEATURE (V8_8A)) /* AIE */ + SYSREG ("amair2_el12", CPENC (3,5,10,3,1), 0, AARCH64_FEATURE (V8_8A)) /* AIE */ + SYSREG ("amair2_el2", CPENC (3,4,10,3,1), 0, AARCH64_FEATURE (V8_8A)) /* AIE */ + SYSREG ("amair2_el3", CPENC (3,6,10,3,1), 0, AARCH64_FEATURE (V8_8A)) /* AIE */ + SYSREG ("amcfgr_el0", CPENC (3,3,13,2,1), F_REG_READ, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amcg1idr_el0", CPENC (3,3,13,2,6), F_REG_READ, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amcgcr_el0", CPENC (3,3,13,2,2), F_REG_READ, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amcntenclr0_el0", CPENC (3,3,13,2,4), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amcntenclr1_el0", CPENC (3,3,13,3,0), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amcntenset0_el0", CPENC (3,3,13,2,5), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amcntenset1_el0", CPENC (3,3,13,3,1), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amcr_el0", CPENC (3,3,13,2,0), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr00_el0", CPENC (3,3,13,4,0), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr01_el0", CPENC (3,3,13,4,1), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr02_el0", CPENC (3,3,13,4,2), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr03_el0", CPENC (3,3,13,4,3), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr10_el0", CPENC (3,3,13,12,0), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr110_el0", CPENC (3,3,13,13,2), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr111_el0", CPENC (3,3,13,13,3), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr112_el0", CPENC (3,3,13,13,4), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr113_el0", CPENC (3,3,13,13,5), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr114_el0", CPENC (3,3,13,13,6), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr115_el0", CPENC (3,3,13,13,7), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr11_el0", CPENC (3,3,13,12,1), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr12_el0", CPENC (3,3,13,12,2), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr13_el0", CPENC (3,3,13,12,3), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr14_el0", CPENC (3,3,13,12,4), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr15_el0", CPENC (3,3,13,12,5), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr16_el0", CPENC (3,3,13,12,6), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr17_el0", CPENC (3,3,13,12,7), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr18_el0", CPENC (3,3,13,13,0), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntr19_el0", CPENC (3,3,13,13,1), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevcntvoff00_el2", CPENC (3,4,13,8,0), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff010_el2", CPENC (3,4,13,9,2), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff011_el2", CPENC (3,4,13,9,3), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff012_el2", CPENC (3,4,13,9,4), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff013_el2", CPENC (3,4,13,9,5), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff014_el2", CPENC (3,4,13,9,6), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff015_el2", CPENC (3,4,13,9,7), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff01_el2", CPENC (3,4,13,8,1), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff02_el2", CPENC (3,4,13,8,2), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff03_el2", CPENC (3,4,13,8,3), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff04_el2", CPENC (3,4,13,8,4), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff05_el2", CPENC (3,4,13,8,5), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff06_el2", CPENC (3,4,13,8,6), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff07_el2", CPENC (3,4,13,8,7), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff08_el2", CPENC (3,4,13,9,0), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff09_el2", CPENC (3,4,13,9,1), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff10_el2", CPENC (3,4,13,10,0), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff110_el2", CPENC (3,4,13,11,2), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff111_el2", CPENC (3,4,13,11,3), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff112_el2", CPENC (3,4,13,11,4), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff113_el2", CPENC (3,4,13,11,5), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff114_el2", CPENC (3,4,13,11,6), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff115_el2", CPENC (3,4,13,11,7), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff11_el2", CPENC (3,4,13,10,1), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff12_el2", CPENC (3,4,13,10,2), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff13_el2", CPENC (3,4,13,10,3), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff14_el2", CPENC (3,4,13,10,4), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff15_el2", CPENC (3,4,13,10,5), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff16_el2", CPENC (3,4,13,10,6), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff17_el2", CPENC (3,4,13,10,7), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff18_el2", CPENC (3,4,13,11,0), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevcntvoff19_el2", CPENC (3,4,13,11,1), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1p1 */ + SYSREG ("amevtyper00_el0", CPENC (3,3,13,6,0), F_REG_READ, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper01_el0", CPENC (3,3,13,6,1), F_REG_READ, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper02_el0", CPENC (3,3,13,6,2), F_REG_READ, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper03_el0", CPENC (3,3,13,6,3), F_REG_READ, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper10_el0", CPENC (3,3,13,14,0), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper110_el0", CPENC (3,3,13,15,2), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper111_el0", CPENC (3,3,13,15,3), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper112_el0", CPENC (3,3,13,15,4), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper113_el0", CPENC (3,3,13,15,5), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper114_el0", CPENC (3,3,13,15,6), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper115_el0", CPENC (3,3,13,15,7), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper11_el0", CPENC (3,3,13,14,1), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper12_el0", CPENC (3,3,13,14,2), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper13_el0", CPENC (3,3,13,14,3), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper14_el0", CPENC (3,3,13,14,4), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper15_el0", CPENC (3,3,13,14,5), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper16_el0", CPENC (3,3,13,14,6), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper17_el0", CPENC (3,3,13,14,7), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper18_el0", CPENC (3,3,13,15,0), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amevtyper19_el0", CPENC (3,3,13,15,1), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("amuserenr_el0", CPENC (3,3,13,2,3), 0, AARCH64_FEATURE (V8_3A)) /* AMUv1 */ + SYSREG ("apdakeyhi_el1", CPENC (3,0,2,2,1), 0, AARCH64_FEATURE (PAUTH)) + SYSREG ("apdakeylo_el1", CPENC (3,0,2,2,0), 0, AARCH64_FEATURE (PAUTH)) + SYSREG ("apdbkeyhi_el1", CPENC (3,0,2,2,3), 0, AARCH64_FEATURE (PAUTH)) + SYSREG ("apdbkeylo_el1", CPENC (3,0,2,2,2), 0, AARCH64_FEATURE (PAUTH)) + SYSREG ("apgakeyhi_el1", CPENC (3,0,2,3,1), 0, AARCH64_FEATURE (PAUTH)) + SYSREG ("apgakeylo_el1", CPENC (3,0,2,3,0), 0, AARCH64_FEATURE (PAUTH)) + SYSREG ("apiakeyhi_el1", CPENC (3,0,2,1,1), 0, AARCH64_FEATURE (PAUTH)) + SYSREG ("apiakeylo_el1", CPENC (3,0,2,1,0), 0, AARCH64_FEATURE (PAUTH)) + SYSREG ("apibkeyhi_el1", CPENC (3,0,2,1,3), 0, AARCH64_FEATURE (PAUTH)) + SYSREG ("apibkeylo_el1", CPENC (3,0,2,1,2), 0, AARCH64_FEATURE (PAUTH)) + SYSREG ("brbcr_el1", CPENC (2,1,9,0,0), 0, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbcr_el12", CPENC (2,5,9,0,0), 0, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbcr_el2", CPENC (2,4,9,0,0), 0, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbfcr_el1", CPENC (2,1,9,0,1), 0, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbidr0_el1", CPENC (2,1,9,2,0), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf0_el1", CPENC (2,1,8,0,0), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf10_el1", CPENC (2,1,8,10,0), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf11_el1", CPENC (2,1,8,11,0), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf12_el1", CPENC (2,1,8,12,0), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf13_el1", CPENC (2,1,8,13,0), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf14_el1", CPENC (2,1,8,14,0), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf15_el1", CPENC (2,1,8,15,0), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf16_el1", CPENC (2,1,8,0,4), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf17_el1", CPENC (2,1,8,1,4), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf18_el1", CPENC (2,1,8,2,4), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf19_el1", CPENC (2,1,8,3,4), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf1_el1", CPENC (2,1,8,1,0), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf20_el1", CPENC (2,1,8,4,4), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf21_el1", CPENC (2,1,8,5,4), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf22_el1", CPENC (2,1,8,6,4), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf23_el1", CPENC (2,1,8,7,4), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf24_el1", CPENC (2,1,8,8,4), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf25_el1", CPENC (2,1,8,9,4), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf26_el1", CPENC (2,1,8,10,4), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf27_el1", CPENC (2,1,8,11,4), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf28_el1", CPENC (2,1,8,12,4), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf29_el1", CPENC (2,1,8,13,4), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf2_el1", CPENC (2,1,8,2,0), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf30_el1", CPENC (2,1,8,14,4), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf31_el1", CPENC (2,1,8,15,4), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf3_el1", CPENC (2,1,8,3,0), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf4_el1", CPENC (2,1,8,4,0), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf5_el1", CPENC (2,1,8,5,0), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf6_el1", CPENC (2,1,8,6,0), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf7_el1", CPENC (2,1,8,7,0), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf8_el1", CPENC (2,1,8,8,0), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinf9_el1", CPENC (2,1,8,9,0), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbinfinj_el1", CPENC (2,1,9,1,0), 0, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc0_el1", CPENC (2,1,8,0,1), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc10_el1", CPENC (2,1,8,10,1), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc11_el1", CPENC (2,1,8,11,1), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc12_el1", CPENC (2,1,8,12,1), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc13_el1", CPENC (2,1,8,13,1), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc14_el1", CPENC (2,1,8,14,1), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc15_el1", CPENC (2,1,8,15,1), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc16_el1", CPENC (2,1,8,0,5), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc17_el1", CPENC (2,1,8,1,5), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc18_el1", CPENC (2,1,8,2,5), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc19_el1", CPENC (2,1,8,3,5), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc1_el1", CPENC (2,1,8,1,1), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc20_el1", CPENC (2,1,8,4,5), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc21_el1", CPENC (2,1,8,5,5), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc22_el1", CPENC (2,1,8,6,5), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc23_el1", CPENC (2,1,8,7,5), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc24_el1", CPENC (2,1,8,8,5), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc25_el1", CPENC (2,1,8,9,5), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc26_el1", CPENC (2,1,8,10,5), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc27_el1", CPENC (2,1,8,11,5), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc28_el1", CPENC (2,1,8,12,5), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc29_el1", CPENC (2,1,8,13,5), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc2_el1", CPENC (2,1,8,2,1), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc30_el1", CPENC (2,1,8,14,5), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc31_el1", CPENC (2,1,8,15,5), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc3_el1", CPENC (2,1,8,3,1), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc4_el1", CPENC (2,1,8,4,1), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc5_el1", CPENC (2,1,8,5,1), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc6_el1", CPENC (2,1,8,6,1), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc7_el1", CPENC (2,1,8,7,1), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc8_el1", CPENC (2,1,8,8,1), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrc9_el1", CPENC (2,1,8,9,1), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbsrcinj_el1", CPENC (2,1,9,1,1), 0, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt0_el1", CPENC (2,1,8,0,2), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt10_el1", CPENC (2,1,8,10,2), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt11_el1", CPENC (2,1,8,11,2), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt12_el1", CPENC (2,1,8,12,2), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt13_el1", CPENC (2,1,8,13,2), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt14_el1", CPENC (2,1,8,14,2), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt15_el1", CPENC (2,1,8,15,2), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt16_el1", CPENC (2,1,8,0,6), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt17_el1", CPENC (2,1,8,1,6), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt18_el1", CPENC (2,1,8,2,6), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt19_el1", CPENC (2,1,8,3,6), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt1_el1", CPENC (2,1,8,1,2), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt20_el1", CPENC (2,1,8,4,6), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt21_el1", CPENC (2,1,8,5,6), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt22_el1", CPENC (2,1,8,6,6), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt23_el1", CPENC (2,1,8,7,6), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt24_el1", CPENC (2,1,8,8,6), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt25_el1", CPENC (2,1,8,9,6), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt26_el1", CPENC (2,1,8,10,6), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt27_el1", CPENC (2,1,8,11,6), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt28_el1", CPENC (2,1,8,12,6), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt29_el1", CPENC (2,1,8,13,6), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt2_el1", CPENC (2,1,8,2,2), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt30_el1", CPENC (2,1,8,14,6), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt31_el1", CPENC (2,1,8,15,6), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt3_el1", CPENC (2,1,8,3,2), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt4_el1", CPENC (2,1,8,4,2), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt5_el1", CPENC (2,1,8,5,2), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt6_el1", CPENC (2,1,8,6,2), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt7_el1", CPENC (2,1,8,7,2), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt8_el1", CPENC (2,1,8,8,2), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgt9_el1", CPENC (2,1,8,9,2), F_REG_READ, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbtgtinj_el1", CPENC (2,1,9,1,2), 0, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("brbts_el1", CPENC (2,1,9,0,2), 0, AARCH64_FEATURE (V9_1A)) /* BRBE */ + SYSREG ("ccsidr2_el1", CPENC (3,1,0,0,2), F_REG_READ, AARCH64_FEATURE (V8_2A)) /* CCIDX */ + SYSREG ("ccsidr_el1", CPENC (3,1,0,0,0), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("clidr_el1", CPENC (3,1,0,0,1), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("cntfrq_el0", CPENC (3,3,14,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("cnthctl_el2", CPENC (3,4,14,1,0), 0, AARCH64_NO_FEATURES) + SYSREG ("cnthp_ctl_el2", CPENC (3,4,14,2,1), 0, AARCH64_NO_FEATURES) + SYSREG ("cnthp_cval_el2", CPENC (3,4,14,2,2), 0, AARCH64_NO_FEATURES) + SYSREG ("cnthp_tval_el2", CPENC (3,4,14,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("cnthps_ctl_el2", CPENC (3,4,14,5,1), 0, AARCH64_FEATURE (V8_3A)) /* SEL2 */ + SYSREG ("cnthps_cval_el2", CPENC (3,4,14,5,2), 0, AARCH64_FEATURE (V8_3A)) /* SEL2 */ + SYSREG ("cnthps_tval_el2", CPENC (3,4,14,5,0), 0, AARCH64_FEATURE (V8_3A)) /* SEL2 */ + SYSREG ("cnthv_ctl_el2", CPENC (3,4,14,3,1), 0, AARCH64_NO_FEATURES) + SYSREG ("cnthv_cval_el2", CPENC (3,4,14,3,2), 0, AARCH64_NO_FEATURES) + SYSREG ("cnthv_tval_el2", CPENC (3,4,14,3,0), 0, AARCH64_NO_FEATURES) + SYSREG ("cnthvs_ctl_el2", CPENC (3,4,14,4,1), 0, AARCH64_FEATURE (V8_3A)) /* SEL2 */ + SYSREG ("cnthvs_cval_el2", CPENC (3,4,14,4,2), 0, AARCH64_FEATURE (V8_3A)) /* SEL2 */ + SYSREG ("cnthvs_tval_el2", CPENC (3,4,14,4,0), 0, AARCH64_FEATURE (V8_3A)) /* SEL2 */ + SYSREG ("cntkctl_el1", CPENC (3,0,14,1,0), 0, AARCH64_NO_FEATURES) + SYSREG ("cntkctl_el12", CPENC (3,5,14,1,0), 0, AARCH64_NO_FEATURES) + SYSREG ("cntp_ctl_el0", CPENC (3,3,14,2,1), 0, AARCH64_NO_FEATURES) + SYSREG ("cntp_ctl_el02", CPENC (3,5,14,2,1), 0, AARCH64_NO_FEATURES) + SYSREG ("cntp_cval_el0", CPENC (3,3,14,2,2), 0, AARCH64_NO_FEATURES) + SYSREG ("cntp_cval_el02", CPENC (3,5,14,2,2), 0, AARCH64_NO_FEATURES) + SYSREG ("cntp_tval_el0", CPENC (3,3,14,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("cntp_tval_el02", CPENC (3,5,14,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("cntpct_el0", CPENC (3,3,14,0,1), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("cntpctss_el0", CPENC (3,3,14,0,5), F_REG_READ, AARCH64_FEATURE (V8_5A)) /* ECV */ + SYSREG ("cntpoff_el2", CPENC (3,4,14,0,6), 0, AARCH64_FEATURE (V8_5A)) /* ECV_POFF */ + SYSREG ("cntps_ctl_el1", CPENC (3,7,14,2,1), 0, AARCH64_NO_FEATURES) + SYSREG ("cntps_cval_el1", CPENC (3,7,14,2,2), 0, AARCH64_NO_FEATURES) + SYSREG ("cntps_tval_el1", CPENC (3,7,14,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("cntv_ctl_el0", CPENC (3,3,14,3,1), 0, AARCH64_NO_FEATURES) + SYSREG ("cntv_ctl_el02", CPENC (3,5,14,3,1), 0, AARCH64_NO_FEATURES) + SYSREG ("cntv_cval_el0", CPENC (3,3,14,3,2), 0, AARCH64_NO_FEATURES) + SYSREG ("cntv_cval_el02", CPENC (3,5,14,3,2), 0, AARCH64_NO_FEATURES) + SYSREG ("cntv_tval_el0", CPENC (3,3,14,3,0), 0, AARCH64_NO_FEATURES) + SYSREG ("cntv_tval_el02", CPENC (3,5,14,3,0), 0, AARCH64_NO_FEATURES) + SYSREG ("cntvct_el0", CPENC (3,3,14,0,2), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("cntvctss_el0", CPENC (3,3,14,0,6), F_REG_READ, AARCH64_FEATURE (V8_5A)) /* ECV */ + SYSREG ("cntvoff_el2", CPENC (3,4,14,0,3), 0, AARCH64_NO_FEATURES) + SYSREG ("contextidr_el1", CPENC (3,0,13,0,1), 0, AARCH64_NO_FEATURES) + SYSREG ("contextidr_el12", CPENC (3,5,13,0,1), 0, AARCH64_NO_FEATURES) + SYSREG ("contextidr_el2", CPENC (3,4,13,0,1), 0, AARCH64_NO_FEATURES) + SYSREG ("cpacr_el1", CPENC (3,0,1,0,2), 0, AARCH64_NO_FEATURES) + SYSREG ("cpacr_el12", CPENC (3,5,1,0,2), 0, AARCH64_NO_FEATURES) + SYSREG ("cpacralias_el1", CPENC (3,0,1,4,4), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("cpacrmask_el1", CPENC (3,0,1,4,2), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("cpacrmask_el12", CPENC (3,5,1,4,2), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("cptr_el2", CPENC (3,4,1,1,2), 0, AARCH64_NO_FEATURES) + SYSREG ("cptr_el3", CPENC (3,6,1,1,2), 0, AARCH64_NO_FEATURES) + SYSREG ("cptrmask_el2", CPENC (3,4,1,4,2), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("csselr_el1", CPENC (3,2,0,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("ctr_el0", CPENC (3,3,0,0,1), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("currentel", CPENC (3,0,4,2,2), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("dacr32_el2", CPENC (3,4,3,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("daif", CPENC (3,3,4,2,1), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgauthstatus_el1", CPENC (2,0,7,14,6), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("dbgbcr0_el1", CPENC (2,0,0,0,5), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbcr10_el1", CPENC (2,0,0,10,5), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbcr11_el1", CPENC (2,0,0,11,5), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbcr12_el1", CPENC (2,0,0,12,5), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbcr13_el1", CPENC (2,0,0,13,5), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbcr14_el1", CPENC (2,0,0,14,5), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbcr15_el1", CPENC (2,0,0,15,5), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbcr1_el1", CPENC (2,0,0,1,5), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbcr2_el1", CPENC (2,0,0,2,5), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbcr3_el1", CPENC (2,0,0,3,5), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbcr4_el1", CPENC (2,0,0,4,5), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbcr5_el1", CPENC (2,0,0,5,5), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbcr6_el1", CPENC (2,0,0,6,5), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbcr7_el1", CPENC (2,0,0,7,5), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbcr8_el1", CPENC (2,0,0,8,5), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbcr9_el1", CPENC (2,0,0,9,5), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbvr0_el1", CPENC (2,0,0,0,4), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbvr10_el1", CPENC (2,0,0,10,4), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbvr11_el1", CPENC (2,0,0,11,4), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbvr12_el1", CPENC (2,0,0,12,4), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbvr13_el1", CPENC (2,0,0,13,4), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbvr14_el1", CPENC (2,0,0,14,4), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbvr15_el1", CPENC (2,0,0,15,4), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbvr1_el1", CPENC (2,0,0,1,4), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbvr2_el1", CPENC (2,0,0,2,4), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbvr3_el1", CPENC (2,0,0,3,4), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbvr4_el1", CPENC (2,0,0,4,4), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbvr5_el1", CPENC (2,0,0,5,4), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbvr6_el1", CPENC (2,0,0,6,4), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbvr7_el1", CPENC (2,0,0,7,4), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbvr8_el1", CPENC (2,0,0,8,4), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgbvr9_el1", CPENC (2,0,0,9,4), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgclaimclr_el1", CPENC (2,0,7,9,6), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgclaimset_el1", CPENC (2,0,7,8,6), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgdtr_el0", CPENC (2,3,0,4,0), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgdtrrx_el0", CPENC (2,3,0,5,0), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("dbgdtrtx_el0", CPENC (2,3,0,5,0), F_REG_WRITE, AARCH64_NO_FEATURES) + SYSREG ("dbgprcr_el1", CPENC (2,0,1,4,4), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgvcr32_el2", CPENC (2,4,0,7,0), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwcr0_el1", CPENC (2,0,0,0,7), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwcr10_el1", CPENC (2,0,0,10,7), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwcr11_el1", CPENC (2,0,0,11,7), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwcr12_el1", CPENC (2,0,0,12,7), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwcr13_el1", CPENC (2,0,0,13,7), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwcr14_el1", CPENC (2,0,0,14,7), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwcr15_el1", CPENC (2,0,0,15,7), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwcr1_el1", CPENC (2,0,0,1,7), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwcr2_el1", CPENC (2,0,0,2,7), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwcr3_el1", CPENC (2,0,0,3,7), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwcr4_el1", CPENC (2,0,0,4,7), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwcr5_el1", CPENC (2,0,0,5,7), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwcr6_el1", CPENC (2,0,0,6,7), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwcr7_el1", CPENC (2,0,0,7,7), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwcr8_el1", CPENC (2,0,0,8,7), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwcr9_el1", CPENC (2,0,0,9,7), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwvr0_el1", CPENC (2,0,0,0,6), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwvr10_el1", CPENC (2,0,0,10,6), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwvr11_el1", CPENC (2,0,0,11,6), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwvr12_el1", CPENC (2,0,0,12,6), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwvr13_el1", CPENC (2,0,0,13,6), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwvr14_el1", CPENC (2,0,0,14,6), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwvr15_el1", CPENC (2,0,0,15,6), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwvr1_el1", CPENC (2,0,0,1,6), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwvr2_el1", CPENC (2,0,0,2,6), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwvr3_el1", CPENC (2,0,0,3,6), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwvr4_el1", CPENC (2,0,0,4,6), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwvr5_el1", CPENC (2,0,0,5,6), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwvr6_el1", CPENC (2,0,0,6,6), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwvr7_el1", CPENC (2,0,0,7,6), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwvr8_el1", CPENC (2,0,0,8,6), 0, AARCH64_NO_FEATURES) + SYSREG ("dbgwvr9_el1", CPENC (2,0,0,9,6), 0, AARCH64_NO_FEATURES) + SYSREG ("dczid_el0", CPENC (3,3,0,0,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("disr_el1", CPENC (3,0,12,1,1), 0, AARCH64_FEATURE (RAS)) + SYSREG ("dit", CPENC (3,3,4,2,5), 0, AARCH64_FEATURE (V8_3A)) /* DIT */ + SYSREG ("dlr_el0", CPENC (3,3,4,5,1), 0, AARCH64_NO_FEATURES) + SYSREG ("dspsr_el0", CPENC (3,3,4,5,0), 0, AARCH64_NO_FEATURES) + SYSREG ("elr_el1", CPENC (3,0,4,0,1), 0, AARCH64_NO_FEATURES) + SYSREG ("elr_el12", CPENC (3,5,4,0,1), 0, AARCH64_NO_FEATURES) + SYSREG ("elr_el2", CPENC (3,4,4,0,1), 0, AARCH64_NO_FEATURES) + SYSREG ("elr_el3", CPENC (3,6,4,0,1), 0, AARCH64_NO_FEATURES) + SYSREG ("erridr_el1", CPENC (3,0,5,3,0), F_REG_READ, AARCH64_FEATURE (RAS)) + SYSREG ("errselr_el1", CPENC (3,0,5,3,1), 0, AARCH64_FEATURE (RAS)) + SYSREG ("erxaddr_el1", CPENC (3,0,5,4,3), 0, AARCH64_FEATURE (RAS)) + SYSREG ("erxctlr_el1", CPENC (3,0,5,4,1), 0, AARCH64_FEATURE (RAS)) + SYSREG ("erxfr_el1", CPENC (3,0,5,4,0), F_REG_READ, AARCH64_FEATURE (RAS)) + SYSREG ("erxgsr_el1", CPENC (3,0,5,3,2), F_REG_READ, AARCH64_FEATURE (RASv2)) + SYSREG ("erxmisc0_el1", CPENC (3,0,5,5,0), 0, AARCH64_FEATURE (RAS)) + SYSREG ("erxmisc1_el1", CPENC (3,0,5,5,1), 0, AARCH64_FEATURE (RAS)) + SYSREG ("erxmisc2_el1", CPENC (3,0,5,5,2), 0, AARCH64_FEATURE (RAS)) /* RASv1p1 */ + SYSREG ("erxmisc3_el1", CPENC (3,0,5,5,3), 0, AARCH64_FEATURE (RAS)) /* RASv1p1 */ + SYSREG ("erxpfgcdn_el1", CPENC (3,0,5,4,6), 0, AARCH64_FEATURE (RAS)) /* RASv1p1 */ + SYSREG ("erxpfgctl_el1", CPENC (3,0,5,4,5), 0, AARCH64_FEATURE (RAS)) /* RASv1p1 */ + SYSREG ("erxpfgf_el1", CPENC (3,0,5,4,4), F_REG_READ, AARCH64_FEATURE (RAS)) /* RASv1p1 */ + SYSREG ("erxstatus_el1", CPENC (3,0,5,4,2), 0, AARCH64_FEATURE (RAS)) + SYSREG ("esr_el1", CPENC (3,0,5,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("esr_el12", CPENC (3,5,5,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("esr_el2", CPENC (3,4,5,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("esr_el3", CPENC (3,6,5,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("far_el1", CPENC (3,0,6,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("far_el12", CPENC (3,5,6,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("far_el2", CPENC (3,4,6,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("far_el3", CPENC (3,6,6,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("fgwte3_el3", CPENC (3,6,1,1,5), 0, AARCH64_FEATURE (V9_4A)) /* FGWTE3 */ + SYSREG ("fpcr", CPENC (3,3,4,4,0), 0, AARCH64_NO_FEATURES) + SYSREG ("fpexc32_el2", CPENC (3,4,5,3,0), 0, AARCH64_NO_FEATURES) + SYSREG ("fpmr", CPENC (3,3,4,4,2), 0, AARCH64_FEATURE (FP8)) + SYSREG ("fpsr", CPENC (3,3,4,4,1), 0, AARCH64_NO_FEATURES) + SYSREG ("gcr_el1", CPENC (3,0,1,0,6), 0, AARCH64_FEATURE (MEMTAG)) + SYSREG ("gcscr_el1", CPENC (3,0,2,5,0), 0, AARCH64_FEATURE (GCS)) + SYSREG ("gcscr_el12", CPENC (3,5,2,5,0), 0, AARCH64_FEATURE (GCS)) + SYSREG ("gcscr_el2", CPENC (3,4,2,5,0), 0, AARCH64_FEATURE (GCS)) + SYSREG ("gcscr_el3", CPENC (3,6,2,5,0), 0, AARCH64_FEATURE (GCS)) + SYSREG ("gcscre0_el1", CPENC (3,0,2,5,2), 0, AARCH64_FEATURE (GCS)) + SYSREG ("gcspr_el0", CPENC (3,3,2,5,1), 0, AARCH64_FEATURE (GCS)) + SYSREG ("gcspr_el1", CPENC (3,0,2,5,1), 0, AARCH64_FEATURE (GCS)) + SYSREG ("gcspr_el12", CPENC (3,5,2,5,1), 0, AARCH64_FEATURE (GCS)) + SYSREG ("gcspr_el2", CPENC (3,4,2,5,1), 0, AARCH64_FEATURE (GCS)) + SYSREG ("gcspr_el3", CPENC (3,6,2,5,1), 0, AARCH64_FEATURE (GCS)) + SYSREG ("gmid_el1", CPENC (3,1,0,0,4), F_REG_READ, AARCH64_FEATURE (MEMTAG)) + SYSREG ("gpcbw_el3", CPENC (3,6,2,1,5), 0, AARCH64_FEATURE (V9_5A)) /* RME_GPC3 */ + SYSREG ("gpccr_el3", CPENC (3,6,2,1,6), 0, AARCH64_FEATURE (V9_1A)) /* RME */ + SYSREG ("gptbr_el3", CPENC (3,6,2,1,4), 0, AARCH64_FEATURE (V9_1A)) /* RME */ + SYSREG ("hacdbsbr_el2", CPENC (3,4,2,3,4), 0, AARCH64_FEATURE (V9_4A)) /* HACDBS */ + SYSREG ("hacdbscons_el2", CPENC (3,4,2,3,5), 0, AARCH64_FEATURE (V9_4A)) /* HACDBS */ + SYSREG ("hacr_el2", CPENC (3,4,1,1,7), 0, AARCH64_NO_FEATURES) + SYSREG ("hafgrtr_el2", CPENC (3,4,3,1,6), 0, AARCH64_FEATURE (V8_5A)) /* AMUv1 && FGT */ + SYSREG ("hcr_el2", CPENC (3,4,1,1,0), 0, AARCH64_NO_FEATURES) + SYSREG ("hcrx_el2", CPENC (3,4,1,2,2), 0, AARCH64_FEATURE (V8_6A)) /* HCX */ + SYSREG ("hdbssbr_el2", CPENC (3,4,2,3,2), 0, AARCH64_FEATURE (V9_4A)) /* HDBSS */ + SYSREG ("hdbssprod_el2", CPENC (3,4,2,3,3), 0, AARCH64_FEATURE (V9_4A)) /* HDBSS */ + SYSREG ("hdfgrtr_el2", CPENC (3,4,3,1,4), 0, AARCH64_FEATURE (V8_5A)) /* FGT */ + SYSREG ("hdfgrtr2_el2", CPENC (3,4,3,1,0), 0, AARCH64_FEATURE (V8_8A)) /* FGT2 */ + SYSREG ("hdfgwtr_el2", CPENC (3,4,3,1,5), 0, AARCH64_FEATURE (V8_5A)) /* FGT */ + SYSREG ("hdfgwtr2_el2", CPENC (3,4,3,1,1), 0, AARCH64_FEATURE (V8_8A)) /* FGT2 */ + SYSREG ("hfgitr_el2", CPENC (3,4,1,1,6), 0, AARCH64_FEATURE (V8_5A)) /* FGT */ + SYSREG ("hfgitr2_el2", CPENC (3,4,3,1,7), 0, AARCH64_FEATURE (V8_8A)) /* FGT2 */ + SYSREG ("hfgrtr_el2", CPENC (3,4,1,1,4), 0, AARCH64_FEATURE (V8_5A)) /* FGT */ + SYSREG ("hfgrtr2_el2", CPENC (3,4,3,1,2), 0, AARCH64_FEATURE (V8_8A)) /* FGT2 */ + SYSREG ("hfgwtr_el2", CPENC (3,4,1,1,5), 0, AARCH64_FEATURE (V8_5A)) /* FGT */ + SYSREG ("hfgwtr2_el2", CPENC (3,4,3,1,3), 0, AARCH64_FEATURE (V8_8A)) /* FGT2 */ + SYSREG ("hpfar_el2", CPENC (3,4,6,0,4), 0, AARCH64_NO_FEATURES) + SYSREG ("hstr_el2", CPENC (3,4,1,1,3), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_ap0r0_el1", CPENC (3,0,12,8,4), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_ap0r1_el1", CPENC (3,0,12,8,5), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_ap0r2_el1", CPENC (3,0,12,8,6), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_ap0r3_el1", CPENC (3,0,12,8,7), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_ap1r0_el1", CPENC (3,0,12,9,0), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_ap1r1_el1", CPENC (3,0,12,9,1), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_ap1r2_el1", CPENC (3,0,12,9,2), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_ap1r3_el1", CPENC (3,0,12,9,3), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_asgi1r_el1", CPENC (3,0,12,11,6), F_REG_WRITE, AARCH64_NO_FEATURES) + SYSREG ("icc_bpr0_el1", CPENC (3,0,12,8,3), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_bpr1_el1", CPENC (3,0,12,12,3), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_ctlr_el1", CPENC (3,0,12,12,4), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_ctlr_el3", CPENC (3,6,12,12,4), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_dir_el1", CPENC (3,0,12,11,1), F_REG_WRITE, AARCH64_NO_FEATURES) + SYSREG ("icc_eoir0_el1", CPENC (3,0,12,8,1), F_REG_WRITE, AARCH64_NO_FEATURES) + SYSREG ("icc_eoir1_el1", CPENC (3,0,12,12,1), F_REG_WRITE, AARCH64_NO_FEATURES) + SYSREG ("icc_hppir0_el1", CPENC (3,0,12,8,2), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("icc_hppir1_el1", CPENC (3,0,12,12,2), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("icc_iar0_el1", CPENC (3,0,12,8,0), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("icc_iar1_el1", CPENC (3,0,12,12,0), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("icc_igrpen0_el1", CPENC (3,0,12,12,6), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_igrpen1_el1", CPENC (3,0,12,12,7), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_igrpen1_el3", CPENC (3,6,12,12,7), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_nmiar1_el1", CPENC (3,0,12,9,5), F_REG_READ, AARCH64_FEATURE (V8_7A)) /* GICv3_NMI */ + SYSREG ("icc_pmr_el1", CPENC (3,0,4,6,0), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_rpr_el1", CPENC (3,0,12,11,3), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("icc_sgi0r_el1", CPENC (3,0,12,11,7), F_REG_WRITE, AARCH64_NO_FEATURES) + SYSREG ("icc_sgi1r_el1", CPENC (3,0,12,11,5), F_REG_WRITE, AARCH64_NO_FEATURES) + SYSREG ("icc_sre_el1", CPENC (3,0,12,12,5), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_sre_el2", CPENC (3,4,12,9,5), 0, AARCH64_NO_FEATURES) + SYSREG ("icc_sre_el3", CPENC (3,6,12,12,5), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_ap0r0_el2", CPENC (3,4,12,8,0), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_ap0r1_el2", CPENC (3,4,12,8,1), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_ap0r2_el2", CPENC (3,4,12,8,2), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_ap0r3_el2", CPENC (3,4,12,8,3), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_ap1r0_el2", CPENC (3,4,12,9,0), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_ap1r1_el2", CPENC (3,4,12,9,1), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_ap1r2_el2", CPENC (3,4,12,9,2), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_ap1r3_el2", CPENC (3,4,12,9,3), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_eisr_el2", CPENC (3,4,12,11,3), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("ich_elrsr_el2", CPENC (3,4,12,11,5), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("ich_hcr_el2", CPENC (3,4,12,11,0), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_lr0_el2", CPENC (3,4,12,12,0), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_lr10_el2", CPENC (3,4,12,13,2), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_lr11_el2", CPENC (3,4,12,13,3), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_lr12_el2", CPENC (3,4,12,13,4), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_lr13_el2", CPENC (3,4,12,13,5), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_lr14_el2", CPENC (3,4,12,13,6), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_lr15_el2", CPENC (3,4,12,13,7), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_lr1_el2", CPENC (3,4,12,12,1), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_lr2_el2", CPENC (3,4,12,12,2), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_lr3_el2", CPENC (3,4,12,12,3), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_lr4_el2", CPENC (3,4,12,12,4), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_lr5_el2", CPENC (3,4,12,12,5), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_lr6_el2", CPENC (3,4,12,12,6), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_lr7_el2", CPENC (3,4,12,12,7), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_lr8_el2", CPENC (3,4,12,13,0), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_lr9_el2", CPENC (3,4,12,13,1), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_misr_el2", CPENC (3,4,12,11,2), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("ich_vmcr_el2", CPENC (3,4,12,11,7), 0, AARCH64_NO_FEATURES) + SYSREG ("ich_vtr_el2", CPENC (3,4,12,11,1), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64afr0_el1", CPENC (3,0,0,5,4), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64afr1_el1", CPENC (3,0,0,5,5), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64dfr0_el1", CPENC (3,0,0,5,0), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64dfr1_el1", CPENC (3,0,0,5,1), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64dfr2_el1", CPENC (3,0,0,5,2), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64fpfr0_el1", CPENC (3,0,0,4,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64isar0_el1", CPENC (3,0,0,6,0), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64isar1_el1", CPENC (3,0,0,6,1), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64isar2_el1", CPENC (3,0,0,6,2), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64isar3_el1", CPENC (3,0,0,6,3), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64mmfr0_el1", CPENC (3,0,0,7,0), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64mmfr1_el1", CPENC (3,0,0,7,1), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64mmfr2_el1", CPENC (3,0,0,7,2), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64mmfr3_el1", CPENC (3,0,0,7,3), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64mmfr4_el1", CPENC (3,0,0,7,4), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64pfr0_el1", CPENC (3,0,0,4,0), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64pfr1_el1", CPENC (3,0,0,4,1), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64pfr2_el1", CPENC (3,0,0,4,2), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64smfr0_el1", CPENC (3,0,0,4,5), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_aa64zfr0_el1", CPENC (3,0,0,4,4), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_afr0_el1", CPENC (3,0,0,1,3), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_dfr0_el1", CPENC (3,0,0,1,2), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_dfr1_el1", CPENC (3,0,0,3,5), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_isar0_el1", CPENC (3,0,0,2,0), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_isar1_el1", CPENC (3,0,0,2,1), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_isar2_el1", CPENC (3,0,0,2,2), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_isar3_el1", CPENC (3,0,0,2,3), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_isar4_el1", CPENC (3,0,0,2,4), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_isar5_el1", CPENC (3,0,0,2,5), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_isar6_el1", CPENC (3,0,0,2,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_mmfr0_el1", CPENC (3,0,0,1,4), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_mmfr1_el1", CPENC (3,0,0,1,5), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_mmfr2_el1", CPENC (3,0,0,1,6), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_mmfr3_el1", CPENC (3,0,0,1,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_mmfr4_el1", CPENC (3,0,0,2,6), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_mmfr5_el1", CPENC (3,0,0,3,6), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_pfr0_el1", CPENC (3,0,0,1,0), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_pfr1_el1", CPENC (3,0,0,1,1), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("id_pfr2_el1", CPENC (3,0,0,3,4), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("ifsr32_el2", CPENC (3,4,5,0,1), 0, AARCH64_NO_FEATURES) + SYSREG ("isr_el1", CPENC (3,0,12,1,0), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("lorc_el1", CPENC (3,0,10,4,3), 0, AARCH64_FEATURE (LOR)) + SYSREG ("lorea_el1", CPENC (3,0,10,4,1), 0, AARCH64_FEATURE (LOR)) + SYSREG ("lorid_el1", CPENC (3,0,10,4,7), F_REG_READ, AARCH64_FEATURE (LOR)) + SYSREG ("lorn_el1", CPENC (3,0,10,4,2), 0, AARCH64_FEATURE (LOR)) + SYSREG ("lorsa_el1", CPENC (3,0,10,4,0), 0, AARCH64_FEATURE (LOR)) + SYSREG ("mair_el1", CPENC (3,0,10,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("mair_el12", CPENC (3,5,10,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("mair_el2", CPENC (3,4,10,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("mair_el3", CPENC (3,6,10,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("mair2_el1", CPENC (3,0,10,2,1), 0, AARCH64_FEATURE (V8_8A)) /* AIE */ + SYSREG ("mair2_el12", CPENC (3,5,10,2,1), 0, AARCH64_FEATURE (V8_8A)) /* AIE */ + SYSREG ("mair2_el2", CPENC (3,4,10,1,1), 0, AARCH64_FEATURE (V8_8A)) /* AIE */ + SYSREG ("mair2_el3", CPENC (3,6,10,1,1), 0, AARCH64_FEATURE (V8_8A)) /* AIE */ + SYSREG ("mdccint_el1", CPENC (2,0,0,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("mdccsr_el0", CPENC (2,3,0,1,0), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("mdcr_el2", CPENC (3,4,1,1,1), 0, AARCH64_NO_FEATURES) + SYSREG ("mdcr_el3", CPENC (3,6,1,3,1), 0, AARCH64_NO_FEATURES) + SYSREG ("mdrar_el1", CPENC (2,0,1,0,0), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("mdscr_el1", CPENC (2,0,0,2,2), 0, AARCH64_NO_FEATURES) + SYSREG ("mdselr_el1", CPENC (2,0,0,4,2), 0, AARCH64_FEATURE (V8_8A)) /* Debugv8p9 */ + SYSREG ("mdstepop_el1", CPENC (2,0,0,5,2), 0, AARCH64_FEATURE (V9_4A)) /* STEP2 */ + SYSREG ("mecid_a0_el2", CPENC (3,4,10,8,1), 0, AARCH64_FEATURE (V9_2A)) /* MEC */ + SYSREG ("mecid_a1_el2", CPENC (3,4,10,8,3), 0, AARCH64_FEATURE (V9_2A)) /* MEC */ + SYSREG ("mecid_p0_el2", CPENC (3,4,10,8,0), 0, AARCH64_FEATURE (V9_2A)) /* MEC */ + SYSREG ("mecid_p1_el2", CPENC (3,4,10,8,2), 0, AARCH64_FEATURE (V9_2A)) /* MEC */ + SYSREG ("mecid_rl_a_el3", CPENC (3,6,10,10,1), 0, AARCH64_FEATURE (V9_2A)) /* MEC */ + SYSREG ("mecidr_el2", CPENC (3,4,10,8,7), F_REG_READ, AARCH64_FEATURE (V9_2A)) /* MEC */ + SYSREG ("mfar_el3", CPENC (3,6,6,0,5), 0, AARCH64_FEATURE (V8_6A)) /* PFAR=>8.8 || RME=>9.1 */ + SYSREG ("midr_el1", CPENC (3,0,0,0,0), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("mpam0_el1", CPENC (3,0,10,5,1), 0, AARCH64_FEATURE (V8_2A)) /* MPAM */ + SYSREG ("mpam1_el1", CPENC (3,0,10,5,0), 0, AARCH64_FEATURE (V8_2A)) /* MPAM */ + SYSREG ("mpam1_el12", CPENC (3,5,10,5,0), 0, AARCH64_FEATURE (V8_2A)) /* MPAM */ + SYSREG ("mpam2_el2", CPENC (3,4,10,5,0), 0, AARCH64_FEATURE (V8_2A)) /* MPAM */ + SYSREG ("mpam3_el3", CPENC (3,6,10,5,0), 0, AARCH64_FEATURE (V8_2A)) /* MPAM */ + SYSREG ("mpambw0_el1", CPENC (3,0,10,5,5), 0, AARCH64_FEATURE (V9_3A)) /* MPAM_PE_BW_CTRL */ + SYSREG ("mpambw1_el1", CPENC (3,0,10,5,4), 0, AARCH64_FEATURE (V9_3A)) /* MPAM_PE_BW_CTRL */ + SYSREG ("mpambw1_el12", CPENC (3,5,10,5,4), 0, AARCH64_FEATURE (V9_3A)) /* MPAM_PE_BW_CTRL */ + SYSREG ("mpambw2_el2", CPENC (3,4,10,5,4), 0, AARCH64_FEATURE (V9_3A)) /* MPAM_PE_BW_CTRL */ + SYSREG ("mpambw3_el3", CPENC (3,6,10,5,4), 0, AARCH64_FEATURE (V9_3A)) /* MPAM_PE_BW_CTRL */ + SYSREG ("mpambwcap_el2", CPENC (3,4,10,5,6), 0, AARCH64_FEATURE (V9_3A)) /* MPAM_PE_BW_CTRL */ + SYSREG ("mpambwidr_el1", CPENC (3,0,10,4,5), F_REG_READ, AARCH64_FEATURE (V9_3A)) /* MPAM_PE_BW_CTRL */ + SYSREG ("mpambwsm_el1", CPENC (3,0,10,5,7), 0, AARCH64_FEATURES (2, SME, V9_3A)) /* SME && MPAM_PE_BW_CTRL */ + SYSREG ("mpamhcr_el2", CPENC (3,4,10,4,0), 0, AARCH64_FEATURE (V8_2A)) /* MPAM */ + SYSREG ("mpamidr_el1", CPENC (3,0,10,4,4), F_REG_READ, AARCH64_FEATURE (V8_2A)) /* MPAM */ + SYSREG ("mpamsm_el1", CPENC (3,0,10,5,3), 0, AARCH64_FEATURES (2, SME, V8_2A)) /* SME && MPAM */ + SYSREG ("mpamvpm0_el2", CPENC (3,4,10,6,0), 0, AARCH64_FEATURE (V8_2A)) /* MPAM */ + SYSREG ("mpamvpm1_el2", CPENC (3,4,10,6,1), 0, AARCH64_FEATURE (V8_2A)) /* MPAM */ + SYSREG ("mpamvpm2_el2", CPENC (3,4,10,6,2), 0, AARCH64_FEATURE (V8_2A)) /* MPAM */ + SYSREG ("mpamvpm3_el2", CPENC (3,4,10,6,3), 0, AARCH64_FEATURE (V8_2A)) /* MPAM */ + SYSREG ("mpamvpm4_el2", CPENC (3,4,10,6,4), 0, AARCH64_FEATURE (V8_2A)) /* MPAM */ + SYSREG ("mpamvpm5_el2", CPENC (3,4,10,6,5), 0, AARCH64_FEATURE (V8_2A)) /* MPAM */ + SYSREG ("mpamvpm6_el2", CPENC (3,4,10,6,6), 0, AARCH64_FEATURE (V8_2A)) /* MPAM */ + SYSREG ("mpamvpm7_el2", CPENC (3,4,10,6,7), 0, AARCH64_FEATURE (V8_2A)) /* MPAM */ + SYSREG ("mpamvpmv_el2", CPENC (3,4,10,4,1), 0, AARCH64_FEATURE (V8_2A)) /* MPAM */ + SYSREG ("mpidr_el1", CPENC (3,0,0,0,5), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("mpuir_el1", CPENC (3,0,0,0,4), F_REG_READ, AARCH64_FEATURE (V8R)) + SYSREG ("mpuir_el2", CPENC (3,4,0,0,4), F_REG_READ, AARCH64_FEATURE (V8R)) + SYSREG ("mvfr0_el1", CPENC (3,0,0,3,0), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("mvfr1_el1", CPENC (3,0,0,3,1), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("mvfr2_el1", CPENC (3,0,0,3,2), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("nzcv", CPENC (3,3,4,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("osdlr_el1", CPENC (2,0,1,3,4), 0, AARCH64_NO_FEATURES) + SYSREG ("osdtrrx_el1", CPENC (2,0,0,0,2), 0, AARCH64_NO_FEATURES) + SYSREG ("osdtrtx_el1", CPENC (2,0,0,3,2), 0, AARCH64_NO_FEATURES) + SYSREG ("oseccr_el1", CPENC (2,0,0,6,2), 0, AARCH64_NO_FEATURES) + SYSREG ("oslar_el1", CPENC (2,0,1,0,4), F_REG_WRITE, AARCH64_NO_FEATURES) + SYSREG ("oslsr_el1", CPENC (2,0,1,1,4), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("pan", CPENC (3,0,4,2,3), 0, AARCH64_FEATURE (PAN)) + SYSREG ("par_el1", CPENC (3,0,7,4,0), F_REG_128, AARCH64_NO_FEATURES) + SYSREG ("pfar_el1", CPENC (3,0,6,0,5), 0, AARCH64_FEATURE (V8_8A)) /* PFAR */ + SYSREG ("pfar_el12", CPENC (3,5,6,0,5), 0, AARCH64_FEATURE (V8_8A)) /* PFAR */ + SYSREG ("pfar_el2", CPENC (3,4,6,0,5), 0, AARCH64_FEATURE (V8_8A)) /* PFAR */ + SYSREG ("pir_el1", CPENC (3,0,10,2,3), 0, AARCH64_FEATURE (V8_8A)) /* S1PIE */ + SYSREG ("pir_el12", CPENC (3,5,10,2,3), 0, AARCH64_FEATURE (V8_8A)) /* S1PIE */ + SYSREG ("pir_el2", CPENC (3,4,10,2,3), 0, AARCH64_FEATURE (V8_8A)) /* S1PIE */ + SYSREG ("pir_el3", CPENC (3,6,10,2,3), 0, AARCH64_FEATURE (V8_8A)) /* S1PIE */ + SYSREG ("pire0_el1", CPENC (3,0,10,2,2), 0, AARCH64_FEATURE (V8_8A)) /* S1PIE */ + SYSREG ("pire0_el12", CPENC (3,5,10,2,2), 0, AARCH64_FEATURE (V8_8A)) /* S1PIE */ + SYSREG ("pire0_el2", CPENC (3,4,10,2,2), 0, AARCH64_FEATURE (V8_8A)) /* S1PIE */ + SYSREG ("pm", CPENC (3,0,4,3,1), 0, AARCH64_FEATURE (V9_3A)) /* EBEP */ + SYSREG ("pmbidr_el1", CPENC (3,0,9,10,7), F_REG_READ, AARCH64_FEATURE (PROFILE)) + SYSREG ("pmblimitr_el1", CPENC (3,0,9,10,0), 0, AARCH64_FEATURE (PROFILE)) + SYSREG ("pmbmar_el1", CPENC (3,0,9,10,5), 0, AARCH64_FEATURES (2, PROFILE, V9_5A)) /* SPE_nVM */ + SYSREG ("pmbptr_el1", CPENC (3,0,9,10,1), 0, AARCH64_FEATURE (PROFILE)) + SYSREG ("pmbsr_el1", CPENC (3,0,9,10,3), 0, AARCH64_FEATURE (PROFILE)) + SYSREG ("pmbsr_el12", CPENC (3,5,9,10,3), 0, AARCH64_FEATURES (2, V9_5A, PROFILE)) /* SPE_EXC */ + SYSREG ("pmbsr_el2", CPENC (3,4,9,10,3), 0, AARCH64_FEATURES (2, V9_5A, PROFILE)) /* SPE_EXC */ + SYSREG ("pmbsr_el3", CPENC (3,6,9,10,3), 0, AARCH64_FEATURES (2, V9_5A, PROFILE)) /* SPE_EXC */ + SYSREG ("pmccfiltr_el0", CPENC (3,3,14,15,7), 0, AARCH64_NO_FEATURES) + SYSREG ("pmccntr_el0", CPENC (3,3,9,13,0), 0, AARCH64_NO_FEATURES) + SYSREG ("pmccntsvr_el1", CPENC (2,0,14,11,7), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmceid0_el0", CPENC (3,3,9,12,6), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("pmceid1_el0", CPENC (3,3,9,12,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("pmcntenclr_el0", CPENC (3,3,9,12,2), 0, AARCH64_NO_FEATURES) + SYSREG ("pmcntenset_el0", CPENC (3,3,9,12,1), 0, AARCH64_NO_FEATURES) + SYSREG ("pmcr_el0", CPENC (3,3,9,12,0), 0, AARCH64_NO_FEATURES) + SYSREG ("pmecr_el1", CPENC (3,0,9,14,5), 0, AARCH64_FEATURE (V8_8A)) /* EBEP || PMUv3_SS */ + SYSREG ("pmevcntr0_el0", CPENC (3,3,14,8,0), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr10_el0", CPENC (3,3,14,9,2), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr11_el0", CPENC (3,3,14,9,3), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr12_el0", CPENC (3,3,14,9,4), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr13_el0", CPENC (3,3,14,9,5), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr14_el0", CPENC (3,3,14,9,6), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr15_el0", CPENC (3,3,14,9,7), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr16_el0", CPENC (3,3,14,10,0), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr17_el0", CPENC (3,3,14,10,1), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr18_el0", CPENC (3,3,14,10,2), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr19_el0", CPENC (3,3,14,10,3), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr1_el0", CPENC (3,3,14,8,1), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr20_el0", CPENC (3,3,14,10,4), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr21_el0", CPENC (3,3,14,10,5), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr22_el0", CPENC (3,3,14,10,6), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr23_el0", CPENC (3,3,14,10,7), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr24_el0", CPENC (3,3,14,11,0), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr25_el0", CPENC (3,3,14,11,1), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr26_el0", CPENC (3,3,14,11,2), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr27_el0", CPENC (3,3,14,11,3), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr28_el0", CPENC (3,3,14,11,4), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr29_el0", CPENC (3,3,14,11,5), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr2_el0", CPENC (3,3,14,8,2), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr30_el0", CPENC (3,3,14,11,6), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr3_el0", CPENC (3,3,14,8,3), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr4_el0", CPENC (3,3,14,8,4), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr5_el0", CPENC (3,3,14,8,5), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr6_el0", CPENC (3,3,14,8,6), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr7_el0", CPENC (3,3,14,8,7), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr8_el0", CPENC (3,3,14,9,0), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntr9_el0", CPENC (3,3,14,9,1), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevcntsvr0_el1", CPENC (2,0,14,8,0), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr10_el1", CPENC (2,0,14,9,2), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr11_el1", CPENC (2,0,14,9,3), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr12_el1", CPENC (2,0,14,9,4), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr13_el1", CPENC (2,0,14,9,5), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr14_el1", CPENC (2,0,14,9,6), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr15_el1", CPENC (2,0,14,9,7), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr16_el1", CPENC (2,0,14,10,0), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr17_el1", CPENC (2,0,14,10,1), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr18_el1", CPENC (2,0,14,10,2), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr19_el1", CPENC (2,0,14,10,3), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr1_el1", CPENC (2,0,14,8,1), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr20_el1", CPENC (2,0,14,10,4), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr21_el1", CPENC (2,0,14,10,5), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr22_el1", CPENC (2,0,14,10,6), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr23_el1", CPENC (2,0,14,10,7), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr24_el1", CPENC (2,0,14,11,0), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr25_el1", CPENC (2,0,14,11,1), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr26_el1", CPENC (2,0,14,11,2), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr27_el1", CPENC (2,0,14,11,3), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr28_el1", CPENC (2,0,14,11,4), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr29_el1", CPENC (2,0,14,11,5), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr2_el1", CPENC (2,0,14,8,2), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr30_el1", CPENC (2,0,14,11,6), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr3_el1", CPENC (2,0,14,8,3), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr4_el1", CPENC (2,0,14,8,4), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr5_el1", CPENC (2,0,14,8,5), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr6_el1", CPENC (2,0,14,8,6), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr7_el1", CPENC (2,0,14,8,7), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr8_el1", CPENC (2,0,14,9,0), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevcntsvr9_el1", CPENC (2,0,14,9,1), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmevtyper0_el0", CPENC (3,3,14,12,0), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper10_el0", CPENC (3,3,14,13,2), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper11_el0", CPENC (3,3,14,13,3), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper12_el0", CPENC (3,3,14,13,4), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper13_el0", CPENC (3,3,14,13,5), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper14_el0", CPENC (3,3,14,13,6), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper15_el0", CPENC (3,3,14,13,7), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper16_el0", CPENC (3,3,14,14,0), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper17_el0", CPENC (3,3,14,14,1), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper18_el0", CPENC (3,3,14,14,2), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper19_el0", CPENC (3,3,14,14,3), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper1_el0", CPENC (3,3,14,12,1), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper20_el0", CPENC (3,3,14,14,4), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper21_el0", CPENC (3,3,14,14,5), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper22_el0", CPENC (3,3,14,14,6), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper23_el0", CPENC (3,3,14,14,7), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper24_el0", CPENC (3,3,14,15,0), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper25_el0", CPENC (3,3,14,15,1), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper26_el0", CPENC (3,3,14,15,2), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper27_el0", CPENC (3,3,14,15,3), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper28_el0", CPENC (3,3,14,15,4), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper29_el0", CPENC (3,3,14,15,5), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper2_el0", CPENC (3,3,14,12,2), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper30_el0", CPENC (3,3,14,15,6), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper3_el0", CPENC (3,3,14,12,3), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper4_el0", CPENC (3,3,14,12,4), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper5_el0", CPENC (3,3,14,12,5), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper6_el0", CPENC (3,3,14,12,6), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper7_el0", CPENC (3,3,14,12,7), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper8_el0", CPENC (3,3,14,13,0), 0, AARCH64_NO_FEATURES) + SYSREG ("pmevtyper9_el0", CPENC (3,3,14,13,1), 0, AARCH64_NO_FEATURES) + SYSREG ("pmiar_el1", CPENC (3,0,9,14,7), 0, AARCH64_FEATURE (V9_3A)) /* SEBEP */ + SYSREG ("pmicfiltr_el0", CPENC (3,3,9,6,0), 0, AARCH64_FEATURE (V8_8A)) /* PMUv3_ICNTR */ + SYSREG ("pmicntr_el0", CPENC (3,3,9,4,0), 0, AARCH64_FEATURE (V8_8A)) /* PMUv3_ICNTR */ + SYSREG ("pmicntsvr_el1", CPENC (2,0,14,12,0), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* PMUv3_ICNTR && PMUv3_SS */ + SYSREG ("pmintenclr_el1", CPENC (3,0,9,14,2), 0, AARCH64_NO_FEATURES) + SYSREG ("pmintenset_el1", CPENC (3,0,9,14,1), 0, AARCH64_NO_FEATURES) + SYSREG ("pmmir_el1", CPENC (3,0,9,14,6), F_REG_READ, AARCH64_FEATURE (V8_3A)) /* PMUv3p4 */ + SYSREG ("pmovsclr_el0", CPENC (3,3,9,12,3), 0, AARCH64_NO_FEATURES) + SYSREG ("pmovsset_el0", CPENC (3,3,9,14,3), 0, AARCH64_NO_FEATURES) + SYSREG ("pmscr_el1", CPENC (3,0,9,9,0), 0, AARCH64_FEATURE (PROFILE)) + SYSREG ("pmscr_el12", CPENC (3,5,9,9,0), 0, AARCH64_FEATURE (PROFILE)) + SYSREG ("pmscr_el2", CPENC (3,4,9,9,0), 0, AARCH64_FEATURE (PROFILE)) + SYSREG ("pmsdsfr_el1", CPENC (3,0,9,10,4), 0, AARCH64_FEATURES (2, PROFILE, V8_8A)) /* SPE_FDS */ + SYSREG ("pmselr_el0", CPENC (3,3,9,12,5), 0, AARCH64_NO_FEATURES) + SYSREG ("pmsevfr_el1", CPENC (3,0,9,9,5), 0, AARCH64_FEATURE (PROFILE)) + SYSREG ("pmsfcr_el1", CPENC (3,0,9,9,4), 0, AARCH64_FEATURE (PROFILE)) + SYSREG ("pmsicr_el1", CPENC (3,0,9,9,2), 0, AARCH64_FEATURE (PROFILE)) + SYSREG ("pmsidr_el1", CPENC (3,0,9,9,7), F_REG_READ, AARCH64_FEATURE (PROFILE)) + SYSREG ("pmsirr_el1", CPENC (3,0,9,9,3), 0, AARCH64_FEATURE (PROFILE)) + SYSREG ("pmslatfr_el1", CPENC (3,0,9,9,6), 0, AARCH64_FEATURE (PROFILE)) + SYSREG ("pmsnevfr_el1", CPENC (3,0,9,9,1), 0, AARCH64_FEATURES (2, PROFILE, V8_6A)) /* SPE_FnE */ + SYSREG ("pmsscr_el1", CPENC (3,0,9,13,3), 0, AARCH64_FEATURE (V8_8A)) /* PMUv3_SS */ + SYSREG ("pmswinc_el0", CPENC (3,3,9,12,4), F_REG_WRITE, AARCH64_NO_FEATURES) + SYSREG ("pmuacr_el1", CPENC (3,0,9,14,4), 0, AARCH64_FEATURE (V8_8A)) /* PMUv3p9 */ + SYSREG ("pmuserenr_el0", CPENC (3,3,9,14,0), 0, AARCH64_NO_FEATURES) + SYSREG ("pmxevcntr_el0", CPENC (3,3,9,13,2), 0, AARCH64_NO_FEATURES) + SYSREG ("pmxevtyper_el0", CPENC (3,3,9,13,1), 0, AARCH64_NO_FEATURES) + SYSREG ("pmzr_el0", CPENC (3,3,9,13,4), F_REG_WRITE, AARCH64_FEATURE (V8_8A)) /* PMUv3p9 */ + SYSREG ("por_el0", CPENC (3,3,10,2,4), 0, AARCH64_FEATURE (V8_8A)) /* S1POE */ + SYSREG ("por_el1", CPENC (3,0,10,2,4), 0, AARCH64_FEATURE (V8_8A)) /* S1POE */ + SYSREG ("por_el12", CPENC (3,5,10,2,4), 0, AARCH64_FEATURE (V8_8A)) /* S1POE */ + SYSREG ("por_el2", CPENC (3,4,10,2,4), 0, AARCH64_FEATURE (V8_8A)) /* S1POE */ + SYSREG ("por_el3", CPENC (3,6,10,2,4), 0, AARCH64_FEATURE (V8_8A)) /* S1POE */ + SYSREG ("prbar10_el1", CPENC (3,0,6,13,0), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar10_el2", CPENC (3,4,6,13,0), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar11_el1", CPENC (3,0,6,13,4), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar11_el2", CPENC (3,4,6,13,4), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar12_el1", CPENC (3,0,6,14,0), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar12_el2", CPENC (3,4,6,14,0), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar13_el1", CPENC (3,0,6,14,4), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar13_el2", CPENC (3,4,6,14,4), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar14_el1", CPENC (3,0,6,15,0), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar14_el2", CPENC (3,4,6,15,0), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar15_el1", CPENC (3,0,6,15,4), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar15_el2", CPENC (3,4,6,15,4), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar1_el1", CPENC (3,0,6,8,4), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar1_el2", CPENC (3,4,6,8,4), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar2_el1", CPENC (3,0,6,9,0), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar2_el2", CPENC (3,4,6,9,0), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar3_el1", CPENC (3,0,6,9,4), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar3_el2", CPENC (3,4,6,9,4), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar4_el1", CPENC (3,0,6,10,0), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar4_el2", CPENC (3,4,6,10,0), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar5_el1", CPENC (3,0,6,10,4), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar5_el2", CPENC (3,4,6,10,4), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar6_el1", CPENC (3,0,6,11,0), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar6_el2", CPENC (3,4,6,11,0), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar7_el1", CPENC (3,0,6,11,4), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar7_el2", CPENC (3,4,6,11,4), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar8_el1", CPENC (3,0,6,12,0), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar8_el2", CPENC (3,4,6,12,0), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar9_el1", CPENC (3,0,6,12,4), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar9_el2", CPENC (3,4,6,12,4), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar_el1", CPENC (3,0,6,8,0), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prbar_el2", CPENC (3,4,6,8,0), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prenr_el1", CPENC (3,0,6,1,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prenr_el2", CPENC (3,4,6,1,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar10_el1", CPENC (3,0,6,13,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar10_el2", CPENC (3,4,6,13,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar11_el1", CPENC (3,0,6,13,5), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar11_el2", CPENC (3,4,6,13,5), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar12_el1", CPENC (3,0,6,14,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar12_el2", CPENC (3,4,6,14,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar13_el1", CPENC (3,0,6,14,5), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar13_el2", CPENC (3,4,6,14,5), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar14_el1", CPENC (3,0,6,15,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar14_el2", CPENC (3,4,6,15,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar15_el1", CPENC (3,0,6,15,5), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar15_el2", CPENC (3,4,6,15,5), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar1_el1", CPENC (3,0,6,8,5), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar1_el2", CPENC (3,4,6,8,5), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar2_el1", CPENC (3,0,6,9,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar2_el2", CPENC (3,4,6,9,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar3_el1", CPENC (3,0,6,9,5), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar3_el2", CPENC (3,4,6,9,5), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar4_el1", CPENC (3,0,6,10,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar4_el2", CPENC (3,4,6,10,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar5_el1", CPENC (3,0,6,10,5), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar5_el2", CPENC (3,4,6,10,5), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar6_el1", CPENC (3,0,6,11,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar6_el2", CPENC (3,4,6,11,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar7_el1", CPENC (3,0,6,11,5), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar7_el2", CPENC (3,4,6,11,5), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar8_el1", CPENC (3,0,6,12,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar8_el2", CPENC (3,4,6,12,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar9_el1", CPENC (3,0,6,12,5), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar9_el2", CPENC (3,4,6,12,5), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar_el1", CPENC (3,0,6,8,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prlar_el2", CPENC (3,4,6,8,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prselr_el1", CPENC (3,0,6,2,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("prselr_el2", CPENC (3,4,6,2,1), 0, AARCH64_FEATURE (V8R)) + SYSREG ("rcwmask_el1", CPENC (3,0,13,0,6), F_REG_128, AARCH64_FEATURE (THE)) + SYSREG ("rcwsmask_el1", CPENC (3,0,13,0,3), F_REG_128, AARCH64_FEATURE (THE)) + SYSREG ("revidr_el1", CPENC (3,0,0,0,6), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("rgsr_el1", CPENC (3,0,1,0,5), 0, AARCH64_FEATURE (MEMTAG)) + SYSREG ("rmr_el1", CPENC (3,0,12,0,2), 0, AARCH64_NO_FEATURES) + SYSREG ("rmr_el2", CPENC (3,4,12,0,2), 0, AARCH64_NO_FEATURES) + SYSREG ("rmr_el3", CPENC (3,6,12,0,2), 0, AARCH64_NO_FEATURES) + SYSREG ("rndr", CPENC (3,3,2,4,0), F_REG_READ, AARCH64_FEATURE (RNG)) + SYSREG ("rndrrs", CPENC (3,3,2,4,1), F_REG_READ, AARCH64_FEATURE (RNG)) + SYSREG ("rvbar_el1", CPENC (3,0,12,0,1), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("rvbar_el2", CPENC (3,4,12,0,1), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("rvbar_el3", CPENC (3,6,12,0,1), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("s2pir_el2", CPENC (3,4,10,2,5), 0, AARCH64_FEATURE (V8_8A)) /* S2PIE */ + SYSREG ("s2por_el1", CPENC (3,0,10,2,5), 0, AARCH64_FEATURE (V8_8A)) /* S2POE */ + SYSREG ("scr_el3", CPENC (3,6,1,1,0), 0, AARCH64_NO_FEATURES) + SYSREG ("sctlr_el1", CPENC (3,0,1,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("sctlr_el12", CPENC (3,5,1,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("sctlr_el2", CPENC (3,4,1,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("sctlr_el3", CPENC (3,6,1,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("sctlr2_el1", CPENC (3,0,1,0,3), 0, AARCH64_NO_FEATURES) + SYSREG ("sctlr2_el12", CPENC (3,5,1,0,3), 0, AARCH64_NO_FEATURES) + SYSREG ("sctlr2_el2", CPENC (3,4,1,0,3), 0, AARCH64_NO_FEATURES) + SYSREG ("sctlr2_el3", CPENC (3,6,1,0,3), 0, AARCH64_NO_FEATURES) + SYSREG ("sctlr2alias_el1", CPENC (3,0,1,4,7), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("sctlr2mask_el1", CPENC (3,0,1,4,3), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("sctlr2mask_el12", CPENC (3,5,1,4,3), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("sctlr2mask_el2", CPENC (3,4,1,4,3), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("sctlralias_el1", CPENC (3,0,1,4,6), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("sctlrmask_el1", CPENC (3,0,1,4,0), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("sctlrmask_el12", CPENC (3,5,1,4,0), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("sctlrmask_el2", CPENC (3,4,1,4,0), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("scxtnum_el0", CPENC (3,3,13,0,7), 0, AARCH64_NO_FEATURES) /* CSV2_2 || CSV2_1p2 */ + SYSREG ("scxtnum_el1", CPENC (3,0,13,0,7), 0, AARCH64_NO_FEATURES) /* CSV2_2 || CSV2_1p2 */ + SYSREG ("scxtnum_el12", CPENC (3,5,13,0,7), 0, AARCH64_NO_FEATURES) /* CSV2_2 || CSV2_1p2 */ + SYSREG ("scxtnum_el2", CPENC (3,4,13,0,7), 0, AARCH64_NO_FEATURES) /* CSV2_2 || CSV2_1p2 */ + SYSREG ("scxtnum_el3", CPENC (3,6,13,0,7), 0, AARCH64_NO_FEATURES) /* CSV2_2 || CSV2_1p2 */ + SYSREG ("sder32_el2", CPENC (3,4,1,3,1), 0, AARCH64_FEATURE (V8_3A)) /* SEL2 */ + SYSREG ("sder32_el3", CPENC (3,6,1,1,1), 0, AARCH64_NO_FEATURES) + SYSREG ("smcr_el1", CPENC (3,0,1,2,6), 0, AARCH64_FEATURE (SME)) + SYSREG ("smcr_el12", CPENC (3,5,1,2,6), 0, AARCH64_FEATURE (SME)) + SYSREG ("smcr_el2", CPENC (3,4,1,2,6), 0, AARCH64_FEATURE (SME)) + SYSREG ("smcr_el3", CPENC (3,6,1,2,6), 0, AARCH64_FEATURE (SME)) + SYSREG ("smidr_el1", CPENC (3,1,0,0,6), F_REG_READ, AARCH64_FEATURE (SME)) + SYSREG ("smpri_el1", CPENC (3,0,1,2,4), 0, AARCH64_FEATURE (SME)) + SYSREG ("smprimap_el2", CPENC (3,4,1,2,5), 0, AARCH64_FEATURE (SME)) + SYSREG ("sp_el0", CPENC (3,0,4,1,0), 0, AARCH64_NO_FEATURES) + SYSREG ("sp_el1", CPENC (3,4,4,1,0), 0, AARCH64_NO_FEATURES) + SYSREG ("sp_el2", CPENC (3,6,4,1,0), 0, AARCH64_NO_FEATURES) + SYSREG ("spmaccessr_el1", CPENC (2,0,9,13,3), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmaccessr_el12", CPENC (2,5,9,13,3), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmaccessr_el2", CPENC (2,4,9,13,3), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmaccessr_el3", CPENC (2,6,9,13,3), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmcfgr_el1", CPENC (2,0,9,13,7), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmcgcr0_el1", CPENC (2,0,9,13,0), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmcgcr1_el1", CPENC (2,0,9,13,1), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmcntenclr_el0", CPENC (2,3,9,12,2), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmcntenset_el0", CPENC (2,3,9,12,1), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmcr_el0", CPENC (2,3,9,12,0), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmdevaff_el1", CPENC (2,0,9,13,6), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmdevarch_el1", CPENC (2,0,9,13,5), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevcntr0_el0", CPENC (2,3,14,0,0), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevcntr1_el0", CPENC (2,3,14,0,1), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevcntr2_el0", CPENC (2,3,14,0,2), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevcntr3_el0", CPENC (2,3,14,0,3), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevcntr4_el0", CPENC (2,3,14,0,4), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevcntr5_el0", CPENC (2,3,14,0,5), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevcntr6_el0", CPENC (2,3,14,0,6), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevcntr7_el0", CPENC (2,3,14,0,7), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevcntr8_el0", CPENC (2,3,14,1,0), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevcntr9_el0", CPENC (2,3,14,1,1), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevcntr10_el0", CPENC (2,3,14,1,2), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevcntr11_el0", CPENC (2,3,14,1,3), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevcntr12_el0", CPENC (2,3,14,1,4), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevcntr13_el0", CPENC (2,3,14,1,5), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevcntr14_el0", CPENC (2,3,14,1,6), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevcntr15_el0", CPENC (2,3,14,1,7), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfilt2r0_el0", CPENC (2,3,14,6,0), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfilt2r1_el0", CPENC (2,3,14,6,1), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfilt2r2_el0", CPENC (2,3,14,6,2), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfilt2r3_el0", CPENC (2,3,14,6,3), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfilt2r4_el0", CPENC (2,3,14,6,4), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfilt2r5_el0", CPENC (2,3,14,6,5), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfilt2r6_el0", CPENC (2,3,14,6,6), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfilt2r7_el0", CPENC (2,3,14,6,7), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfilt2r8_el0", CPENC (2,3,14,7,0), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfilt2r9_el0", CPENC (2,3,14,7,1), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfilt2r10_el0", CPENC (2,3,14,7,2), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfilt2r11_el0", CPENC (2,3,14,7,3), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfilt2r12_el0", CPENC (2,3,14,7,4), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfilt2r13_el0", CPENC (2,3,14,7,5), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfilt2r14_el0", CPENC (2,3,14,7,6), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfilt2r15_el0", CPENC (2,3,14,7,7), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfiltr0_el0", CPENC (2,3,14,4,0), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfiltr1_el0", CPENC (2,3,14,4,1), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfiltr2_el0", CPENC (2,3,14,4,2), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfiltr3_el0", CPENC (2,3,14,4,3), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfiltr4_el0", CPENC (2,3,14,4,4), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfiltr5_el0", CPENC (2,3,14,4,5), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfiltr6_el0", CPENC (2,3,14,4,6), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfiltr7_el0", CPENC (2,3,14,4,7), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfiltr8_el0", CPENC (2,3,14,5,0), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfiltr9_el0", CPENC (2,3,14,5,1), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfiltr10_el0", CPENC (2,3,14,5,2), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfiltr11_el0", CPENC (2,3,14,5,3), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfiltr12_el0", CPENC (2,3,14,5,4), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfiltr13_el0", CPENC (2,3,14,5,5), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfiltr14_el0", CPENC (2,3,14,5,6), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevfiltr15_el0", CPENC (2,3,14,5,7), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevtyper0_el0", CPENC (2,3,14,2,0), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevtyper1_el0", CPENC (2,3,14,2,1), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevtyper2_el0", CPENC (2,3,14,2,2), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevtyper3_el0", CPENC (2,3,14,2,3), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevtyper4_el0", CPENC (2,3,14,2,4), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevtyper5_el0", CPENC (2,3,14,2,5), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevtyper6_el0", CPENC (2,3,14,2,6), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevtyper7_el0", CPENC (2,3,14,2,7), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevtyper8_el0", CPENC (2,3,14,3,0), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevtyper9_el0", CPENC (2,3,14,3,1), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevtyper10_el0", CPENC (2,3,14,3,2), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevtyper11_el0", CPENC (2,3,14,3,3), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevtyper12_el0", CPENC (2,3,14,3,4), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevtyper13_el0", CPENC (2,3,14,3,5), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevtyper14_el0", CPENC (2,3,14,3,6), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmevtyper15_el0", CPENC (2,3,14,3,7), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmiidr_el1", CPENC (2,0,9,13,4), F_REG_READ, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmintenclr_el1", CPENC (2,0,9,14,2), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmintenset_el1", CPENC (2,0,9,14,1), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmovsclr_el0", CPENC (2,3,9,12,3), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmovsset_el0", CPENC (2,3,9,14,3), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmrootcr_el3", CPENC (2,6,9,14,7), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmscr_el1", CPENC (2,7,9,14,7), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmselr_el0", CPENC (2,3,9,12,5), 0, AARCH64_FEATURE (V8_8A)) /* SPMU */ + SYSREG ("spmzr_el0", CPENC (2,3,9,12,4), F_REG_WRITE, AARCH64_FEATURE (V9_4A)) /* SPMU2 */ + SYSREG ("spsel", CPENC (3,0,4,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("spsr_abt", CPENC (3,4,4,3,1), 0, AARCH64_NO_FEATURES) + SYSREG ("spsr_el1", CPENC (3,0,4,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("spsr_el12", CPENC (3,5,4,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("spsr_el2", CPENC (3,4,4,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("spsr_el3", CPENC (3,6,4,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("spsr_fiq", CPENC (3,4,4,3,3), 0, AARCH64_NO_FEATURES) + SYSREG ("spsr_hyp", CPENC (3,4,4,0,0), F_DEPRECATED, AARCH64_NO_FEATURES) + SYSREG ("spsr_irq", CPENC (3,4,4,3,0), 0, AARCH64_NO_FEATURES) + SYSREG ("spsr_svc", CPENC (3,0,4,0,0), F_DEPRECATED, AARCH64_NO_FEATURES) + SYSREG ("spsr_und", CPENC (3,4,4,3,2), 0, AARCH64_NO_FEATURES) + SYSREG ("ssbs", CPENC (3,3,4,2,6), 0, AARCH64_FEATURE (SSBS)) + SYSREG ("svcr", CPENC (3,3,4,2,2), 0, AARCH64_FEATURE (SME)) + SYSREG ("tco", CPENC (3,3,4,2,7), 0, AARCH64_FEATURE (MEMTAG)) + SYSREG ("tcr_el1", CPENC (3,0,2,0,2), 0, AARCH64_NO_FEATURES) + SYSREG ("tcr_el12", CPENC (3,5,2,0,2), 0, AARCH64_NO_FEATURES) + SYSREG ("tcr_el2", CPENC (3,4,2,0,2), 0, AARCH64_NO_FEATURES) + SYSREG ("tcr_el3", CPENC (3,6,2,0,2), 0, AARCH64_NO_FEATURES) + SYSREG ("tcr2_el1", CPENC (3,0,2,0,3), 0, AARCH64_NO_FEATURES) + SYSREG ("tcr2_el12", CPENC (3,5,2,0,3), 0, AARCH64_NO_FEATURES) + SYSREG ("tcr2_el2", CPENC (3,4,2,0,3), 0, AARCH64_NO_FEATURES) + SYSREG ("tcr2alias_el1", CPENC (3,0,2,7,7), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("tcr2mask_el1", CPENC (3,0,2,7,3), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("tcr2mask_el12", CPENC (3,5,2,7,3), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("tcr2mask_el2", CPENC (3,4,2,7,3), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("tcralias_el1", CPENC (3,0,2,7,6), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("tcrmask_el1", CPENC (3,0,2,7,2), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("tcrmask_el12", CPENC (3,5,2,7,2), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("tcrmask_el2", CPENC (3,4,2,7,2), 0, AARCH64_FEATURE (V9_5A)) /* SRMASK */ + SYSREG ("tfsr_el1", CPENC (3,0,5,6,0), 0, AARCH64_FEATURE (MEMTAG)) + SYSREG ("tfsr_el12", CPENC (3,5,5,6,0), 0, AARCH64_FEATURE (MEMTAG)) + SYSREG ("tfsr_el2", CPENC (3,4,5,6,0), 0, AARCH64_FEATURE (MEMTAG)) + SYSREG ("tfsr_el3", CPENC (3,6,5,6,0), 0, AARCH64_FEATURE (MEMTAG)) + SYSREG ("tfsre0_el1", CPENC (3,0,5,6,1), 0, AARCH64_FEATURE (MEMTAG)) + SYSREG ("tpidr2_el0", CPENC (3,3,13,0,5), 0, AARCH64_FEATURE (SME)) + SYSREG ("tpidr_el0", CPENC (3,3,13,0,2), 0, AARCH64_NO_FEATURES) + SYSREG ("tpidr_el1", CPENC (3,0,13,0,4), 0, AARCH64_NO_FEATURES) + SYSREG ("tpidr_el2", CPENC (3,4,13,0,2), 0, AARCH64_NO_FEATURES) + SYSREG ("tpidr_el3", CPENC (3,6,13,0,2), 0, AARCH64_NO_FEATURES) + SYSREG ("tpidrro_el0", CPENC (3,3,13,0,3), 0, AARCH64_NO_FEATURES) + SYSREG ("trbbaser_el1", CPENC (3,0,9,11,2), 0, AARCH64_FEATURE (V9A)) /* TRBE */ + SYSREG ("trbidr_el1", CPENC (3,0,9,11,7), F_REG_READ, AARCH64_FEATURE (V9A)) /* TRBE */ + SYSREG ("trblimitr_el1", CPENC (3,0,9,11,0), 0, AARCH64_FEATURE (V9A)) /* TRBE */ + SYSREG ("trbmar_el1", CPENC (3,0,9,11,4), 0, AARCH64_FEATURE (V9A)) /* TRBE */ + SYSREG ("trbmpam_el1", CPENC (3,0,9,11,5), 0, AARCH64_FEATURE (V9_3A)) /* TRBE_MPAM */ + SYSREG ("trbptr_el1", CPENC (3,0,9,11,1), 0, AARCH64_FEATURE (V9A)) /* TRBE */ + SYSREG ("trbsr_el1", CPENC (3,0,9,11,3), 0, AARCH64_FEATURE (V9A)) /* TRBE */ + SYSREG ("trbsr_el12", CPENC (3,5,9,11,3), 0, AARCH64_FEATURE (V9_5A)) /* TRBE_EXC */ + SYSREG ("trbsr_el2", CPENC (3,4,9,11,3), 0, AARCH64_FEATURE (V9_5A)) /* TRBE_EXC */ + SYSREG ("trbsr_el3", CPENC (3,6,9,11,3), 0, AARCH64_FEATURE (V9_5A)) /* TRBE_EXC */ + SYSREG ("trbtrg_el1", CPENC (3,0,9,11,6), 0, AARCH64_FEATURE (V9A)) /* TRBE */ + SYSREG ("trcacatr0", CPENC (2,1,2,0,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacatr1", CPENC (2,1,2,2,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacatr10", CPENC (2,1,2,4,3), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacatr11", CPENC (2,1,2,6,3), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacatr12", CPENC (2,1,2,8,3), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacatr13", CPENC (2,1,2,10,3), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacatr14", CPENC (2,1,2,12,3), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacatr15", CPENC (2,1,2,14,3), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacatr2", CPENC (2,1,2,4,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacatr3", CPENC (2,1,2,6,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacatr4", CPENC (2,1,2,8,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacatr5", CPENC (2,1,2,10,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacatr6", CPENC (2,1,2,12,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacatr7", CPENC (2,1,2,14,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacatr8", CPENC (2,1,2,0,3), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacatr9", CPENC (2,1,2,2,3), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacvr0", CPENC (2,1,2,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacvr1", CPENC (2,1,2,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacvr10", CPENC (2,1,2,4,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacvr11", CPENC (2,1,2,6,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacvr12", CPENC (2,1,2,8,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacvr13", CPENC (2,1,2,10,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacvr14", CPENC (2,1,2,12,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacvr15", CPENC (2,1,2,14,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacvr2", CPENC (2,1,2,4,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacvr3", CPENC (2,1,2,6,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacvr4", CPENC (2,1,2,8,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacvr5", CPENC (2,1,2,10,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacvr6", CPENC (2,1,2,12,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacvr7", CPENC (2,1,2,14,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacvr8", CPENC (2,1,2,0,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcacvr9", CPENC (2,1,2,2,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcauthstatus", CPENC (2,1,7,14,6), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcauxctlr", CPENC (2,1,0,6,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcbbctlr", CPENC (2,1,0,15,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcccctlr", CPENC (2,1,0,14,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trccidcctlr0", CPENC (2,1,3,0,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trccidcctlr1", CPENC (2,1,3,1,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trccidcvr0", CPENC (2,1,3,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trccidcvr1", CPENC (2,1,3,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trccidcvr2", CPENC (2,1,3,4,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trccidcvr3", CPENC (2,1,3,6,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trccidcvr4", CPENC (2,1,3,8,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trccidcvr5", CPENC (2,1,3,10,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trccidcvr6", CPENC (2,1,3,12,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trccidcvr7", CPENC (2,1,3,14,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trccidr0", CPENC (2,1,7,12,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trccidr1", CPENC (2,1,7,13,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trccidr2", CPENC (2,1,7,14,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trccidr3", CPENC (2,1,7,15,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcclaimclr", CPENC (2,1,7,9,6), 0, AARCH64_NO_FEATURES) + SYSREG ("trcclaimset", CPENC (2,1,7,8,6), 0, AARCH64_NO_FEATURES) + SYSREG ("trccntctlr0", CPENC (2,1,0,4,5), 0, AARCH64_NO_FEATURES) + SYSREG ("trccntctlr1", CPENC (2,1,0,5,5), 0, AARCH64_NO_FEATURES) + SYSREG ("trccntctlr2", CPENC (2,1,0,6,5), 0, AARCH64_NO_FEATURES) + SYSREG ("trccntctlr3", CPENC (2,1,0,7,5), 0, AARCH64_NO_FEATURES) + SYSREG ("trccntrldvr0", CPENC (2,1,0,0,5), 0, AARCH64_NO_FEATURES) + SYSREG ("trccntrldvr1", CPENC (2,1,0,1,5), 0, AARCH64_NO_FEATURES) + SYSREG ("trccntrldvr2", CPENC (2,1,0,2,5), 0, AARCH64_NO_FEATURES) + SYSREG ("trccntrldvr3", CPENC (2,1,0,3,5), 0, AARCH64_NO_FEATURES) + SYSREG ("trccntvr0", CPENC (2,1,0,8,5), 0, AARCH64_NO_FEATURES) + SYSREG ("trccntvr1", CPENC (2,1,0,9,5), 0, AARCH64_NO_FEATURES) + SYSREG ("trccntvr2", CPENC (2,1,0,10,5), 0, AARCH64_NO_FEATURES) + SYSREG ("trccntvr3", CPENC (2,1,0,11,5), 0, AARCH64_NO_FEATURES) + SYSREG ("trcconfigr", CPENC (2,1,0,4,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcdevaff0", CPENC (2,1,7,10,6), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcdevaff1", CPENC (2,1,7,11,6), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcdevarch", CPENC (2,1,7,15,6), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcdevid", CPENC (2,1,7,2,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcdevtype", CPENC (2,1,7,3,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcdvcmr0", CPENC (2,1,2,0,6), 0, AARCH64_NO_FEATURES) + SYSREG ("trcdvcmr1", CPENC (2,1,2,4,6), 0, AARCH64_NO_FEATURES) + SYSREG ("trcdvcmr2", CPENC (2,1,2,8,6), 0, AARCH64_NO_FEATURES) + SYSREG ("trcdvcmr3", CPENC (2,1,2,12,6), 0, AARCH64_NO_FEATURES) + SYSREG ("trcdvcmr4", CPENC (2,1,2,0,7), 0, AARCH64_NO_FEATURES) + SYSREG ("trcdvcmr5", CPENC (2,1,2,4,7), 0, AARCH64_NO_FEATURES) + SYSREG ("trcdvcmr6", CPENC (2,1,2,8,7), 0, AARCH64_NO_FEATURES) + SYSREG ("trcdvcmr7", CPENC (2,1,2,12,7), 0, AARCH64_NO_FEATURES) + SYSREG ("trcdvcvr0", CPENC (2,1,2,0,4), 0, AARCH64_NO_FEATURES) + SYSREG ("trcdvcvr1", CPENC (2,1,2,4,4), 0, AARCH64_NO_FEATURES) + SYSREG ("trcdvcvr2", CPENC (2,1,2,8,4), 0, AARCH64_NO_FEATURES) + SYSREG ("trcdvcvr3", CPENC (2,1,2,12,4), 0, AARCH64_NO_FEATURES) + SYSREG ("trcdvcvr4", CPENC (2,1,2,0,5), 0, AARCH64_NO_FEATURES) + SYSREG ("trcdvcvr5", CPENC (2,1,2,4,5), 0, AARCH64_NO_FEATURES) + SYSREG ("trcdvcvr6", CPENC (2,1,2,8,5), 0, AARCH64_NO_FEATURES) + SYSREG ("trcdvcvr7", CPENC (2,1,2,12,5), 0, AARCH64_NO_FEATURES) + SYSREG ("trceventctl0r", CPENC (2,1,0,8,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trceventctl1r", CPENC (2,1,0,9,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcextinselr", CPENC (2,1,0,8,4), F_REG_ALIAS, AARCH64_NO_FEATURES) + SYSREG ("trcextinselr0", CPENC (2,1,0,8,4), 0, AARCH64_NO_FEATURES) + SYSREG ("trcextinselr1", CPENC (2,1,0,9,4), 0, AARCH64_NO_FEATURES) + SYSREG ("trcextinselr2", CPENC (2,1,0,10,4), 0, AARCH64_NO_FEATURES) + SYSREG ("trcextinselr3", CPENC (2,1,0,11,4), 0, AARCH64_NO_FEATURES) + SYSREG ("trcidr0", CPENC (2,1,0,8,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcidr1", CPENC (2,1,0,9,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcidr10", CPENC (2,1,0,2,6), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcidr11", CPENC (2,1,0,3,6), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcidr12", CPENC (2,1,0,4,6), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcidr13", CPENC (2,1,0,5,6), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcidr2", CPENC (2,1,0,10,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcidr3", CPENC (2,1,0,11,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcidr4", CPENC (2,1,0,12,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcidr5", CPENC (2,1,0,13,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcidr6", CPENC (2,1,0,14,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcidr7", CPENC (2,1,0,15,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcidr8", CPENC (2,1,0,0,6), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcidr9", CPENC (2,1,0,1,6), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcimspec0", CPENC (2,1,0,0,7), 0, AARCH64_NO_FEATURES) + SYSREG ("trcimspec1", CPENC (2,1,0,1,7), 0, AARCH64_NO_FEATURES) + SYSREG ("trcimspec2", CPENC (2,1,0,2,7), 0, AARCH64_NO_FEATURES) + SYSREG ("trcimspec3", CPENC (2,1,0,3,7), 0, AARCH64_NO_FEATURES) + SYSREG ("trcimspec4", CPENC (2,1,0,4,7), 0, AARCH64_NO_FEATURES) + SYSREG ("trcimspec5", CPENC (2,1,0,5,7), 0, AARCH64_NO_FEATURES) + SYSREG ("trcimspec6", CPENC (2,1,0,6,7), 0, AARCH64_NO_FEATURES) + SYSREG ("trcimspec7", CPENC (2,1,0,7,7), 0, AARCH64_NO_FEATURES) + SYSREG ("trcitctrl", CPENC (2,1,7,0,4), 0, AARCH64_NO_FEATURES) + SYSREG ("trcitecr_el1", CPENC (3,0,1,2,3), 0, AARCH64_FEATURE (ITE)) + SYSREG ("trcitecr_el12", CPENC (3,5,1,2,3), 0, AARCH64_FEATURE (ITE)) + SYSREG ("trcitecr_el2", CPENC (3,4,1,2,3), 0, AARCH64_FEATURE (ITE)) + SYSREG ("trciteedcr", CPENC (2,1,0,2,1), 0, AARCH64_FEATURE (ITE)) + SYSREG ("trclar", CPENC (2,1,7,12,6), F_REG_WRITE, AARCH64_NO_FEATURES) + SYSREG ("trclsr", CPENC (2,1,7,13,6), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcoslar", CPENC (2,1,1,0,4), F_REG_WRITE, AARCH64_NO_FEATURES) + SYSREG ("trcoslsr", CPENC (2,1,1,1,4), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcpdcr", CPENC (2,1,1,4,4), 0, AARCH64_NO_FEATURES) + SYSREG ("trcpdsr", CPENC (2,1,1,5,4), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcpidr0", CPENC (2,1,7,8,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcpidr1", CPENC (2,1,7,9,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcpidr2", CPENC (2,1,7,10,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcpidr3", CPENC (2,1,7,11,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcpidr4", CPENC (2,1,7,4,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcpidr5", CPENC (2,1,7,5,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcpidr6", CPENC (2,1,7,6,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcpidr7", CPENC (2,1,7,7,7), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcprgctlr", CPENC (2,1,0,1,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcprocselr", CPENC (2,1,0,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcqctlr", CPENC (2,1,0,1,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr10", CPENC (2,1,1,10,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr11", CPENC (2,1,1,11,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr12", CPENC (2,1,1,12,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr13", CPENC (2,1,1,13,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr14", CPENC (2,1,1,14,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr15", CPENC (2,1,1,15,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr16", CPENC (2,1,1,0,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr17", CPENC (2,1,1,1,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr18", CPENC (2,1,1,2,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr19", CPENC (2,1,1,3,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr2", CPENC (2,1,1,2,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr20", CPENC (2,1,1,4,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr21", CPENC (2,1,1,5,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr22", CPENC (2,1,1,6,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr23", CPENC (2,1,1,7,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr24", CPENC (2,1,1,8,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr25", CPENC (2,1,1,9,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr26", CPENC (2,1,1,10,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr27", CPENC (2,1,1,11,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr28", CPENC (2,1,1,12,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr29", CPENC (2,1,1,13,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr3", CPENC (2,1,1,3,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr30", CPENC (2,1,1,14,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr31", CPENC (2,1,1,15,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr4", CPENC (2,1,1,4,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr5", CPENC (2,1,1,5,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr6", CPENC (2,1,1,6,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr7", CPENC (2,1,1,7,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr8", CPENC (2,1,1,8,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsctlr9", CPENC (2,1,1,9,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcrsr", CPENC (2,1,0,10,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcseqevr0", CPENC (2,1,0,0,4), 0, AARCH64_NO_FEATURES) + SYSREG ("trcseqevr1", CPENC (2,1,0,1,4), 0, AARCH64_NO_FEATURES) + SYSREG ("trcseqevr2", CPENC (2,1,0,2,4), 0, AARCH64_NO_FEATURES) + SYSREG ("trcseqrstevr", CPENC (2,1,0,6,4), 0, AARCH64_NO_FEATURES) + SYSREG ("trcseqstr", CPENC (2,1,0,7,4), 0, AARCH64_NO_FEATURES) + SYSREG ("trcssccr0", CPENC (2,1,1,0,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcssccr1", CPENC (2,1,1,1,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcssccr2", CPENC (2,1,1,2,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcssccr3", CPENC (2,1,1,3,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcssccr4", CPENC (2,1,1,4,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcssccr5", CPENC (2,1,1,5,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcssccr6", CPENC (2,1,1,6,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcssccr7", CPENC (2,1,1,7,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcsscsr0", CPENC (2,1,1,8,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcsscsr1", CPENC (2,1,1,9,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcsscsr2", CPENC (2,1,1,10,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcsscsr3", CPENC (2,1,1,11,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcsscsr4", CPENC (2,1,1,12,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcsscsr5", CPENC (2,1,1,13,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcsscsr6", CPENC (2,1,1,14,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcsscsr7", CPENC (2,1,1,15,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcsspcicr0", CPENC (2,1,1,0,3), 0, AARCH64_NO_FEATURES) + SYSREG ("trcsspcicr1", CPENC (2,1,1,1,3), 0, AARCH64_NO_FEATURES) + SYSREG ("trcsspcicr2", CPENC (2,1,1,2,3), 0, AARCH64_NO_FEATURES) + SYSREG ("trcsspcicr3", CPENC (2,1,1,3,3), 0, AARCH64_NO_FEATURES) + SYSREG ("trcsspcicr4", CPENC (2,1,1,4,3), 0, AARCH64_NO_FEATURES) + SYSREG ("trcsspcicr5", CPENC (2,1,1,5,3), 0, AARCH64_NO_FEATURES) + SYSREG ("trcsspcicr6", CPENC (2,1,1,6,3), 0, AARCH64_NO_FEATURES) + SYSREG ("trcsspcicr7", CPENC (2,1,1,7,3), 0, AARCH64_NO_FEATURES) + SYSREG ("trcstallctlr", CPENC (2,1,0,11,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcstatr", CPENC (2,1,0,3,0), F_REG_READ, AARCH64_NO_FEATURES) + SYSREG ("trcsyncpr", CPENC (2,1,0,13,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trctraceidr", CPENC (2,1,0,0,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trctsctlr", CPENC (2,1,0,12,0), 0, AARCH64_NO_FEATURES) + SYSREG ("trcvdarcctlr", CPENC (2,1,0,10,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcvdctlr", CPENC (2,1,0,8,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcvdsacctlr", CPENC (2,1,0,9,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcvictlr", CPENC (2,1,0,0,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcviiectlr", CPENC (2,1,0,1,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcvipcssctlr", CPENC (2,1,0,3,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcvissctlr", CPENC (2,1,0,2,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcvmidcctlr0", CPENC (2,1,3,2,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcvmidcctlr1", CPENC (2,1,3,3,2), 0, AARCH64_NO_FEATURES) + SYSREG ("trcvmidcvr0", CPENC (2,1,3,0,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcvmidcvr1", CPENC (2,1,3,2,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcvmidcvr2", CPENC (2,1,3,4,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcvmidcvr3", CPENC (2,1,3,6,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcvmidcvr4", CPENC (2,1,3,8,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcvmidcvr5", CPENC (2,1,3,10,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcvmidcvr6", CPENC (2,1,3,12,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trcvmidcvr7", CPENC (2,1,3,14,1), 0, AARCH64_NO_FEATURES) + SYSREG ("trfcr_el1", CPENC (3,0,1,2,1), 0, AARCH64_FEATURE (V8_3A)) /* TRF */ + SYSREG ("trfcr_el12", CPENC (3,5,1,2,1), 0, AARCH64_FEATURE (V8_3A)) /* TRF */ + SYSREG ("trfcr_el2", CPENC (3,4,1,2,1), 0, AARCH64_FEATURE (V8_3A)) /* TRF */ + SYSREG ("ttbr0_el1", CPENC (3,0,2,0,0), F_REG_128, AARCH64_NO_FEATURES) + SYSREG ("ttbr0_el12", CPENC (3,5,2,0,0), F_REG_128, AARCH64_NO_FEATURES) + SYSREG ("ttbr0_el2", CPENC (3,4,2,0,0), F_REG_128, AARCH64_FEATURE (V8A)) + SYSREG ("ttbr0_el3", CPENC (3,6,2,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("ttbr1_el1", CPENC (3,0,2,0,1), F_REG_128, AARCH64_NO_FEATURES) + SYSREG ("ttbr1_el12", CPENC (3,5,2,0,1), F_REG_128, AARCH64_NO_FEATURES) + SYSREG ("ttbr1_el2", CPENC (3,4,2,0,1), F_REG_128, AARCH64_FEATURE (V8A)) + SYSREG ("uao", CPENC (3,0,4,2,4), 0, AARCH64_FEATURE (V8_1A)) /* UAO */ + SYSREG ("vbar_el1", CPENC (3,0,12,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("vbar_el12", CPENC (3,5,12,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("vbar_el2", CPENC (3,4,12,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("vbar_el3", CPENC (3,6,12,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("vdisr_el2", CPENC (3,4,12,1,1), 0, AARCH64_FEATURE (RAS)) + SYSREG ("vdisr_el3", CPENC (3,6,12,1,1), 0, AARCH64_FEATURE (V9_4A)) /* E3DSE */ + SYSREG ("vmecid_a_el2", CPENC (3,4,10,9,1), 0, AARCH64_FEATURE (V9_2A)) /* MEC */ + SYSREG ("vmecid_p_el2", CPENC (3,4,10,9,0), 0, AARCH64_FEATURE (V9_2A)) /* MEC */ + SYSREG ("vmpidr_el2", CPENC (3,4,0,0,5), 0, AARCH64_NO_FEATURES) + SYSREG ("vncr_el2", CPENC (3,4,2,2,0), 0, AARCH64_FEATURE (V8_3A)) /* NV2 */ + SYSREG ("vpidr_el2", CPENC (3,4,0,0,0), 0, AARCH64_NO_FEATURES) + SYSREG ("vsctlr_el2", CPENC (3,4,2,0,0), 0, AARCH64_FEATURE (V8R)) + SYSREG ("vsesr_el2", CPENC (3,4,5,2,3), 0, AARCH64_FEATURE (RAS)) + SYSREG ("vsesr_el3", CPENC (3,6,5,2,3), 0, AARCH64_FEATURE (V9_4A)) /* E3DSE */ + SYSREG ("vstcr_el2", CPENC (3,4,2,6,2), 0, AARCH64_FEATURE (V8_3A)) /* SEL2 */ + SYSREG ("vsttbr_el2", CPENC (3,4,2,6,0), 0, AARCH64_FEATURES (2, V8A, V8_3A)) /* SEL2 */ + SYSREG ("vtcr_el2", CPENC (3,4,2,1,2), 0, AARCH64_NO_FEATURES) + SYSREG ("vttbr_el2", CPENC (3,4,2,1,0), F_REG_128, AARCH64_FEATURE (V8A)) + SYSREG ("zcr_el1", CPENC (3,0,1,2,0), 0, AARCH64_FEATURE (SVE)) + SYSREG ("zcr_el12", CPENC (3,5,1,2,0), 0, AARCH64_FEATURE (SVE)) + SYSREG ("zcr_el2", CPENC (3,4,1,2,0), 0, AARCH64_FEATURE (SVE)) + SYSREG ("zcr_el3", CPENC (3,6,1,2,0), 0, AARCH64_FEATURE (SVE)) diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc index 9d2c343..f28d670 100644 --- a/gcc/config/aarch64/aarch64.cc +++ b/gcc/config/aarch64/aarch64.cc @@ -519,7 +519,6 @@ typedef struct { #define F_DEPRECATED (1 << 1) #define F_REG_READ (1 << 2) #define F_REG_WRITE (1 << 3) -#define F_ARCHEXT (1 << 4) /* Flag indicating register name is alias for another system register. */ #define F_REG_ALIAS (1 << 5) /* Flag indicatinig registers which may be implemented with 128-bits. */ diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h index 2b6075d..2cd929d 100644 --- a/gcc/config/aarch64/aarch64.h +++ b/gcc/config/aarch64/aarch64.h @@ -240,44 +240,22 @@ constexpr auto AARCH64_FL_DEFAULT_ISA_MODE ATTRIBUTE_UNUSED #define TARGET_SIMD (TARGET_BASE_SIMD && TARGET_NON_STREAMING) #define TARGET_FLOAT AARCH64_HAVE_ISA (FP) -/* AARCH64_FL options necessary for system register implementation. */ - /* Define AARCH64_FL aliases for architectural features which are protected by -march flags in binutils but which receive no special treatment by GCC. + These features are used in the aarch64-sys-regs.def file, which is copied + from Binutils. - Such flags are inherited from the Binutils definition of system registers - and are mapped to the architecture in which the feature is implemented. */ + We should try to eliminate these inconsistencies in future. */ #define AARCH64_FL_RAS AARCH64_FL_V8A #define AARCH64_FL_LOR AARCH64_FL_V8_1A #define AARCH64_FL_PAN AARCH64_FL_V8_1A -#define AARCH64_FL_AMU AARCH64_FL_V8_4A -#define AARCH64_FL_SCXTNUM AARCH64_FL_V8_5A -#define AARCH64_FL_ID_PFR2 AARCH64_FL_V8_5A - -/* Armv8.9-A extension feature bits defined in Binutils but absent from GCC, - aliased to their base architecture. */ -#define AARCH64_FL_AIE AARCH64_FL_V8_9A -#define AARCH64_FL_DEBUGv8p9 AARCH64_FL_V8_9A -#define AARCH64_FL_FGT2 AARCH64_FL_V8_9A #define AARCH64_FL_ITE AARCH64_FL_V8_9A -#define AARCH64_FL_PFAR AARCH64_FL_V8_9A -#define AARCH64_FL_PMUv3_ICNTR AARCH64_FL_V8_9A -#define AARCH64_FL_PMUv3_SS AARCH64_FL_V8_9A -#define AARCH64_FL_PMUv3p9 AARCH64_FL_V8_9A #define AARCH64_FL_RASv2 AARCH64_FL_V8_9A -#define AARCH64_FL_S1PIE AARCH64_FL_V8_9A -#define AARCH64_FL_S1POE AARCH64_FL_V8_9A -#define AARCH64_FL_S2PIE AARCH64_FL_V8_9A -#define AARCH64_FL_S2POE AARCH64_FL_V8_9A -#define AARCH64_FL_SCTLR2 AARCH64_FL_V8_9A -#define AARCH64_FL_SEBEP AARCH64_FL_V8_9A -#define AARCH64_FL_SPE_FDS AARCH64_FL_V8_9A -#define AARCH64_FL_TCR2 AARCH64_FL_V8_9A + #define TARGET_V8R AARCH64_HAVE_ISA (V8R) #define TARGET_V9A AARCH64_HAVE_ISA (V9A) - /* SHA2 is an optional extension to AdvSIMD. */ #define TARGET_SHA2 AARCH64_HAVE_ISA (SHA2) diff --git a/gcc/config/arm/arm.cc b/gcc/config/arm/arm.cc index da28d96..6df2fa0 100644 --- a/gcc/config/arm/arm.cc +++ b/gcc/config/arm/arm.cc @@ -5696,6 +5696,22 @@ arm_canonicalize_comparison (int *code, rtx *op0, rtx *op1, maxval = (HOST_WIDE_INT_1U << (GET_MODE_BITSIZE (mode) - 1)) - 1; + /* For floating-point comparisons, prefer >= and > over <= and < since + the former are supported by VSEL on some architectures. Only do this + if both operands are registers. */ + if (GET_MODE_CLASS (mode) == MODE_FLOAT + && (*code == LE + || *code == LT + || *code == UNGT + || *code == UNGE) + && register_operand (*op0, mode) + && register_operand (*op1, mode)) + { + std::swap (*op0, *op1); + *code = (int) swap_condition ((rtx_code)*code); + return; + } + /* For DImode, we have GE/LT/GEU/LTU comparisons (with cmp/sbc). In ARM mode we can also use cmp/cmpeq for GTU/LEU. GT/LE must be either reversed or (for constant OP1) adjusted to GE/LT. @@ -32478,6 +32494,9 @@ arm_validize_comparison (rtx *comparison, rtx * op1, rtx * op2) case E_HFmode: if (!TARGET_VFP_FP16INST) break; + if (!arm_vsel_comparison_operator (*comparison, mode)) + return false; + /* FP16 comparisons are done in SF mode. */ mode = SFmode; *op1 = convert_to_mode (mode, *op1, 1); diff --git a/gcc/config/arm/iterators.md b/gcc/config/arm/iterators.md index 0c163ed..eb519e7 100644 --- a/gcc/config/arm/iterators.md +++ b/gcc/config/arm/iterators.md @@ -3014,3 +3014,20 @@ ;; Define iterators for VCMLA operations as MUL (define_int_iterator VCMUL_OP [UNSPEC_VCMUL UNSPEC_VCMUL_CONJ]) + +(define_int_attr VxCIQ_carry [(VADCIQ_U "VADCIQ_U_carry") + (VADCIQ_S "VADCIQ_S_carry") + (VSBCIQ_U "VSBCIQ_U_carry") + (VSBCIQ_S "VSBCIQ_S_carry")]) +(define_int_attr VxCIQ_M_carry [(VADCIQ_M_U "VADCIQ_M_U_carry") + (VADCIQ_M_S "VADCIQ_M_S_carry") + (VSBCIQ_M_U "VSBCIQ_M_U_carry") + (VSBCIQ_M_S "VSBCIQ_M_S_carry")]) +(define_int_attr VxCQ_carry [(VADCQ_U "VADCQ_U_carry") + (VADCQ_S "VADCQ_S_carry") + (VSBCQ_U "VSBCQ_U_carry") + (VSBCQ_S "VSBCQ_S_carry")]) +(define_int_attr VxCQ_M_carry [(VADCQ_M_U "VADCQ_M_U_carry") + (VADCQ_M_S "VADCQ_M_S_carry") + (VSBCQ_M_U "VSBCQ_M_U_carry") + (VSBCQ_M_S "VSBCQ_M_S_carry")]) diff --git a/gcc/config/arm/mve.md b/gcc/config/arm/mve.md index bd3db24..87b45b2 100644 --- a/gcc/config/arm/mve.md +++ b/gcc/config/arm/mve.md @@ -3965,14 +3965,14 @@ (define_insn "get_fpscr_nzcvqc" [(set (match_operand:SI 0 "register_operand" "=r") - (unspec_volatile:SI [(reg:SI VFPCC_REGNUM)] UNSPEC_GET_FPSCR_NZCVQC))] + (unspec:SI [(reg:SI VFPCC_REGNUM)] UNSPEC_GET_FPSCR_NZCVQC))] "TARGET_HAVE_MVE" "vmrs\\t%0, FPSCR_nzcvqc" [(set_attr "type" "mve_move")]) (define_insn "set_fpscr_nzcvqc" [(set (reg:SI VFPCC_REGNUM) - (unspec_volatile:SI [(match_operand:SI 0 "register_operand" "r")] + (unspec:SI [(match_operand:SI 0 "register_operand" "r")] VUNSPEC_SET_FPSCR_NZCVQC))] "TARGET_HAVE_MVE" "vmsr\\tFPSCR_nzcvqc, %0" @@ -3988,8 +3988,9 @@ (match_operand:V4SI 2 "s_register_operand" "w")] VxCIQ)) (set (reg:SI VFPCC_REGNUM) - (unspec:SI [(const_int 0)] - VxCIQ)) + (unspec:SI [(match_dup 1) + (match_dup 2)] + <VxCIQ_carry>)) ] "TARGET_HAVE_MVE" "<mve_insn>.i32\t%q0, %q1, %q2" @@ -4009,8 +4010,11 @@ (match_operand:V4BI 4 "vpr_register_operand" "Up")] VxCIQ_M)) (set (reg:SI VFPCC_REGNUM) - (unspec:SI [(const_int 0)] - VxCIQ_M)) + (unspec:SI [(match_dup 1) + (match_dup 2) + (match_dup 3) + (match_dup 4)] + <VxCIQ_M_carry>)) ] "TARGET_HAVE_MVE" "vpst\;<mve_insn>t.i32\t%q0, %q2, %q3" @@ -4025,11 +4029,14 @@ (define_insn "@mve_<mve_insn>q_<supf>v4si" [(set (match_operand:V4SI 0 "s_register_operand" "=w") (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "w") - (match_operand:V4SI 2 "s_register_operand" "w")] + (match_operand:V4SI 2 "s_register_operand" "w") + (reg:SI VFPCC_REGNUM)] VxCQ)) (set (reg:SI VFPCC_REGNUM) - (unspec:SI [(reg:SI VFPCC_REGNUM)] - VxCQ)) + (unspec:SI [(match_dup 1) + (match_dup 2) + (reg:SI VFPCC_REGNUM)] + <VxCQ_carry>)) ] "TARGET_HAVE_MVE" "<mve_insn>.i32\t%q0, %q1, %q2" @@ -4047,11 +4054,16 @@ (unspec:V4SI [(match_operand:V4SI 1 "s_register_operand" "0") (match_operand:V4SI 2 "s_register_operand" "w") (match_operand:V4SI 3 "s_register_operand" "w") - (match_operand:V4BI 4 "vpr_register_operand" "Up")] + (match_operand:V4BI 4 "vpr_register_operand" "Up") + (reg:SI VFPCC_REGNUM)] VxCQ_M)) (set (reg:SI VFPCC_REGNUM) - (unspec:SI [(reg:SI VFPCC_REGNUM)] - VxCQ_M)) + (unspec:SI [(match_dup 1) + (match_dup 2) + (match_dup 3) + (match_dup 4) + (reg:SI VFPCC_REGNUM)] + <VxCQ_M_carry>)) ] "TARGET_HAVE_MVE" "vpst\;<mve_insn>t.i32\t%q0, %q2, %q3" diff --git a/gcc/config/arm/unspecs.md b/gcc/config/arm/unspecs.md index c1ee972..17af152 100644 --- a/gcc/config/arm/unspecs.md +++ b/gcc/config/arm/unspecs.md @@ -1160,21 +1160,37 @@ VLDRGBWBQ VLDRGBWBQ_Z VADCQ_U + VADCQ_U_carry VADCQ_M_U + VADCQ_M_U_carry VADCQ_S + VADCQ_S_carry VADCQ_M_S + VADCQ_M_S_carry VSBCIQ_U + VSBCIQ_U_carry VSBCIQ_S + VSBCIQ_S_carry VSBCIQ_M_U + VSBCIQ_M_U_carry VSBCIQ_M_S + VSBCIQ_M_S_carry VSBCQ_U + VSBCQ_U_carry VSBCQ_S + VSBCQ_S_carry VSBCQ_M_U + VSBCQ_M_U_carry VSBCQ_M_S + VSBCQ_M_S_carry VADCIQ_U + VADCIQ_U_carry VADCIQ_M_U + VADCIQ_M_U_carry VADCIQ_S + VADCIQ_S_carry VADCIQ_M_S + VADCIQ_M_S_carry VLD2Q VLD4Q VST2Q diff --git a/gcc/config/gcn/gcn-devices.def b/gcc/config/gcn/gcn-devices.def index b27385b..9277d41 100644 --- a/gcc/config/gcn/gcn-devices.def +++ b/gcc/config/gcn/gcn-devices.def @@ -179,7 +179,7 @@ GCN_DEVICE(gfx942, GFX942, 0x4c, ISA_CDNA3, /* Max ISA VGPRs */ 512, /* Generic code obj version */ 0, /* non-generic */ /* Architecture Family */ GFX9, - /* Generic Name */ NONE + /* Generic Name */ GFX9_4_GENERIC ) GCN_DEVICE(gfx950, GFX950, 0x4f, ISA_CDNA3, @@ -190,7 +190,7 @@ GCN_DEVICE(gfx950, GFX950, 0x4f, ISA_CDNA3, /* Max ISA VGPRs */ 512, /* Generic code obj version */ 0, /* non-generic */ /* Architecture Family */ GFX9, - /* Generic Name */ NONE + /* Generic Name */ GFX9_4_GENERIC ) GCN_DEVICE(gfx9-generic, GFX9_GENERIC, 0x051, ISA_GCN5, diff --git a/gcc/config/gcn/t-omp-device b/gcc/config/gcn/t-omp-device index cae6bd3..38d99d0 100644 --- a/gcc/config/gcn/t-omp-device +++ b/gcc/config/gcn/t-omp-device @@ -1,4 +1,4 @@ omp-device-properties-gcn: $(srcdir)/config/gcn/gcn-devices.def echo kind: gpu > $@ echo arch: amdgcn gcn >> $@ - echo isa: `grep -o -P '(?<=GCN_DEVICE\()gfx[0-9a-f]+(?=,)' $<` >> $@ + echo isa: `grep -o -P '(?<=GCN_DEVICE\()gfx[-0-9a-f]+(|-generic)(?=,)' $<` >> $@ diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h index fbd8d9a..3a66d78 100644 --- a/gcc/config/i386/i386.h +++ b/gcc/config/i386/i386.h @@ -2481,12 +2481,12 @@ constexpr wide_int_bitmask PTA_CLEARWATERFOREST = (PTA_SIERRAFOREST & (~(PTA_KL | PTA_WIDEKL))) | PTA_AVXVNNIINT16 | PTA_SHA512 | PTA_SM3 | PTA_SM4 | PTA_USER_MSR | PTA_PREFETCHI; constexpr wide_int_bitmask PTA_PANTHERLAKE = - (PTA_ARROWLAKE_S & (~(PTA_KL | PTA_WIDEKL))) | PTA_PREFETCHI; + (PTA_ARROWLAKE_S & (~(PTA_KL | PTA_WIDEKL))); constexpr wide_int_bitmask PTA_DIAMONDRAPIDS = PTA_GRANITERAPIDS_D | PTA_AVXIFMA | PTA_AVXNECONVERT | PTA_AVXVNNIINT16 | PTA_AVXVNNIINT8 | PTA_CMPCCXADD | PTA_SHA512 | PTA_SM3 | PTA_SM4 | PTA_AVX10_2 | PTA_APX_F | PTA_AMX_AVX512 | PTA_AMX_FP8 | PTA_AMX_TF32 | PTA_MOVRS - | PTA_AMX_MOVRS | PTA_USER_MSR; + | PTA_AMX_MOVRS; constexpr wide_int_bitmask PTA_BDVER1 = PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_SSE4A | PTA_CX16 | PTA_POPCNT | PTA_LZCNT diff --git a/gcc/config/riscv/autovec-opt.md b/gcc/config/riscv/autovec-opt.md index 3d6e0a1..d2705cf 100644 --- a/gcc/config/riscv/autovec-opt.md +++ b/gcc/config/riscv/autovec-opt.md @@ -1913,7 +1913,7 @@ (define_insn_and_split "*widen_waddu_wx_<mode>" [(set (match_operand:VWEXTI_D 0 "register_operand") - (any_widen_binop:VWEXTI_D + (plus:VWEXTI_D (vec_duplicate:VWEXTI_D (any_extend:<VEL> (match_operand:<VSUBEL> 2 "register_operand"))) @@ -1933,7 +1933,7 @@ (define_insn_and_split "*widen_wsubu_wx_<mode>" [(set (match_operand:VWEXTI_D 0 "register_operand") - (any_widen_binop:VWEXTI_D + (minus:VWEXTI_D (match_operand:VWEXTI_D 1 "register_operand") (vec_duplicate:VWEXTI_D (any_extend:<VEL> diff --git a/gcc/config/rs6000/aix.h b/gcc/config/rs6000/aix.h index 9e7edbb..c83eace 100644 --- a/gcc/config/rs6000/aix.h +++ b/gcc/config/rs6000/aix.h @@ -281,4 +281,6 @@ #undef SUBTARGET_DRIVER_SELF_SPECS #define SUBTARGET_DRIVER_SELF_SPECS \ "%{m64:-maix64} %<m64", \ -"%{m32:-maix32} %<m32" +"%{m32:-maix32} %<m32", \ +"%{fstack-protector*: %<fstack-protector* \ + %estack-protector not supported on AIX}" diff --git a/gcc/cp/ChangeLog b/gcc/cp/ChangeLog index 4e41b69..ecf3986 100644 --- a/gcc/cp/ChangeLog +++ b/gcc/cp/ChangeLog @@ -1,3 +1,10 @@ +2025-10-14 Patrick Palka <ppalka@redhat.com> + + PR c++/122192 + * parser.cc (cp_parser_mem_initializer_id): Pass class_type + instead of typename_type to cp_parser_class_name in the + nested-name-specifier case. + 2025-10-13 Jakub Jelinek <jakub@redhat.com> PR c++/122228 diff --git a/gcc/cp/parser.cc b/gcc/cp/parser.cc index 1ed2f37..9280632 100644 --- a/gcc/cp/parser.cc +++ b/gcc/cp/parser.cc @@ -19091,7 +19091,7 @@ cp_parser_mem_initializer_id (cp_parser* parser) return cp_parser_class_name (parser, /*typename_keyword_p=*/true, /*template_keyword_p=*/template_p, - typename_type, + class_type, /*check_dependency_p=*/true, /*class_head_p=*/false, /*is_declaration=*/true); diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi index 6bd5128..3f53986 100644 --- a/gcc/doc/invoke.texi +++ b/gcc/doc/invoke.texi @@ -659,7 +659,7 @@ Objective-C and Objective-C++ Dialects}. -ftree-phiprop -ftree-loop-distribution -ftree-loop-distribute-patterns -ftree-loop-ivcanon -ftree-loop-linear -ftree-loop-optimize -ftree-loop-vectorize --ftree-parallelize-loops=@var{n} -ftree-pre -ftree-partial-pre -ftree-pta +-ftree-parallelize-loops[=@var{n}] -ftree-pre -ftree-partial-pre -ftree-pta -ftree-reassoc -ftree-scev-cprop -ftree-sink -ftree-slsr -ftree-sra -ftree-switch-conversion -ftree-tail-merge -ftree-ter -ftree-vectorize -ftree-vrp -ftrivial-auto-var-init @@ -14691,8 +14691,9 @@ variable merging and induction variable elimination) on trees. Enabled by default at @option{-O1} and higher. @opindex ftree-parallelize-loops -@item -ftree-parallelize-loops=n -Parallelize loops, i.e., split their iteration space to run in n threads. +@item -ftree-parallelize-loops +@itemx -ftree-parallelize-loops=@var{n} +Parallelize loops, i.e., split their iteration space to run in multiple threads. This is only possible for loops whose iterations are independent and can be arbitrarily reordered. The optimization is only profitable on multiprocessor machines, for loops that are CPU-intensive, @@ -14700,6 +14701,17 @@ rather than constrained e.g.@: by memory bandwidth. This option implies @option{-pthread}, and thus is only supported on targets that have support for @option{-pthread}. +When a positive value @var{n} is specified, the number of threads is fixed +at compile time and cannot be changed after compilation. The compiler +generates ``#pragma omp parallel num_threads(@var{n})''. + +When used without @code{=@var{n}} (i.e., @option{-ftree-parallelize-loops}), +the number of threads is determined at program execution time via the +@env{OMP_NUM_THREADS} environment variable. If @env{OMP_NUM_THREADS} is not +set, the OpenMP runtime automatically detects the number of available +processors and uses that value. This enables creating binaries that +adapt to different hardware configurations without recompilation. + @opindex ftree-pta @item -ftree-pta Perform function-local points-to analysis on trees. This flag is @@ -35143,13 +35155,14 @@ AVXIFMA, AVXVNNIINT8, AVXNECONVERT, CMPCCXADD, AVXVNNIINT16, SHA512, SM3 and SM4 instruction set support. @item pantherlake -Intel Panther Lake CPU with 64-bit extensions, MOVBE, MMX, SSE, SSE2, SSE3, -SSSE3, SSE4.1, SSE4.2, POPCNT, AES, PREFETCHW, PCLMUL, RDRND, XSAVE, XSAVEC, -XSAVES, XSAVEOPT, FSGSBASE, PTWRITE, RDPID, SGX, GFNI-SSE, CLWB, MOVDIRI, -MOVDIR64B, WAITPKG, ADCX, AVX, AVX2, BMI, BMI2, F16C, FMA, LZCNT, PCONFIG, PKU, -VAES, VPCLMULQDQ, SERIALIZE, HRESET, AVX-VNNI, UINTR, AVXIFMA, AVXVNNIINT8, -AVXNECONVERT, CMPCCXADD, AVXVNNIINT16, SHA512, SM3, SM4 and PREFETCHI -instruction set support. +@itemx wildcatlake +Intel Panther Lake/Wildcat Lake CPU with 64-bit extensions, MOVBE, MMX, SSE, +SSE2, SSE3, SSSE3, SSE4.1, SSE4.2, POPCNT, AES, PREFETCHW, PCLMUL, RDRND, +XSAVE, XSAVEC, XSAVES, XSAVEOPT, FSGSBASE, PTWRITE, RDPID, SGX, GFNI-SSE, +CLWB, MOVDIRI, MOVDIR64B, WAITPKG, ADCX, AVX, AVX2, BMI, BMI2, F16C, FMA, +LZCNT, PCONFIG, PKU, VAES, VPCLMULQDQ, SERIALIZE, HRESET, AVX-VNNI, UINTR, +AVXIFMA, AVXVNNIINT8, AVXNECONVERT, CMPCCXADD, AVXVNNIINT16, SHA512, SM3 and +SM4 instruction set support. @item sapphirerapids @itemx emeraldrapids @@ -35196,8 +35209,7 @@ MOVDIRI, MOVDIR64B, ENQCMD, CLDEMOTE, PTWRITE, WAITPKG, SERIALIZE, TSXLDTRK, UINTR, AMX-BF16, AMX-TILE, AMX-INT8, AVX-VNNI, AVX512FP16, AVX512BF16, AMX-FP16, PREFETCHI, AMX-COMPLEX, AVX10.1-512, AVX-IFMA, AVX-NE-CONVERT, AVX-VNNI-INT16, AVX-VNNI-INT8, CMPccXADD, SHA512, SM3, SM4, AVX10.2-512, APX_F, AMX-AVX512, -AMX-FP8, AMX-TF32, AMX-TRANSPOSE, MOVRS, AMX-MOVRS and USER_MSR instruction set -support. +AMX-FP8, AMX-TF32, MOVRS and AMX-MOVRS instruction set support. @item bonnell @itemx atom diff --git a/gcc/ipa-fnsummary.cc b/gcc/ipa-fnsummary.cc index dd41de9..28f79aa 100644 --- a/gcc/ipa-fnsummary.cc +++ b/gcc/ipa-fnsummary.cc @@ -2356,7 +2356,8 @@ param_change_prob (ipa_func_body_info *fbi, gimple *stmt, int i) /* Lookup the most frequent update of the value and believe that it dominates all the other; precise analysis here is difficult. */ EXECUTE_IF_SET_IN_BITMAP (info.bb_set, 0, index, bi) - max = max.max (BASIC_BLOCK_FOR_FN (cfun, index)->count); + max = profile_count::max_prefer_initialized + (max, BASIC_BLOCK_FOR_FN (cfun, index)->count); if (dump_file) { fprintf (dump_file, " Set with count "); diff --git a/gcc/ipa-inline-transform.cc b/gcc/ipa-inline-transform.cc index a204854..99969aa 100644 --- a/gcc/ipa-inline-transform.cc +++ b/gcc/ipa-inline-transform.cc @@ -833,7 +833,8 @@ inline_transform (struct cgraph_node *node) FOR_ALL_BB_FN (bb, cfun) { bb->count = bb->count.apply_scale (num, den); - cfun->cfg->count_max = cfun->cfg->count_max.max (bb->count); + cfun->cfg->count_max = profile_count::max_prefer_initialized + (cfun->cfg->count_max, bb->count); } ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = node->count; } diff --git a/gcc/predict.cc b/gcc/predict.cc index 895c5f9..d937cc6 100644 --- a/gcc/predict.cc +++ b/gcc/predict.cc @@ -3849,7 +3849,7 @@ update_max_bb_count (void) basic_block bb; FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, next_bb) - true_count_max = true_count_max.max (bb->count); + true_count_max = profile_count::max_prefer_initialized (true_count_max, bb->count); cfun->cfg->count_max = true_count_max; @@ -4162,7 +4162,9 @@ estimate_bb_frequencies () executed, then preserve this info. */ if (!(bb->count == profile_count::zero ())) bb->count = count.guessed_local ().combine_with_ipa_count (ipa_count); - cfun->cfg->count_max = cfun->cfg->count_max.max (bb->count); + cfun->cfg->count_max + = profile_count::max_prefer_initialized (cfun->cfg->count_max, + bb->count); } free_aux_for_blocks (); @@ -4473,7 +4475,9 @@ rebuild_frequencies (void) cfun->cfg->count_max = profile_count::uninitialized (); FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, next_bb) { - cfun->cfg->count_max = cfun->cfg->count_max.max (bb->count); + cfun->cfg->count_max + = profile_count::max_prefer_initialized (cfun->cfg->count_max, + bb->count); if (bb->count.nonzero_p () && bb->count.quality () >= AFDO) feedback_found = true; /* Uninitialized count may be result of inlining or an omision in an diff --git a/gcc/print-tree.cc b/gcc/print-tree.cc index f84be76..7b8a680 100644 --- a/gcc/print-tree.cc +++ b/gcc/print-tree.cc @@ -747,6 +747,14 @@ print_node (FILE *file, const char *prefix, tree node, int indent, case tcc_reference: case tcc_statement: case tcc_vl_exp: + if ((code == MEM_REF || code == TARGET_MEM_REF) + && MR_DEPENDENCE_CLIQUE (node) != 0) + { + indent_to (file, indent + 4); + fprintf (file, "clique: %d base: %d", + MR_DEPENDENCE_CLIQUE (node), + MR_DEPENDENCE_BASE (node)); + } if (code == BIND_EXPR) { print_node (file, "vars", TREE_OPERAND (node, 0), indent + 4); diff --git a/gcc/profile-count.h b/gcc/profile-count.h index 89746c6..85e601e 100644 --- a/gcc/profile-count.h +++ b/gcc/profile-count.h @@ -1109,30 +1109,23 @@ public: /* Make counter forcibly nonzero. */ profile_count force_nonzero () const; - profile_count max (profile_count other) const - { - profile_count val = *this; - /* Always prefer nonzero IPA counts over local counts. */ - if (ipa ().nonzero_p () || other.ipa ().nonzero_p ()) - { - val = ipa (); - other = other.ipa (); - } - if (!initialized_p ()) - return other; - if (!other.initialized_p ()) - return *this; - if (*this == zero ()) - return other; - if (other == zero ()) - return *this; - gcc_checking_assert (compatible_p (other)); - if (val.m_val < other.m_val || (m_val == other.m_val - && val.m_quality < other.m_quality)) - return other; - return *this; - } + /* Return maximum of A and B. If one of values is uninitialized return the + other. */ + + static profile_count + max_prefer_initialized (const profile_count a, const profile_count b) + { + if (!a.initialized_p ()) + return b; + if (!b.initialized_p ()) + return a; + profile_count ret; + gcc_checking_assert (a.compatible_p (b)); + ret.m_val = MAX (a.m_val, b.m_val); + ret.m_quality = MIN (a.m_quality, b.m_quality); + return ret; + } /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter accordingly. */ diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog index 7b857ec..d910bc0 100644 --- a/gcc/testsuite/ChangeLog +++ b/gcc/testsuite/ChangeLog @@ -1,3 +1,107 @@ +2025-10-15 Andrew MacLeod <amacleod@redhat.com> + + PR tree-optimization/121468 + PR tree-optimization/121206 + PR tree-optimization/122200 + * gcc.dg/pr121468.c: New. + * gcc.dg/pr122200.c: New. + +2025-10-15 Richard Earnshaw <rearnsha@arm.com> + + PR target/118460 + * gcc.target/arm/armv8_2-fp16-move-1.c: Adjust expected output. + * gcc.target/arm/armv8_2-fp16-move-2.c: Likewise. + +2025-10-15 Andrew Pinski <andrew.pinski@oss.qualcomm.com> + + PR tree-optimization/122037 + * gcc.dg/tree-ssa/vla-1.c: New test. + +2025-10-15 Alice Carlotti <alice.carlotti@arm.com> + + * gcc.target/aarch64/acle/rwsr-armv8p9.c: Fix incorrect encoding. + +2025-10-15 Sebastian Pop <spop@nvidia.com> + + * gcc.dg/autopar/runtime-auto.c: New test. + +2025-10-15 Christophe Lyon <christophe.lyon@linaro.org> + + PR target/122189 + * gcc.target/arm/mve/intrinsics/vadcq-check-carry.c: New test. + * gcc.target/arm/mve/intrinsics/vadcq_m_s32.c: Adjust instructions + order. + * gcc.target/arm/mve/intrinsics/vadcq_m_u32.c: Likewise. + * gcc.target/arm/mve/intrinsics/vsbcq_m_s32.c: Likewise. + * gcc.target/arm/mve/intrinsics/vsbcq_m_u32.c: Likewise. + +2025-10-15 Roger Sayle <roger@nextmovesoftware.com> + + PR rtl-optimization/122266 + * gcc.target/i386/pr122266.c: New test case. + +2025-10-14 Patrick Palka <ppalka@redhat.com> + + PR c++/122192 + * g++.dg/template/dependent-base6.C: Verify mem-initializer-id + qualified name lookup is type-only too. + +2025-10-14 Tamar Christina <tamar.christina@arm.com> + + PR tree-optimization/121949 + * gcc.dg/vect/pr121949_1.c: New test. + * gcc.dg/vect/pr121949_2.c: New test. + * gcc.dg/vect/pr121949_3.c: New test. + +2025-10-14 Richard Biener <rguenther@suse.de> + + * gcc.dg/vect/pr120687-1.c: Adjust. + * gcc.dg/vect/pr120687-2.c: Likewise. + * gcc.dg/vect/pr120687-3.c: Likewise. + +2025-10-14 Haochen Jiang <haochen.jiang@intel.com> + + * g++.dg/other/i386-2.C: Remove AMX-TRANSPOSE test. + * g++.dg/other/i386-3.C: Ditto. + * gcc.target/i386/amx-check.h: Ditto. + * gcc.target/i386/amxmovrs-asmatt-1.c: Ditto. + * gcc.target/i386/amxmovrs-asmintel-1.c: Ditto. + * gcc.target/i386/funcspec-56.inc: Ditto. + * gcc.target/i386/sse-12.c: Ditto. + * gcc.target/i386/sse-13.c: Ditto. + * gcc.target/i386/sse-14.c: Ditto. + * gcc.target/i386/sse-22.c: Ditto. + * gcc.target/i386/sse-23.c: Ditto. + * lib/target-supports.exp: Ditto. + * gcc.target/i386/amxmovrs-2rpntlvwrs-2.c: Removed. + * gcc.target/i386/amxtranspose-2rpntlvw-2.c: Removed. + * gcc.target/i386/amxtranspose-asmatt-1.c: Removed. + * gcc.target/i386/amxtranspose-asmintel-1.c: Removed. + * gcc.target/i386/amxtranspose-conjtcmmimfp16ps-2.c: Removed. + * gcc.target/i386/amxtranspose-conjtfp16-2.c: Removed. + * gcc.target/i386/amxtranspose-tcmmimfp16ps-2.c: Removed. + * gcc.target/i386/amxtranspose-tcmmrlfp16ps-2.c: Removed. + * gcc.target/i386/amxtranspose-tdpbf16ps-2.c: Removed. + * gcc.target/i386/amxtranspose-tdpfp16ps-2.c: Removed. + * gcc.target/i386/amxtranspose-tmmultf32ps-2.c: Removed. + * gcc.target/i386/amxtranspose-transposed-2.c: Removed. + +2025-10-14 Andrew Pinski <andrew.pinski@oss.qualcomm.com> + + PR tree-optimization/122178 + * g++.dg/tree-ssa/cselim-1.C: New test. + +2025-10-14 Zhongyao Chen <chenzhongyao.hit@gmail.com> + + * gcc.target/riscv/predef-profiles-1.c: New test for __riscv_rvi20u64. + * gcc.target/riscv/predef-profiles-2.c: New test for __riscv_rvi20u32. + * gcc.target/riscv/predef-profiles-3.c: New test for __riscv_rva20u64. + * gcc.target/riscv/predef-profiles-4.c: New test for __riscv_rva22u64. + * gcc.target/riscv/predef-profiles-5.c: New test for __riscv_rva23u64. + * gcc.target/riscv/predef-profiles-6.c: New test for __riscv_rva23s64. + * gcc.target/riscv/predef-profiles-7.c: New test for __riscv_rvb23u64. + * gcc.target/riscv/predef-profiles-8.c: New test for __riscv_rvb23s64. + 2025-10-13 Eric Botcazou <ebotcazou@adacore.com> * gcc.dg/cpp/cpp.exp: Process .i files. diff --git a/gcc/testsuite/g++.dg/template/dependent-base6.C b/gcc/testsuite/g++.dg/template/dependent-base6.C index b4bc5c2..9f2a7a2 100644 --- a/gcc/testsuite/g++.dg/template/dependent-base6.C +++ b/gcc/testsuite/g++.dg/template/dependent-base6.C @@ -8,5 +8,7 @@ struct A { struct S1 : A::B { }; // OK -template<class T> struct S2 : T::B { }; // OK, used to fail +template<class T> struct S2 : T::B { // OK, used to fail + S2() : T::B() { } // Also OK +}; template struct S2<A>; diff --git a/gcc/testsuite/gcc.dg/autopar/runtime-auto.c b/gcc/testsuite/gcc.dg/autopar/runtime-auto.c new file mode 100644 index 0000000..c1a3131 --- /dev/null +++ b/gcc/testsuite/gcc.dg/autopar/runtime-auto.c @@ -0,0 +1,53 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -ftree-parallelize-loops -fdump-tree-parloops2-details" } */ + +void abort (void); + +#define N 1000 + +int a[N], b[N], c[N]; + +void +test_parallel_loop (void) +{ + int i; + + /* This loop should be auto-parallelized when -ftree-parallelize-loops + (without =number) is used for runtime thread detection via OMP_NUM_THREADS. */ + for (i = 0; i < N; i++) + a[i] = b[i] + c[i]; +} + +int +main (void) +{ + int i; + + for (i = 0; i < N; i++) + { + b[i] = i; + c[i] = i * 2; + } + + test_parallel_loop (); + + for (i = 0; i < N; i++) + { + if (a[i] != b[i] + c[i]) + abort (); + } + + return 0; +} + +/* Check that the loop is parallelized with runtime thread detection. */ +/* { dg-final { scan-tree-dump "parallelizing" "parloops2" } } */ + +/* Check that "#pragma omp parallel" is generated. */ +/* { dg-final { scan-tree-dump "pragma omp parallel" "parloops2" } } */ + +/* Check that instead of generating a num_threads(x) clause, the compiler calls + "__builtin_omp_get_num_threads" that will set the number of threads at + program execution time. */ +/* { dg-final { scan-tree-dump "__builtin_omp_get_num_threads" "parloops2" } } */ + diff --git a/gcc/testsuite/gcc.dg/pr121468.c b/gcc/testsuite/gcc.dg/pr121468.c new file mode 100644 index 0000000..07df274 --- /dev/null +++ b/gcc/testsuite/gcc.dg/pr121468.c @@ -0,0 +1,30 @@ +/* { dg-do compile } */ +/* { dg-options "-Os" } */ +int e, f, n; +static int a () { return e; } +void b () { while (a()); } +static int d () { return e; } +static void g (int h) { + if (e) + c: + if (d()) + goto i; + do { + if (f) + goto c; + goto k; + i: + h = 2147483647; + k: + e = 2147483646; + e = 6 + e; + do { + b (); + } while (1784828957 / f + e + (808 + h) > 0); + } while (1 % h); +} +void m () { g (-2); } +int main () { + if (n) + g (-1); +} diff --git a/gcc/testsuite/gcc.dg/pr122200.c b/gcc/testsuite/gcc.dg/pr122200.c new file mode 100644 index 0000000..cd770fc --- /dev/null +++ b/gcc/testsuite/gcc.dg/pr122200.c @@ -0,0 +1,23 @@ +/* { dg-do compile } */ +/* { dg-options "-O3" } */ +/* { dg-additional-options "-mavx" { target { x86_64-*-* i?86-*-* } } } */ + + +int a, b; + +void f(float g[][5]) { + int c; + + for (c = 0; c != a; c++) + g[1][c] = c; + + for (int d; d; d++) + for (int e = 1; e != b; e++) { + for (c = 0; c != a; c++) { + g[0][1] = 1; + + if (g[1][c]) + g[1][c] = 1; + } + } +} diff --git a/gcc/testsuite/gcc.dg/tree-ssa/vla-1.c b/gcc/testsuite/gcc.dg/tree-ssa/vla-1.c new file mode 100644 index 0000000..37b7514 --- /dev/null +++ b/gcc/testsuite/gcc.dg/tree-ssa/vla-1.c @@ -0,0 +1,15 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -fdump-tree-optimized -fdump-tree-cddce1-details" } */ +/* PR tree-optimization/122037 */ + +void bar1 (char *, int) __attribute__((noreturn)); +void foo1 (int size) +{ + char temp[size]; + temp[size-1] = '\0'; + bar1 (temp, size); +} + +/* The call to __builtin_stack_save should have been removed. */ +/* { dg-final { scan-tree-dump "Deleting : __builtin_stack_save" "cddce1" } } */ +/* { dg-final { scan-tree-dump-not "__builtin_stack_save " "optimized" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/pr120687-1.c b/gcc/testsuite/gcc.dg/vect/pr120687-1.c index ce9cf63..ac684c0 100644 --- a/gcc/testsuite/gcc.dg/vect/pr120687-1.c +++ b/gcc/testsuite/gcc.dg/vect/pr120687-1.c @@ -11,6 +11,6 @@ frd (unsigned *p, unsigned *lastone) return sum; } -/* { dg-final { scan-tree-dump "reduction: detected reduction chain" "vect" } } */ +/* { dg-final { scan-tree-dump "Starting SLP discovery of reduction chain" "vect" } } */ /* { dg-final { scan-tree-dump-not "SLP discovery of reduction chain failed" "vect" } } */ /* { dg-final { scan-tree-dump "optimized: loop vectorized" "vect" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/pr120687-2.c b/gcc/testsuite/gcc.dg/vect/pr120687-2.c index dfc6dc7..25f0355 100644 --- a/gcc/testsuite/gcc.dg/vect/pr120687-2.c +++ b/gcc/testsuite/gcc.dg/vect/pr120687-2.c @@ -12,6 +12,6 @@ frd (float *p, float *lastone) return sum; } -/* { dg-final { scan-tree-dump "reduction: detected reduction chain" "vect" } } */ +/* { dg-final { scan-tree-dump "Starting SLP discovery of reduction chain" "vect" } } */ /* { dg-final { scan-tree-dump-not "SLP discovery of reduction chain failed" "vect" } } */ /* { dg-final { scan-tree-dump "optimized: loop vectorized" "vect" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/pr120687-3.c b/gcc/testsuite/gcc.dg/vect/pr120687-3.c index f20a66a..31a6c94 100644 --- a/gcc/testsuite/gcc.dg/vect/pr120687-3.c +++ b/gcc/testsuite/gcc.dg/vect/pr120687-3.c @@ -11,6 +11,6 @@ frd (float *p, float *lastone) return sum; } -/* { dg-final { scan-tree-dump "reduction: detected reduction chain" "vect" } } */ +/* { dg-final { scan-tree-dump "Starting SLP discovery of reduction chain" "vect" } } */ /* { dg-final { scan-tree-dump-not "SLP discovery of reduction chain failed" "vect" } } */ /* { dg-final { scan-tree-dump "optimized: loop vectorized" "vect" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/pr121949_1.c b/gcc/testsuite/gcc.dg/vect/pr121949_1.c new file mode 100644 index 0000000..9e8d41e --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/pr121949_1.c @@ -0,0 +1,45 @@ +#ifndef TYPE +#define TYPE short +#define MAX 16 +#define IV_TYPE char +#endif + +#include "tree-vect.h" + +__attribute__((noipa)) +void f(TYPE* acc) +{ + for (IV_TYPE row = 0; row < MAX; ++row) + acc[row] = acc[row] << row; +} + +__attribute__((noipa)) +void g(TYPE* acc) +{ +#pragma GCC novector + for (IV_TYPE row = 0; row < MAX; ++row) + acc[row] = acc[row] << row; +} + +int main () +{ + + check_vect (); + + TYPE acc1[MAX] = {}; + TYPE acc2[MAX] = {}; +#pragma GCC novector + for (int i = 0; i < MAX; i++) + acc1[i] = acc2[i] = i; + + f (acc1); + f (acc2); + +#pragma GCC novector + for (int i = 0; i < MAX; i++) + if (acc1[i] != acc2[i]) + __builtin_abort (); +} + +/* { dg-final { scan-tree-dump "LOOP VECTORIZED" "vect" { target { vect_var_shift && vect_int } } } } */ +/* { dg-final { scan-tree-dump "vect_recog_over_widening_pattern: detected" "vect" { target { vect_var_shift && vect_int } } } } */ diff --git a/gcc/testsuite/gcc.dg/vect/pr121949_2.c b/gcc/testsuite/gcc.dg/vect/pr121949_2.c new file mode 100644 index 0000000..f448eb6 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/pr121949_2.c @@ -0,0 +1,45 @@ +#ifndef TYPE +#define TYPE int +#define MAX 32 +#define IV_TYPE short +#endif + +#include "tree-vect.h" + +__attribute__((noipa)) +void f(TYPE* acc) +{ + for (IV_TYPE row = 0; row < MAX; ++row) + acc[row] = acc[row] << row; +} + +__attribute__((noipa)) +void g(TYPE* acc) +{ +#pragma GCC novector + for (IV_TYPE row = 0; row < MAX; ++row) + acc[row] = acc[row] << row; +} + +int main () +{ + + check_vect (); + + TYPE acc1[MAX] = {}; + TYPE acc2[MAX] = {}; +#pragma GCC novector + for (int i = 0; i < MAX; i++) + acc1[i] = acc2[i] = i; + + f (acc1); + f (acc2); + +#pragma GCC novector + for (int i = 0; i < MAX; i++) + if (acc1[i] != acc2[i]) + __builtin_abort (); +} + +/* { dg-final { scan-tree-dump "LOOP VECTORIZED" "vect" { target { vect_var_shift && vect_int } } } } */ +/* { dg-final { scan-tree-dump-not "vect_recog_over_widening_pattern: detected" "vect" { target { vect_var_shift && vect_int } } } } */ diff --git a/gcc/testsuite/gcc.dg/vect/pr121949_3.c b/gcc/testsuite/gcc.dg/vect/pr121949_3.c new file mode 100644 index 0000000..b7e6a3d --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/pr121949_3.c @@ -0,0 +1,45 @@ +#ifndef TYPE +#define TYPE long long +#define MAX 64 +#define IV_TYPE int +#endif + +#include "tree-vect.h" + +__attribute__((noipa)) +void f(TYPE* acc) +{ + for (IV_TYPE row = 0; row < MAX; ++row) + acc[row] = acc[row] << row; +} + +__attribute__((noipa)) +void g(TYPE* acc) +{ +#pragma GCC novector + for (IV_TYPE row = 0; row < MAX; ++row) + acc[row] = acc[row] << row; +} + +int main () +{ + + check_vect (); + + TYPE acc1[MAX] = {}; + TYPE acc2[MAX] = {}; +#pragma GCC novector + for (int i = 0; i < MAX; i++) + acc1[i] = acc2[i] = i; + + f (acc1); + f (acc2); + +#pragma GCC novector + for (int i = 0; i < MAX; i++) + if (acc1[i] != acc2[i]) + __builtin_abort (); +} + +/* { dg-final { scan-tree-dump "LOOP VECTORIZED" "vect" { target { vect_var_shift && vect_int } } } } */ +/* { dg-final { scan-tree-dump "vect_recog_vector_vector_shift_pattern: detected" "vect" { target { vect_var_shift && vect_int } } } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/acle/rwsr-armv8p9.c b/gcc/testsuite/gcc.target/aarch64/acle/rwsr-armv8p9.c index c49fbb5..1ff51de 100644 --- a/gcc/testsuite/gcc.target/aarch64/acle/rwsr-armv8p9.c +++ b/gcc/testsuite/gcc.target/aarch64/acle/rwsr-armv8p9.c @@ -72,7 +72,7 @@ readwrite_armv8p9a_sysregs (long long int a) a = __arm_rsr64 ("pmicfiltr_el0"); /* { { dg-final { scan-assembler "mrs\tx0, s3_3_c9_c6_0" } } */ a = __arm_rsr64 ("pmicntr_el0"); /* { { dg-final { scan-assembler "mrs\tx0, s3_3_c9_c4_0" } } */ a = __arm_rsr64 ("pmicntsvr_el1"); /* { { dg-final { scan-assembler "mrs\tx0, s2_0_c14_c12_0" } } */ - a = __arm_rsr64 ("pmsdsfr_el1"); /* { { dg-final { scan-assembler "mrs\tx0, s3_4_c9_c10_4" } } */ + a = __arm_rsr64 ("pmsdsfr_el1"); /* { { dg-final { scan-assembler "mrs\tx0, s3_0_c9_c10_4" } } */ a = __arm_rsr64 ("pmsscr_el1"); /* { { dg-final { scan-assembler "mrs\tx0, s3_0_c9_c13_3" } } */ a = __arm_rsr64 ("pmuacr_el1"); /* { { dg-final { scan-assembler "mrs\tx0, s3_0_c9_c14_4" } } */ a = __arm_rsr64 ("por_el0"); /* { { dg-final { scan-assembler "mrs\tx0, s3_3_c10_c2_4" } } */ diff --git a/gcc/testsuite/gcc.target/arm/armv8_2-fp16-move-1.c b/gcc/testsuite/gcc.target/arm/armv8_2-fp16-move-1.c index 444c4a3..02d7b51 100644 --- a/gcc/testsuite/gcc.target/arm/armv8_2-fp16-move-1.c +++ b/gcc/testsuite/gcc.target/arm/armv8_2-fp16-move-1.c @@ -134,8 +134,8 @@ test_select_8 (__fp16 a, __fp16 b, __fp16 c) } /* { dg-final { scan-assembler-times {vseleq\.f16\ts[0-9]+, s[0-9]+, s[0-9]+} 4 } } */ -/* { dg-final { scan-assembler-times {vselgt\.f16\ts[0-9]+, s[0-9]+, s[0-9]+} 1 } } */ -/* { dg-final { scan-assembler-times {vselge\.f16\ts[0-9]+, s[0-9]+, s[0-9]+} 1 } } */ +/* { dg-final { scan-assembler-times {vselgt\.f16\ts[0-9]+, s[0-9]+, s[0-9]+} 2 } } */ +/* { dg-final { scan-assembler-times {vselge\.f16\ts[0-9]+, s[0-9]+, s[0-9]+} 2 } } */ /* { dg-final { scan-assembler-not {vmov\.f16} } } */ diff --git a/gcc/testsuite/gcc.target/arm/armv8_2-fp16-move-2.c b/gcc/testsuite/gcc.target/arm/armv8_2-fp16-move-2.c index dff57ac..a249d71 100644 --- a/gcc/testsuite/gcc.target/arm/armv8_2-fp16-move-2.c +++ b/gcc/testsuite/gcc.target/arm/armv8_2-fp16-move-2.c @@ -8,4 +8,4 @@ test_select (__fp16 a, __fp16 b, __fp16 c) { return (a < b) ? b : c; } -/* { dg-final { scan-assembler "bx?(mi|pl)" } } */ +/* { dg-final { scan-assembler "vselgt\.f16\t" } } */ diff --git a/gcc/testsuite/gcc.target/arm/mve/intrinsics/vadcq-check-carry.c b/gcc/testsuite/gcc.target/arm/mve/intrinsics/vadcq-check-carry.c new file mode 100644 index 0000000..3a9b8de --- /dev/null +++ b/gcc/testsuite/gcc.target/arm/mve/intrinsics/vadcq-check-carry.c @@ -0,0 +1,48 @@ +/* { dg-do run } */ +/* { dg-require-effective-target arm_v8_1m_mve_ok } */ +/* { dg-require-effective-target arm_mve_hw } */ +/* { dg-options "-O2" } */ +/* { dg-add-options arm_v8_1m_mve } */ + +#include "arm_mve.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#include <inttypes.h> +#include <stdio.h> + +__attribute((noinline)) void print_uint32x4_t(const char *name, uint32x4_t val) +{ + printf("%s: %u, %u, %u, %u\n", + name, + vgetq_lane_u32(val, 0), + vgetq_lane_u32(val, 1), + vgetq_lane_u32(val, 2), + vgetq_lane_u32(val, 3)); +} + +void __attribute__ ((noinline)) test_2(void) +{ + uint32x4_t v12, v18, v108; + unsigned v17 = 0; + v12 = vdupq_n_u32(1); + v18 = vadcq_u32(v12, v12, &v17); + v17 = 1; + v108 = vadcq_u32(v12, v12, &v17); + print_uint32x4_t("v108", v108); +} + +int main() +{ + test_2(); + return 0; +} + +#ifdef __cplusplus +} +#endif + +/* { dg-output "v108: 3, 2, 2, 2" } */ +/* { dg-final { scan-assembler-times {\tvmrs\t(?:ip|fp|r[0-9]+), FPSCR_nzcvqc} 3 } } */ diff --git a/gcc/testsuite/gcc.target/arm/mve/intrinsics/vadcq_m_s32.c b/gcc/testsuite/gcc.target/arm/mve/intrinsics/vadcq_m_s32.c index 0d4cb77..c5a5878 100644 --- a/gcc/testsuite/gcc.target/arm/mve/intrinsics/vadcq_m_s32.c +++ b/gcc/testsuite/gcc.target/arm/mve/intrinsics/vadcq_m_s32.c @@ -14,12 +14,12 @@ extern "C" { ** ... ** vmrs (?:ip|fp|r[0-9]+), FPSCR_nzcvqc(?: @.*|) ** ... +** vmsr p0, (?:ip|fp|r[0-9]+)(?: @.*|) +** ... ** bfi (?:ip|fp|r[0-9]+), (?:ip|fp|r[0-9]+), #29, #1(?: @.*|) ** ... ** vmsr FPSCR_nzcvqc, (?:ip|fp|r[0-9]+)(?: @.*|) ** ... -** vmsr p0, (?:ip|fp|r[0-9]+)(?: @.*|) -** ... ** vpst(?: @.*|) ** ... ** vadct.i32 q[0-9]+, q[0-9]+, q[0-9]+(?: @.*|) @@ -41,12 +41,12 @@ foo (int32x4_t inactive, int32x4_t a, int32x4_t b, unsigned *carry, mve_pred16_t ** ... ** vmrs (?:ip|fp|r[0-9]+), FPSCR_nzcvqc(?: @.*|) ** ... +** vmsr p0, (?:ip|fp|r[0-9]+)(?: @.*|) +** ... ** bfi (?:ip|fp|r[0-9]+), (?:ip|fp|r[0-9]+), #29, #1(?: @.*|) ** ... ** vmsr FPSCR_nzcvqc, (?:ip|fp|r[0-9]+)(?: @.*|) ** ... -** vmsr p0, (?:ip|fp|r[0-9]+)(?: @.*|) -** ... ** vpst(?: @.*|) ** ... ** vadct.i32 q[0-9]+, q[0-9]+, q[0-9]+(?: @.*|) diff --git a/gcc/testsuite/gcc.target/arm/mve/intrinsics/vadcq_m_u32.c b/gcc/testsuite/gcc.target/arm/mve/intrinsics/vadcq_m_u32.c index a0ba682..23908a4 100644 --- a/gcc/testsuite/gcc.target/arm/mve/intrinsics/vadcq_m_u32.c +++ b/gcc/testsuite/gcc.target/arm/mve/intrinsics/vadcq_m_u32.c @@ -14,12 +14,12 @@ extern "C" { ** ... ** vmrs (?:ip|fp|r[0-9]+), FPSCR_nzcvqc(?: @.*|) ** ... +** vmsr p0, (?:ip|fp|r[0-9]+)(?: @.*|) +** ... ** bfi (?:ip|fp|r[0-9]+), (?:ip|fp|r[0-9]+), #29, #1(?: @.*|) ** ... ** vmsr FPSCR_nzcvqc, (?:ip|fp|r[0-9]+)(?: @.*|) ** ... -** vmsr p0, (?:ip|fp|r[0-9]+)(?: @.*|) -** ... ** vpst(?: @.*|) ** ... ** vadct.i32 q[0-9]+, q[0-9]+, q[0-9]+(?: @.*|) @@ -41,12 +41,12 @@ foo (uint32x4_t inactive, uint32x4_t a, uint32x4_t b, unsigned *carry, mve_pred1 ** ... ** vmrs (?:ip|fp|r[0-9]+), FPSCR_nzcvqc(?: @.*|) ** ... +** vmsr p0, (?:ip|fp|r[0-9]+)(?: @.*|) +** ... ** bfi (?:ip|fp|r[0-9]+), (?:ip|fp|r[0-9]+), #29, #1(?: @.*|) ** ... ** vmsr FPSCR_nzcvqc, (?:ip|fp|r[0-9]+)(?: @.*|) ** ... -** vmsr p0, (?:ip|fp|r[0-9]+)(?: @.*|) -** ... ** vpst(?: @.*|) ** ... ** vadct.i32 q[0-9]+, q[0-9]+, q[0-9]+(?: @.*|) diff --git a/gcc/testsuite/gcc.target/arm/mve/intrinsics/vsbcq_m_s32.c b/gcc/testsuite/gcc.target/arm/mve/intrinsics/vsbcq_m_s32.c index 7a33261..940e2ed 100644 --- a/gcc/testsuite/gcc.target/arm/mve/intrinsics/vsbcq_m_s32.c +++ b/gcc/testsuite/gcc.target/arm/mve/intrinsics/vsbcq_m_s32.c @@ -14,12 +14,12 @@ extern "C" { ** ... ** vmrs (?:ip|fp|r[0-9]+), FPSCR_nzcvqc(?: @.*|) ** ... +** vmsr p0, (?:ip|fp|r[0-9]+)(?: @.*|) +** ... ** bfi (?:ip|fp|r[0-9]+), (?:ip|fp|r[0-9]+), #29, #1(?: @.*|) ** ... ** vmsr FPSCR_nzcvqc, (?:ip|fp|r[0-9]+)(?: @.*|) ** ... -** vmsr p0, (?:ip|fp|r[0-9]+)(?: @.*|) -** ... ** vpst(?: @.*|) ** ... ** vsbct.i32 q[0-9]+, q[0-9]+, q[0-9]+(?: @.*|) @@ -41,12 +41,12 @@ foo (int32x4_t inactive, int32x4_t a, int32x4_t b, unsigned *carry, mve_pred16_t ** ... ** vmrs (?:ip|fp|r[0-9]+), FPSCR_nzcvqc(?: @.*|) ** ... +** vmsr p0, (?:ip|fp|r[0-9]+)(?: @.*|) +** ... ** bfi (?:ip|fp|r[0-9]+), (?:ip|fp|r[0-9]+), #29, #1(?: @.*|) ** ... ** vmsr FPSCR_nzcvqc, (?:ip|fp|r[0-9]+)(?: @.*|) ** ... -** vmsr p0, (?:ip|fp|r[0-9]+)(?: @.*|) -** ... ** vpst(?: @.*|) ** ... ** vsbct.i32 q[0-9]+, q[0-9]+, q[0-9]+(?: @.*|) diff --git a/gcc/testsuite/gcc.target/arm/mve/intrinsics/vsbcq_m_u32.c b/gcc/testsuite/gcc.target/arm/mve/intrinsics/vsbcq_m_u32.c index 6090219..478b938 100644 --- a/gcc/testsuite/gcc.target/arm/mve/intrinsics/vsbcq_m_u32.c +++ b/gcc/testsuite/gcc.target/arm/mve/intrinsics/vsbcq_m_u32.c @@ -14,12 +14,12 @@ extern "C" { ** ... ** vmrs (?:ip|fp|r[0-9]+), FPSCR_nzcvqc(?: @.*|) ** ... +** vmsr p0, (?:ip|fp|r[0-9]+)(?: @.*|) +** ... ** bfi (?:ip|fp|r[0-9]+), (?:ip|fp|r[0-9]+), #29, #1(?: @.*|) ** ... ** vmsr FPSCR_nzcvqc, (?:ip|fp|r[0-9]+)(?: @.*|) ** ... -** vmsr p0, (?:ip|fp|r[0-9]+)(?: @.*|) -** ... ** vpst(?: @.*|) ** ... ** vsbct.i32 q[0-9]+, q[0-9]+, q[0-9]+(?: @.*|) @@ -41,12 +41,12 @@ foo (uint32x4_t inactive, uint32x4_t a, uint32x4_t b, unsigned *carry, mve_pred1 ** ... ** vmrs (?:ip|fp|r[0-9]+), FPSCR_nzcvqc(?: @.*|) ** ... +** vmsr p0, (?:ip|fp|r[0-9]+)(?: @.*|) +** ... ** bfi (?:ip|fp|r[0-9]+), (?:ip|fp|r[0-9]+), #29, #1(?: @.*|) ** ... ** vmsr FPSCR_nzcvqc, (?:ip|fp|r[0-9]+)(?: @.*|) ** ... -** vmsr p0, (?:ip|fp|r[0-9]+)(?: @.*|) -** ... ** vpst(?: @.*|) ** ... ** vsbct.i32 q[0-9]+, q[0-9]+, q[0-9]+(?: @.*|) diff --git a/gcc/testsuite/gcc.target/i386/pr122266.c b/gcc/testsuite/gcc.target/i386/pr122266.c new file mode 100644 index 0000000..4e31a6a --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr122266.c @@ -0,0 +1,10 @@ +/* { dg-do compile { target int128 } } */ +/* { dg-options "-O2" } */ + +signed __int128 foo(signed __int128 x) { + signed __int128 t = x >> 127; + return ((x^t)>>1)^t; +} + +/* { dg-final { scan-assembler-times "xorq" 4 } } */ +/* { dg-final { scan-assembler-times "sarq" 2 } } */ diff --git a/gcc/tree-parloops.cc b/gcc/tree-parloops.cc index 666c6a1..7361828 100644 --- a/gcc/tree-parloops.cc +++ b/gcc/tree-parloops.cc @@ -2601,10 +2601,19 @@ create_parallel_loop (class loop *loop, tree loop_fn, tree data, gsi = gsi_last_bb (paral_bb); gcc_checking_assert (n_threads != 0); - t = build_omp_clause (loc, OMP_CLAUSE_NUM_THREADS); - OMP_CLAUSE_NUM_THREADS_EXPR (t) - = build_int_cst (integer_type_node, n_threads); - omp_par_stmt = gimple_build_omp_parallel (NULL, t, loop_fn, data); + if (n_threads == INT_MAX) + /* No hardcoded thread count, let OpenMP runtime decide. */ + omp_par_stmt = gimple_build_omp_parallel (NULL, NULL_TREE, loop_fn, + data); + else + { + /* Build the OMP_CLAUSE_NUM_THREADS clause only if we have a fixed + thread count. */ + t = build_omp_clause (loc, OMP_CLAUSE_NUM_THREADS); + OMP_CLAUSE_NUM_THREADS_EXPR (t) + = build_int_cst (integer_type_node, n_threads); + omp_par_stmt = gimple_build_omp_parallel (NULL, t, loop_fn, data); + } gimple_set_location (omp_par_stmt, loc); gsi_insert_after (&gsi, omp_par_stmt, GSI_NEW_STMT); @@ -2812,7 +2821,6 @@ gen_parallel_loop (class loop *loop, struct clsn_data clsn_data; location_t loc; gimple *cond_stmt; - unsigned int m_p_thread=2; /* From @@ -2885,15 +2893,14 @@ gen_parallel_loop (class loop *loop, if (!oacc_kernels_p) { - if (loop->inner) - m_p_thread=2; - else - m_p_thread=MIN_PER_THREAD; - gcc_checking_assert (n_threads != 0); + /* For runtime thread detection, use a conservative estimate of 2 threads + for the many iterations condition check. */ + unsigned threads = (n_threads == INT_MAX) ? 2 : n_threads; + unsigned m_p_thread = loop->inner ? 2 : MIN_PER_THREAD; many_iterations_cond = fold_build2 (GE_EXPR, boolean_type_node, - nit, build_int_cst (type, m_p_thread * n_threads - 1)); + nit, build_int_cst (type, m_p_thread * threads - 1)); many_iterations_cond = fold_build2 (TRUTH_AND_EXPR, boolean_type_node, @@ -3905,14 +3912,15 @@ parallelize_loops (bool oacc_kernels_p) estimated = estimated_loop_iterations_int (loop); if (estimated == -1) estimated = get_likely_max_loop_iterations_int (loop); + /* For runtime thread detection, use an estimate of 2 threads. */ + unsigned threads = (n_threads == INT_MAX) ? 2 : n_threads; + unsigned m_p_thread = loop->inner ? 2 : MIN_PER_THREAD; /* FIXME: Bypass this check as graphite doesn't update the count and frequency correctly now. */ if (!flag_loop_parallelize_all && !oacc_kernels_p && ((estimated != -1 - && (estimated - < ((HOST_WIDE_INT) n_threads - * (loop->inner ? 2 : MIN_PER_THREAD) - 1))) + && (estimated < ((HOST_WIDE_INT) threads * m_p_thread - 1))) /* Do not bother with loops in cold areas. */ || optimize_loop_nest_for_size_p (loop))) continue; diff --git a/gcc/tree-ssa-dce.cc b/gcc/tree-ssa-dce.cc index 4386908..160c28c 100644 --- a/gcc/tree-ssa-dce.cc +++ b/gcc/tree-ssa-dce.cc @@ -1682,9 +1682,12 @@ eliminate_unnecessary_stmts (bool aggressive) update_stmt (call_stmt); release_ssa_name (name); + /* __builtin_stack_save without lhs is not needed. */ + if (gimple_call_builtin_p (call_stmt, BUILT_IN_STACK_SAVE)) + remove_dead_stmt (&gsi, bb, to_remove_edges); /* GOMP_SIMD_LANE (unless three argument) or ASAN_POISON without lhs is not needed. */ - if (gimple_call_internal_p (call_stmt)) + else if (gimple_call_internal_p (call_stmt)) switch (gimple_call_internal_fn (call_stmt)) { case IFN_GOMP_SIMD_LANE: diff --git a/gcc/tree-ssa-loop-unswitch.cc b/gcc/tree-ssa-loop-unswitch.cc index c5ca4ff..dc0cb24 100644 --- a/gcc/tree-ssa-loop-unswitch.cc +++ b/gcc/tree-ssa-loop-unswitch.cc @@ -136,7 +136,8 @@ struct unswitch_predicate tree rhs = gimple_cond_rhs (stmt); enum tree_code code = gimple_cond_code (stmt); condition = build2 (code, boolean_type_node, lhs, rhs); - count = EDGE_SUCC (bb, 0)->count ().max (EDGE_SUCC (bb, 1)->count ()); + count = profile_count::max_prefer_initialized (EDGE_SUCC (bb, 0)->count (), + EDGE_SUCC (bb, 1)->count ()); if (irange::supports_p (TREE_TYPE (lhs))) { auto range_op = range_op_handler (code); diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc index 73398e5..568353a 100644 --- a/gcc/tree-vect-loop.cc +++ b/gcc/tree-vect-loop.cc @@ -161,7 +161,7 @@ along with GCC; see the file COPYING3. If not see static void vect_estimate_min_profitable_iters (loop_vec_info, int *, int *, unsigned *); static stmt_vec_info vect_is_simple_reduction (loop_vec_info, stmt_vec_info, - gphi **, bool *, bool); + gphi **); /* Function vect_is_simple_iv_evolution. @@ -341,8 +341,7 @@ vect_phi_first_order_recurrence_p (loop_vec_info loop_vinfo, class loop *loop, slp analyses or not. */ static void -vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, class loop *loop, - bool slp) +vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, class loop *loop) { basic_block bb = loop->header; auto_vec<stmt_vec_info, 64> worklist; @@ -425,19 +424,15 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, class loop *loop, && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_unknown_def_type); gphi *double_reduc; - bool reduc_chain; stmt_vec_info reduc_stmt_info - = vect_is_simple_reduction (loop_vinfo, stmt_vinfo, &double_reduc, - &reduc_chain, slp); + = vect_is_simple_reduction (loop_vinfo, stmt_vinfo, &double_reduc); if (reduc_stmt_info && double_reduc) { - bool inner_chain; stmt_vec_info inner_phi_info = loop_vinfo->lookup_stmt (double_reduc); /* ??? Pass down flag we're the inner loop of a double reduc. */ stmt_vec_info inner_reduc_info - = vect_is_simple_reduction (loop_vinfo, inner_phi_info, - NULL, &inner_chain, slp); + = vect_is_simple_reduction (loop_vinfo, inner_phi_info, NULL); if (inner_reduc_info) { STMT_VINFO_REDUC_DEF (stmt_vinfo) = reduc_stmt_info; @@ -478,12 +473,7 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, class loop *loop, STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_reduction_def; STMT_VINFO_DEF_TYPE (reduc_stmt_info) = vect_reduction_def; - /* Store the reduction cycles for possible vectorization in - loop-aware SLP if it was not detected as reduction - chain. */ - if (! reduc_chain) - LOOP_VINFO_REDUCTIONS (loop_vinfo).safe_push - (reduc_stmt_info); + LOOP_VINFO_REDUCTIONS (loop_vinfo).safe_push (reduc_stmt_info); } } else if (vect_phi_first_order_recurrence_p (loop_vinfo, loop, phi)) @@ -518,11 +508,11 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, class loop *loop, a[i] = i; */ static void -vect_analyze_scalar_cycles (loop_vec_info loop_vinfo, bool slp) +vect_analyze_scalar_cycles (loop_vec_info loop_vinfo) { class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); - vect_analyze_scalar_cycles_1 (loop_vinfo, loop, slp); + vect_analyze_scalar_cycles_1 (loop_vinfo, loop); /* When vectorizing an outer-loop, the inner-loop is executed sequentially. Reductions in such inner-loop therefore have different properties than @@ -534,87 +524,7 @@ vect_analyze_scalar_cycles (loop_vec_info loop_vinfo, bool slp) current checks are too strict. */ if (loop->inner) - vect_analyze_scalar_cycles_1 (loop_vinfo, loop->inner, slp); -} - -/* Transfer group and reduction information from STMT_INFO to its - pattern stmt. */ - -static void -vect_fixup_reduc_chain (stmt_vec_info stmt_info) -{ - stmt_vec_info firstp = STMT_VINFO_RELATED_STMT (stmt_info); - stmt_vec_info stmtp; - gcc_assert (!REDUC_GROUP_FIRST_ELEMENT (firstp) - && REDUC_GROUP_FIRST_ELEMENT (stmt_info)); - REDUC_GROUP_SIZE (firstp) = REDUC_GROUP_SIZE (stmt_info); - do - { - stmtp = STMT_VINFO_RELATED_STMT (stmt_info); - gcc_checking_assert (STMT_VINFO_DEF_TYPE (stmtp) - == STMT_VINFO_DEF_TYPE (stmt_info)); - REDUC_GROUP_FIRST_ELEMENT (stmtp) = firstp; - stmt_info = REDUC_GROUP_NEXT_ELEMENT (stmt_info); - if (stmt_info) - REDUC_GROUP_NEXT_ELEMENT (stmtp) - = STMT_VINFO_RELATED_STMT (stmt_info); - } - while (stmt_info); -} - -/* Fixup scalar cycles that now have their stmts detected as patterns. */ - -static void -vect_fixup_scalar_cycles_with_patterns (loop_vec_info loop_vinfo) -{ - stmt_vec_info first; - unsigned i; - - FOR_EACH_VEC_ELT (LOOP_VINFO_REDUCTION_CHAINS (loop_vinfo), i, first) - { - stmt_vec_info next = REDUC_GROUP_NEXT_ELEMENT (first); - while (next) - { - if ((STMT_VINFO_IN_PATTERN_P (next) - != STMT_VINFO_IN_PATTERN_P (first)) - || STMT_VINFO_REDUC_IDX (vect_stmt_to_vectorize (next)) == -1) - break; - next = REDUC_GROUP_NEXT_ELEMENT (next); - } - /* If all reduction chain members are well-formed patterns adjust - the group to group the pattern stmts instead. */ - if (! next - && STMT_VINFO_REDUC_IDX (vect_stmt_to_vectorize (first)) != -1) - { - if (STMT_VINFO_IN_PATTERN_P (first)) - { - vect_fixup_reduc_chain (first); - LOOP_VINFO_REDUCTION_CHAINS (loop_vinfo)[i] - = STMT_VINFO_RELATED_STMT (first); - } - } - /* If not all stmt in the chain are patterns or if we failed - to update STMT_VINFO_REDUC_IDX dissolve the chain and handle - it as regular reduction instead. */ - else - { - stmt_vec_info vinfo = first; - stmt_vec_info last = NULL; - while (vinfo) - { - next = REDUC_GROUP_NEXT_ELEMENT (vinfo); - REDUC_GROUP_FIRST_ELEMENT (vinfo) = NULL; - REDUC_GROUP_NEXT_ELEMENT (vinfo) = NULL; - last = vinfo; - vinfo = next; - } - STMT_VINFO_DEF_TYPE (vect_stmt_to_vectorize (first)) - = vect_internal_def; - loop_vinfo->reductions.safe_push (vect_stmt_to_vectorize (last)); - LOOP_VINFO_REDUCTION_CHAINS (loop_vinfo).unordered_remove (i); - --i; - } - } + vect_analyze_scalar_cycles_1 (loop_vinfo, loop->inner); } /* Function vect_get_loop_niters. @@ -2264,12 +2174,10 @@ vect_analyze_loop_2 (loop_vec_info loop_vinfo, bool &fatal, /* Classify all cross-iteration scalar data-flow cycles. Cross-iteration cycles caused by virtual phis are analyzed separately. */ - vect_analyze_scalar_cycles (loop_vinfo, !force_single_lane); + vect_analyze_scalar_cycles (loop_vinfo); vect_pattern_recog (loop_vinfo); - vect_fixup_scalar_cycles_with_patterns (loop_vinfo); - /* Analyze the access patterns of the data-refs in the loop (consecutive, complex, etc.). FORNOW: Only handle consecutive access pattern. */ @@ -2678,10 +2586,6 @@ again: if (applying_suggested_uf) return ok; - /* If there are reduction chains re-trying will fail anyway. */ - if (! LOOP_VINFO_REDUCTION_CHAINS (loop_vinfo).is_empty ()) - return ok; - /* Likewise if the grouped loads or stores in the SLP cannot be handled via interleaving or lane instructions. */ slp_instance instance; @@ -3756,7 +3660,7 @@ check_reduction_path (dump_user_location_t loc, loop_p loop, gphi *phi, static stmt_vec_info vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info, - gphi **double_reduc, bool *reduc_chain_p, bool slp) + gphi **double_reduc) { gphi *phi = as_a <gphi *> (phi_info->stmt); gimple *phi_use_stmt = NULL; @@ -3768,7 +3672,6 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info, bool inner_loop_of_double_reduc = double_reduc == NULL; if (double_reduc) *double_reduc = NULL; - *reduc_chain_p = false; STMT_VINFO_REDUC_TYPE (phi_info) = TREE_CODE_REDUCTION; tree phi_name = PHI_RESULT (phi); @@ -3918,12 +3821,8 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info, if (code == COND_EXPR && !nested_in_vect_loop) STMT_VINFO_REDUC_TYPE (phi_info) = COND_REDUCTION; - /* Fill in STMT_VINFO_REDUC_IDX and gather stmts for an SLP - reduction chain for which the additional restriction is that - all operations in the chain are the same. */ - auto_vec<stmt_vec_info, 8> reduc_chain; + /* Fill in STMT_VINFO_REDUC_IDX. */ unsigned i; - bool is_slp_reduc = !nested_in_vect_loop && code != COND_EXPR; for (i = path.length () - 1; i >= 1; --i) { gimple *stmt = USE_STMT (path[i].second); @@ -3940,39 +3839,8 @@ vect_is_simple_reduction (loop_vec_info loop_info, stmt_vec_info phi_info, STMT_VINFO_REDUC_IDX (stmt_info) = path[i].second->use - gimple_call_arg_ptr (call, 0); } - bool leading_conversion = (CONVERT_EXPR_CODE_P (op.code) - && (i == 1 || i == path.length () - 1)); - if ((op.code != code && !leading_conversion) - /* We can only handle the final value in epilogue - generation for reduction chains. */ - || (i != 1 && !has_single_use (gimple_get_lhs (stmt)))) - is_slp_reduc = false; - /* For reduction chains we support a trailing/leading - conversions. We do not store those in the actual chain. */ - if (leading_conversion) - continue; - reduc_chain.safe_push (stmt_info); } - if (slp && is_slp_reduc && reduc_chain.length () > 1) - { - for (unsigned i = 0; i < reduc_chain.length () - 1; ++i) - { - REDUC_GROUP_FIRST_ELEMENT (reduc_chain[i]) = reduc_chain[0]; - REDUC_GROUP_NEXT_ELEMENT (reduc_chain[i]) = reduc_chain[i+1]; - } - REDUC_GROUP_FIRST_ELEMENT (reduc_chain.last ()) = reduc_chain[0]; - REDUC_GROUP_NEXT_ELEMENT (reduc_chain.last ()) = NULL; - - /* Save the chain for further analysis in SLP detection. */ - LOOP_VINFO_REDUCTION_CHAINS (loop_info).safe_push (reduc_chain[0]); - REDUC_GROUP_SIZE (reduc_chain[0]) = reduc_chain.length (); - - *reduc_chain_p = true; - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "reduction: detected reduction chain\n"); - } - else if (dump_enabled_p ()) + if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, "reduction: detected reduction\n"); @@ -5390,7 +5258,6 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo, tree new_temp = NULL_TREE, new_name, new_scalar_dest; gimple *epilog_stmt = NULL; gimple *exit_phi; - tree bitsize; tree def; tree orig_name, scalar_result; imm_use_iterator imm_iter; @@ -5405,8 +5272,7 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo, # b1 = phi <b2, b0> a2 = operation (a1) b2 = operation (b1) */ - const bool slp_reduc - = SLP_INSTANCE_KIND (slp_node_instance) != slp_inst_kind_reduc_chain; + const bool slp_reduc = !reduc_info->is_reduc_chain; tree induction_index = NULL_TREE; unsigned int group_size = SLP_TREE_LANES (slp_node); @@ -5608,7 +5474,6 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo, scalar_results.truncate (0); scalar_results.reserve_exact (group_size); new_scalar_dest = vect_create_destination_var (scalar_dest, NULL); - bitsize = TYPE_SIZE (scalar_type); /* True if we should implement SLP_REDUC using native reduction operations instead of scalar operations. */ @@ -6030,6 +5895,7 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo, if (reduce_with_shift && (!slp_reduc || group_size == 1)) { + tree bitsize = TYPE_SIZE (TREE_TYPE (vectype1)); int element_bitsize = tree_to_uhwi (bitsize); /* Enforced by vectorizable_reduction, which disallows SLP reductions for variable-length vectors and also requires direct target support @@ -6098,9 +5964,10 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo, dump_printf_loc (MSG_NOTE, vect_location, "Reduce using scalar code.\n"); + tree compute_type = TREE_TYPE (vectype1); + tree bitsize = TYPE_SIZE (compute_type); int vec_size_in_bits = tree_to_uhwi (TYPE_SIZE (vectype1)); int element_bitsize = tree_to_uhwi (bitsize); - tree compute_type = TREE_TYPE (vectype); gimple_seq stmts = NULL; FOR_EACH_VEC_ELT (reduc_inputs, i, vec_temp) { @@ -6956,8 +6823,6 @@ vectorizable_reduction (loop_vec_info loop_vinfo, bool single_defuse_cycle = false; tree cr_index_scalar_type = NULL_TREE, cr_index_vector_type = NULL_TREE; tree cond_reduc_val = NULL_TREE; - const bool reduc_chain - = SLP_INSTANCE_KIND (slp_node_instance) == slp_inst_kind_reduc_chain; /* Make sure it was already recognized as a reduction computation. */ if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_reduction_def @@ -7019,6 +6884,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo, double_reduc = true; } + const bool reduc_chain = reduc_info->is_reduc_chain; slp_node_instance->reduc_phis = slp_node; /* ??? We're leaving slp_node to point to the PHIs, we only need it to get at the number of vector stmts which wasn't @@ -7030,33 +6896,28 @@ vectorizable_reduction (loop_vec_info loop_vinfo, /* Verify following REDUC_IDX from the latch def leads us back to the PHI and compute the reduction chain length. Discover the real - reduction operation stmt on the way (stmt_info and slp_for_stmt_info). */ - tree reduc_def - = PHI_ARG_DEF_FROM_EDGE (reduc_def_phi, loop_latch_edge (loop)); + reduction operation stmt on the way (slp_for_stmt_info). */ unsigned reduc_chain_length = 0; - bool only_slp_reduc_chain = true; stmt_info = NULL; slp_tree slp_for_stmt_info = NULL; slp_tree vdef_slp = slp_node_instance->root; - /* For double-reductions we start SLP analysis at the inner loop LC PHI - which is the def of the outer loop live stmt. */ - if (double_reduc) - vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[0]; - while (reduc_def != PHI_RESULT (reduc_def_phi)) + while (vdef_slp != slp_node) { - stmt_vec_info def = loop_vinfo->lookup_def (reduc_def); - stmt_vec_info vdef = vect_stmt_to_vectorize (def); - int reduc_idx = STMT_VINFO_REDUC_IDX (vdef); - if (STMT_VINFO_REDUC_IDX (vdef) == -1 - || SLP_TREE_REDUC_IDX (vdef_slp) == -1) + int reduc_idx = SLP_TREE_REDUC_IDX (vdef_slp); + if (reduc_idx == -1) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, "reduction chain broken by patterns.\n"); return false; } - if (!REDUC_GROUP_FIRST_ELEMENT (vdef)) - only_slp_reduc_chain = false; + stmt_vec_info vdef = SLP_TREE_REPRESENTATIVE (vdef_slp); + if (is_a <gphi *> (vdef->stmt)) + { + vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[reduc_idx]; + /* Do not count PHIs towards the chain length. */ + continue; + } gimple_match_op op; if (!gimple_extract_op (vdef->stmt, &op)) { @@ -7080,11 +6941,8 @@ vectorizable_reduction (loop_vec_info loop_vinfo, else { /* First non-conversion stmt. */ - if (!stmt_info) - { - stmt_info = vdef; - slp_for_stmt_info = vdef_slp; - } + if (!slp_for_stmt_info) + slp_for_stmt_info = vdef_slp; if (lane_reducing_op_p (op.code)) { @@ -7116,29 +6974,15 @@ vectorizable_reduction (loop_vec_info loop_vinfo, } else if (!vectype_in) vectype_in = SLP_TREE_VECTYPE (slp_node); - if (!REDUC_GROUP_FIRST_ELEMENT (vdef)) - { - gcc_assert (reduc_idx == SLP_TREE_REDUC_IDX (vdef_slp)); - vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[reduc_idx]; - } + vdef_slp = SLP_TREE_CHILDREN (vdef_slp)[reduc_idx]; } - - reduc_def = op.ops[reduc_idx]; reduc_chain_length++; } + stmt_info = SLP_TREE_REPRESENTATIVE (slp_for_stmt_info); + /* PHIs should not participate in patterns. */ gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info)); - /* STMT_VINFO_REDUC_DEF doesn't point to the first but the last - element. */ - if (REDUC_GROUP_FIRST_ELEMENT (stmt_info)) - { - gcc_assert (!REDUC_GROUP_NEXT_ELEMENT (stmt_info)); - stmt_info = REDUC_GROUP_FIRST_ELEMENT (stmt_info); - } - if (REDUC_GROUP_FIRST_ELEMENT (stmt_info)) - gcc_assert (REDUC_GROUP_FIRST_ELEMENT (stmt_info) == stmt_info); - /* 1. Is vectorizable reduction? */ /* Not supportable if the reduction variable is used in the loop, unless it's a reduction chain. */ @@ -7453,8 +7297,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo, { /* When vectorizing a reduction chain w/o SLP the reduction PHI is not directy used in stmt. */ - if (!only_slp_reduc_chain - && reduc_chain_length != 1) + if (reduc_chain_length != 1) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -7789,22 +7632,18 @@ vectorizable_reduction (loop_vec_info loop_vinfo, /* All but single defuse-cycle optimized and fold-left reductions go through their own vectorizable_* routines. */ + stmt_vec_info tem + = SLP_TREE_REPRESENTATIVE (SLP_INSTANCE_TREE (slp_node_instance)); if (!single_defuse_cycle && reduction_type != FOLD_LEFT_REDUCTION) + STMT_VINFO_DEF_TYPE (tem) = vect_internal_def; + else { - stmt_vec_info tem - = vect_stmt_to_vectorize (STMT_VINFO_REDUC_DEF (phi_info)); - if (REDUC_GROUP_FIRST_ELEMENT (tem)) - { - gcc_assert (!REDUC_GROUP_NEXT_ELEMENT (tem)); - tem = REDUC_GROUP_FIRST_ELEMENT (tem); - } - STMT_VINFO_DEF_TYPE (vect_orig_stmt (tem)) = vect_internal_def; - STMT_VINFO_DEF_TYPE (tem) = vect_internal_def; + STMT_VINFO_DEF_TYPE (tem) = vect_reduction_def; + if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)) + vect_reduction_update_partial_vector_usage (loop_vinfo, reduc_info, + slp_node, op.code, op.type, + vectype_in); } - else if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)) - vect_reduction_update_partial_vector_usage (loop_vinfo, reduc_info, - slp_node, op.code, op.type, - vectype_in); return true; } @@ -7913,7 +7752,9 @@ vect_transform_reduction (loop_vec_info loop_vinfo, assumption is not true: we use reduc_index to record the index of the reduction variable. */ int reduc_index = SLP_TREE_REDUC_IDX (slp_node); - tree vectype_in = SLP_TREE_VECTYPE (SLP_TREE_CHILDREN (slp_node)[0]); + tree vectype_in = SLP_TREE_VECTYPE (slp_node); + if (lane_reducing_op_p (op.code)) + vectype_in = SLP_TREE_VECTYPE (SLP_TREE_CHILDREN (slp_node)[0]); vec_num = vect_get_num_copies (loop_vinfo, SLP_TREE_CHILDREN (slp_node)[0]); @@ -8238,8 +8079,6 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo, int i; bool nested_cycle = false; int vec_num; - const bool reduc_chain - = SLP_INSTANCE_KIND (slp_node_instance) == slp_inst_kind_reduc_chain; if (nested_in_vect_loop_p (loop, stmt_info)) { @@ -8308,7 +8147,7 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo, vec<stmt_vec_info> &stmts = SLP_TREE_SCALAR_STMTS (slp_node); unsigned int num_phis = stmts.length (); - if (reduc_chain) + if (reduc_info->is_reduc_chain) num_phis = 1; initial_values.reserve (num_phis); for (unsigned int i = 0; i < num_phis; ++i) diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc index 74a9a19..becee62 100644 --- a/gcc/tree-vect-patterns.cc +++ b/gcc/tree-vect-patterns.cc @@ -1022,13 +1022,11 @@ vect_reassociating_reduction_p (vec_info *vinfo, if (loop && nested_in_vect_loop_p (loop, stmt_info)) return false; - if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def) - { - if (needs_fold_left_reduction_p (TREE_TYPE (gimple_assign_lhs (assign)), - code)) - return false; - } - else if (REDUC_GROUP_FIRST_ELEMENT (stmt_info) == NULL) + if (!vect_is_reduction (stmt_info)) + return false; + + if (needs_fold_left_reduction_p (TREE_TYPE (gimple_assign_lhs (assign)), + code)) return false; *op0_out = gimple_assign_rhs1 (assign); @@ -4087,10 +4085,13 @@ vect_recog_vector_vector_shift_pattern (vec_info *vinfo, != TYPE_PRECISION (TREE_TYPE (oprnd0))) return NULL; - stmt_vec_info def_vinfo = vect_get_internal_def (vinfo, oprnd1); - if (!def_vinfo) + stmt_vec_info def_vinfo = vinfo->lookup_def (oprnd1); + if (!def_vinfo || STMT_VINFO_DEF_TYPE (def_vinfo) == vect_external_def) return NULL; + def_vinfo = vect_stmt_to_vectorize (def_vinfo); + gcc_assert (def_vinfo); + *type_out = get_vectype_for_scalar_type (vinfo, TREE_TYPE (oprnd0)); if (*type_out == NULL_TREE) return NULL; diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc index f553e8f..13a2995 100644 --- a/gcc/tree-vect-slp.cc +++ b/gcc/tree-vect-slp.cc @@ -53,6 +53,9 @@ along with GCC; see the file COPYING3. If not see #include "sreal.h" #include "predict.h" +#define REDUC_GROUP_FIRST_ELEMENT(S) \ + (gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element) + static bool vect_transform_slp_perm_load_1 (vec_info *, slp_tree, load_permutation_t &, const vec<tree> &, @@ -4187,41 +4190,60 @@ vect_build_slp_instance (vec_info *vinfo, Return FALSE if SLP build fails. */ static bool -vect_analyze_slp_reduc_chain (vec_info *vinfo, +vect_analyze_slp_reduc_chain (loop_vec_info vinfo, scalar_stmts_to_slp_tree_map_t *bst_map, - stmt_vec_info stmt_info, + stmt_vec_info scalar_stmt, unsigned max_tree_size, unsigned *limit) { - vec<stmt_vec_info> scalar_stmts; + vec<stmt_vec_info> scalar_stmts = vNULL; - /* Collect the reduction stmts and store them in scalar_stmts. */ - scalar_stmts.create (REDUC_GROUP_SIZE (stmt_info)); - stmt_vec_info next_info = stmt_info; - while (next_info) + bool fail = false; + /* ??? We could leave operation code checking to SLP discovery. */ + code_helper code = STMT_VINFO_REDUC_CODE (STMT_VINFO_REDUC_DEF + (vect_orig_stmt (scalar_stmt))); + bool first = true; + stmt_vec_info next_stmt = scalar_stmt; + do { - scalar_stmts.quick_push (vect_stmt_to_vectorize (next_info)); - next_info = REDUC_GROUP_NEXT_ELEMENT (next_info); + stmt_vec_info stmt = next_stmt; + gimple_match_op op; + if (!gimple_extract_op (STMT_VINFO_STMT (stmt), &op)) + gcc_unreachable (); + tree reduc_def = gimple_arg (STMT_VINFO_STMT (stmt), + STMT_VINFO_REDUC_IDX (stmt)); + next_stmt = vect_stmt_to_vectorize (vinfo->lookup_def (reduc_def)); + gcc_assert (is_a <gphi *> (STMT_VINFO_STMT (next_stmt)) + || STMT_VINFO_REDUC_IDX (next_stmt) != -1); + if (!gimple_extract_op (STMT_VINFO_STMT (vect_orig_stmt (stmt)), &op)) + gcc_unreachable (); + if (CONVERT_EXPR_CODE_P (op.code) + && (first + || is_a <gphi *> (STMT_VINFO_STMT (next_stmt)))) + ; + else if (code != op.code) + { + fail = true; + break; + } + else + scalar_stmts.safe_push (stmt); + first = false; } - /* Mark the first element of the reduction chain as reduction to properly - transform the node. In the reduction analysis phase only the last - element of the chain is marked as reduction. */ - STMT_VINFO_DEF_TYPE (stmt_info) - = STMT_VINFO_DEF_TYPE (scalar_stmts.last ()); - STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info)) - = STMT_VINFO_REDUC_DEF (vect_orig_stmt (scalar_stmts.last ())); + while (!is_a <gphi *> (STMT_VINFO_STMT (next_stmt))); + if (fail || scalar_stmts.length () <= 1) + return false; + + scalar_stmts.reverse (); + stmt_vec_info reduc_phi_info = next_stmt; /* Build the tree for the SLP instance. */ vec<stmt_vec_info> root_stmt_infos = vNULL; vec<tree> remain = vNULL; - /* If there's no budget left bail out early. */ - if (*limit == 0) - return false; - if (dump_enabled_p ()) { dump_printf_loc (MSG_NOTE, vect_location, - "Starting SLP discovery for\n"); + "Starting SLP discovery of reduction chain for\n"); for (unsigned i = 0; i < scalar_stmts.length (); ++i) dump_printf_loc (MSG_NOTE, vect_location, " %G", scalar_stmts[i]->stmt); @@ -4233,136 +4255,195 @@ vect_analyze_slp_reduc_chain (vec_info *vinfo, poly_uint64 max_nunits = 1; unsigned tree_size = 0; + /* ??? We need this only for SLP discovery. */ + for (unsigned i = 0; i < scalar_stmts.length (); ++i) + REDUC_GROUP_FIRST_ELEMENT (scalar_stmts[i]) = scalar_stmts[0]; + slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts, group_size, &max_nunits, matches, limit, &tree_size, bst_map); + + for (unsigned i = 0; i < scalar_stmts.length (); ++i) + REDUC_GROUP_FIRST_ELEMENT (scalar_stmts[i]) = NULL; + if (node != NULL) { - /* Calculate the unrolling factor based on the smallest type. */ - poly_uint64 unrolling_factor - = calculate_unrolling_factor (max_nunits, group_size); + /* Create a new SLP instance. */ + slp_instance new_instance = XNEW (class _slp_instance); + SLP_INSTANCE_TREE (new_instance) = node; + SLP_INSTANCE_LOADS (new_instance) = vNULL; + SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos; + SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain; + SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_chain; + new_instance->reduc_phis = NULL; + new_instance->cost_vec = vNULL; + new_instance->subgraph_entries = vNULL; - if (maybe_ne (unrolling_factor, 1U) - && is_a <bb_vec_info> (vinfo)) + vect_reduc_info reduc_info = info_for_reduction (vinfo, node); + reduc_info->is_reduc_chain = true; + + if (dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, + "SLP size %u vs. limit %u.\n", + tree_size, max_tree_size); + + /* Fixup SLP reduction chains. If this is a reduction chain with + a conversion in front amend the SLP tree with a node for that. */ + gimple *scalar_def = STMT_VINFO_REDUC_DEF (reduc_phi_info)->stmt; + if (is_gimple_assign (scalar_def) + && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (scalar_def))) + { + stmt_vec_info conv_info = vect_stmt_to_vectorize + (STMT_VINFO_REDUC_DEF (reduc_phi_info)); + scalar_stmts = vNULL; + scalar_stmts.create (group_size); + for (unsigned i = 0; i < group_size; ++i) + scalar_stmts.quick_push (conv_info); + slp_tree conv = vect_create_new_slp_node (scalar_stmts, 1); + SLP_TREE_VECTYPE (conv) + = get_vectype_for_scalar_type (vinfo, + TREE_TYPE + (gimple_assign_lhs (scalar_def)), + group_size); + SLP_TREE_REDUC_IDX (conv) = 0; + conv->cycle_info.id = node->cycle_info.id; + SLP_TREE_CHILDREN (conv).quick_push (node); + SLP_INSTANCE_TREE (new_instance) = conv; + } + /* Fill the backedge child of the PHI SLP node. The + general matching code cannot find it because the + scalar code does not reflect how we vectorize the + reduction. */ + use_operand_p use_p; + imm_use_iterator imm_iter; + class loop *loop = LOOP_VINFO_LOOP (vinfo); + FOR_EACH_IMM_USE_FAST (use_p, imm_iter, + gimple_get_lhs (scalar_def)) + /* There are exactly two non-debug uses, the reduction + PHI and the loop-closed PHI node. */ + if (!is_gimple_debug (USE_STMT (use_p)) + && gimple_bb (USE_STMT (use_p)) == loop->header) + { + auto_vec<stmt_vec_info, 64> phis (group_size); + stmt_vec_info phi_info = vinfo->lookup_stmt (USE_STMT (use_p)); + for (unsigned i = 0; i < group_size; ++i) + phis.quick_push (phi_info); + slp_tree *phi_node = bst_map->get (phis); + unsigned dest_idx = loop_latch_edge (loop)->dest_idx; + SLP_TREE_CHILDREN (*phi_node)[dest_idx] + = SLP_INSTANCE_TREE (new_instance); + SLP_INSTANCE_TREE (new_instance)->refcnt++; + } + + vinfo->slp_instances.safe_push (new_instance); + + /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with + the number of scalar stmts in the root in a few places. + Verify that assumption holds. */ + gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance)) + .length () == group_size); + + if (dump_enabled_p ()) { - unsigned HOST_WIDE_INT const_max_nunits; - if (!max_nunits.is_constant (&const_max_nunits) - || const_max_nunits > group_size) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "Build SLP failed: store group " - "size not a multiple of the vector size " - "in basic block SLP\n"); - vect_free_slp_tree (node); - return false; - } - /* Fatal mismatch. */ - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "SLP discovery succeeded but node needs " - "splitting\n"); - memset (matches, true, group_size); - matches[group_size / const_max_nunits * const_max_nunits] = false; - vect_free_slp_tree (node); + dump_printf_loc (MSG_NOTE, vect_location, + "Final SLP tree for instance %p:\n", + (void *) new_instance); + vect_print_slp_graph (MSG_NOTE, vect_location, + SLP_INSTANCE_TREE (new_instance)); } - else - { - /* Create a new SLP instance. */ - slp_instance new_instance = XNEW (class _slp_instance); - SLP_INSTANCE_TREE (new_instance) = node; - SLP_INSTANCE_LOADS (new_instance) = vNULL; - SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos; - SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain; - SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_chain; - new_instance->reduc_phis = NULL; - new_instance->cost_vec = vNULL; - new_instance->subgraph_entries = vNULL; - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "SLP size %u vs. limit %u.\n", - tree_size, max_tree_size); + return true; + } - /* Fixup SLP reduction chains. If this is a reduction chain with - a conversion in front amend the SLP tree with a node for that. */ - gimple *scalar_def - = vect_orig_stmt (scalar_stmts[group_size - 1])->stmt; - if (STMT_VINFO_DEF_TYPE (scalar_stmts[0]) != vect_reduction_def) - { - /* Get at the conversion stmt - we know it's the single use - of the last stmt of the reduction chain. */ - use_operand_p use_p; - bool r = single_imm_use (gimple_assign_lhs (scalar_def), - &use_p, &scalar_def); - gcc_assert (r); - stmt_vec_info next_info = vinfo->lookup_stmt (scalar_def); - next_info = vect_stmt_to_vectorize (next_info); - scalar_stmts = vNULL; - scalar_stmts.create (group_size); - for (unsigned i = 0; i < group_size; ++i) - scalar_stmts.quick_push (next_info); - slp_tree conv = vect_create_new_slp_node (scalar_stmts, 1); - SLP_TREE_VECTYPE (conv) - = get_vectype_for_scalar_type (vinfo, - TREE_TYPE - (gimple_assign_lhs (scalar_def)), - group_size); - SLP_TREE_REDUC_IDX (conv) = 0; - conv->cycle_info.id = node->cycle_info.id; - SLP_TREE_CHILDREN (conv).quick_push (node); - SLP_INSTANCE_TREE (new_instance) = conv; - /* We also have to fake this conversion stmt as SLP reduction - group so we don't have to mess with too much code - elsewhere. */ - REDUC_GROUP_FIRST_ELEMENT (next_info) = next_info; - REDUC_GROUP_NEXT_ELEMENT (next_info) = NULL; - } - /* Fill the backedge child of the PHI SLP node. The - general matching code cannot find it because the - scalar code does not reflect how we vectorize the - reduction. */ - use_operand_p use_p; - imm_use_iterator imm_iter; - class loop *loop = LOOP_VINFO_LOOP (as_a <loop_vec_info> (vinfo)); - FOR_EACH_IMM_USE_FAST (use_p, imm_iter, - gimple_get_lhs (scalar_def)) - /* There are exactly two non-debug uses, the reduction - PHI and the loop-closed PHI node. */ - if (!is_gimple_debug (USE_STMT (use_p)) - && gimple_bb (USE_STMT (use_p)) == loop->header) - { - auto_vec<stmt_vec_info, 64> phis (group_size); - stmt_vec_info phi_info - = vinfo->lookup_stmt (USE_STMT (use_p)); - for (unsigned i = 0; i < group_size; ++i) - phis.quick_push (phi_info); - slp_tree *phi_node = bst_map->get (phis); - unsigned dest_idx = loop_latch_edge (loop)->dest_idx; - SLP_TREE_CHILDREN (*phi_node)[dest_idx] - = SLP_INSTANCE_TREE (new_instance); - SLP_INSTANCE_TREE (new_instance)->refcnt++; - } + /* Failed to SLP. */ + scalar_stmts.release (); + if (dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, + "SLP discovery of reduction chain failed\n"); + return false; +} - vinfo->slp_instances.safe_push (new_instance); +/* Analyze an SLP instance starting from SCALAR_STMTS which are a group + of KIND. Return true if successful. */ - /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with - the number of scalar stmts in the root in a few places. - Verify that assumption holds. */ - gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance)) - .length () == group_size); +static bool +vect_analyze_slp_reduction (loop_vec_info vinfo, + stmt_vec_info scalar_stmt, + unsigned max_tree_size, unsigned *limit, + scalar_stmts_to_slp_tree_map_t *bst_map, + bool force_single_lane) +{ + slp_instance_kind kind = slp_inst_kind_reduc_group; - if (dump_enabled_p ()) - { - dump_printf_loc (MSG_NOTE, vect_location, - "Final SLP tree for instance %p:\n", - (void *) new_instance); - vect_print_slp_graph (MSG_NOTE, vect_location, - SLP_INSTANCE_TREE (new_instance)); - } + /* If there's no budget left bail out early. */ + if (*limit == 0) + return false; - return true; + /* Try to gather a reduction chain. */ + if (! force_single_lane + && STMT_VINFO_DEF_TYPE (scalar_stmt) == vect_reduction_def + && vect_analyze_slp_reduc_chain (vinfo, bst_map, scalar_stmt, + max_tree_size, limit)) + return true; + + vec<stmt_vec_info> scalar_stmts; + scalar_stmts.create (1); + scalar_stmts.quick_push (scalar_stmt); + + if (dump_enabled_p ()) + { + dump_printf_loc (MSG_NOTE, vect_location, + "Starting SLP discovery for\n"); + for (unsigned i = 0; i < scalar_stmts.length (); ++i) + dump_printf_loc (MSG_NOTE, vect_location, + " %G", scalar_stmts[i]->stmt); + } + + /* Build the tree for the SLP instance. */ + unsigned int group_size = scalar_stmts.length (); + bool *matches = XALLOCAVEC (bool, group_size); + poly_uint64 max_nunits = 1; + unsigned tree_size = 0; + + slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts, group_size, + &max_nunits, matches, limit, + &tree_size, bst_map); + if (node != NULL) + { + /* Create a new SLP instance. */ + slp_instance new_instance = XNEW (class _slp_instance); + SLP_INSTANCE_TREE (new_instance) = node; + SLP_INSTANCE_LOADS (new_instance) = vNULL; + SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL; + SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL; + SLP_INSTANCE_KIND (new_instance) = kind; + new_instance->reduc_phis = NULL; + new_instance->cost_vec = vNULL; + new_instance->subgraph_entries = vNULL; + + if (dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, + "SLP size %u vs. limit %u.\n", + tree_size, max_tree_size); + + vinfo->slp_instances.safe_push (new_instance); + + /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with + the number of scalar stmts in the root in a few places. + Verify that assumption holds. */ + gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance)) + .length () == group_size); + + if (dump_enabled_p ()) + { + dump_printf_loc (MSG_NOTE, vect_location, + "Final SLP tree for instance %p:\n", + (void *) new_instance); + vect_print_slp_graph (MSG_NOTE, vect_location, + SLP_INSTANCE_TREE (new_instance)); } + + return true; } /* Failed to SLP. */ @@ -5256,40 +5337,6 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size, if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo)) { - /* Find SLP sequences starting from reduction chains. */ - FOR_EACH_VEC_ELT (loop_vinfo->reduction_chains, i, first_element) - if (! STMT_VINFO_RELEVANT_P (first_element) - && ! STMT_VINFO_LIVE_P (first_element)) - ; - else if (force_single_lane - || ! vect_analyze_slp_reduc_chain (vinfo, bst_map, - first_element, - max_tree_size, &limit)) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "SLP discovery of reduction chain failed\n"); - /* Dissolve reduction chain group. */ - stmt_vec_info vinfo = first_element; - stmt_vec_info last = NULL; - while (vinfo) - { - stmt_vec_info next = REDUC_GROUP_NEXT_ELEMENT (vinfo); - REDUC_GROUP_FIRST_ELEMENT (vinfo) = NULL; - REDUC_GROUP_NEXT_ELEMENT (vinfo) = NULL; - last = vinfo; - vinfo = next; - } - STMT_VINFO_DEF_TYPE (first_element) = vect_internal_def; - /* ??? When there's a conversion around the reduction - chain 'last' isn't the entry of the reduction. */ - if (STMT_VINFO_DEF_TYPE (last) != vect_reduction_def) - return opt_result::failure_at (vect_location, - "SLP build failed.\n"); - /* It can be still vectorized as part of an SLP reduction. */ - loop_vinfo->reductions.safe_push (last); - } - /* Find SLP sequences starting from groups of reductions. */ if (loop_vinfo->reductions.length () > 0) { @@ -5315,23 +5362,13 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size, if (!force_single_lane && !lane_reducing_stmt_p (STMT_VINFO_STMT (next_info))) scalar_stmts.quick_push (next_info); - else - { - /* Do SLP discovery for single-lane reductions. */ - vec<stmt_vec_info> stmts; - vec<stmt_vec_info> roots = vNULL; - vec<tree> remain = vNULL; - stmts.create (1); - stmts.quick_push (next_info); - if (! vect_build_slp_instance (vinfo, - slp_inst_kind_reduc_group, - stmts, roots, remain, - max_tree_size, &limit, - bst_map, - force_single_lane)) - return opt_result::failure_at (vect_location, - "SLP build failed.\n"); - } + /* Do SLP discovery for single-lane reductions. */ + else if (! vect_analyze_slp_reduction (loop_vinfo, next_info, + max_tree_size, &limit, + bst_map, + force_single_lane)) + return opt_result::failure_at (vect_location, + "SLP build failed.\n"); } } /* Save for re-processing on failure. */ @@ -5349,20 +5386,13 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size, scalar_stmts.release (); /* Do SLP discovery for single-lane reductions. */ for (auto stmt_info : saved_stmts) - { - vec<stmt_vec_info> stmts; - vec<stmt_vec_info> roots = vNULL; - vec<tree> remain = vNULL; - stmts.create (1); - stmts.quick_push (vect_stmt_to_vectorize (stmt_info)); - if (! vect_build_slp_instance (vinfo, - slp_inst_kind_reduc_group, - stmts, roots, remain, - max_tree_size, &limit, - bst_map, force_single_lane)) - return opt_result::failure_at (vect_location, - "SLP build failed.\n"); - } + if (! vect_analyze_slp_reduction (loop_vinfo, + vect_stmt_to_vectorize + (stmt_info), + max_tree_size, &limit, + bst_map, force_single_lane)) + return opt_result::failure_at (vect_location, + "SLP build failed.\n"); } saved_stmts.release (); } diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc index dcb2522..83acbb3 100644 --- a/gcc/tree-vect-stmts.cc +++ b/gcc/tree-vect-stmts.cc @@ -2062,16 +2062,13 @@ vector_vector_composition_type (tree vtype, poly_uint64 nelts, tree *ptype, VECTYPE is the vector type that the vectorized statements will use. If ELSVALS is nonzero the supported else values will be stored in the - vector ELSVALS points to. - - For loads PERM_OK indicates whether we can code generate a - SLP_TREE_LOAD_PERMUTATION on the node. */ + vector ELSVALS points to. */ static bool get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype, slp_tree slp_node, bool masked_p, vec_load_store_type vls_type, - bool perm_ok, vect_load_store_data *ls) + vect_load_store_data *ls) { vect_memory_access_type *memory_access_type = &ls->memory_access_type; poly_int64 *poffset = &ls->poffset; @@ -2081,6 +2078,8 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, internal_fn *lanes_ifn = &ls->lanes_ifn; vec<int> *elsvals = &ls->elsvals; tree *ls_type = &ls->ls_type; + bool *slp_perm = &ls->slp_perm; + unsigned *n_perms = &ls->n_perms; loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo); poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype); class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL; @@ -2093,6 +2092,15 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, *misalignment = DR_MISALIGNMENT_UNKNOWN; *poffset = 0; *ls_type = NULL_TREE; + *slp_perm = false; + *n_perms = -1U; + + bool perm_ok = true; + poly_int64 vf = loop_vinfo ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1; + + if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()) + perm_ok = vect_transform_slp_perm_load (vinfo, slp_node, vNULL, NULL, + vf, true, n_perms); if (STMT_VINFO_GROUPED_ACCESS (stmt_info)) { @@ -2534,7 +2542,7 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, poly_uint64 read_amount = vf * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))); if (STMT_VINFO_GROUPED_ACCESS (stmt_info)) - read_amount *= DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (stmt_info)); + read_amount *= group_size; auto target_alignment = DR_TARGET_ALIGNMENT (STMT_VINFO_DR_INFO (stmt_info)); @@ -2627,6 +2635,60 @@ get_load_store_type (vec_info *vinfo, stmt_vec_info stmt_info, if (!loop_vinfo && *memory_access_type == VMAT_ELEMENTWISE) return false; + /* Some loads need to explicitly permute the loaded data if there + is a load permutation. Among those are: + - VMAT_ELEMENTWISE. + - VMAT_STRIDED_SLP. + - VMAT_GATHER_SCATTER: + - Strided gather (fallback for VMAT_STRIDED_SLP if #lanes == 1). + - Grouped strided gather (ditto but for #lanes > 1). + + For VMAT_ELEMENTWISE we can fold the load permutation into the + individual indices we access directly, eliding the permutation. + Strided gather only allows load permutations for the + single-element case. */ + + if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists () + && !(*memory_access_type == VMAT_ELEMENTWISE + || (mat_gather_scatter_p (*memory_access_type) + && SLP_TREE_LANES (slp_node) == 1 + && single_element_p))) + { + if (!loop_vinfo) + { + /* In BB vectorization we may not actually use a loaded vector + accessing elements in excess of DR_GROUP_SIZE. */ + stmt_vec_info group_info = SLP_TREE_SCALAR_STMTS (slp_node)[0]; + group_info = DR_GROUP_FIRST_ELEMENT (group_info); + unsigned HOST_WIDE_INT nunits; + unsigned j, k, maxk = 0; + FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (slp_node), j, k) + if (k > maxk) + maxk = k; + tree vectype = SLP_TREE_VECTYPE (slp_node); + if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits) + || maxk >= (DR_GROUP_SIZE (group_info) & ~(nunits - 1))) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "BB vectorization with gaps at the end of " + "a load is not supported\n"); + return false; + } + } + + if (!perm_ok) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, + vect_location, + "unsupported load permutation\n"); + return false; + } + + *slp_perm = true; + } + return true; } @@ -8009,7 +8071,7 @@ vectorizable_store (vec_info *vinfo, vect_load_store_data &ls = slp_node->get_data (_ls_data); if (cost_vec && !get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node, - vls_type, false, &_ls_data)) + vls_type, &_ls_data)) return false; /* Temporary aliases to analysis data, should not be modified through these. */ @@ -9454,7 +9516,6 @@ vectorizable_load (vec_info *vinfo, bool compute_in_loop = false; class loop *at_loop; int vec_num; - bool slp_perm = false; bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo); poly_uint64 vf; tree aggr_type; @@ -9592,17 +9653,11 @@ vectorizable_load (vec_info *vinfo, else group_size = 1; - bool perm_ok = true; - unsigned n_perms = -1U; - if (cost_vec && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()) - perm_ok = vect_transform_slp_perm_load (vinfo, slp_node, vNULL, NULL, vf, - true, &n_perms); - vect_load_store_data _ls_data{}; vect_load_store_data &ls = slp_node->get_data (_ls_data); if (cost_vec && !get_load_store_type (vinfo, stmt_info, vectype, slp_node, mask_node, - VLS_LOAD, perm_ok, &ls)) + VLS_LOAD, &ls)) return false; /* Temporary aliases to analysis data, should not be modified through these. */ @@ -9623,56 +9678,6 @@ vectorizable_load (vec_info *vinfo, bool type_mode_padding_p = TYPE_PRECISION (scalar_type) < GET_MODE_PRECISION (GET_MODE_INNER (mode)); - /* ??? The following checks should really be part of - get_load_store_type. */ - if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists () - && !(memory_access_type == VMAT_ELEMENTWISE - || (mat_gather_scatter_p (memory_access_type) - && SLP_TREE_LANES (slp_node) == 1 - && (!grouped_load - || !DR_GROUP_NEXT_ELEMENT (first_stmt_info))))) - { - slp_perm = true; - - if (!loop_vinfo && cost_vec) - { - /* In BB vectorization we may not actually use a loaded vector - accessing elements in excess of DR_GROUP_SIZE. */ - stmt_vec_info group_info = SLP_TREE_SCALAR_STMTS (slp_node)[0]; - group_info = DR_GROUP_FIRST_ELEMENT (group_info); - unsigned HOST_WIDE_INT nunits; - unsigned j, k, maxk = 0; - FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (slp_node), j, k) - if (k > maxk) - maxk = k; - tree vectype = SLP_TREE_VECTYPE (slp_node); - if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits) - || maxk >= (DR_GROUP_SIZE (group_info) & ~(nunits - 1))) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "BB vectorization with gaps at the end of " - "a load is not supported\n"); - return false; - } - } - - if (cost_vec) - { - if (!perm_ok) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, - vect_location, - "unsupported load permutation\n"); - return false; - } - ls.n_perms = n_perms; - } - else - n_perms = ls.n_perms; - } - if (slp_node->ldst_lanes && memory_access_type != VMAT_LOAD_STORE_LANES) { @@ -10027,7 +10032,7 @@ vectorizable_load (vec_info *vinfo, not only the number of vector stmts the permutation result fits in. */ int ncopies; - if (slp_perm) + if (ls.slp_perm) { gcc_assert (memory_access_type != VMAT_ELEMENTWISE); /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for @@ -10135,18 +10140,18 @@ vectorizable_load (vec_info *vinfo, if (!costing_p) { - if (slp_perm) + if (ls.slp_perm) dr_chain.quick_push (gimple_assign_lhs (new_stmt)); else slp_node->push_vec_def (new_stmt); } } - if (slp_perm) + if (ls.slp_perm) { if (costing_p) { - gcc_assert (n_perms != -1U); - inside_cost += record_stmt_cost (cost_vec, n_perms, vec_perm, + gcc_assert (ls.n_perms != -1U); + inside_cost += record_stmt_cost (cost_vec, ls.n_perms, vec_perm, slp_node, 0, vect_body); } else @@ -10154,7 +10159,7 @@ vectorizable_load (vec_info *vinfo, unsigned n_perms2; vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, gsi, vf, false, &n_perms2); - gcc_assert (n_perms == n_perms2); + gcc_assert (ls.n_perms == n_perms2); } } @@ -10219,7 +10224,7 @@ vectorizable_load (vec_info *vinfo, instead the access is contiguous but it might be permuted. No gap adjustment is needed though. */ ; - else if (slp_perm + else if (ls.slp_perm && (group_size != scalar_lanes || !multiple_p (nunits, group_size))) { @@ -10568,7 +10573,7 @@ vectorizable_load (vec_info *vinfo, if (mat_gather_scatter_p (memory_access_type)) { - gcc_assert ((!grouped_load && !slp_perm) || ls.ls_type); + gcc_assert ((!grouped_load && !ls.slp_perm) || ls.ls_type); /* If we pun the original vectype the loads as well as costing, length, etc. is performed with the new type. After loading we VIEW_CONVERT @@ -10930,14 +10935,14 @@ vectorizable_load (vec_info *vinfo, /* Store vector loads in the corresponding SLP_NODE. */ if (!costing_p) { - if (slp_perm) + if (ls.slp_perm) dr_chain.quick_push (gimple_assign_lhs (new_stmt)); else slp_node->push_vec_def (new_stmt); } } - if (slp_perm) + if (ls.slp_perm) { if (costing_p) { @@ -11034,7 +11039,7 @@ vectorizable_load (vec_info *vinfo, stmt_info, bump); } - if (grouped_load || slp_perm) + if (grouped_load || ls.slp_perm) dr_chain.create (vec_num); gimple *new_stmt = NULL; @@ -11531,11 +11536,11 @@ vectorizable_load (vec_info *vinfo, /* Collect vector loads and later create their permutation in vect_transform_slp_perm_load. */ - if (!costing_p && (grouped_load || slp_perm)) + if (!costing_p && (grouped_load || ls.slp_perm)) dr_chain.quick_push (new_temp); /* Store vector loads in the corresponding SLP_NODE. */ - if (!costing_p && !slp_perm) + if (!costing_p && !ls.slp_perm) slp_node->push_vec_def (new_stmt); /* With SLP permutation we load the gaps as well, without @@ -11544,7 +11549,7 @@ vectorizable_load (vec_info *vinfo, group_elt += nunits; if (!costing_p && maybe_ne (group_gap_adj, 0U) - && !slp_perm + && !ls.slp_perm && known_eq (group_elt, group_size - group_gap_adj)) { poly_wide_int bump_val @@ -11561,7 +11566,7 @@ vectorizable_load (vec_info *vinfo, elements loaded for a permuted SLP load. */ if (!costing_p && maybe_ne (group_gap_adj, 0U) - && slp_perm) + && ls.slp_perm) { poly_wide_int bump_val = (wi::to_wide (TYPE_SIZE_UNIT (elem_type)) * group_gap_adj); @@ -11572,7 +11577,7 @@ vectorizable_load (vec_info *vinfo, stmt_info, bump); } - if (slp_perm) + if (ls.slp_perm) { /* For SLP we know we've seen all possible uses of dr_chain so direct vect_transform_slp_perm_load to DCE the unused parts. @@ -11580,9 +11585,9 @@ vectorizable_load (vec_info *vinfo, in PR101120 and friends. */ if (costing_p) { - gcc_assert (n_perms != -1U); - if (n_perms != 0) - inside_cost = record_stmt_cost (cost_vec, n_perms, vec_perm, + gcc_assert (ls.n_perms != -1U); + if (ls.n_perms != 0) + inside_cost = record_stmt_cost (cost_vec, ls.n_perms, vec_perm, slp_node, 0, vect_body); } else @@ -11591,7 +11596,7 @@ vectorizable_load (vec_info *vinfo, bool ok = vect_transform_slp_perm_load (vinfo, slp_node, dr_chain, gsi, vf, false, &n_perms2, nullptr, true); - gcc_assert (ok && n_perms == n_perms2); + gcc_assert (ok && ls.n_perms == n_perms2); } dr_chain.release (); } diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 52bc0d6..4785cbd 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -290,6 +290,8 @@ struct vect_load_store_data : vect_data { tree strided_offset_vectype; // VMAT_GATHER_SCATTER_IFN, originally strided tree ls_type; // VMAT_GATHER_SCATTER_IFN auto_vec<int> elsvals; + /* True if the load requires a load permutation. */ + bool slp_perm; // SLP_TREE_LOAD_PERMUTATION unsigned n_perms; // SLP_TREE_LOAD_PERMUTATION }; @@ -844,6 +846,9 @@ public: following land-reducing operation would be assigned to. */ unsigned int reduc_result_pos; + /* Whether this represents a reduction chain. */ + bool is_reduc_chain; + /* Whether we force a single cycle PHI during reduction vectorization. */ bool force_single_cycle; @@ -1066,10 +1071,6 @@ public: /* Reduction cycles detected in the loop. Used in loop-aware SLP. */ auto_vec<stmt_vec_info> reductions; - /* All reduction chains in the loop, represented by the first - stmt in the chain. */ - auto_vec<stmt_vec_info> reduction_chains; - /* Defs that could not be analyzed such as OMP SIMD calls without a LHS. */ auto_vec<stmt_vec_info> alternate_defs; @@ -1290,7 +1291,6 @@ public: #define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances #define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor #define LOOP_VINFO_REDUCTIONS(L) (L)->reductions -#define LOOP_VINFO_REDUCTION_CHAINS(L) (L)->reduction_chains #define LOOP_VINFO_PEELING_FOR_GAPS(L) (L)->peeling_for_gaps #define LOOP_VINFO_PEELING_FOR_NITER(L) (L)->peeling_for_niter #define LOOP_VINFO_EARLY_BREAKS(L) (L)->early_breaks @@ -1538,7 +1538,7 @@ public: /* Whether the stmt is SLPed, loop-based vectorized, or both. */ enum slp_vect_type slp_type; - /* Interleaving and reduction chains info. */ + /* Interleaving chains info. */ /* First element in the group. */ stmt_vec_info first_element; /* Pointer to the next element in the group. */ @@ -1711,13 +1711,6 @@ struct gather_scatter_info { #define DR_GROUP_GAP(S) \ (gcc_checking_assert ((S)->dr_aux.dr), (S)->gap) -#define REDUC_GROUP_FIRST_ELEMENT(S) \ - (gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element) -#define REDUC_GROUP_NEXT_ELEMENT(S) \ - (gcc_checking_assert (!(S)->dr_aux.dr), (S)->next_element) -#define REDUC_GROUP_SIZE(S) \ - (gcc_checking_assert (!(S)->dr_aux.dr), (S)->size) - #define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope) #define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp) diff --git a/gcc/value-range.cc b/gcc/value-range.cc index d34a262..f93a7e5 100644 --- a/gcc/value-range.cc +++ b/gcc/value-range.cc @@ -55,6 +55,93 @@ irange_bitmask::irange_bitmask (tree type, } } +// Return a range in R of TYPE for this bitmask which encompasses +// a set of valid values which are allowable for this bitmask/value +// combination. If false is returned, no range was set. + +bool +irange_bitmask::range_from_mask (irange &r, tree type) const +{ + if (unknown_p ()) + return false; + + gcc_checking_assert ((value () & mask ()) == 0); + unsigned popcount = wi::popcount (mask ()); + + // For 0, 1 or 2 bits set, create a range with only the allowed values. + if (popcount <= 2) + { + // VALUE is always a valid range. + r.set (type, value (), value ()); + // If there are bits in mask, (VALUE | MASK) is also valid. + if (popcount >= 1) + r.union_ (int_range<1> (type, value () | mask (), value () | mask ())); + // If there are 2 bits set, add the other 2 possible values. + if (popcount == 2) + { + // Extract the two 1-bit masks into lb and ub. + wide_int lb = mask () & -mask (); // Lowest set bit. + wide_int ub = mask () & (mask () - 1); // The other bit. + r.union_ (int_range<1> (type, value () | lb, value () | lb)); + r.union_ (int_range<1> (type, value () | ub, value () | ub)); + } + return true; + } + + // Otherwise, calculate the valid range allowed by the bitmask. + int prec = TYPE_PRECISION (type); + wide_int ub = mask () | value (); + wide_int sign_bit = wi::one (prec) << (prec - 1); + wide_int sign_mask = mask () & sign_bit; + wide_int sign_value = value () & sign_bit; + // Create a lower and upper bound. + // If unsigned, or the sign is known to be positive, create [lb, ub] + if (TYPE_SIGN (type) == UNSIGNED || (sign_mask == 0 && sign_value == 0)) + r.set (type, value (), mask () | value ()); + // If the sign bit is KNOWN to be 1, we have a completely negative range. + else if (sign_mask == 0 && sign_value != 0) + r.set (type, value (), value () | (mask () & ~sign_bit)); + else + { + // Otherwise there are 2 ranges, a negative and positive interval. + wide_int neg_base = value () | sign_bit; + wide_int pos_mask = mask () & ~sign_bit; + r.set (type, neg_base , neg_base | pos_mask); + r.union_ (int_range<1> (type, value (), value () | pos_mask)); + } + + // If the mask doesn't have a trailing zero, there is nothing else to filter. + int z = wi::ctz (mask ()); + if (z == 0) + return true; + + // Remove the [0, X] values which the trailing-zero mask rules out. + // For example, if z == 4, the mask is 0xFFF0, and the lowest 4 bits + // define the range [0, 15]. Only (value & low_mask) is allowed. + ub = (wi::one (prec) << z) - 1; // Upper bound of range. + int_range<4> mask_range (type, wi::zero (prec), ub); + // Remove the valid value from the excluded range and form an anti-range. + wide_int allow = value () & ub; + mask_range.intersect (int_range<2> (type, allow, allow, VR_ANTI_RANGE)); + mask_range.invert (); + r.intersect (mask_range); + + if (TYPE_SIGN (type) == SIGNED) + { + // For signed negative values, find the lowest value with trailing zeros. + // This forms a range such as [-512, -1] for z=9. + wide_int lb = -(wi::one (prec) << z); + int_range<4> mask_range (type, lb, wi::minus_one (prec)); + // Remove the one allowed value from that set. + wide_int allow = value () | lb; + mask_range.intersect (int_range<2> (type, allow, allow, VR_ANTI_RANGE)); + mask_range.invert (); + r.intersect (mask_range); + } + return true; +} + + void irange::accept (const vrange_visitor &v) const { @@ -2275,47 +2362,57 @@ irange::invert () // This routine will take the bounds [LB, UB], and apply the bitmask to those // values such that both bounds satisfy the bitmask. TRUE is returned // if either bound changes, and they are returned as [NEW_LB, NEW_UB]. -// if NEW_UB < NEW_LB, then the entire bound is to be removed as none of -// the values are valid. +// If there is an overflow, or if (NEW_UB < NEW_LB), then the entire bound is +// to be removed as none of the values are valid. This is indicated by +// teturning TRUE in OVF. False indicates the bounds are fine. // ie, [4, 14] MASK 0xFFFE VALUE 0x1 -// means all values must be odd, the new bounds returned will be [5, 13]. +// means all values must be odd, the new bounds returned will be [5, 13] with +// OVF set to FALSE. // ie, [4, 4] MASK 0xFFFE VALUE 0x1 -// would return [1, 0] and as the LB < UB, the entire subrange is invalid -// and should be removed. +// would return TRUE and OVF == TRUE. The entire subrange should be removed. bool irange::snap (const wide_int &lb, const wide_int &ub, - wide_int &new_lb, wide_int &new_ub) + wide_int &new_lb, wide_int &new_ub, bool &ovf) { + ovf = false; int z = wi::ctz (m_bitmask.mask ()); if (z == 0) return false; + // Shortcircuit check for values that are already good. + if ((((lb ^ m_bitmask.value ()) | (ub ^ m_bitmask.value ())) + & ~m_bitmask.mask ()) == 0) + return false; + const wide_int step = (wi::one (TYPE_PRECISION (type ())) << z); const wide_int match_mask = step - 1; const wide_int value = m_bitmask.value () & match_mask; - bool ovf = false; - wide_int rem_lb = lb & match_mask; wide_int offset = (value - rem_lb) & match_mask; new_lb = lb + offset; // Check for overflows at +INF if (wi::lt_p (new_lb, lb, TYPE_SIGN (type ()))) - ovf = true; + { + ovf = true; + return true; + } wide_int rem_ub = ub & match_mask; wide_int offset_ub = (rem_ub - value) & match_mask; new_ub = ub - offset_ub; // Check for underflows at -INF if (wi::gt_p (new_ub, ub, TYPE_SIGN (type ()))) - ovf = true; + { + ovf = true; + return true; + } - // Overflow or inverted range = invalid - if (ovf || wi::lt_p (new_ub, new_lb, TYPE_SIGN (type ()))) + // If inverted range is invalid, set overflow to TRUE. + if (wi::lt_p (new_ub, new_lb, TYPE_SIGN (type ()))) { - new_lb = wi::one (lb.get_precision ()); - new_ub = wi::zero (ub.get_precision ()); + ovf = true; return true; } return (new_lb != lb) || (new_ub != ub); @@ -2334,11 +2431,12 @@ irange::snap_subranges () wide_int lb, ub; for (x = 0; x < m_num_ranges; x++) { - if (snap (lower_bound (x), upper_bound (x), lb, ub)) + bool ovf; + if (snap (lower_bound (x), upper_bound (x), lb, ub, ovf)) { changed = true; - // This subrange is to be completely removed. - if (wi::lt_p (ub, lb, TYPE_SIGN (type ()))) + // Check if this subrange is to be completely removed. + if (ovf) { int_range<1> tmp (type (), lower_bound (x), upper_bound (x)); invalid.union_ (tmp); @@ -2359,109 +2457,25 @@ irange::snap_subranges () return changed; } -// If the mask can be trivially converted to a range, do so. -// Otherwise attempt to remove the lower bits from the range. -// Return true if the range changed in any way. +// If the bitmask has a range representation, intersect this range with +// the bitmasks range. Then ensure all enpoints match the bitmask. +// Return TRUE if the range changes at all. bool irange::set_range_from_bitmask () { gcc_checking_assert (!undefined_p ()); - if (m_bitmask.unknown_p ()) - return false; - - // If all the bits are known, this is a singleton. - if (m_bitmask.mask () == 0) - { - // Make sure the singleton is within the range. - if (contains_p (m_bitmask.value ())) - set (m_type, m_bitmask.value (), m_bitmask.value ()); - else - set_undefined (); - return true; - } - - unsigned popcount = wi::popcount (m_bitmask.get_nonzero_bits ()); - - // If we have only one bit set in the mask, we can figure out the - // range immediately. - if (popcount == 1) - { - // Make sure we don't pessimize the range. - if (!contains_p (m_bitmask.get_nonzero_bits ())) - return false; - - bool has_zero = contains_zero_p (*this); - wide_int nz = m_bitmask.get_nonzero_bits (); - set (m_type, nz, nz); - m_bitmask.set_nonzero_bits (nz); - if (has_zero) - { - int_range<2> zero; - zero.set_zero (m_type); - union_ (zero); - } - if (flag_checking) - verify_range (); - return true; - } - else if (popcount == 0) - { - set_zero (m_type); - return true; - } + // Snap subranmges when bitmask is first set. + snap_subranges (); + if (undefined_p ()) + return true; - // If the mask doesn't have a trailing zero, theres nothing to filter. - int z = wi::ctz (m_bitmask.mask ()); - if (!z) + // Calculate the set of ranges valid for the bitmask. + int_range_max allow; + if (!m_bitmask.range_from_mask (allow, m_type)) return false; - - int prec = TYPE_PRECISION (m_type); - wide_int value = m_bitmask.value (); - wide_int mask = m_bitmask.mask (); - - // Remove the [0, X] values which the trailing-zero mask rules out. - // For example, if z == 4, the mask is 0xFFF0, and the lowest 4 bits - // define the range [0, 15]. Only one of which (value & low_mask) is allowed. - wide_int ub = (wi::one (prec) << z) - 1; // Upper bound of affected range. - int_range_max mask_range (m_type, wi::zero (prec), ub); - - // Remove the one valid value from the excluded range and form an anti-range. - wide_int allow = value & ub; - mask_range.intersect (int_range<2> (m_type, allow, allow, VR_ANTI_RANGE)); - - // Invert it to get the allowed values and intersect it with the main range. - mask_range.invert (); - bool changed = intersect (mask_range); - - // Now handle the rest of the domain — the upper side for positive values, - // or [-X, -1] for signed negatives. - // Compute the maximum value representable under the mask/value constraint. - ub = mask | value; - - // If value is non-negative, adjust the upper limit to remove values above - // UB that conflict with known fixed bits. - if (TYPE_SIGN (m_type) == UNSIGNED || wi::clz (ub) > 0) - mask_range = int_range<1> (m_type, wi::zero (prec), ub); - else - { - // For signed negative values, find the lowest value with trailing zeros. - // This forms a range such as [-512, -1] for z=9. - wide_int lb = -(wi::one (prec) << z); - mask_range = int_range<2> (m_type, lb, wi::minus_one (prec)); - - // Remove the one allowed value from that set. - allow = value | lb; - mask_range.intersect (int_range<2> (m_type, allow, allow, VR_ANTI_RANGE)); - mask_range.invert (); - } - - // Make sure we call intersect, so do it first. - changed = intersect (mask_range) | changed; - // Now make sure each subrange endpoint matches the bitmask. - changed |= snap_subranges (); - - return changed; + // And intersect that set of ranges with the current set. + return intersect (allow); } void @@ -2932,7 +2946,7 @@ test_irange_snap_bounds () tree u1 = build_nonstandard_integer_type (1, /*unsigned=*/ 1); // Basic aligned range: even-only - assert_snap_result (5, 15, 6, 14, 0xFFFFFFFE, 0x0, u32); + assert_snap_result (5, 15, 6, 14, 0xE, 0x0, u32); // Singleton that doesn't match mask: undefined. assert_snap_result (7, 7, 1, 0, 0xFFFFFFFE, 0x0, u32); // 8-bit signed char, mask 0xF0 (i.e. step of 16). @@ -3204,15 +3218,12 @@ range_tests_misc () static void range_tests_nonzero_bits () { - int_range<2> r0, r1; + int_range<8> r0, r1; // Adding nonzero bits to a varying drops the varying. r0.set_varying (integer_type_node); r0.set_nonzero_bits (INT (255)); ASSERT_TRUE (!r0.varying_p ()); - // Dropping the nonzero bits brings us back to varying. - r0.set_nonzero_bits (INT (-1)); - ASSERT_TRUE (r0.varying_p ()); // Test contains_p with nonzero bits. r0.set_zero (integer_type_node); @@ -3244,17 +3255,6 @@ range_tests_nonzero_bits () r0.intersect (r1); ASSERT_TRUE (r0.get_nonzero_bits () == 0xff); - // The union of a mask of 0xff..ffff00 with a mask of 0xff spans the - // entire domain, and makes the range a varying. - r0.set_varying (integer_type_node); - wide_int x = wi::shwi (0xff, TYPE_PRECISION (integer_type_node)); - x = wi::bit_not (x); - r0.set_nonzero_bits (x); // 0xff..ff00 - r1.set_varying (integer_type_node); - r1.set_nonzero_bits (INT (0xff)); - r0.union_ (r1); - ASSERT_TRUE (r0.varying_p ()); - // Test that setting a nonzero bit of 1 does not pessimize the range. r0.set_zero (integer_type_node); r0.set_nonzero_bits (INT (1)); diff --git a/gcc/value-range.h b/gcc/value-range.h index 3bc02db..6ae46e1 100644 --- a/gcc/value-range.h +++ b/gcc/value-range.h @@ -150,6 +150,7 @@ public: bool operator!= (const irange_bitmask &src) const { return !(*this == src); } void verify_mask () const; void dump (FILE *) const; + bool range_from_mask (irange &r, tree type) const; bool member_p (const wide_int &val) const; @@ -346,7 +347,8 @@ private: bool union_bitmask (const irange &r); bool set_range_from_bitmask (); bool snap_subranges (); - bool snap (const wide_int &, const wide_int &, wide_int &, wide_int &); + bool snap (const wide_int &, const wide_int &, wide_int &, wide_int &, + bool &); bool intersect (const wide_int& lb, const wide_int& ub); bool union_append (const irange &r); |