From 631c5e818ef8bb0f61fd3bb44cc4449be2142e2b Mon Sep 17 00:00:00 2001 From: MbjYjbpivj Date: Wed, 17 Apr 2024 20:15:30 +0800 Subject: [PATCH 001/714] [mlir] fix intNEQValue summary (#89029) Fix the summary of intNEQValue. --- mlir/include/mlir/IR/CommonAttrConstraints.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlir/include/mlir/IR/CommonAttrConstraints.td b/mlir/include/mlir/IR/CommonAttrConstraints.td index 0312ac7ec1d8..0d69bb0717a5 100644 --- a/mlir/include/mlir/IR/CommonAttrConstraints.td +++ b/mlir/include/mlir/IR/CommonAttrConstraints.td @@ -755,7 +755,7 @@ class AllAttrOf constraints> : AttrConstraint< class IntNEQValue : AttrConstraint< CPred<"::llvm::cast<::mlir::IntegerAttr>($_self).getInt() != " # n>, - "whose minimum value is " # n>; + "whose value is not " # n>; class IntMinValue : AttrConstraint< CPred<"::llvm::cast<::mlir::IntegerAttr>($_self).getInt() >= " # n>, -- GitLab From d57907d0b4f292f148310695ed011fe5a0585d6b Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Wed, 17 Apr 2024 08:21:48 -0400 Subject: [PATCH 002/714] [libc++] Add missing iterator requirement checks in the PSTL (#88127) Also add tests for those, and add a few missing requirements to testing iterators in the test suite. --- .../__algorithm/pstl_any_all_none_of.h | 6 +- libcxx/include/__algorithm/pstl_copy.h | 13 ++ libcxx/include/__algorithm/pstl_count.h | 5 + libcxx/include/__algorithm/pstl_equal.h | 9 + libcxx/include/__algorithm/pstl_fill.h | 6 +- libcxx/include/__algorithm/pstl_find.h | 6 +- libcxx/include/__algorithm/pstl_for_each.h | 4 +- libcxx/include/__algorithm/pstl_generate.h | 5 +- .../include/__algorithm/pstl_is_partitioned.h | 2 + libcxx/include/__algorithm/pstl_merge.h | 5 + libcxx/include/__algorithm/pstl_move.h | 5 + libcxx/include/__algorithm/pstl_replace.h | 13 ++ libcxx/include/__algorithm/pstl_rotate_copy.h | 5 + libcxx/include/__algorithm/pstl_sort.h | 3 + libcxx/include/__algorithm/pstl_stable_sort.h | 2 + libcxx/include/__algorithm/pstl_transform.h | 16 +- .../__iterator/cpp17_iterator_concepts.h | 38 ++-- libcxx/include/__numeric/pstl_reduce.h | 3 + .../include/__numeric/pstl_transform_reduce.h | 6 + .../cpp17_iterator_concepts.verify.cpp | 125 ++++++------ .../pstl.iterator-requirements.verify.cpp | 192 ++++++++++++++++++ libcxx/test/support/test_iterators.h | 16 +- 22 files changed, 383 insertions(+), 102 deletions(-) create mode 100644 libcxx/test/libcxx/algorithms/pstl.iterator-requirements.verify.cpp diff --git a/libcxx/include/__algorithm/pstl_any_all_none_of.h b/libcxx/include/__algorithm/pstl_any_all_none_of.h index 4b1e0e61b542..911a7e42b3fa 100644 --- a/libcxx/include/__algorithm/pstl_any_all_none_of.h +++ b/libcxx/include/__algorithm/pstl_any_all_none_of.h @@ -60,7 +60,7 @@ template , int> = 0> _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI bool any_of(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "any_of requires a ForwardIterator"); auto __res = std::__any_of(__policy, std::move(__first), std::move(__last), std::move(__pred)); if (!__res) std::__throw_bad_alloc(); @@ -99,7 +99,7 @@ template , int> = 0> _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI bool all_of(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "all_of requires a ForwardIterator"); auto __res = std::__all_of(__policy, std::move(__first), std::move(__last), std::move(__pred)); if (!__res) std::__throw_bad_alloc(); @@ -136,7 +136,7 @@ template , int> = 0> _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI bool none_of(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "none_of requires a ForwardIterator"); auto __res = std::__none_of(__policy, std::move(__first), std::move(__last), std::move(__pred)); if (!__res) std::__throw_bad_alloc(); diff --git a/libcxx/include/__algorithm/pstl_copy.h b/libcxx/include/__algorithm/pstl_copy.h index 1069dcec0e11..f35bb9713ef1 100644 --- a/libcxx/include/__algorithm/pstl_copy.h +++ b/libcxx/include/__algorithm/pstl_copy.h @@ -16,6 +16,7 @@ #include <__config> #include <__functional/identity.h> #include <__iterator/concepts.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_constant_evaluated.h> #include <__type_traits/is_execution_policy.h> @@ -67,6 +68,12 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI _ForwardOutIterator copy(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _ForwardOutIterator __result) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR( + _ForwardIterator, "copy(first, last, result) requires [first, last) to be ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR( + _ForwardOutIterator, "copy(first, last, result) requires result to be a ForwardIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( + _ForwardOutIterator, decltype(*__first), "copy(first, last, result) requires result to be an OutputIterator"); auto __res = std::__copy(__policy, std::move(__first), std::move(__last), std::move(__result)); if (!__res) std::__throw_bad_alloc(); @@ -106,6 +113,12 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI _ForwardOutIterator copy_n(_ExecutionPolicy&& __policy, _ForwardIterator __first, _Size __n, _ForwardOutIterator __result) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR( + _ForwardIterator, "copy_n(first, n, result) requires first to be a ForwardIterator"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR( + _ForwardOutIterator, "copy_n(first, n, result) requires result to be a ForwardIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( + _ForwardOutIterator, decltype(*__first), "copy_n(first, n, result) requires result to be an OutputIterator"); auto __res = std::__copy_n(__policy, std::move(__first), std::move(__n), std::move(__result)); if (!__res) std::__throw_bad_alloc(); diff --git a/libcxx/include/__algorithm/pstl_count.h b/libcxx/include/__algorithm/pstl_count.h index 2781f6bfd3c9..6ff57cac334e 100644 --- a/libcxx/include/__algorithm/pstl_count.h +++ b/libcxx/include/__algorithm/pstl_count.h @@ -17,6 +17,7 @@ #include <__atomic/atomic.h> #include <__config> #include <__functional/operations.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> #include <__numeric/pstl_transform_reduce.h> #include <__type_traits/enable_if.h> @@ -70,6 +71,8 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI __iter_diff_t<_ForwardIterator> count_if(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR( + _ForwardIterator, "count_if(first, last, pred) requires [first, last) to be ForwardIterators"); auto __res = std::__count_if(__policy, std::move(__first), std::move(__last), std::move(__pred)); if (!__res) std::__throw_bad_alloc(); @@ -106,6 +109,8 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI __iter_diff_t<_ForwardIterator> count(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR( + _ForwardIterator, "count(first, last, val) requires [first, last) to be ForwardIterators"); auto __res = std::__count(__policy, std::move(__first), std::move(__last), __value); if (!__res) std::__throw_bad_alloc(); diff --git a/libcxx/include/__algorithm/pstl_equal.h b/libcxx/include/__algorithm/pstl_equal.h index d235c0f4f419..0b38197d7f63 100644 --- a/libcxx/include/__algorithm/pstl_equal.h +++ b/libcxx/include/__algorithm/pstl_equal.h @@ -13,6 +13,7 @@ #include <__algorithm/pstl_frontend_dispatch.h> #include <__config> #include <__functional/operations.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> #include <__numeric/pstl_transform_reduce.h> #include <__utility/move.h> @@ -74,6 +75,8 @@ equal(_ExecutionPolicy&& __policy, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _Pred __pred) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "equal requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "equal requires ForwardIterators"); auto __res = std::__equal(__policy, std::move(__first1), std::move(__last1), std::move(__first2), std::move(__pred)); if (!__res) std::__throw_bad_alloc(); @@ -86,6 +89,8 @@ template >, int> = 0> _LIBCPP_HIDE_FROM_ABI bool equal(_ExecutionPolicy&& __policy, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "equal requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "equal requires ForwardIterators"); return std::equal(__policy, std::move(__first1), std::move(__last1), std::move(__first2), std::equal_to{}); } @@ -145,6 +150,8 @@ equal(_ExecutionPolicy&& __policy, _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Pred __pred) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "equal requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "equal requires ForwardIterators"); auto __res = std::__equal( __policy, std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), std::move(__pred)); if (!__res) @@ -162,6 +169,8 @@ equal(_ExecutionPolicy&& __policy, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _ForwardIterator2 __last2) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "equal requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "equal requires ForwardIterators"); return std::equal( __policy, std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), std::equal_to{}); } diff --git a/libcxx/include/__algorithm/pstl_fill.h b/libcxx/include/__algorithm/pstl_fill.h index 488b49a0feec..fd248506bc4b 100644 --- a/libcxx/include/__algorithm/pstl_fill.h +++ b/libcxx/include/__algorithm/pstl_fill.h @@ -43,7 +43,6 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI optional<__empty> __fill(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) noexcept { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); return std::__pstl_frontend_dispatch( _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_fill, _RawPolicy), [&](_ForwardIterator __g_first, _ForwardIterator __g_last, const _Tp& __g_value) { @@ -63,7 +62,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI void fill(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "fill requires ForwardIterators"); if (!std::__fill(__policy, std::move(__first), std::move(__last), __value)) std::__throw_bad_alloc(); } @@ -79,7 +78,6 @@ template , int> = 0> [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> __fill_n(_ExecutionPolicy&& __policy, _ForwardIterator&& __first, _SizeT&& __n, const _Tp& __value) noexcept { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); return std::__pstl_frontend_dispatch( _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_fill_n, _RawPolicy), [&](_ForwardIterator __g_first, _SizeT __g_n, const _Tp& __g_value) { @@ -102,7 +100,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI void fill_n(_ExecutionPolicy&& __policy, _ForwardIterator __first, _SizeT __n, const _Tp& __value) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "fill_n requires ForwardIterators"); if (!std::__fill_n(__policy, std::move(__first), std::move(__n), __value)) std::__throw_bad_alloc(); } diff --git a/libcxx/include/__algorithm/pstl_find.h b/libcxx/include/__algorithm/pstl_find.h index 5b694db68aea..3b30a7bc9b45 100644 --- a/libcxx/include/__algorithm/pstl_find.h +++ b/libcxx/include/__algorithm/pstl_find.h @@ -50,7 +50,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI _ForwardIterator find_if(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "find_if requires ForwardIterators"); auto __res = std::__find_if(__policy, std::move(__first), std::move(__last), std::move(__pred)); if (!__res) std::__throw_bad_alloc(); @@ -88,7 +88,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI _ForwardIterator find_if_not(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "find_if_not requires ForwardIterators"); auto __res = std::__find_if_not(__policy, std::move(__first), std::move(__last), std::move(__pred)); if (!__res) std::__throw_bad_alloc(); @@ -125,7 +125,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI _ForwardIterator find(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "find requires ForwardIterators"); auto __res = std::__find(__policy, std::move(__first), std::move(__last), __value); if (!__res) std::__throw_bad_alloc(); diff --git a/libcxx/include/__algorithm/pstl_for_each.h b/libcxx/include/__algorithm/pstl_for_each.h index bb7b5a61a6dc..a9ebed74a62f 100644 --- a/libcxx/include/__algorithm/pstl_for_each.h +++ b/libcxx/include/__algorithm/pstl_for_each.h @@ -53,7 +53,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI void for_each(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Function __func) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "for_each requires ForwardIterators"); if (!std::__for_each(__policy, std::move(__first), std::move(__last), std::move(__func))) std::__throw_bad_alloc(); } @@ -93,7 +93,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI void for_each_n(_ExecutionPolicy&& __policy, _ForwardIterator __first, _Size __size, _Function __func) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "for_each_n requires a ForwardIterator"); auto __res = std::__for_each_n(__policy, std::move(__first), std::move(__size), std::move(__func)); if (!__res) std::__throw_bad_alloc(); diff --git a/libcxx/include/__algorithm/pstl_generate.h b/libcxx/include/__algorithm/pstl_generate.h index 7133c6f4f4c6..886af290d7f2 100644 --- a/libcxx/include/__algorithm/pstl_generate.h +++ b/libcxx/include/__algorithm/pstl_generate.h @@ -42,7 +42,6 @@ template , int> = 0> [[nodiscard]] _LIBCPP_HIDE_FROM_ABI optional<__empty> __generate(_ExecutionPolicy&& __policy, _ForwardIterator&& __first, _ForwardIterator&& __last, _Generator&& __gen) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); return std::__pstl_frontend_dispatch( _LIBCPP_PSTL_CUSTOMIZATION_POINT(__pstl_generate, _RawPolicy), [&__policy](_ForwardIterator __g_first, _ForwardIterator __g_last, _Generator __g_gen) { @@ -63,7 +62,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI void generate(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Generator __gen) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "generate requires ForwardIterators"); if (!std::__generate(__policy, std::move(__first), std::move(__last), std::move(__gen))) std::__throw_bad_alloc(); } @@ -100,7 +99,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI void generate_n(_ExecutionPolicy&& __policy, _ForwardIterator __first, _Size __n, _Generator __gen) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "generate_n requires a ForwardIterator"); if (!std::__generate_n(__policy, std::move(__first), std::move(__n), std::move(__gen))) std::__throw_bad_alloc(); } diff --git a/libcxx/include/__algorithm/pstl_is_partitioned.h b/libcxx/include/__algorithm/pstl_is_partitioned.h index b65430212207..108bb1e43252 100644 --- a/libcxx/include/__algorithm/pstl_is_partitioned.h +++ b/libcxx/include/__algorithm/pstl_is_partitioned.h @@ -14,6 +14,7 @@ #include <__algorithm/pstl_find.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__config> +#include <__iterator/cpp17_iterator_concepts.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> @@ -62,6 +63,7 @@ template , int> = 0> _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI bool is_partitioned(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "is_partitioned requires ForwardIterators"); auto __res = std::__is_partitioned(__policy, std::move(__first), std::move(__last), std::move(__pred)); if (!__res) std::__throw_bad_alloc(); diff --git a/libcxx/include/__algorithm/pstl_merge.h b/libcxx/include/__algorithm/pstl_merge.h index 3d262db6bc0c..d03cd8c7fbd5 100644 --- a/libcxx/include/__algorithm/pstl_merge.h +++ b/libcxx/include/__algorithm/pstl_merge.h @@ -12,6 +12,7 @@ #include <__algorithm/pstl_backend.h> #include <__config> #include <__functional/operations.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> @@ -70,6 +71,10 @@ merge(_ExecutionPolicy&& __policy, _ForwardIterator2 __last2, _ForwardOutIterator __result, _Comp __comp = {}) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "merge requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "merge requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, decltype(*__first1), "merge requires an OutputIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, decltype(*__first2), "merge requires an OutputIterator"); auto __res = std::__merge( __policy, std::move(__first1), diff --git a/libcxx/include/__algorithm/pstl_move.h b/libcxx/include/__algorithm/pstl_move.h index d8441f1a6c2e..f4c8c1fbb2e8 100644 --- a/libcxx/include/__algorithm/pstl_move.h +++ b/libcxx/include/__algorithm/pstl_move.h @@ -15,6 +15,7 @@ #include <__algorithm/pstl_transform.h> #include <__config> #include <__functional/identity.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_constant_evaluated.h> @@ -69,6 +70,10 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI _ForwardOutIterator move(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last, _ForwardOutIterator __result) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "move requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "move requires an OutputIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( + _ForwardOutIterator, decltype(std::move(*__first)), "move requires an OutputIterator"); auto __res = std::__move(__policy, std::move(__first), std::move(__last), std::move(__result)); if (!__res) std::__throw_bad_alloc(); diff --git a/libcxx/include/__algorithm/pstl_replace.h b/libcxx/include/__algorithm/pstl_replace.h index b1caf3fd4ac0..73ac11cda26a 100644 --- a/libcxx/include/__algorithm/pstl_replace.h +++ b/libcxx/include/__algorithm/pstl_replace.h @@ -14,6 +14,7 @@ #include <__algorithm/pstl_frontend_dispatch.h> #include <__algorithm/pstl_transform.h> #include <__config> +#include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> #include <__type_traits/enable_if.h> #include <__type_traits/remove_cvref.h> @@ -74,6 +75,7 @@ replace_if(_ExecutionPolicy&& __policy, _ForwardIterator __last, _Pred __pred, const _Tp& __new_value) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "replace_if requires ForwardIterators"); auto __res = std::__replace_if(__policy, std::move(__first), std::move(__last), std::move(__pred), __new_value); if (!__res) std::__throw_bad_alloc(); @@ -121,6 +123,7 @@ replace(_ExecutionPolicy&& __policy, _ForwardIterator __last, const _Tp& __old_value, const _Tp& __new_value) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "replace requires ForwardIterators"); if (!std::__replace(__policy, std::move(__first), std::move(__last), __old_value, __new_value)) std::__throw_bad_alloc(); } @@ -177,6 +180,11 @@ _LIBCPP_HIDE_FROM_ABI void replace_copy_if( _ForwardOutIterator __result, _Pred __pred, const _Tp& __new_value) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "replace_copy_if requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "replace_copy_if requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( + _ForwardOutIterator, decltype(*__first), "replace_copy_if requires an OutputIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, const _Tp&, "replace_copy requires an OutputIterator"); if (!std::__replace_copy_if( __policy, std::move(__first), std::move(__last), std::move(__result), std::move(__pred), __new_value)) std::__throw_bad_alloc(); @@ -233,6 +241,11 @@ _LIBCPP_HIDE_FROM_ABI void replace_copy( _ForwardOutIterator __result, const _Tp& __old_value, const _Tp& __new_value) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "replace_copy requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "replace_copy requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( + _ForwardOutIterator, decltype(*__first), "replace_copy requires an OutputIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, const _Tp&, "replace_copy requires an OutputIterator"); if (!std::__replace_copy( __policy, std::move(__first), std::move(__last), std::move(__result), __old_value, __new_value)) std::__throw_bad_alloc(); diff --git a/libcxx/include/__algorithm/pstl_rotate_copy.h b/libcxx/include/__algorithm/pstl_rotate_copy.h index 346aab1d4a55..adab3958fe31 100644 --- a/libcxx/include/__algorithm/pstl_rotate_copy.h +++ b/libcxx/include/__algorithm/pstl_rotate_copy.h @@ -12,6 +12,7 @@ #include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_copy.h> #include <__algorithm/pstl_frontend_dispatch.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__type_traits/is_execution_policy.h> #include @@ -69,6 +70,10 @@ _LIBCPP_HIDE_FROM_ABI _ForwardOutIterator rotate_copy( _ForwardIterator __middle, _ForwardIterator __last, _ForwardOutIterator __result) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "rotate_copy requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "rotate_copy requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( + _ForwardOutIterator, decltype(*__first), "rotate_copy requires an OutputIterator"); auto __res = std::__rotate_copy(__policy, std::move(__first), std::move(__middle), std::move(__last), std::move(__result)); if (!__res) diff --git a/libcxx/include/__algorithm/pstl_sort.h b/libcxx/include/__algorithm/pstl_sort.h index a931f768111a..65bc794ca6f4 100644 --- a/libcxx/include/__algorithm/pstl_sort.h +++ b/libcxx/include/__algorithm/pstl_sort.h @@ -14,6 +14,7 @@ #include <__algorithm/pstl_stable_sort.h> #include <__config> #include <__functional/operations.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> #include <__utility/empty.h> @@ -60,6 +61,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI void sort(_ExecutionPolicy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) { + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(_RandomAccessIterator, "sort requires RandomAccessIterators"); if (!std::__sort(__policy, std::move(__first), std::move(__last), std::move(__comp))) std::__throw_bad_alloc(); } @@ -70,6 +72,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI void sort(_ExecutionPolicy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last) { + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(_RandomAccessIterator, "sort requires RandomAccessIterators"); std::sort(std::forward<_ExecutionPolicy>(__policy), std::move(__first), std::move(__last), less{}); } diff --git a/libcxx/include/__algorithm/pstl_stable_sort.h b/libcxx/include/__algorithm/pstl_stable_sort.h index 8ea0bb3f9a8d..79b94557e3dc 100644 --- a/libcxx/include/__algorithm/pstl_stable_sort.h +++ b/libcxx/include/__algorithm/pstl_stable_sort.h @@ -12,6 +12,7 @@ #include <__algorithm/pstl_backend.h> #include <__config> #include <__functional/operations.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> @@ -48,6 +49,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI void stable_sort( _ExecutionPolicy&& __policy, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp = {}) { + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(_RandomAccessIterator, "stable_sort requires RandomAccessIterators"); if (!std::__stable_sort(__policy, std::move(__first), std::move(__last), std::move(__comp))) std::__throw_bad_alloc(); } diff --git a/libcxx/include/__algorithm/pstl_transform.h b/libcxx/include/__algorithm/pstl_transform.h index f95938782fc3..a01a64a43cf1 100644 --- a/libcxx/include/__algorithm/pstl_transform.h +++ b/libcxx/include/__algorithm/pstl_transform.h @@ -58,9 +58,10 @@ _LIBCPP_HIDE_FROM_ABI _ForwardOutIterator transform( _ForwardIterator __last, _ForwardOutIterator __result, _UnaryOperation __op) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator); - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator); - _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, decltype(__op(*__first))); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "transform requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "transform requires an OutputIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( + _ForwardOutIterator, decltype(__op(*__first)), "transform requires an OutputIterator"); auto __res = std::__transform(__policy, std::move(__first), std::move(__last), std::move(__result), std::move(__op)); if (!__res) std::__throw_bad_alloc(); @@ -100,10 +101,11 @@ _LIBCPP_HIDE_FROM_ABI _ForwardOutIterator transform( _ForwardIterator2 __first2, _ForwardOutIterator __result, _BinaryOperation __op) { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1); - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2); - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator); - _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(_ForwardOutIterator, decltype(__op(*__first1, *__first2))); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "transform requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "transform requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardOutIterator, "transform requires an OutputIterator"); + _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR( + _ForwardOutIterator, decltype(__op(*__first1, *__first2)), "transform requires an OutputIterator"); auto __res = std::__transform( __policy, std::move(__first1), std::move(__last1), std::move(__first2), std::move(__result), std::move(__op)); if (!__res) diff --git a/libcxx/include/__iterator/cpp17_iterator_concepts.h b/libcxx/include/__iterator/cpp17_iterator_concepts.h index cdb561e68452..9d5a392582da 100644 --- a/libcxx/include/__iterator/cpp17_iterator_concepts.h +++ b/libcxx/include/__iterator/cpp17_iterator_concepts.h @@ -157,29 +157,31 @@ concept __cpp17_random_access_iterator = _LIBCPP_END_NAMESPACE_STD # ifndef _LIBCPP_DISABLE_ITERATOR_CHECKS -# define _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(iter_t) static_assert(::std::__cpp17_input_iterator); -# define _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(iter_t, write_t) \ - static_assert(::std::__cpp17_output_iterator); -# define _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(iter_t) static_assert(::std::__cpp17_forward_iterator); -# define _LIBCPP_REQUIRE_CPP17_BIDIRECTIONAL_ITERATOR(iter_t) \ - static_assert(::std::__cpp17_bidirectional_iterator); -# define _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(iter_t) \ - static_assert(::std::__cpp17_random_access_iterator); +# define _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(iter_t, message) \ + static_assert(::std::__cpp17_input_iterator, message) +# define _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(iter_t, write_t, message) \ + static_assert(::std::__cpp17_output_iterator, message) +# define _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(iter_t, message) \ + static_assert(::std::__cpp17_forward_iterator, message) +# define _LIBCPP_REQUIRE_CPP17_BIDIRECTIONAL_ITERATOR(iter_t, message) \ + static_assert(::std::__cpp17_bidirectional_iterator, message) +# define _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(iter_t, message) \ + static_assert(::std::__cpp17_random_access_iterator, message) # else -# define _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(iter_t) -# define _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(iter_t, write_t) -# define _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(iter_t) -# define _LIBCPP_REQUIRE_CPP17_BIDIRECTIONAL_ITERATOR(iter_t) -# define _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(iter_t) +# define _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(iter_t, message) static_assert(true) +# define _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(iter_t, write_t, message) static_assert(true) +# define _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(iter_t, message) static_assert(true) +# define _LIBCPP_REQUIRE_CPP17_BIDIRECTIONAL_ITERATOR(iter_t, message) static_assert(true) +# define _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(iter_t, message) static_assert(true) # endif #else // _LIBCPP_STD_VER >= 20 -# define _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(iter_t) -# define _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(iter_t, write_t) -# define _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(iter_t) -# define _LIBCPP_REQUIRE_CPP17_BIDIRECTIONAL_ITERATOR(iter_t) -# define _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(iter_t) +# define _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(iter_t, message) static_assert(true) +# define _LIBCPP_REQUIRE_CPP17_OUTPUT_ITERATOR(iter_t, write_t, message) static_assert(true) +# define _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(iter_t, message) static_assert(true) +# define _LIBCPP_REQUIRE_CPP17_BIDIRECTIONAL_ITERATOR(iter_t, message) static_assert(true) +# define _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(iter_t, message) static_assert(true) #endif // _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__numeric/pstl_reduce.h b/libcxx/include/__numeric/pstl_reduce.h index f9f666c2bb38..d678b9480070 100644 --- a/libcxx/include/__numeric/pstl_reduce.h +++ b/libcxx/include/__numeric/pstl_reduce.h @@ -12,6 +12,7 @@ #include <__algorithm/pstl_frontend_dispatch.h> #include <__config> #include <__functional/identity.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> #include <__numeric/pstl_transform_reduce.h> #include <__type_traits/is_execution_policy.h> @@ -66,6 +67,7 @@ reduce(_ExecutionPolicy&& __policy, _ForwardIterator __last, _Tp __init, _BinaryOperation __op = {}) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "reduce requires ForwardIterators"); auto __res = std::__reduce(__policy, std::move(__first), std::move(__last), std::move(__init), std::move(__op)); if (!__res) std::__throw_bad_alloc(); @@ -94,6 +96,7 @@ template , int> = 0> _LIBCPP_HIDE_FROM_ABI __iter_value_type<_ForwardIterator> reduce(_ExecutionPolicy&& __policy, _ForwardIterator __first, _ForwardIterator __last) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "reduce requires ForwardIterators"); auto __res = std::__reduce(__policy, std::move(__first), std::move(__last)); if (!__res) std::__throw_bad_alloc(); diff --git a/libcxx/include/__numeric/pstl_transform_reduce.h b/libcxx/include/__numeric/pstl_transform_reduce.h index 07ecf0d9956b..2d2621dc8dad 100644 --- a/libcxx/include/__numeric/pstl_transform_reduce.h +++ b/libcxx/include/__numeric/pstl_transform_reduce.h @@ -13,6 +13,7 @@ #include <__algorithm/pstl_frontend_dispatch.h> #include <__config> #include <__functional/operations.h> +#include <__iterator/cpp17_iterator_concepts.h> #include <__numeric/transform_reduce.h> #include <__type_traits/is_execution_policy.h> #include <__utility/move.h> @@ -72,6 +73,8 @@ _LIBCPP_HIDE_FROM_ABI _Tp transform_reduce( _Tp __init, _BinaryOperation1 __reduce, _BinaryOperation2 __transform) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "transform_reduce requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "transform_reduce requires ForwardIterators"); auto __res = std::__transform_reduce( __policy, std::move(__first1), @@ -99,6 +102,8 @@ _LIBCPP_HIDE_FROM_ABI _Tp transform_reduce( _ForwardIterator1 __last1, _ForwardIterator2 __first2, _Tp __init) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator1, "transform_reduce requires ForwardIterators"); + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator2, "transform_reduce requires ForwardIterators"); return std::transform_reduce(__policy, __first1, __last1, __first2, __init, plus{}, multiplies{}); } @@ -140,6 +145,7 @@ _LIBCPP_HIDE_FROM_ABI _Tp transform_reduce( _Tp __init, _BinaryOperation __reduce, _UnaryOperation __transform) { + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(_ForwardIterator, "transform_reduce requires ForwardIterators"); auto __res = std::__transform_reduce( __policy, std::move(__first), std::move(__last), std::move(__init), std::move(__reduce), std::move(__transform)); if (!__res) diff --git a/libcxx/test/libcxx/algorithms/cpp17_iterator_concepts.verify.cpp b/libcxx/test/libcxx/algorithms/cpp17_iterator_concepts.verify.cpp index 344543d5f19f..544a9744b790 100644 --- a/libcxx/test/libcxx/algorithms/cpp17_iterator_concepts.verify.cpp +++ b/libcxx/test/libcxx/algorithms/cpp17_iterator_concepts.verify.cpp @@ -16,29 +16,29 @@ #include struct missing_deref { - using difference_type = std::ptrdiff_t; + using difference_type = std::ptrdiff_t; using iterator_category = std::input_iterator_tag; - using value_type = int; - using reference = int&; + using value_type = int; + using reference = int&; missing_deref& operator++(); }; struct missing_preincrement { - using difference_type = std::ptrdiff_t; + using difference_type = std::ptrdiff_t; using iterator_category = std::input_iterator_tag; - using value_type = int; - using reference = int&; + using value_type = int; + using reference = int&; int& operator*(); }; template struct valid_iterator { - using difference_type = std::ptrdiff_t; + using difference_type = std::ptrdiff_t; using iterator_category = std::input_iterator_tag; - using value_type = int; - using reference = int&; + using value_type = int; + using reference = int&; int& operator*() const; Derived& operator++(); @@ -51,30 +51,30 @@ struct valid_iterator { }; struct not_move_constructible : valid_iterator { - not_move_constructible(const not_move_constructible&) = default; - not_move_constructible(not_move_constructible&&) = delete; - not_move_constructible& operator=(not_move_constructible&&) = default; + not_move_constructible(const not_move_constructible&) = default; + not_move_constructible(not_move_constructible&&) = delete; + not_move_constructible& operator=(not_move_constructible&&) = default; not_move_constructible& operator=(const not_move_constructible&) = default; }; struct not_copy_constructible : valid_iterator { - not_copy_constructible(const not_copy_constructible&) = delete; - not_copy_constructible(not_copy_constructible&&) = default; - not_copy_constructible& operator=(not_copy_constructible&&) = default; + not_copy_constructible(const not_copy_constructible&) = delete; + not_copy_constructible(not_copy_constructible&&) = default; + not_copy_constructible& operator=(not_copy_constructible&&) = default; not_copy_constructible& operator=(const not_copy_constructible&) = default; }; struct not_move_assignable : valid_iterator { - not_move_assignable(const not_move_assignable&) = default; - not_move_assignable(not_move_assignable&&) = default; - not_move_assignable& operator=(not_move_assignable&&) = delete; + not_move_assignable(const not_move_assignable&) = default; + not_move_assignable(not_move_assignable&&) = default; + not_move_assignable& operator=(not_move_assignable&&) = delete; not_move_assignable& operator=(const not_move_assignable&) = default; }; struct not_copy_assignable : valid_iterator { - not_copy_assignable(const not_copy_assignable&) = default; - not_copy_assignable(not_copy_assignable&&) = default; - not_copy_assignable& operator=(not_copy_assignable&&) = default; + not_copy_assignable(const not_copy_assignable&) = default; + not_copy_assignable(not_copy_assignable&&) = default; + not_copy_assignable& operator=(not_copy_assignable&&) = default; not_copy_assignable& operator=(const not_copy_assignable&) = delete; }; @@ -89,7 +89,6 @@ void check_iterator_requirements() { static_assert(std::__cpp17_iterator); // expected-error {{static assertion failed}} // expected-note@*:* {{cannot increment value of type 'missing_preincrement'}} - static_assert(std::__cpp17_iterator); // expected-error {{static assertion failed}} // expected-note@*:* {{because 'not_move_constructible' does not satisfy '__cpp17_move_constructible'}} @@ -115,11 +114,13 @@ bool operator==(not_unequality_comparable, not_unequality_comparable); bool operator!=(not_unequality_comparable, not_unequality_comparable) = delete; void check_input_iterator_requirements() { - _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(not_equality_comparable); // expected-error {{static assertion failed}} + // clang-format off + _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(not_equality_comparable, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{'__lhs == __rhs' would be invalid: overload resolution selected deleted operator '=='}} - _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(not_unequality_comparable); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_INPUT_ITERATOR(not_unequality_comparable, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{'__lhs != __rhs' would be invalid: overload resolution selected deleted operator '!='}} + // clang-format on } template @@ -138,9 +139,9 @@ struct postincrement_not_ref : valid_iterator {}; bool operator==(postincrement_not_ref, postincrement_not_ref); void check_forward_iterator_requirements() { - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(not_default_constructible); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(not_default_constructible, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because 'not_default_constructible' does not satisfy '__cpp17_default_constructible'}} - _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(postincrement_not_ref); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_FORWARD_ITERATOR(postincrement_not_ref, ""); // expected-error {{static assertion failed}} #ifndef _AIX // expected-note@*:* {{because type constraint 'convertible_to::Proxy, const postincrement_not_ref &>' was not satisfied}} #endif @@ -155,7 +156,6 @@ struct missing_postdecrement : valid_forward_iterator { }; struct not_returning_iter_reference : valid_forward_iterator { - struct Proxy { operator const not_returning_iter_reference&(); @@ -167,12 +167,14 @@ struct not_returning_iter_reference : valid_forward_iterator >' was not satisfied}} + // clang-format on } template @@ -246,7 +248,8 @@ struct missing_minus_const_iter_const_iter : valid_random_access_iterator { @@ -359,62 +362,64 @@ struct missing_const_const_greater_eq : valid_random_access_iterator __iter' would be invalid: overload resolution selected deleted operator '>'}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_mut_greater); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_mut_greater, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because 'std::as_const(__iter) > __iter' would be invalid: overload resolution selected deleted operator '>'}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_mut_const_greater); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_mut_const_greater, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because '__iter > std::as_const(__iter)' would be invalid: overload resolution selected deleted operator '>'}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_const_greater); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_const_greater, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because 'std::as_const(__iter) > std::as_const(__iter)' would be invalid: overload resolution selected deleted operator '>'}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_less_eq); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_less_eq, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because '__iter <= __iter' would be invalid: overload resolution selected deleted operator '<='}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_mut_less_eq); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_mut_less_eq, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because 'std::as_const(__iter) <= __iter' would be invalid: overload resolution selected deleted operator '<='}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_mut_const_less_eq); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_mut_const_less_eq, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because '__iter <= std::as_const(__iter)' would be invalid: overload resolution selected deleted operator '<='}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_const_less_eq); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_const_less_eq, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because 'std::as_const(__iter) <= std::as_const(__iter)' would be invalid: overload resolution selected deleted operator '<='}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_greater_eq); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_greater_eq, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because '__iter >= __iter' would be invalid: overload resolution selected deleted operator '>='}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_mut_greater_eq); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_mut_greater_eq, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because 'std::as_const(__iter) >= __iter' would be invalid: overload resolution selected deleted operator '>='}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_mut_const_greater_eq); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_mut_const_greater_eq, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because '__iter >= std::as_const(__iter)' would be invalid: overload resolution selected deleted operator '>='}} - _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_const_greater_eq); // expected-error {{static assertion failed}} + _LIBCPP_REQUIRE_CPP17_RANDOM_ACCESS_ITERATOR(missing_const_const_greater_eq, ""); // expected-error {{static assertion failed}} // expected-note@*:* {{because 'std::as_const(__iter) >= std::as_const(__iter)' would be invalid: overload resolution selected deleted operator '>='}} + // clang-format on } diff --git a/libcxx/test/libcxx/algorithms/pstl.iterator-requirements.verify.cpp b/libcxx/test/libcxx/algorithms/pstl.iterator-requirements.verify.cpp new file mode 100644 index 000000000000..98e3509752e1 --- /dev/null +++ b/libcxx/test/libcxx/algorithms/pstl.iterator-requirements.verify.cpp @@ -0,0 +1,192 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14 +// REQUIRES: stdlib=libc++ +// UNSUPPORTED: libcpp-has-no-incomplete-pstl + +// +// + +// Make sure that all PSTL algorithms contain checks for iterator requirements. +// This is not a requirement from the Standard, but we strive to catch misuse in +// the PSTL both because we can, and because iterator category mistakes in the +// PSTL can lead to subtle bugs. + +// Ignore spurious errors after the initial static_assert failure. +// ADDITIONAL_COMPILE_FLAGS: -Xclang -verify-ignore-unexpected=error + +// We only diagnose this in C++20 and above because we implement the checks with concepts. +// UNSUPPORTED: c++17 + +#include +#include +#include + +#include "test_iterators.h" + +using non_forward_iterator = cpp17_input_iterator; +struct non_output_iterator : forward_iterator { + constexpr int const& operator*() const; // prevent it from being an output iterator +}; + +void f(non_forward_iterator non_fwd, non_output_iterator non_output, std::execution::sequenced_policy pol) { + auto pred = [](auto&&...) -> bool { return true; }; + auto func = [](auto&&...) -> int { return 1; }; + int* it = nullptr; + int* out = nullptr; + std::size_t n = 0; + int val = 0; + + { + (void)std::any_of(pol, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: any_of}} + (void)std::all_of(pol, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: all_of}} + (void)std::none_of(pol, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: none_of}} + } + + { + (void)std::copy(pol, non_fwd, non_fwd, it); // expected-error@*:* {{static assertion failed: copy}} + (void)std::copy(pol, it, it, non_fwd); // expected-error@*:* {{static assertion failed: copy}} + (void)std::copy(pol, it, it, non_output); // expected-error@*:* {{static assertion failed: copy}} + } + { + (void)std::copy_n(pol, non_fwd, n, it); // expected-error@*:* {{static assertion failed: copy_n}} + (void)std::copy_n(pol, it, n, non_fwd); // expected-error@*:* {{static assertion failed: copy_n}} + (void)std::copy_n(pol, it, n, non_output); // expected-error@*:* {{static assertion failed: copy_n}} + } + + { + (void)std::count(pol, non_fwd, non_fwd, val); // expected-error@*:* {{static assertion failed: count}} + (void)std::count_if(pol, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: count_if}} + } + + { + (void)std::equal(pol, non_fwd, non_fwd, it); // expected-error@*:* {{static assertion failed: equal}} + (void)std::equal(pol, it, it, non_fwd); // expected-error@*:* {{static assertion failed: equal}} + (void)std::equal(pol, non_fwd, non_fwd, it, pred); // expected-error@*:* {{static assertion failed: equal}} + (void)std::equal(pol, it, it, non_fwd, pred); // expected-error@*:* {{static assertion failed: equal}} + + (void)std::equal(pol, non_fwd, non_fwd, it, it); // expected-error@*:* {{static assertion failed: equal}} + (void)std::equal(pol, it, it, non_fwd, non_fwd); // expected-error@*:* {{static assertion failed: equal}} + (void)std::equal(pol, non_fwd, non_fwd, it, it, pred); // expected-error@*:* {{static assertion failed: equal}} + (void)std::equal(pol, it, it, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: equal}} + } + + { + (void)std::fill(pol, non_fwd, non_fwd, val); // expected-error@*:* {{static assertion failed: fill}} + (void)std::fill_n(pol, non_fwd, n, val); // expected-error@*:* {{static assertion failed: fill_n}} + } + + { + (void)std::find(pol, non_fwd, non_fwd, val); // expected-error@*:* {{static assertion failed: find}} + (void)std::find_if(pol, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: find_if}} + (void)std::find_if_not(pol, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: find_if_not}} + } + + { + (void)std::for_each(pol, non_fwd, non_fwd, func); // expected-error@*:* {{static assertion failed: for_each}} + (void)std::for_each_n(pol, non_fwd, n, func); // expected-error@*:* {{static assertion failed: for_each_n}} + } + + { + (void)std::generate(pol, non_fwd, non_fwd, func); // expected-error@*:* {{static assertion failed: generate}} + (void)std::generate_n(pol, non_fwd, n, func); // expected-error@*:* {{static assertion failed: generate_n}} + } + + { + (void)std::is_partitioned( + pol, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: is_partitioned}} + } + + { + (void)std::merge(pol, non_fwd, non_fwd, it, it, out); // expected-error@*:* {{static assertion failed: merge}} + (void)std::merge(pol, it, it, non_fwd, non_fwd, out); // expected-error@*:* {{static assertion failed: merge}} + (void)std::merge(pol, it, it, it, it, non_output); // expected-error@*:* {{static assertion failed: merge}} + + (void)std::merge(pol, non_fwd, non_fwd, it, it, out, pred); // expected-error@*:* {{static assertion failed: merge}} + (void)std::merge(pol, it, it, non_fwd, non_fwd, out, pred); // expected-error@*:* {{static assertion failed: merge}} + (void)std::merge(pol, it, it, it, it, non_output, pred); // expected-error@*:* {{static assertion failed: merge}} + } + + { + (void)std::move(pol, non_fwd, non_fwd, out); // expected-error@*:* {{static assertion failed: move}} + (void)std::move(pol, it, it, non_fwd); // expected-error@*:* {{static assertion failed: move}} + (void)std::move(pol, it, it, non_output); // expected-error@*:* {{static assertion failed: move}} + } + + { + (void)std::replace_if( + pol, non_fwd, non_fwd, pred, val); // expected-error@*:* {{static assertion failed: replace_if}} + (void)std::replace(pol, non_fwd, non_fwd, val, val); // expected-error@*:* {{static assertion failed: replace}} + + (void)std::replace_copy_if( + pol, non_fwd, non_fwd, out, pred, val); // expected-error@*:* {{static assertion failed: replace_copy_if}} + (void)std::replace_copy_if( + pol, it, it, non_fwd, pred, val); // expected-error@*:* {{static assertion failed: replace_copy_if}} + (void)std::replace_copy_if( + pol, it, it, non_output, pred, val); // expected-error@*:* {{static assertion failed: replace_copy_if}} + + (void)std::replace_copy( + pol, non_fwd, non_fwd, out, val, val); // expected-error@*:* {{static assertion failed: replace_copy}} + (void)std::replace_copy( + pol, it, it, non_fwd, val, val); // expected-error@*:* {{static assertion failed: replace_copy}} + (void)std::replace_copy( + pol, it, it, non_output, val, val); // expected-error@*:* {{static assertion failed: replace_copy}} + } + + { + (void)std::rotate_copy( + pol, non_fwd, non_fwd, non_fwd, out); // expected-error@*:* {{static assertion failed: rotate_copy}} + (void)std::rotate_copy(pol, it, it, it, non_fwd); // expected-error@*:* {{static assertion failed: rotate_copy}} + (void)std::rotate_copy(pol, it, it, it, non_output); // expected-error@*:* {{static assertion failed: rotate_copy}} + } + + { + (void)std::sort(pol, non_fwd, non_fwd); // expected-error@*:* {{static assertion failed: sort}} + (void)std::sort(pol, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: sort}} + } + + { + (void)std::stable_sort(pol, non_fwd, non_fwd); // expected-error@*:* {{static assertion failed: stable_sort}} + (void)std::stable_sort(pol, non_fwd, non_fwd, pred); // expected-error@*:* {{static assertion failed: stable_sort}} + } + + { + (void)std::transform(pol, non_fwd, non_fwd, out, func); // expected-error@*:* {{static assertion failed: transform}} + (void)std::transform(pol, it, it, non_fwd, func); // expected-error@*:* {{static assertion failed: transform}} + (void)std::transform(pol, it, it, non_output, func); // expected-error@*:* {{static assertion failed: transform}} + + (void)std::transform( + pol, non_fwd, non_fwd, it, out, func); // expected-error@*:* {{static assertion failed: transform}} + (void)std::transform(pol, it, it, non_fwd, out, func); // expected-error@*:* {{static assertion failed: transform}} + (void)std::transform(pol, it, it, it, non_fwd, func); // expected-error@*:* {{static assertion failed: transform}} + (void)std::transform( + pol, it, it, it, non_output, func); // expected-error@*:* {{static assertion failed: transform}} + } + + { + (void)std::reduce(pol, non_fwd, non_fwd); // expected-error@*:* {{static assertion failed: reduce}} + (void)std::reduce(pol, non_fwd, non_fwd, val); // expected-error@*:* {{static assertion failed: reduce}} + (void)std::reduce(pol, non_fwd, non_fwd, val, func); // expected-error@*:* {{static assertion failed: reduce}} + } + + { + (void)std::transform_reduce( + pol, non_fwd, non_fwd, it, val); // expected-error@*:* {{static assertion failed: transform_reduce}} + (void)std::transform_reduce( + pol, it, it, non_fwd, val); // expected-error@*:* {{static assertion failed: transform_reduce}} + + (void)std::transform_reduce( + pol, non_fwd, non_fwd, it, val, func, func); // expected-error@*:* {{static assertion failed: transform_reduce}} + (void)std::transform_reduce( + pol, it, it, non_fwd, val, func, func); // expected-error@*:* {{static assertion failed: transform_reduce}} + + (void)std::transform_reduce( + pol, non_fwd, non_fwd, val, func, func); // expected-error@*:* {{static assertion failed: transform_reduce}} + } +} diff --git a/libcxx/test/support/test_iterators.h b/libcxx/test/support/test_iterators.h index 7ffb74990fa4..aa819ecd4733 100644 --- a/libcxx/test/support/test_iterators.h +++ b/libcxx/test/support/test_iterators.h @@ -1484,9 +1484,14 @@ public: return tmp; } - iterator_wrapper& operator+=(difference_type i) { + Derived& operator+=(difference_type i) { iter_ += i; - return *this; + return static_cast(*this); + } + + Derived& operator-=(difference_type i) { + iter_ -= i; + return static_cast(*this); } friend decltype(iter_ - iter_) operator-(const iterator_wrapper& lhs, const iterator_wrapper& rhs) { @@ -1503,8 +1508,15 @@ public: return iter; } + friend Derived operator+(difference_type i, Derived iter) { return iter + i; } + friend bool operator==(const iterator_wrapper& lhs, const iterator_wrapper& rhs) { return lhs.iter_ == rhs.iter_; } friend bool operator!=(const iterator_wrapper& lhs, const iterator_wrapper& rhs) { return lhs.iter_ != rhs.iter_; } + + friend bool operator>(const iterator_wrapper& lhs, const iterator_wrapper& rhs) { return lhs.iter_ > rhs.iter_; } + friend bool operator<(const iterator_wrapper& lhs, const iterator_wrapper& rhs) { return lhs.iter_ < rhs.iter_; } + friend bool operator<=(const iterator_wrapper& lhs, const iterator_wrapper& rhs) { return lhs.iter_ <= rhs.iter_; } + friend bool operator>=(const iterator_wrapper& lhs, const iterator_wrapper& rhs) { return lhs.iter_ >= rhs.iter_; } }; class iterator_error : std::runtime_error { -- GitLab From 6c7853080451a7f9ba612f57b41a076061b2a1a9 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 17 Apr 2024 13:18:55 +0100 Subject: [PATCH 003/714] [X86] vector-shuffle-combining-sse41.ll - add missing AVX1/2/512 check prefixes --- .../X86/vector-shuffle-combining-sse41.ll | 133 +++++++++++++++++- 1 file changed, 129 insertions(+), 4 deletions(-) diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-sse41.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-sse41.ll index 33851f56fe8d..8d213d257743 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-combining-sse41.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-sse41.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE -; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX -; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX -; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX +; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE +; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1 +; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2 +; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512 ; Combine tests involving SSE41 target shuffles (BLEND,INSERTPS,MOVZX) @@ -29,6 +29,30 @@ define <4 x i32> @combine_blend_of_permutes_v4i32(<2 x i64> %a0, <2 x i64> %a1) ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1] ; SSE-NEXT: pblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3],xmm2[4,5],xmm0[6,7] ; SSE-NEXT: retq +; +; AVX1-LABEL: combine_blend_of_permutes_v4i32: +; AVX1: # %bb.0: +; AVX1-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,3,0,1] +; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm1[2,3,0,1] +; AVX1-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] +; AVX1-NEXT: retq +; +; AVX2-LABEL: combine_blend_of_permutes_v4i32: +; AVX2: # %bb.0: +; AVX2-NEXT: vshufps {{.*#+}} xmm0 = xmm0[2,3,0,1] +; AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm1[2,3,0,1] +; AVX2-NEXT: vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] +; AVX2-NEXT: retq +; +; AVX512-LABEL: combine_blend_of_permutes_v4i32: +; AVX512: # %bb.0: +; AVX512-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 +; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 +; AVX512-NEXT: vpmovsxbd {{.*#+}} xmm2 = [2,19,0,17] +; AVX512-NEXT: vpermt2d %zmm1, %zmm2, %zmm0 +; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 +; AVX512-NEXT: vzeroupper +; AVX512-NEXT: retq %s0 = shufflevector <2 x i64> %a0, <2 x i64> undef, <2 x i32> %s1 = shufflevector <2 x i64> %a1, <2 x i64> undef, <2 x i32> %x0 = bitcast <2 x i64> %s0 to <4 x i32> @@ -71,6 +95,107 @@ define <16 x i8> @PR50049(ptr %p1, ptr %p2) { ; SSE-NEXT: pand %xmm5, %xmm1 ; SSE-NEXT: packuswb %xmm1, %xmm0 ; SSE-NEXT: retq +; +; AVX1-LABEL: PR50049: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [1,4,7,10,13,128,128,128,128,128,128,u,u,u,u,u] +; AVX1-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,128,128,128,0,3,6,9,12,15,u,u,u,u,u] +; AVX1-NEXT: vpshufb %xmm4, %xmm0, %xmm0 +; AVX1-NEXT: vpor %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa (%rsi), %xmm2 +; AVX1-NEXT: vmovdqa 16(%rsi), %xmm5 +; AVX1-NEXT: vmovdqa 32(%rsi), %xmm6 +; AVX1-NEXT: vpshufb %xmm3, %xmm6, %xmm3 +; AVX1-NEXT: vpshufb %xmm4, %xmm2, %xmm2 +; AVX1-NEXT: vpor %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [5,6,7,8,9,10,128,128,128,128,128,0,1,2,3,4] +; AVX1-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,128,128,128,128,2,5,8,11,14,128,128,128,128,128] +; AVX1-NEXT: vpshufb %xmm4, %xmm5, %xmm5 +; AVX1-NEXT: vpor %xmm5, %xmm2, %xmm2 +; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] +; AVX1-NEXT: vpshufb %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpshufb %xmm4, %xmm1, %xmm1 +; AVX1-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15] +; AVX1-NEXT: vpmullw %xmm5, %xmm1, %xmm1 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255] +; AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX1-NEXT: vpmullw %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: retq +; +; AVX2-LABEL: PR50049: +; AVX2: # %bb.0: +; AVX2-NEXT: vmovdqa (%rdi), %xmm0 +; AVX2-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX2-NEXT: vmovdqa 32(%rdi), %xmm2 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [1,4,7,10,13,128,128,128,128,128,128,u,u,u,u,u] +; AVX2-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,128,128,128,0,3,6,9,12,15,u,u,u,u,u] +; AVX2-NEXT: vpshufb %xmm4, %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vmovdqa (%rsi), %xmm2 +; AVX2-NEXT: vmovdqa 16(%rsi), %xmm5 +; AVX2-NEXT: vmovdqa 32(%rsi), %xmm6 +; AVX2-NEXT: vpshufb %xmm3, %xmm6, %xmm3 +; AVX2-NEXT: vpshufb %xmm4, %xmm2, %xmm2 +; AVX2-NEXT: vpor %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [5,6,7,8,9,10,128,128,128,128,128,0,1,2,3,4] +; AVX2-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; AVX2-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,128,128,128,128,2,5,8,11,14,128,128,128,128,128] +; AVX2-NEXT: vpshufb %xmm4, %xmm5, %xmm5 +; AVX2-NEXT: vpor %xmm5, %xmm2, %xmm2 +; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero +; AVX2-NEXT: vpshufb %xmm3, %xmm0, %xmm0 +; AVX2-NEXT: vpshufb %xmm4, %xmm1, %xmm1 +; AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX2-NEXT: vpmullw %ymm2, %ymm0, %ymm0 +; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 +; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0 +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq +; +; AVX512-LABEL: PR50049: +; AVX512: # %bb.0: +; AVX512-NEXT: vmovdqa (%rdi), %xmm0 +; AVX512-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX512-NEXT: vmovdqa 32(%rdi), %xmm2 +; AVX512-NEXT: vmovdqa {{.*#+}} xmm3 = [1,4,7,10,13,128,128,128,128,128,128,u,u,u,u,u] +; AVX512-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; AVX512-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,128,128,128,0,3,6,9,12,15,u,u,u,u,u] +; AVX512-NEXT: vpshufb %xmm4, %xmm0, %xmm0 +; AVX512-NEXT: vpor %xmm2, %xmm0, %xmm0 +; AVX512-NEXT: vmovdqa (%rsi), %xmm2 +; AVX512-NEXT: vmovdqa 16(%rsi), %xmm5 +; AVX512-NEXT: vmovdqa 32(%rsi), %xmm6 +; AVX512-NEXT: vpshufb %xmm3, %xmm6, %xmm3 +; AVX512-NEXT: vpshufb %xmm4, %xmm2, %xmm2 +; AVX512-NEXT: vpor %xmm3, %xmm2, %xmm2 +; AVX512-NEXT: vmovdqa {{.*#+}} xmm3 = [5,6,7,8,9,10,128,128,128,128,128,0,1,2,3,4] +; AVX512-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; AVX512-NEXT: vmovdqa {{.*#+}} xmm4 = [128,128,128,128,128,128,2,5,8,11,14,128,128,128,128,128] +; AVX512-NEXT: vpshufb %xmm4, %xmm5, %xmm5 +; AVX512-NEXT: vpor %xmm5, %xmm2, %xmm2 +; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero +; AVX512-NEXT: vpshufb %xmm3, %xmm0, %xmm0 +; AVX512-NEXT: vpshufb %xmm4, %xmm1, %xmm1 +; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0 +; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero +; AVX512-NEXT: vpmullw %ymm2, %ymm0, %ymm0 +; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero +; AVX512-NEXT: vpmovdb %zmm0, %xmm0 +; AVX512-NEXT: vzeroupper +; AVX512-NEXT: retq %x1 = load <48 x i8>, ptr %p1, align 16 %x2 = load <48 x i8>, ptr %p2, align 16 %s1 = shufflevector <48 x i8> %x1, <48 x i8> poison, <16 x i32> -- GitLab From 37b26bf48b9894ed0c13fd1aede23472660fb75e Mon Sep 17 00:00:00 2001 From: "Oleksandr \"Alex\" Zinenko" Date: Wed, 17 Apr 2024 14:24:51 +0200 Subject: [PATCH 004/714] [mlir] transform.apply_patterns support more config options (#88484) Greedy rewrite driver has options to control the number of rewrites applies. Expose those via the corresponding transform op. --- .../mlir/Dialect/Transform/IR/TransformOps.td | 5 +++- .../lib/Dialect/Transform/IR/TransformOps.cpp | 7 +++++ .../Transform/test-pattern-application.mlir | 30 +++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td b/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td index 21c9595860d4..fbac1ffb621f 100644 --- a/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td +++ b/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td @@ -331,7 +331,10 @@ def ApplyPatternsOp : TransformDialectOp<"apply_patterns", }]; let arguments = (ins - TransformHandleTypeInterface:$target, UnitAttr:$apply_cse); + TransformHandleTypeInterface:$target, + UnitAttr:$apply_cse, + DefaultValuedAttr(-1)">:$max_iterations, + DefaultValuedAttr(-1)">:$max_num_rewrites); let results = (outs); let regions = (region MaxSizedRegion<1>:$patterns); diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp index dc19022219e5..53f958caa0bd 100644 --- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp +++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp @@ -396,6 +396,13 @@ DiagnosedSilenceableFailure transform::ApplyPatternsOp::applyToOne( static_cast(rewriter.getListener()); FrozenRewritePatternSet frozenPatterns(std::move(patterns)); + config.maxIterations = getMaxIterations() == static_cast(-1) + ? GreedyRewriteConfig::kNoLimit + : getMaxIterations(); + config.maxNumRewrites = getMaxNumRewrites() == static_cast(-1) + ? GreedyRewriteConfig::kNoLimit + : getMaxNumRewrites(); + // Apply patterns and CSE repetitively until a fixpoint is reached. If no CSE // was requested, apply the greedy pattern rewrite only once. (The greedy // pattern rewrite driver already iterates to a fixpoint internally.) diff --git a/mlir/test/Dialect/Transform/test-pattern-application.mlir b/mlir/test/Dialect/Transform/test-pattern-application.mlir index fa8a555af921..f78b4b6f6798 100644 --- a/mlir/test/Dialect/Transform/test-pattern-application.mlir +++ b/mlir/test/Dialect/Transform/test-pattern-application.mlir @@ -26,6 +26,36 @@ module attributes {transform.with_named_sequence} { // ----- +// CHECK-LABEL: @limited_updates +func.func @limited_updates() { + "test.container"() ({ + // Only one is replaced. + // CHECK: "test.foo"() {replace_with_new_op = "test.foo"} + // CHECK: "test.foo"() : () + %0 = "test.foo"() {replace_with_new_op = "test.foo"} : () -> (i32) + %1 = "test.foo"() {replace_with_new_op = "test.foo"} : () -> (i32) + }) : () -> () + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op) { + // Pattern application will fail because of the upper limit, wrap in + // sequence to suppress the error message. + transform.sequence %arg0 : !transform.any_op failures(suppress) { + ^bb0(%arg1: !transform.any_op): + %0 = transform.structured.match ops{["test.container"]} in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = transform.structured.match ops{["test.foo"]} in %arg1 : (!transform.any_op) -> !transform.any_op + transform.apply_patterns to %0 { + transform.apply_patterns.transform.test_patterns + } {max_num_rewrites = 1} : !transform.any_op + } + transform.yield + } +} + +// ----- + func.func @replacement_op_not_found() { "test.container"() ({ // expected-note @below {{[0] replaced op}} -- GitLab From 4536ad47579d8d61f372ab85128bcfaed58a1256 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 17 Apr 2024 20:24:59 +0800 Subject: [PATCH 005/714] [RISCV] Fix clang-tidy warning about else after return. NFC --- llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp index 6e45f0c703ce..aab91adbb64b 100644 --- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp +++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp @@ -1551,7 +1551,9 @@ void RISCVInsertVSETVLI::doLocalPostpass(MachineBasicBlock &MBB) { ToDelete.push_back(&MI); // Leave NextMI unchanged continue; - } else if (canMutatePriorConfig(MI, *NextMI, Used, *MRI)) { + } + + if (canMutatePriorConfig(MI, *NextMI, Used, *MRI)) { if (!isVLPreservingConfig(*NextMI)) { MI.getOperand(0).setReg(NextMI->getOperand(0).getReg()); MI.getOperand(0).setIsDead(false); -- GitLab From 86a78284e7ce2ecc7a9283c7d141566a32371492 Mon Sep 17 00:00:00 2001 From: Quentin Dian Date: Wed, 17 Apr 2024 20:27:09 +0800 Subject: [PATCH 006/714] [TailDuplicator] Add maximum predecessors and successors to consider tail duplicating blocks (#78582) Fixes #78578. Duplicating a BB which has both multiple predecessors and successors will result in a complex CFG and also may cause huge amount of PHI nodes. See https://github.com/llvm/llvm-project/issues/78578#issuecomment-1962363580 for a detailed description of the limit. --- llvm/lib/CodeGen/TailDuplicator.cpp | 20 ++ .../CodeGen/X86/tail-dup-pred-succ-size.mir | 260 ++++++++++++++++++ 2 files changed, 280 insertions(+) create mode 100644 llvm/test/CodeGen/X86/tail-dup-pred-succ-size.mir diff --git a/llvm/lib/CodeGen/TailDuplicator.cpp b/llvm/lib/CodeGen/TailDuplicator.cpp index 5ed67bd0a121..f5dd21cb9270 100644 --- a/llvm/lib/CodeGen/TailDuplicator.cpp +++ b/llvm/lib/CodeGen/TailDuplicator.cpp @@ -68,6 +68,18 @@ static cl::opt TailDupIndirectBranchSize( "end with indirect branches."), cl::init(20), cl::Hidden); +static cl::opt + TailDupPredSize("tail-dup-pred-size", + cl::desc("Maximum predecessors (maximum successors at the " + "same time) to consider tail duplicating blocks."), + cl::init(16), cl::Hidden); + +static cl::opt + TailDupSuccSize("tail-dup-succ-size", + cl::desc("Maximum successors (maximum predecessors at the " + "same time) to consider tail duplicating blocks."), + cl::init(16), cl::Hidden); + static cl::opt TailDupVerify("tail-dup-verify", cl::desc("Verify sanity of PHI instructions during taildup"), @@ -565,6 +577,14 @@ bool TailDuplicator::shouldTailDuplicate(bool IsSimple, if (TailBB.isSuccessor(&TailBB)) return false; + // Duplicating a BB which has both multiple predecessors and successors will + // result in a complex CFG and also may cause huge amount of PHI nodes. If we + // want to remove this limitation, we have to address + // https://github.com/llvm/llvm-project/issues/78578. + if (TailBB.pred_size() > TailDupPredSize && + TailBB.succ_size() > TailDupSuccSize) + return false; + // Set the limit on the cost to duplicate. When optimizing for size, // duplicate only one, because one branch instruction can be eliminated to // compensate for the duplication. diff --git a/llvm/test/CodeGen/X86/tail-dup-pred-succ-size.mir b/llvm/test/CodeGen/X86/tail-dup-pred-succ-size.mir new file mode 100644 index 000000000000..67f8cc72e0d7 --- /dev/null +++ b/llvm/test/CodeGen/X86/tail-dup-pred-succ-size.mir @@ -0,0 +1,260 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 +# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=early-tailduplication -tail-dup-pred-size=3 -tail-dup-succ-size=3 %s -o - | FileCheck %s -check-prefix=LIMIT +# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=early-tailduplication -tail-dup-pred-size=4 -tail-dup-succ-size=4 %s -o - | FileCheck %s -check-prefix=NOLIMIT + +--- +name: foo +tracksRegLiveness: true +jumpTable: + kind: block-address + entries: + - id: 0 + blocks: [ '%bb.2', '%bb.3', '%bb.4', '%bb.5' ] + - id: 1 + blocks: [ '%bb.9', '%bb.10', '%bb.11', '%bb.12' ] +body: | + ; LIMIT-LABEL: name: foo + ; LIMIT: bb.0: + ; LIMIT-NEXT: successors: %bb.2(0x20000000), %bb.3(0x20000000), %bb.4(0x20000000), %bb.5(0x20000000) + ; LIMIT-NEXT: liveins: $rdi, $esi + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $esi + ; LIMIT-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi + ; LIMIT-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 1, implicit-def dead $eflags + ; LIMIT-NEXT: [[AND32ri:%[0-9]+]]:gr32 = AND32ri [[SHR32ri]], 7, implicit-def dead $eflags + ; LIMIT-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gr64_nosp = SUBREG_TO_REG 0, killed [[AND32ri]], %subreg.sub_32bit + ; LIMIT-NEXT: JMP64m $noreg, 8, [[SUBREG_TO_REG]], %jump-table.0, $noreg + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.2: + ; LIMIT-NEXT: successors: %bb.7(0x80000000) + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; LIMIT-NEXT: JMP_1 %bb.7 + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.3: + ; LIMIT-NEXT: successors: %bb.7(0x80000000) + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; LIMIT-NEXT: [[SHR32ri1:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm1]], 1, implicit-def dead $eflags + ; LIMIT-NEXT: JMP_1 %bb.7 + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.4: + ; LIMIT-NEXT: successors: %bb.7(0x80000000) + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[MOV32rm2:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; LIMIT-NEXT: [[SHR32ri2:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm2]], 2, implicit-def dead $eflags + ; LIMIT-NEXT: JMP_1 %bb.7 + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.5: + ; LIMIT-NEXT: successors: %bb.7(0x80000000) + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[MOV32rm3:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; LIMIT-NEXT: [[SHR32ri3:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm3]], 3, implicit-def dead $eflags + ; LIMIT-NEXT: JMP_1 %bb.7 + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.6: + ; LIMIT-NEXT: successors: + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.7: + ; LIMIT-NEXT: successors: %bb.9(0x20000000), %bb.10(0x20000000), %bb.11(0x20000000), %bb.12(0x20000000) + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[PHI:%[0-9]+]]:gr32 = PHI [[SHR32ri3]], %bb.5, [[SHR32ri2]], %bb.4, [[SHR32ri1]], %bb.3, [[MOV32rm]], %bb.2 + ; LIMIT-NEXT: [[SHR32ri4:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 2, implicit-def dead $eflags + ; LIMIT-NEXT: [[AND32ri1:%[0-9]+]]:gr32 = AND32ri [[SHR32ri4]], 7, implicit-def dead $eflags + ; LIMIT-NEXT: [[SUBREG_TO_REG1:%[0-9]+]]:gr64_nosp = SUBREG_TO_REG 0, killed [[AND32ri1]], %subreg.sub_32bit + ; LIMIT-NEXT: JMP64m $noreg, 8, [[SUBREG_TO_REG1]], %jump-table.1, $noreg + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.9: + ; LIMIT-NEXT: successors: %bb.13(0x80000000) + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[MOV32rm4:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; LIMIT-NEXT: JMP_1 %bb.13 + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.10: + ; LIMIT-NEXT: successors: %bb.13(0x80000000) + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[MOV32rm5:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; LIMIT-NEXT: [[SHR32ri5:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm5]], 1, implicit-def dead $eflags + ; LIMIT-NEXT: JMP_1 %bb.13 + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.11: + ; LIMIT-NEXT: successors: %bb.13(0x80000000) + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[MOV32rm6:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; LIMIT-NEXT: [[SHR32ri6:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm6]], 2, implicit-def dead $eflags + ; LIMIT-NEXT: JMP_1 %bb.13 + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.12: + ; LIMIT-NEXT: successors: %bb.13(0x80000000) + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: [[MOV32rm7:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; LIMIT-NEXT: [[SHR32ri7:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm7]], 6, implicit-def dead $eflags + ; LIMIT-NEXT: {{ $}} + ; LIMIT-NEXT: bb.13: + ; LIMIT-NEXT: [[PHI1:%[0-9]+]]:gr32 = PHI [[SHR32ri7]], %bb.12, [[SHR32ri6]], %bb.11, [[SHR32ri5]], %bb.10, [[MOV32rm4]], %bb.9 + ; LIMIT-NEXT: [[OR32rr:%[0-9]+]]:gr32 = OR32rr [[PHI1]], [[PHI]], implicit-def dead $eflags + ; LIMIT-NEXT: $eax = COPY [[OR32rr]] + ; LIMIT-NEXT: RET 0, $eax + ; + ; NOLIMIT-LABEL: name: foo + ; NOLIMIT: bb.0: + ; NOLIMIT-NEXT: successors: %bb.2(0x20000000), %bb.3(0x20000000), %bb.4(0x20000000), %bb.5(0x20000000) + ; NOLIMIT-NEXT: liveins: $rdi, $esi + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $esi + ; NOLIMIT-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdi + ; NOLIMIT-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 1, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[AND32ri:%[0-9]+]]:gr32 = AND32ri [[SHR32ri]], 7, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gr64_nosp = SUBREG_TO_REG 0, killed [[AND32ri]], %subreg.sub_32bit + ; NOLIMIT-NEXT: JMP64m $noreg, 8, [[SUBREG_TO_REG]], %jump-table.0, $noreg + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.2: + ; NOLIMIT-NEXT: successors: %bb.9(0x20000000), %bb.10(0x20000000), %bb.11(0x20000000), %bb.12(0x20000000) + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; NOLIMIT-NEXT: [[SHR32ri1:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 2, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[AND32ri1:%[0-9]+]]:gr32 = AND32ri [[SHR32ri1]], 7, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[SUBREG_TO_REG1:%[0-9]+]]:gr64_nosp = SUBREG_TO_REG 0, [[AND32ri1]], %subreg.sub_32bit + ; NOLIMIT-NEXT: JMP64m $noreg, 8, [[SUBREG_TO_REG1]], %jump-table.1, $noreg + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.3: + ; NOLIMIT-NEXT: successors: %bb.9(0x20000000), %bb.10(0x20000000), %bb.11(0x20000000), %bb.12(0x20000000) + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; NOLIMIT-NEXT: [[SHR32ri2:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm1]], 1, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[SHR32ri3:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 2, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[AND32ri2:%[0-9]+]]:gr32 = AND32ri [[SHR32ri3]], 7, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[SUBREG_TO_REG2:%[0-9]+]]:gr64_nosp = SUBREG_TO_REG 0, [[AND32ri2]], %subreg.sub_32bit + ; NOLIMIT-NEXT: JMP64m $noreg, 8, [[SUBREG_TO_REG2]], %jump-table.1, $noreg + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.4: + ; NOLIMIT-NEXT: successors: %bb.9(0x20000000), %bb.10(0x20000000), %bb.11(0x20000000), %bb.12(0x20000000) + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: [[MOV32rm2:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; NOLIMIT-NEXT: [[SHR32ri4:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm2]], 2, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[SHR32ri5:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 2, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[AND32ri3:%[0-9]+]]:gr32 = AND32ri [[SHR32ri5]], 7, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[SUBREG_TO_REG3:%[0-9]+]]:gr64_nosp = SUBREG_TO_REG 0, [[AND32ri3]], %subreg.sub_32bit + ; NOLIMIT-NEXT: JMP64m $noreg, 8, [[SUBREG_TO_REG3]], %jump-table.1, $noreg + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.5: + ; NOLIMIT-NEXT: successors: %bb.9(0x20000000), %bb.10(0x20000000), %bb.11(0x20000000), %bb.12(0x20000000) + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: [[MOV32rm3:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; NOLIMIT-NEXT: [[SHR32ri6:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm3]], 3, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[SHR32ri7:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 2, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[AND32ri4:%[0-9]+]]:gr32 = AND32ri [[SHR32ri7]], 7, implicit-def dead $eflags + ; NOLIMIT-NEXT: [[SUBREG_TO_REG4:%[0-9]+]]:gr64_nosp = SUBREG_TO_REG 0, [[AND32ri4]], %subreg.sub_32bit + ; NOLIMIT-NEXT: JMP64m $noreg, 8, [[SUBREG_TO_REG4]], %jump-table.1, $noreg + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.6: + ; NOLIMIT-NEXT: successors: + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.9: + ; NOLIMIT-NEXT: successors: %bb.13(0x80000000) + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: [[PHI:%[0-9]+]]:gr32 = PHI [[MOV32rm]], %bb.2, [[SHR32ri2]], %bb.3, [[SHR32ri4]], %bb.4, [[SHR32ri6]], %bb.5 + ; NOLIMIT-NEXT: [[MOV32rm4:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; NOLIMIT-NEXT: JMP_1 %bb.13 + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.10: + ; NOLIMIT-NEXT: successors: %bb.13(0x80000000) + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: [[PHI1:%[0-9]+]]:gr32 = PHI [[MOV32rm]], %bb.2, [[SHR32ri2]], %bb.3, [[SHR32ri4]], %bb.4, [[SHR32ri6]], %bb.5 + ; NOLIMIT-NEXT: [[MOV32rm5:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; NOLIMIT-NEXT: [[SHR32ri8:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm5]], 1, implicit-def dead $eflags + ; NOLIMIT-NEXT: JMP_1 %bb.13 + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.11: + ; NOLIMIT-NEXT: successors: %bb.13(0x80000000) + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: [[PHI2:%[0-9]+]]:gr32 = PHI [[MOV32rm]], %bb.2, [[SHR32ri2]], %bb.3, [[SHR32ri4]], %bb.4, [[SHR32ri6]], %bb.5 + ; NOLIMIT-NEXT: [[MOV32rm6:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; NOLIMIT-NEXT: [[SHR32ri9:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm6]], 2, implicit-def dead $eflags + ; NOLIMIT-NEXT: JMP_1 %bb.13 + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.12: + ; NOLIMIT-NEXT: successors: %bb.13(0x80000000) + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: [[PHI3:%[0-9]+]]:gr32 = PHI [[MOV32rm]], %bb.2, [[SHR32ri2]], %bb.3, [[SHR32ri4]], %bb.4, [[SHR32ri6]], %bb.5 + ; NOLIMIT-NEXT: [[MOV32rm7:%[0-9]+]]:gr32 = MOV32rm [[COPY1]], 1, $noreg, 0, $noreg + ; NOLIMIT-NEXT: [[SHR32ri10:%[0-9]+]]:gr32 = SHR32ri [[MOV32rm7]], 6, implicit-def dead $eflags + ; NOLIMIT-NEXT: {{ $}} + ; NOLIMIT-NEXT: bb.13: + ; NOLIMIT-NEXT: [[PHI4:%[0-9]+]]:gr32 = PHI [[PHI]], %bb.9, [[PHI1]], %bb.10, [[PHI2]], %bb.11, [[PHI3]], %bb.12 + ; NOLIMIT-NEXT: [[PHI5:%[0-9]+]]:gr32 = PHI [[SHR32ri10]], %bb.12, [[SHR32ri9]], %bb.11, [[SHR32ri8]], %bb.10, [[MOV32rm4]], %bb.9 + ; NOLIMIT-NEXT: [[OR32rr:%[0-9]+]]:gr32 = OR32rr [[PHI5]], [[PHI4]], implicit-def dead $eflags + ; NOLIMIT-NEXT: $eax = COPY [[OR32rr]] + ; NOLIMIT-NEXT: RET 0, $eax + bb.0: + liveins: $rdi, $esi + + %11:gr32 = COPY $esi + %10:gr64 = COPY $rdi + %13:gr32 = SHR32ri %11, 1, implicit-def dead $eflags + %14:gr32 = AND32ri %13, 7, implicit-def dead $eflags + %12:gr64_nosp = SUBREG_TO_REG 0, killed %14, %subreg.sub_32bit + + bb.1: + successors: %bb.2, %bb.3, %bb.4, %bb.5 + + JMP64m $noreg, 8, %12, %jump-table.0, $noreg + + bb.2: + %0:gr32 = MOV32rm %10, 1, $noreg, 0, $noreg + JMP_1 %bb.7 + + bb.3: + %17:gr32 = MOV32rm %10, 1, $noreg, 0, $noreg + %1:gr32 = SHR32ri %17, 1, implicit-def dead $eflags + JMP_1 %bb.7 + + bb.4: + %16:gr32 = MOV32rm %10, 1, $noreg, 0, $noreg + %2:gr32 = SHR32ri %16, 2, implicit-def dead $eflags + JMP_1 %bb.7 + + bb.5: + %15:gr32 = MOV32rm %10, 1, $noreg, 0, $noreg + %3:gr32 = SHR32ri %15, 3, implicit-def dead $eflags + JMP_1 %bb.7 + + bb.6: + successors: + + bb.7: + %4:gr32 = PHI %3, %bb.5, %2, %bb.4, %1, %bb.3, %0, %bb.2 + %19:gr32 = SHR32ri %11, 2, implicit-def dead $eflags + %20:gr32 = AND32ri %19, 7, implicit-def dead $eflags + %18:gr64_nosp = SUBREG_TO_REG 0, killed %20, %subreg.sub_32bit + + bb.8: + successors: %bb.9, %bb.10, %bb.11, %bb.12 + + JMP64m $noreg, 8, %18, %jump-table.1, $noreg + + bb.9: + %5:gr32 = MOV32rm %10, 1, $noreg, 0, $noreg + JMP_1 %bb.13 + + bb.10: + %23:gr32 = MOV32rm %10, 1, $noreg, 0, $noreg + %6:gr32 = SHR32ri %23, 1, implicit-def dead $eflags + JMP_1 %bb.13 + + bb.11: + %22:gr32 = MOV32rm %10, 1, $noreg, 0, $noreg + %7:gr32 = SHR32ri %22, 2, implicit-def dead $eflags + JMP_1 %bb.13 + + bb.12: + %21:gr32 = MOV32rm %10, 1, $noreg, 0, $noreg + %8:gr32 = SHR32ri %21, 6, implicit-def dead $eflags + + bb.13: + %9:gr32 = PHI %8, %bb.12, %7, %bb.11, %6, %bb.10, %5, %bb.9 + %24:gr32 = OR32rr %9, %4, implicit-def dead $eflags + $eax = COPY %24 + RET 0, $eax + +... -- GitLab From 915c84b1480bb3c6d2e44ca83822d2c2304b763a Mon Sep 17 00:00:00 2001 From: Pavel Labath Date: Wed, 17 Apr 2024 14:33:18 +0200 Subject: [PATCH 007/714] [lldb] Fix evaluation of expressions with static initializers (#89063) After 281d71604f418eb952e967d9dc4b26241b7f96a, llvm generates 32-bit relocations, which overflow when we load these objects into high memory. Interestingly, setting the code model to "large" does not help here (perhaps it is the default?). I'm not completely sure that this is the right thing to do, but it doesn't seem to cause any ill effects. I'll follow up with the author of that patch about the expected behavior here. --- lldb/source/Expression/IRExecutionUnit.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lldb/source/Expression/IRExecutionUnit.cpp b/lldb/source/Expression/IRExecutionUnit.cpp index cb9bee8733e1..7ad0e5ff22b2 100644 --- a/lldb/source/Expression/IRExecutionUnit.cpp +++ b/lldb/source/Expression/IRExecutionUnit.cpp @@ -13,6 +13,7 @@ #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" +#include "llvm/Support/CodeGen.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/raw_ostream.h" @@ -279,10 +280,13 @@ void IRExecutionUnit::GetRunnableInfo(Status &error, lldb::addr_t &func_addr, llvm::EngineBuilder builder(std::move(m_module_up)); llvm::Triple triple(m_module->getTargetTriple()); + // PIC needed for ELF to avoid generating 32-bit relocations (which overflow + // if the object is loaded into high memory). + bool want_pic = triple.isOSBinFormatMachO() || triple.isOSBinFormatELF(); + builder.setEngineKind(llvm::EngineKind::JIT) .setErrorStr(&error_string) - .setRelocationModel(triple.isOSBinFormatMachO() ? llvm::Reloc::PIC_ - : llvm::Reloc::Static) + .setRelocationModel(want_pic ? llvm::Reloc::PIC_ : llvm::Reloc::Static) .setMCJITMemoryManager(std::make_unique(*this)) .setOptLevel(llvm::CodeGenOptLevel::Less); -- GitLab From 79726ef5d20f3bf6510b67d7dd3378b8a41db768 Mon Sep 17 00:00:00 2001 From: "Kevin P. Neal" Date: Wed, 17 Apr 2024 08:34:25 -0400 Subject: [PATCH 008/714] [VP] Correct lowering of predicated fma and faddmul to avoid strictfp. (#85272) Correct missing cases in a switch that result in @llvm.vp.fma.v4f32 getting lowered to a constrained fma intrinsic. Vector predicated lowering to contrained intrinsics is not supported currently, and there's no consensus on the path forward. We certainly shouldn't be introducing constrained intrinsics into a function that isn't strictfp. Problem found with D146845. --- llvm/include/llvm/IR/Intrinsics.h | 4 + llvm/lib/CodeGen/ExpandVectorPredication.cpp | 12 +- llvm/lib/IR/Function.cpp | 22 ++- .../Generic/expand-vp-fp-intrinsics.ll | 176 ++++++++++++++++++ 4 files changed, 203 insertions(+), 11 deletions(-) create mode 100644 llvm/test/CodeGen/Generic/expand-vp-fp-intrinsics.ll diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h index 0dfe9f029f9b..92eae344ce72 100644 --- a/llvm/include/llvm/IR/Intrinsics.h +++ b/llvm/include/llvm/IR/Intrinsics.h @@ -105,6 +105,10 @@ namespace Intrinsic { /// Map a MS builtin name to an intrinsic ID. ID getIntrinsicForMSBuiltin(const char *Prefix, StringRef BuiltinName); + /// Returns true if the intrinsic ID is for one of the "Constrained + /// Floating-Point Intrinsics". + bool isConstrainedFPIntrinsic(ID QID); + /// This is a type descriptor which explains the type requirements of an /// intrinsic. This is returned by getIntrinsicInfoTableEntries. struct IITDescriptor { diff --git a/llvm/lib/CodeGen/ExpandVectorPredication.cpp b/llvm/lib/CodeGen/ExpandVectorPredication.cpp index 0fe4cfefdb16..8e623c85b737 100644 --- a/llvm/lib/CodeGen/ExpandVectorPredication.cpp +++ b/llvm/lib/CodeGen/ExpandVectorPredication.cpp @@ -340,6 +340,8 @@ Value *CachingVPExpander::expandPredicationToFPCall( replaceOperation(*NewOp, VPI); return NewOp; } + case Intrinsic::fma: + case Intrinsic::fmuladd: case Intrinsic::experimental_constrained_fma: case Intrinsic::experimental_constrained_fmuladd: { Value *Op0 = VPI.getOperand(0); @@ -347,8 +349,12 @@ Value *CachingVPExpander::expandPredicationToFPCall( Value *Op2 = VPI.getOperand(2); Function *Fn = Intrinsic::getDeclaration( VPI.getModule(), UnpredicatedIntrinsicID, {VPI.getType()}); - Value *NewOp = - Builder.CreateConstrainedFPCall(Fn, {Op0, Op1, Op2}, VPI.getName()); + Value *NewOp; + if (Intrinsic::isConstrainedFPIntrinsic(UnpredicatedIntrinsicID)) + NewOp = + Builder.CreateConstrainedFPCall(Fn, {Op0, Op1, Op2}, VPI.getName()); + else + NewOp = Builder.CreateCall(Fn, {Op0, Op1, Op2}, VPI.getName()); replaceOperation(*NewOp, VPI); return NewOp; } @@ -731,6 +737,8 @@ Value *CachingVPExpander::expandPredication(VPIntrinsic &VPI) { case Intrinsic::vp_minnum: case Intrinsic::vp_maximum: case Intrinsic::vp_minimum: + case Intrinsic::vp_fma: + case Intrinsic::vp_fmuladd: return expandPredicationToFPCall(Builder, VPI, VPI.getFunctionalIntrinsicID().value()); case Intrinsic::vp_load: diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp index 96953ac49c19..818a167560de 100644 --- a/llvm/lib/IR/Function.cpp +++ b/llvm/lib/IR/Function.cpp @@ -499,15 +499,7 @@ static MutableArrayRef makeArgArray(Argument *Args, size_t Count) { } bool Function::isConstrainedFPIntrinsic() const { - switch (getIntrinsicID()) { -#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ - case Intrinsic::INTRINSIC: -#include "llvm/IR/ConstrainedOps.def" - return true; -#undef INSTRUCTION - default: - return false; - } + return Intrinsic::isConstrainedFPIntrinsic(getIntrinsicID()); } void Function::clearArguments() { @@ -1486,6 +1478,18 @@ Function *Intrinsic::getDeclaration(Module *M, ID id, ArrayRef Tys) { #include "llvm/IR/IntrinsicImpl.inc" #undef GET_LLVM_INTRINSIC_FOR_MS_BUILTIN +bool Intrinsic::isConstrainedFPIntrinsic(ID QID) { + switch (QID) { +#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ + case Intrinsic::INTRINSIC: +#include "llvm/IR/ConstrainedOps.def" + return true; +#undef INSTRUCTION + default: + return false; + } +} + using DeferredIntrinsicMatchPair = std::pair>; diff --git a/llvm/test/CodeGen/Generic/expand-vp-fp-intrinsics.ll b/llvm/test/CodeGen/Generic/expand-vp-fp-intrinsics.ll new file mode 100644 index 000000000000..bc89ddea6b85 --- /dev/null +++ b/llvm/test/CodeGen/Generic/expand-vp-fp-intrinsics.ll @@ -0,0 +1,176 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -expandvp -S < %s | FileCheck %s + +define void @vp_fadd_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind { +; CHECK-LABEL: define void @vp_fadd_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i32 [[VP:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[RES1:%.*]] = fadd <4 x float> [[A0]], [[A1]] +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.fadd.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x i1> , i32 %vp) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.fadd.v4f32(<4 x float>, <4 x float>, <4 x i1>, i32) + +define void @vp_fsub_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind { +; CHECK-LABEL: define void @vp_fsub_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i32 [[VP:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES1:%.*]] = fsub <4 x float> [[A0]], [[A1]] +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.fsub.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x i1> , i32 %vp) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.fsub.v4f32(<4 x float>, <4 x float>, <4 x i1>, i32) + +define void @vp_fmul_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind { +; CHECK-LABEL: define void @vp_fmul_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i32 [[VP:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES1:%.*]] = fmul <4 x float> [[A0]], [[A1]] +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.fmul.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x i1> , i32 %vp) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.fmul.v4f32(<4 x float>, <4 x float>, <4 x i1>, i32) + +define void @vp_fdiv_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind { +; CHECK-LABEL: define void @vp_fdiv_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i32 [[VP:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES1:%.*]] = fdiv <4 x float> [[A0]], [[A1]] +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.fdiv.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x i1> , i32 %vp) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.fdiv.v4f32(<4 x float>, <4 x float>, <4 x i1>, i32) + +define void @vp_frem_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind { +; CHECK-LABEL: define void @vp_frem_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i32 [[VP:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES1:%.*]] = frem <4 x float> [[A0]], [[A1]] +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.frem.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x i1> , i32 %vp) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.frem.v4f32(<4 x float>, <4 x float>, <4 x i1>, i32) + +define void @vp_fabs_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind { +; CHECK-LABEL: define void @vp_fabs_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i32 [[VP:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES1:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[A0]]) +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.fabs.v4f32(<4 x float> %a0, <4 x i1> , i32 %vp) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.fabs.v4f32(<4 x float>, <4 x i1>, i32) + +define void @vp_sqrt_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind { +; CHECK-LABEL: define void @vp_sqrt_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i32 [[VP:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES1:%.*]] = call <4 x float> @llvm.sqrt.v4f32(<4 x float> [[A0]]) +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.sqrt.v4f32(<4 x float> %a0, <4 x i1> , i32 %vp) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.sqrt.v4f32(<4 x float>, <4 x i1>, i32) + +define void @vp_fneg_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i32 %vp) nounwind { +; CHECK-LABEL: define void @vp_fneg_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i32 [[VP:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES1:%.*]] = fneg <4 x float> [[A0]] +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.fneg.v4f32(<4 x float> %a0, <4 x i1> , i32 %vp) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.fneg.v4f32(<4 x float>, <4 x i1>, i32) + +define void @vp_fma_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i4 %a5) nounwind { +; CHECK-LABEL: define void @vp_fma_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i4 [[A5:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> [[A1]]) +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.fma.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %a1, <4 x i1> , i32 4) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.fma.v4f32(<4 x float>, <4 x float>, <4 x float>, <4 x i1>, i32) + +define void @vp_fmuladd_v4f32(<4 x float> %a0, <4 x float> %a1, ptr %out, i4 %a5) nounwind { +; CHECK-LABEL: define void @vp_fmuladd_v4f32( +; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], ptr [[OUT:%.*]], i4 [[A5:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[RES1:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[A0]], <4 x float> [[A1]], <4 x float> [[A1]]) +; CHECK-NEXT: store <4 x float> [[RES1]], ptr [[OUT]], align 16 +; CHECK-NEXT: ret void +; + %res = call <4 x float> @llvm.vp.fmuladd.v4f32(<4 x float> %a0, <4 x float> %a1, <4 x float> %a1, <4 x i1> , i32 4) + store <4 x float> %res, ptr %out + ret void +} +declare <4 x float> @llvm.vp.fmuladd.v4f32(<4 x float>, <4 x float>, <4 x float>, <4 x i1>, i32) + +declare <4 x float> @llvm.vp.maxnum.v4f32(<4 x float>, <4 x float>, <4 x i1>, i32) +define <4 x float> @vfmax_vv_v4f32(<4 x float> %va, <4 x float> %vb, <4 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: define <4 x float> @vfmax_vv_v4f32( +; CHECK-SAME: <4 x float> [[VA:%.*]], <4 x float> [[VB:%.*]], <4 x i1> [[M:%.*]], i32 zeroext [[EVL:%.*]]) { +; CHECK-NEXT: [[V1:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[VA]], <4 x float> [[VB]]) +; CHECK-NEXT: ret <4 x float> [[V1]] +; + %v = call <4 x float> @llvm.vp.maxnum.v4f32(<4 x float> %va, <4 x float> %vb, <4 x i1> %m, i32 %evl) + ret <4 x float> %v +} + +declare <8 x float> @llvm.vp.maxnum.v8f32(<8 x float>, <8 x float>, <8 x i1>, i32) +define <8 x float> @vfmax_vv_v8f32(<8 x float> %va, <8 x float> %vb, <8 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: define <8 x float> @vfmax_vv_v8f32( +; CHECK-SAME: <8 x float> [[VA:%.*]], <8 x float> [[VB:%.*]], <8 x i1> [[M:%.*]], i32 zeroext [[EVL:%.*]]) { +; CHECK-NEXT: [[V1:%.*]] = call <8 x float> @llvm.maxnum.v8f32(<8 x float> [[VA]], <8 x float> [[VB]]) +; CHECK-NEXT: ret <8 x float> [[V1]] +; + %v = call <8 x float> @llvm.vp.maxnum.v8f32(<8 x float> %va, <8 x float> %vb, <8 x i1> %m, i32 %evl) + ret <8 x float> %v +} + +declare <4 x float> @llvm.vp.minnum.v4f32(<4 x float>, <4 x float>, <4 x i1>, i32) +define <4 x float> @vfmin_vv_v4f32(<4 x float> %va, <4 x float> %vb, <4 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: define <4 x float> @vfmin_vv_v4f32( +; CHECK-SAME: <4 x float> [[VA:%.*]], <4 x float> [[VB:%.*]], <4 x i1> [[M:%.*]], i32 zeroext [[EVL:%.*]]) { +; CHECK-NEXT: [[V1:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[VA]], <4 x float> [[VB]]) +; CHECK-NEXT: ret <4 x float> [[V1]] +; + %v = call <4 x float> @llvm.vp.minnum.v4f32(<4 x float> %va, <4 x float> %vb, <4 x i1> %m, i32 %evl) + ret <4 x float> %v +} + +declare <8 x float> @llvm.vp.minnum.v8f32(<8 x float>, <8 x float>, <8 x i1>, i32) +define <8 x float> @vfmin_vv_v8f32(<8 x float> %va, <8 x float> %vb, <8 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: define <8 x float> @vfmin_vv_v8f32( +; CHECK-SAME: <8 x float> [[VA:%.*]], <8 x float> [[VB:%.*]], <8 x i1> [[M:%.*]], i32 zeroext [[EVL:%.*]]) { +; CHECK-NEXT: [[V1:%.*]] = call <8 x float> @llvm.minnum.v8f32(<8 x float> [[VA]], <8 x float> [[VB]]) +; CHECK-NEXT: ret <8 x float> [[V1]] +; + %v = call <8 x float> @llvm.vp.minnum.v8f32(<8 x float> %va, <8 x float> %vb, <8 x i1> %m, i32 %evl) + ret <8 x float> %v +} -- GitLab From 7b8625ec16efcb6d11e14a80e08c65dbfe68db9f Mon Sep 17 00:00:00 2001 From: Fabian Ritter Date: Wed, 17 Apr 2024 14:54:14 +0200 Subject: [PATCH 009/714] [AMDGPU][Docs] Fix broken link to HRF memory model reference (#88696) The link to the Heterogeneous-race-free Memory Models ASPLOS'14 paper by Hower et al. pointed to a bogus website, probably because the domain ownership has changed. This patch updates it to a version hosted on research.cs.wisc.edu. --- llvm/docs/AMDGPUUsage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst index 22c1d1f186ea..7da5d8e41f6f 100644 --- a/llvm/docs/AMDGPUUsage.rst +++ b/llvm/docs/AMDGPUUsage.rst @@ -16024,7 +16024,7 @@ Additional Documentation .. [CLANG-ATTR] `Attributes in Clang `__ .. [DWARF] `DWARF Debugging Information Format `__ .. [ELF] `Executable and Linkable Format (ELF) `__ -.. [HRF] `Heterogeneous-race-free Memory Models `__ +.. [HRF] `Heterogeneous-race-free Memory Models `__ .. [HSA] `Heterogeneous System Architecture (HSA) Foundation `__ .. [MsgPack] `Message Pack `__ .. [OpenCL] `The OpenCL Specification Version 2.0 `__ -- GitLab From 971ec1f0eea324d4a1eec6709e2c97e1798a6002 Mon Sep 17 00:00:00 2001 From: DianQK Date: Wed, 17 Apr 2024 20:54:17 +0800 Subject: [PATCH 010/714] [Inline] Regenerate inline-switch-default-2.ll (NFC) --- .../Inline/inline-switch-default-2.ll | 176 ------------------ 1 file changed, 176 deletions(-) diff --git a/llvm/test/Transforms/Inline/inline-switch-default-2.ll b/llvm/test/Transforms/Inline/inline-switch-default-2.ll index 8d3e24c798df..82dae1c27648 100644 --- a/llvm/test/Transforms/Inline/inline-switch-default-2.ll +++ b/llvm/test/Transforms/Inline/inline-switch-default-2.ll @@ -4,50 +4,6 @@ ; Check for scenarios without TTI. define i64 @foo1(i64 %a) { -; LOOKUPTABLE-LABEL: define i64 @foo1( -; LOOKUPTABLE-SAME: i64 [[TMP0:%.*]]) { -; LOOKUPTABLE-NEXT: switch i64 [[TMP0]], label [[DEFAULT_BRANCH_I:%.*]] [ -; LOOKUPTABLE-NEXT: i64 0, label [[BRANCH_0_I:%.*]] -; LOOKUPTABLE-NEXT: i64 2, label [[BRANCH_2_I:%.*]] -; LOOKUPTABLE-NEXT: i64 4, label [[BRANCH_4_I:%.*]] -; LOOKUPTABLE-NEXT: i64 6, label [[BRANCH_6_I:%.*]] -; LOOKUPTABLE-NEXT: ] -; LOOKUPTABLE: branch_0.i: -; LOOKUPTABLE-NEXT: br label [[BAR1_EXIT:%.*]] -; LOOKUPTABLE: branch_2.i: -; LOOKUPTABLE-NEXT: br label [[BAR1_EXIT]] -; LOOKUPTABLE: branch_4.i: -; LOOKUPTABLE-NEXT: br label [[BAR1_EXIT]] -; LOOKUPTABLE: branch_6.i: -; LOOKUPTABLE-NEXT: br label [[BAR1_EXIT]] -; LOOKUPTABLE: default_branch.i: -; LOOKUPTABLE-NEXT: br label [[BAR1_EXIT]] -; LOOKUPTABLE: bar1.exit: -; LOOKUPTABLE-NEXT: [[TMP2:%.*]] = phi i64 [ 5, [[BRANCH_0_I]] ], [ 9, [[BRANCH_2_I]] ], [ 2, [[BRANCH_4_I]] ], [ 7, [[BRANCH_6_I]] ], [ 3, [[DEFAULT_BRANCH_I]] ] -; LOOKUPTABLE-NEXT: ret i64 [[TMP2]] -; -; SWITCH-LABEL: define i64 @foo1( -; SWITCH-SAME: i64 [[TMP0:%.*]]) { -; SWITCH-NEXT: switch i64 [[TMP0]], label [[DEFAULT_BRANCH_I:%.*]] [ -; SWITCH-NEXT: i64 0, label [[BRANCH_0_I:%.*]] -; SWITCH-NEXT: i64 2, label [[BRANCH_2_I:%.*]] -; SWITCH-NEXT: i64 4, label [[BRANCH_4_I:%.*]] -; SWITCH-NEXT: i64 6, label [[BRANCH_6_I:%.*]] -; SWITCH-NEXT: ] -; SWITCH: branch_0.i: -; SWITCH-NEXT: br label [[BAR1_EXIT:%.*]] -; SWITCH: branch_2.i: -; SWITCH-NEXT: br label [[BAR1_EXIT]] -; SWITCH: branch_4.i: -; SWITCH-NEXT: br label [[BAR1_EXIT]] -; SWITCH: branch_6.i: -; SWITCH-NEXT: br label [[BAR1_EXIT]] -; SWITCH: default_branch.i: -; SWITCH-NEXT: br label [[BAR1_EXIT]] -; SWITCH: bar1.exit: -; SWITCH-NEXT: [[TMP2:%.*]] = phi i64 [ 5, [[BRANCH_0_I]] ], [ 9, [[BRANCH_2_I]] ], [ 2, [[BRANCH_4_I]] ], [ 7, [[BRANCH_6_I]] ], [ 3, [[DEFAULT_BRANCH_I]] ] -; SWITCH-NEXT: ret i64 [[TMP2]] -; ; CHECK-LABEL: define i64 @foo1( ; CHECK-SAME: i64 [[A:%.*]]) { ; CHECK-NEXT: [[B:%.*]] = call i64 @bar1(i64 [[A]]) @@ -58,50 +14,6 @@ define i64 @foo1(i64 %a) { } define i64 @foo2(i64 %a) { -; LOOKUPTABLE-LABEL: define i64 @foo2( -; LOOKUPTABLE-SAME: i64 [[TMP0:%.*]]) { -; LOOKUPTABLE-NEXT: switch i64 [[TMP0]], label [[UNREACHABLEDEFAULT_I:%.*]] [ -; LOOKUPTABLE-NEXT: i64 0, label [[BRANCH_0_I:%.*]] -; LOOKUPTABLE-NEXT: i64 2, label [[BRANCH_2_I:%.*]] -; LOOKUPTABLE-NEXT: i64 4, label [[BRANCH_4_I:%.*]] -; LOOKUPTABLE-NEXT: i64 6, label [[BRANCH_6_I:%.*]] -; LOOKUPTABLE-NEXT: ] -; LOOKUPTABLE: branch_0.i: -; LOOKUPTABLE-NEXT: br label [[BAR2_EXIT:%.*]] -; LOOKUPTABLE: branch_2.i: -; LOOKUPTABLE-NEXT: br label [[BAR2_EXIT]] -; LOOKUPTABLE: branch_4.i: -; LOOKUPTABLE-NEXT: br label [[BAR2_EXIT]] -; LOOKUPTABLE: branch_6.i: -; LOOKUPTABLE-NEXT: br label [[BAR2_EXIT]] -; LOOKUPTABLE: unreachabledefault.i: -; LOOKUPTABLE-NEXT: unreachable -; LOOKUPTABLE: bar2.exit: -; LOOKUPTABLE-NEXT: [[TMP2:%.*]] = phi i64 [ 5, [[BRANCH_0_I]] ], [ 9, [[BRANCH_2_I]] ], [ 2, [[BRANCH_4_I]] ], [ 7, [[BRANCH_6_I]] ] -; LOOKUPTABLE-NEXT: ret i64 [[TMP2]] -; -; SWITCH-LABEL: define i64 @foo2( -; SWITCH-SAME: i64 [[TMP0:%.*]]) { -; SWITCH-NEXT: switch i64 [[TMP0]], label [[UNREACHABLEDEFAULT_I:%.*]] [ -; SWITCH-NEXT: i64 0, label [[BRANCH_0_I:%.*]] -; SWITCH-NEXT: i64 2, label [[BRANCH_2_I:%.*]] -; SWITCH-NEXT: i64 4, label [[BRANCH_4_I:%.*]] -; SWITCH-NEXT: i64 6, label [[BRANCH_6_I:%.*]] -; SWITCH-NEXT: ] -; SWITCH: branch_0.i: -; SWITCH-NEXT: br label [[BAR2_EXIT:%.*]] -; SWITCH: branch_2.i: -; SWITCH-NEXT: br label [[BAR2_EXIT]] -; SWITCH: branch_4.i: -; SWITCH-NEXT: br label [[BAR2_EXIT]] -; SWITCH: branch_6.i: -; SWITCH-NEXT: br label [[BAR2_EXIT]] -; SWITCH: unreachabledefault.i: -; SWITCH-NEXT: unreachable -; SWITCH: bar2.exit: -; SWITCH-NEXT: [[TMP2:%.*]] = phi i64 [ 5, [[BRANCH_0_I]] ], [ 9, [[BRANCH_2_I]] ], [ 2, [[BRANCH_4_I]] ], [ 7, [[BRANCH_6_I]] ] -; SWITCH-NEXT: ret i64 [[TMP2]] -; ; CHECK-LABEL: define i64 @foo2( ; CHECK-SAME: i64 [[A:%.*]]) { ; CHECK-NEXT: switch i64 [[A]], label [[UNREACHABLEDEFAULT_I:%.*]] [ @@ -129,50 +41,6 @@ define i64 @foo2(i64 %a) { } define i64 @bar1(i64 %a) { -; LOOKUPTABLE-LABEL: define i64 @bar1( -; LOOKUPTABLE-SAME: i64 [[TMP0:%.*]]) { -; LOOKUPTABLE-NEXT: switch i64 [[TMP0]], label [[DEFAULT_BRANCH:%.*]] [ -; LOOKUPTABLE-NEXT: i64 0, label [[BRANCH_0:%.*]] -; LOOKUPTABLE-NEXT: i64 2, label [[BRANCH_2:%.*]] -; LOOKUPTABLE-NEXT: i64 4, label [[BRANCH_4:%.*]] -; LOOKUPTABLE-NEXT: i64 6, label [[BRANCH_6:%.*]] -; LOOKUPTABLE-NEXT: ] -; LOOKUPTABLE: branch_0: -; LOOKUPTABLE-NEXT: br label [[EXIT:%.*]] -; LOOKUPTABLE: branch_2: -; LOOKUPTABLE-NEXT: br label [[EXIT]] -; LOOKUPTABLE: branch_4: -; LOOKUPTABLE-NEXT: br label [[EXIT]] -; LOOKUPTABLE: branch_6: -; LOOKUPTABLE-NEXT: br label [[EXIT]] -; LOOKUPTABLE: default_branch: -; LOOKUPTABLE-NEXT: br label [[EXIT]] -; LOOKUPTABLE: exit: -; LOOKUPTABLE-NEXT: [[TMP2:%.*]] = phi i64 [ 5, [[BRANCH_0]] ], [ 9, [[BRANCH_2]] ], [ 2, [[BRANCH_4]] ], [ 7, [[BRANCH_6]] ], [ 3, [[DEFAULT_BRANCH]] ] -; LOOKUPTABLE-NEXT: ret i64 [[TMP2]] -; -; SWITCH-LABEL: define i64 @bar1( -; SWITCH-SAME: i64 [[TMP0:%.*]]) { -; SWITCH-NEXT: switch i64 [[TMP0]], label [[DEFAULT_BRANCH:%.*]] [ -; SWITCH-NEXT: i64 0, label [[BRANCH_0:%.*]] -; SWITCH-NEXT: i64 2, label [[BRANCH_2:%.*]] -; SWITCH-NEXT: i64 4, label [[BRANCH_4:%.*]] -; SWITCH-NEXT: i64 6, label [[BRANCH_6:%.*]] -; SWITCH-NEXT: ] -; SWITCH: branch_0: -; SWITCH-NEXT: br label [[EXIT:%.*]] -; SWITCH: branch_2: -; SWITCH-NEXT: br label [[EXIT]] -; SWITCH: branch_4: -; SWITCH-NEXT: br label [[EXIT]] -; SWITCH: branch_6: -; SWITCH-NEXT: br label [[EXIT]] -; SWITCH: default_branch: -; SWITCH-NEXT: br label [[EXIT]] -; SWITCH: exit: -; SWITCH-NEXT: [[TMP2:%.*]] = phi i64 [ 5, [[BRANCH_0]] ], [ 9, [[BRANCH_2]] ], [ 2, [[BRANCH_4]] ], [ 7, [[BRANCH_6]] ], [ 3, [[DEFAULT_BRANCH]] ] -; SWITCH-NEXT: ret i64 [[TMP2]] -; ; CHECK-LABEL: define i64 @bar1( ; CHECK-SAME: i64 [[A:%.*]]) { ; CHECK-NEXT: switch i64 [[A]], label [[DEFAULT_BRANCH:%.*]] [ @@ -223,50 +91,6 @@ exit: } define i64 @bar2(i64 %a) { -; LOOKUPTABLE-LABEL: define i64 @bar2( -; LOOKUPTABLE-SAME: i64 [[TMP0:%.*]]) { -; LOOKUPTABLE-NEXT: switch i64 [[TMP0]], label [[UNREACHABLEDEFAULT:%.*]] [ -; LOOKUPTABLE-NEXT: i64 0, label [[BRANCH_0:%.*]] -; LOOKUPTABLE-NEXT: i64 2, label [[BRANCH_2:%.*]] -; LOOKUPTABLE-NEXT: i64 4, label [[BRANCH_4:%.*]] -; LOOKUPTABLE-NEXT: i64 6, label [[BRANCH_6:%.*]] -; LOOKUPTABLE-NEXT: ] -; LOOKUPTABLE: branch_0: -; LOOKUPTABLE-NEXT: br label [[EXIT:%.*]] -; LOOKUPTABLE: branch_2: -; LOOKUPTABLE-NEXT: br label [[EXIT]] -; LOOKUPTABLE: branch_4: -; LOOKUPTABLE-NEXT: br label [[EXIT]] -; LOOKUPTABLE: branch_6: -; LOOKUPTABLE-NEXT: br label [[EXIT]] -; LOOKUPTABLE: unreachabledefault: -; LOOKUPTABLE-NEXT: unreachable -; LOOKUPTABLE: exit: -; LOOKUPTABLE-NEXT: [[TMP2:%.*]] = phi i64 [ 5, [[BRANCH_0]] ], [ 9, [[BRANCH_2]] ], [ 2, [[BRANCH_4]] ], [ 7, [[BRANCH_6]] ] -; LOOKUPTABLE-NEXT: ret i64 [[TMP2]] -; -; SWITCH-LABEL: define i64 @bar2( -; SWITCH-SAME: i64 [[TMP0:%.*]]) { -; SWITCH-NEXT: switch i64 [[TMP0]], label [[UNREACHABLEDEFAULT:%.*]] [ -; SWITCH-NEXT: i64 0, label [[BRANCH_0:%.*]] -; SWITCH-NEXT: i64 2, label [[BRANCH_2:%.*]] -; SWITCH-NEXT: i64 4, label [[BRANCH_4:%.*]] -; SWITCH-NEXT: i64 6, label [[BRANCH_6:%.*]] -; SWITCH-NEXT: ] -; SWITCH: branch_0: -; SWITCH-NEXT: br label [[EXIT:%.*]] -; SWITCH: branch_2: -; SWITCH-NEXT: br label [[EXIT]] -; SWITCH: branch_4: -; SWITCH-NEXT: br label [[EXIT]] -; SWITCH: branch_6: -; SWITCH-NEXT: br label [[EXIT]] -; SWITCH: unreachabledefault: -; SWITCH-NEXT: unreachable -; SWITCH: exit: -; SWITCH-NEXT: [[TMP2:%.*]] = phi i64 [ 5, [[BRANCH_0]] ], [ 9, [[BRANCH_2]] ], [ 2, [[BRANCH_4]] ], [ 7, [[BRANCH_6]] ] -; SWITCH-NEXT: ret i64 [[TMP2]] -; ; CHECK-LABEL: define i64 @bar2( ; CHECK-SAME: i64 [[A:%.*]]) { ; CHECK-NEXT: switch i64 [[A]], label [[UNREACHABLEDEFAULT:%.*]] [ -- GitLab From 73140daebbf522dbb14dc4b2f3c67dc0aa1a62dd Mon Sep 17 00:00:00 2001 From: "Oleksandr \"Alex\" Zinenko" Date: Wed, 17 Apr 2024 15:01:59 +0200 Subject: [PATCH 011/714] [mlir] expose transform dialect symbol merge to python (#87690) This functionality is available in C++, make it available in Python directly to operate on transform modules. --- .../mlir-c/Dialect/Transform/Interpreter.h | 12 ++- .../Bindings/Python/TransformInterpreter.cpp | 15 ++++ .../lib/CAPI/Dialect/TransformInterpreter.cpp | 9 +++ .../transform/interpreter/__init__.py | 10 ++- .../python/dialects/transform_interpreter.py | 76 +++++++++++++++++++ 5 files changed, 120 insertions(+), 2 deletions(-) diff --git a/mlir/include/mlir-c/Dialect/Transform/Interpreter.h b/mlir/include/mlir-c/Dialect/Transform/Interpreter.h index 00095d5040a0..fa320324234e 100644 --- a/mlir/include/mlir-c/Dialect/Transform/Interpreter.h +++ b/mlir/include/mlir-c/Dialect/Transform/Interpreter.h @@ -60,7 +60,7 @@ MLIR_CAPI_EXPORTED void mlirTransformOptionsDestroy(MlirTransformOptions transformOptions); //----------------------------------------------------------------------------// -// Transform interpreter. +// Transform interpreter and utilities. //----------------------------------------------------------------------------// /// Applies the transformation script starting at the given transform root @@ -72,6 +72,16 @@ MLIR_CAPI_EXPORTED MlirLogicalResult mlirTransformApplyNamedSequence( MlirOperation payload, MlirOperation transformRoot, MlirOperation transformModule, MlirTransformOptions transformOptions); +/// Merge the symbols from `other` into `target`, potentially renaming them to +/// avoid conflicts. Private symbols may be renamed during the merge, public +/// symbols must have at most one declaration. A name conflict in public symbols +/// is reported as an error before returning a failure. +/// +/// Note that this clones the `other` operation unlike the C++ counterpart that +/// takes ownership. +MLIR_CAPI_EXPORTED MlirLogicalResult +mlirMergeSymbolsIntoFromClone(MlirOperation target, MlirOperation other); + #ifdef __cplusplus } #endif diff --git a/mlir/lib/Bindings/Python/TransformInterpreter.cpp b/mlir/lib/Bindings/Python/TransformInterpreter.cpp index 6517f8c39dfa..f6b4532b1b6b 100644 --- a/mlir/lib/Bindings/Python/TransformInterpreter.cpp +++ b/mlir/lib/Bindings/Python/TransformInterpreter.cpp @@ -82,6 +82,21 @@ static void populateTransformInterpreterSubmodule(py::module &m) { py::arg("payload_root"), py::arg("transform_root"), py::arg("transform_module"), py::arg("transform_options") = PyMlirTransformOptions()); + + m.def( + "copy_symbols_and_merge_into", + [](MlirOperation target, MlirOperation other) { + mlir::python::CollectDiagnosticsToStringScope scope( + mlirOperationGetContext(target)); + + MlirLogicalResult result = mlirMergeSymbolsIntoFromClone(target, other); + if (mlirLogicalResultIsFailure(result)) { + throw py::value_error( + "Failed to merge symbols.\nDiagnostic message " + + scope.takeMessage()); + } + }, + py::arg("target"), py::arg("other")); } PYBIND11_MODULE(_mlirTransformInterpreter, m) { diff --git a/mlir/lib/CAPI/Dialect/TransformInterpreter.cpp b/mlir/lib/CAPI/Dialect/TransformInterpreter.cpp index eb6951dc5584..145455e1c1b3 100644 --- a/mlir/lib/CAPI/Dialect/TransformInterpreter.cpp +++ b/mlir/lib/CAPI/Dialect/TransformInterpreter.cpp @@ -15,6 +15,7 @@ #include "mlir/CAPI/IR.h" #include "mlir/CAPI/Support.h" #include "mlir/CAPI/Wrap.h" +#include "mlir/Dialect/Transform/IR/Utils.h" #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Dialect/Transform/Transforms/TransformInterpreterUtils.h" @@ -71,4 +72,12 @@ MlirLogicalResult mlirTransformApplyNamedSequence( unwrap(payload), unwrap(transformRoot), cast(unwrap(transformModule)), *unwrap(transformOptions))); } + +MlirLogicalResult mlirMergeSymbolsIntoFromClone(MlirOperation target, + MlirOperation other) { + OwningOpRef otherOwning(unwrap(other)->clone()); + LogicalResult result = transform::detail::mergeSymbolsInto( + unwrap(target), std::move(otherOwning)); + return wrap(result); +} } diff --git a/mlir/python/mlir/dialects/transform/interpreter/__init__.py b/mlir/python/mlir/dialects/transform/interpreter/__init__.py index 6145b99224eb..34cdc43cb617 100644 --- a/mlir/python/mlir/dialects/transform/interpreter/__init__.py +++ b/mlir/python/mlir/dialects/transform/interpreter/__init__.py @@ -5,7 +5,6 @@ from ....ir import Operation from ...._mlir_libs import _mlirTransformInterpreter as _cextTransformInterpreter - TransformOptions = _cextTransformInterpreter.TransformOptions @@ -31,3 +30,12 @@ def apply_named_sequence( _cextTransformInterpreter.apply_named_sequence(*args) else: _cextTransformInterpreter(*args, transform_options) + + +def copy_symbols_and_merge_into(target, other): + """Copies symbols from other into target, renaming private symbols to avoid + duplicates. Raises an error if copying would lead to duplicate public + symbols.""" + _cextTransformInterpreter.copy_symbols_and_merge_into( + _unpack_operation(target), _unpack_operation(other) + ) diff --git a/mlir/test/python/dialects/transform_interpreter.py b/mlir/test/python/dialects/transform_interpreter.py index 740c49f76a26..807a98c49327 100644 --- a/mlir/test/python/dialects/transform_interpreter.py +++ b/mlir/test/python/dialects/transform_interpreter.py @@ -54,3 +54,79 @@ def failed(): assert ( "must implement TransformOpInterface to be used as transform root" in str(e) ) + + +print_root_via_include_module = """ +module @print_root_via_include_module attributes {transform.with_named_sequence} { + transform.named_sequence private @callee1(%root: !transform.any_op {transform.readonly}) + transform.named_sequence private @callee2(%root: !transform.any_op {transform.readonly}) + transform.named_sequence @__transform_main(%root: !transform.any_op) { + transform.include @callee2 failures(propagate) + (%root) : (!transform.any_op) -> () + transform.yield + } +}""" + +callee2_definition = """ +module attributes {transform.with_named_sequence} { + transform.named_sequence private @callee1(%root: !transform.any_op {transform.readonly}) + transform.named_sequence @callee2(%root: !transform.any_op {transform.readonly}) { + transform.include @callee1 failures(propagate) + (%root) : (!transform.any_op) -> () + transform.yield + } +} +""" + +callee1_definition = """ +module attributes {transform.with_named_sequence} { + transform.named_sequence @callee1(%root: !transform.any_op {transform.readonly}) { + transform.print %root { name = \"from interpreter\" }: !transform.any_op + transform.yield + } +} +""" + + +@test_in_context +def include(): + main = ir.Module.parse(print_root_via_include_module) + callee1 = ir.Module.parse(callee1_definition) + callee2 = ir.Module.parse(callee2_definition) + interp.copy_symbols_and_merge_into(main, callee1) + interp.copy_symbols_and_merge_into(main, callee2) + + # CHECK: @print_root_via_include_module + # CHECK: transform.named_sequence @__transform_main + # CHECK: transform.include @callee2 + # + # CHECK: transform.named_sequence @callee1 + # CHECK: transform.print + # + # CHECK: transform.named_sequence @callee2 + # CHECK: transform.include @callee1 + interp.apply_named_sequence(main, main.body.operations[0], main) + + +@test_in_context +def partial_include(): + main = ir.Module.parse(print_root_via_include_module) + callee2 = ir.Module.parse(callee2_definition) + interp.copy_symbols_and_merge_into(main, callee2) + + try: + interp.apply_named_sequence(main, main.body.operations[0], main) + except ValueError as e: + assert "Failed to apply" in str(e) + + +@test_in_context +def repeated_include(): + main = ir.Module.parse(print_root_via_include_module) + callee2 = ir.Module.parse(callee2_definition) + interp.copy_symbols_and_merge_into(main, callee2) + + try: + interp.copy_symbols_and_merge_into(main, callee2) + except ValueError as e: + assert "doubly defined symbol @callee2" in str(e) -- GitLab From 20d653fdb2d4d6eafa4575cd954beaf7ecad113a Mon Sep 17 00:00:00 2001 From: Krzysztof Parzyszek Date: Wed, 17 Apr 2024 08:04:19 -0500 Subject: [PATCH 012/714] [LLVM][CodeGen] Fix register lane liveness tracking in RegisterPressure (#88892) Re-enable an old assertion in `decreaseSetPressure`. --- llvm/lib/CodeGen/RegisterPressure.cpp | 39 ++++++++++++++------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/llvm/lib/CodeGen/RegisterPressure.cpp b/llvm/lib/CodeGen/RegisterPressure.cpp index f86aa3a16720..3fa22447f416 100644 --- a/llvm/lib/CodeGen/RegisterPressure.cpp +++ b/llvm/lib/CodeGen/RegisterPressure.cpp @@ -64,7 +64,7 @@ static void increaseSetPressure(std::vector &CurrSetPressure, static void decreaseSetPressure(std::vector &CurrSetPressure, const MachineRegisterInfo &MRI, Register Reg, LaneBitmask PrevMask, LaneBitmask NewMask) { - //assert((NewMask & !PrevMask) == 0 && "Must not add bits"); + assert((NewMask & ~PrevMask).none() && "Must not add bits"); if (NewMask.any() || PrevMask.none()) return; @@ -617,17 +617,11 @@ void RegisterOperands::adjustLaneLiveness(const LiveIntervals &LIS, ++I; } } - for (auto *I = Uses.begin(); I != Uses.end();) { - LaneBitmask LiveBefore = getLiveLanesAt(LIS, MRI, true, I->RegUnit, - Pos.getBaseIndex()); - LaneBitmask LaneMask = I->LaneMask & LiveBefore; - if (LaneMask.none()) { - I = Uses.erase(I); - } else { - I->LaneMask = LaneMask; - ++I; - } - } + + // For uses just copy the information from LIS. + for (auto &[RegUnit, LaneMask] : Uses) + LaneMask = getLiveLanesAt(LIS, MRI, true, RegUnit, Pos.getBaseIndex()); + if (AddFlagsMI != nullptr) { for (const RegisterMaskPair &P : DeadDefs) { Register RegUnit = P.RegUnit; @@ -1060,18 +1054,27 @@ void RegPressureTracker::bumpUpwardPressure(const MachineInstr *MI) { // Kill liveness at live defs. for (const RegisterMaskPair &P : RegOpers.Defs) { Register Reg = P.RegUnit; - LaneBitmask LiveLanes = LiveRegs.contains(Reg); + LaneBitmask LiveAfter = LiveRegs.contains(Reg); LaneBitmask UseLanes = getRegLanes(RegOpers.Uses, Reg); LaneBitmask DefLanes = P.LaneMask; - LaneBitmask LiveAfter = (LiveLanes & ~DefLanes) | UseLanes; - decreaseRegPressure(Reg, LiveLanes, LiveAfter); + LaneBitmask LiveBefore = (LiveAfter & ~DefLanes) | UseLanes; + + // There may be parts of the register that were dead before the + // instruction, but became live afterwards. Similarly, some parts + // may have been killed in this instruction. + decreaseRegPressure(Reg, LiveAfter, LiveAfter & LiveBefore); + increaseRegPressure(Reg, LiveAfter, ~LiveAfter & LiveBefore); } // Generate liveness for uses. for (const RegisterMaskPair &P : RegOpers.Uses) { Register Reg = P.RegUnit; - LaneBitmask LiveLanes = LiveRegs.contains(Reg); - LaneBitmask LiveAfter = LiveLanes | P.LaneMask; - increaseRegPressure(Reg, LiveLanes, LiveAfter); + // If this register was also in a def operand, we've handled it + // with defs. + if (getRegLanes(RegOpers.Defs, Reg).any()) + continue; + LaneBitmask LiveAfter = LiveRegs.contains(Reg); + LaneBitmask LiveBefore = LiveAfter | P.LaneMask; + increaseRegPressure(Reg, LiveAfter, LiveBefore); } } -- GitLab From 1fc72dbc807fb138cafd05501e2e31beaa574693 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 17 Apr 2024 21:10:56 +0800 Subject: [PATCH 013/714] [RISCV] Add test for doLocalPostpass issue not checking if VL was modified. NFC --- .../test/CodeGen/RISCV/rvv/vsetvli-insert.mir | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir index 1850abe6363b..d9a87c2cb12a 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir +++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir @@ -76,6 +76,10 @@ ret void } + define void @postpass_modify_vl() { + ret void + } + declare @llvm.riscv.vadd.nxv1i64.nxv1i64.i64(, , , i64) #1 declare @llvm.riscv.vle.nxv1i64.i64(, ptr nocapture, i64) #4 @@ -503,3 +507,24 @@ body: | %5:vr = PseudoVMV_V_I_MF2 $noreg, 1, 2, 5, 0 PseudoRET ... +--- +name: postpass_modify_vl +tracksRegLiveness: true +body: | + bb.0: + liveins: $x1 + ; CHECK-LABEL: name: postpass_modify_vl + ; CHECK: liveins: $x1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: $x0 = PseudoVSETIVLI 3, 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype + ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $vtype + ; CHECK-NEXT: $vl = COPY $x1 + ; CHECK-NEXT: [[PseudoVADD_VV_M1_:%[0-9]+]]:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 3, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype + ; CHECK-NEXT: PseudoRET + dead $x0 = PseudoVSETIVLI 3, 216, implicit-def $vl, implicit-def $vtype + %1:gpr = COPY $vtype + $vl = COPY $x1 + dead $x0 = PseudoVSETIVLI 3, 216, implicit-def $vl, implicit-def $vtype + %4:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 3, 6, 0 + PseudoRET +... -- GitLab From 76ad2897480a85532eee93daf041246881772693 Mon Sep 17 00:00:00 2001 From: Zaara Syeda Date: Wed, 17 Apr 2024 09:24:53 -0400 Subject: [PATCH 014/714] [PowerPC] 32-bit large code-model support for toc-data (#85129) This patch adds the pseudo op ADDItocL for 32-bit large code-model support for toc-data. --- llvm/lib/Target/PowerPC/P10InstrResources.td | 2 +- llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp | 61 ++++++++++++------ llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp | 35 +++++++--- llvm/lib/Target/PowerPC/PPCInstrInfo.cpp | 1 + llvm/lib/Target/PowerPC/PPCInstrInfo.td | 4 +- llvm/lib/Target/PowerPC/PPCMacroFusion.def | 10 +-- llvm/test/CodeGen/PowerPC/toc-data.ll | 67 ++++++++++++++++++++ 7 files changed, 144 insertions(+), 36 deletions(-) diff --git a/llvm/lib/Target/PowerPC/P10InstrResources.td b/llvm/lib/Target/PowerPC/P10InstrResources.td index 5015ba887d0b..32cebb65cb56 100644 --- a/llvm/lib/Target/PowerPC/P10InstrResources.td +++ b/llvm/lib/Target/PowerPC/P10InstrResources.td @@ -881,7 +881,7 @@ def : InstRW<[P10W_FX_3C, P10W_DISP_ANY], // 3 Cycles ALU operations, 1 input operands def : InstRW<[P10W_FX_3C, P10W_DISP_ANY, P10FX_Read], (instrs - ADDI, ADDI8, ADDIdtprelL32, ADDItlsldLADDR32, ADDItocL8, LI, LI8, + ADDI, ADDI8, ADDIdtprelL32, ADDItlsldLADDR32, ADDItocL, ADDItocL8, LI, LI8, ADDIC, ADDIC8, ADDIS, ADDIS8, ADDISdtprelHA32, ADDIStocHA, ADDIStocHA8, LIS, LIS8, ADDME, ADDME8, diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp index 1c57b92057ff..6e1002c45d81 100644 --- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp +++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp @@ -1148,15 +1148,27 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) { MCSymbolRefExpr::VariantKind VK = GetVKForMO(MO); - // Always use TOC on AIX. Map the global address operand to be a reference - // to the TOC entry we will synthesize later. 'TOCEntry' is a label used to - // reference the storage allocated in the TOC which contains the address of - // 'MOSymbol'. - MCSymbol *TOCEntry = - lookUpOrCreateTOCEntry(MOSymbol, getTOCEntryTypeForMO(MO), VK); - const MCExpr *Exp = MCSymbolRefExpr::create(TOCEntry, - MCSymbolRefExpr::VK_PPC_U, - OutContext); + // If the symbol isn't toc-data then use the TOC on AIX. + // Map the global address operand to be a reference to the TOC entry we + // will synthesize later. 'TOCEntry' is a label used to reference the + // storage allocated in the TOC which contains the address of 'MOSymbol'. + // If the toc-data attribute is used, the TOC entry contains the data + // rather than the address of the MOSymbol. + if (![](const MachineOperand &MO) { + if (!MO.isGlobal()) + return false; + + const GlobalVariable *GV = dyn_cast(MO.getGlobal()); + if (!GV) + return false; + + return GV->hasAttribute("toc-data"); + }(MO)) { + MOSymbol = lookUpOrCreateTOCEntry(MOSymbol, getTOCEntryTypeForMO(MO), VK); + } + + const MCExpr *Exp = MCSymbolRefExpr::create( + MOSymbol, MCSymbolRefExpr::VK_PPC_U, OutContext); TmpInst.getOperand(2) = MCOperand::createExpr(Exp); EmitToStreamer(*OutStreamer, TmpInst); return; @@ -1273,25 +1285,32 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) { EmitToStreamer(*OutStreamer, TmpInst); return; } + case PPC::ADDItocL: case PPC::ADDItocL8: { - // Transform %xd = ADDItocL8 %xs, @sym + // Transform %xd = ADDItocL %xs, @sym LowerPPCMachineInstrToMCInst(MI, TmpInst, *this); - // Change the opcode to ADDI8. If the global address is external, then - // generate a TOC entry and reference that. Otherwise, reference the - // symbol directly. - TmpInst.setOpcode(PPC::ADDI8); + unsigned Op = MI->getOpcode(); + + // Change the opcode to load address for tocdata + TmpInst.setOpcode(Op == PPC::ADDItocL8 ? PPC::ADDI8 : PPC::LA); const MachineOperand &MO = MI->getOperand(2); - assert((MO.isGlobal() || MO.isCPI()) && "Invalid operand for ADDItocL8."); + assert((Op == PPC::ADDItocL8) + ? (MO.isGlobal() || MO.isCPI()) + : MO.isGlobal() && "Invalid operand for ADDItocL8."); + assert(!(MO.isGlobal() && Subtarget->isGVIndirectSymbol(MO.getGlobal())) && + "Interposable definitions must use indirect accesses."); - LLVM_DEBUG(assert( - !(MO.isGlobal() && Subtarget->isGVIndirectSymbol(MO.getGlobal())) && - "Interposable definitions must use indirect access.")); + // Map the operand to its corresponding MCSymbol. + const MCSymbol *const MOSymbol = getMCSymbolForTOCPseudoMO(MO, *this); + + const MCExpr *Exp = MCSymbolRefExpr::create( + MOSymbol, + Op == PPC::ADDItocL8 ? MCSymbolRefExpr::VK_PPC_TOC_LO + : MCSymbolRefExpr::VK_PPC_L, + OutContext); - const MCExpr *Exp = - MCSymbolRefExpr::create(getMCSymbolForTOCPseudoMO(MO, *this), - MCSymbolRefExpr::VK_PPC_TOC_LO, OutContext); TmpInst.getOperand(2) = MCOperand::createExpr(Exp); EmitToStreamer(*OutStreamer, TmpInst); return; diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp index af82b6cdb180..2f647daa4bcb 100644 --- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp @@ -510,7 +510,7 @@ SDNode *PPCDAGToDAGISel::getGlobalBaseReg() { } // Check if a SDValue has the toc-data attribute. -static bool hasTocDataAttr(SDValue Val, unsigned PointerSize) { +static bool hasTocDataAttr(SDValue Val) { GlobalAddressSDNode *GA = dyn_cast(Val); if (!GA) return false; @@ -6115,8 +6115,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) { assert(isAIXABI && "ELF ABI already handled"); - if (hasTocDataAttr(N->getOperand(0), - CurDAG->getDataLayout().getPointerSize())) { + if (hasTocDataAttr(N->getOperand(0))) { replaceWith(PPC::ADDItoc, N, MVT::i32); return; } @@ -6128,8 +6127,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) { if (isPPC64 && CModel == CodeModel::Small) { assert(isAIXABI && "ELF ABI handled in common SelectCode"); - if (hasTocDataAttr(N->getOperand(0), - CurDAG->getDataLayout().getPointerSize())) { + if (hasTocDataAttr(N->getOperand(0))) { replaceWith(PPC::ADDItoc8, N, MVT::i64); return; } @@ -6144,9 +6142,10 @@ void PPCDAGToDAGISel::Select(SDNode *N) { " ELF/AIX or 32-bit AIX in the following."); // Transforms the ISD::TOC_ENTRY node for 32-bit AIX large code model mode - // or 64-bit medium (ELF-only) or large (ELF and AIX) code model code. We - // generate two instructions as described below. The first source operand - // is a symbol reference. If it must be toc-referenced according to + // or 64-bit medium (ELF-only) or large (ELF and AIX) code model code non + // toc-data symbols. + // We generate two instructions as described below. The first source + // operand is a symbol reference. If it must be toc-referenced according to // Subtarget, we generate: // [32-bit AIX] // LWZtocL(@sym, ADDIStocHA(%r2, @sym)) @@ -6154,6 +6153,13 @@ void PPCDAGToDAGISel::Select(SDNode *N) { // LDtocL(@sym, ADDIStocHA8(%x2, @sym)) // Otherwise we generate: // ADDItocL8(ADDIStocHA8(%x2, @sym), @sym) + + // For large code model toc-data symbols we generate: + // [32-bit AIX] + // ADDItocL(ADDIStocHA(%x2, @sym), @sym) + // [64-bit AIX] + // Currently not supported. + SDValue GA = N->getOperand(0); SDValue TOCbase = N->getOperand(1); @@ -6161,6 +6167,19 @@ void PPCDAGToDAGISel::Select(SDNode *N) { SDNode *Tmp = CurDAG->getMachineNode( isPPC64 ? PPC::ADDIStocHA8 : PPC::ADDIStocHA, dl, VT, TOCbase, GA); + // On AIX if the symbol has the toc-data attribute it will be defined + // in the TOC entry, so we use an ADDItocL similar to the medium code + // model ELF abi. + if (isAIXABI && hasTocDataAttr(GA)) { + if (isPPC64) + report_fatal_error( + "64-bit large code model toc-data not yet supported"); + + ReplaceNode(N, CurDAG->getMachineNode(PPC::ADDItocL, dl, VT, + SDValue(Tmp, 0), GA)); + return; + } + if (PPCLowering->isAccessedAsGotIndirect(GA)) { // If it is accessed as got-indirect, we need an extra LWZ/LD to load // the address. diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp index 93874d65531a..b32f178ca38e 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp @@ -1090,6 +1090,7 @@ bool PPCInstrInfo::isReallyTriviallyReMaterializable( case PPC::LIS8: case PPC::ADDIStocHA: case PPC::ADDIStocHA8: + case PPC::ADDItocL: case PPC::ADDItocL8: case PPC::LOAD_STACK_GUARD: case PPC::PPCLdFixedAddr: diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td index 6423e692d88c..43e3902cf407 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td @@ -3346,11 +3346,13 @@ def ADDIStocHA : PPCEmitTimePseudo<(outs gprc:$rD), (ins gprc_nor0:$reg, tocentr "#ADDIStocHA", [(set i32:$rD, (PPCtoc_entry i32:$reg, tglobaladdr:$disp))]>; -// Local Data Transform +// TOC Data Transform AIX def ADDItoc : PPCEmitTimePseudo<(outs gprc:$rD), (ins tocentry32:$disp, gprc:$reg), "#ADDItoc", [(set i32:$rD, (PPCtoc_entry tglobaladdr:$disp, i32:$reg))]>; +def ADDItocL : PPCEmitTimePseudo<(outs gprc:$rD), (ins gprc_nor0:$reg, tocentry32:$disp), + "#ADDItocL", []>; // Get Global (GOT) Base Register offset, from the word immediately preceding // the function label. diff --git a/llvm/lib/Target/PowerPC/PPCMacroFusion.def b/llvm/lib/Target/PowerPC/PPCMacroFusion.def index fb6e656edb8b..1a61ae23e5d7 100644 --- a/llvm/lib/Target/PowerPC/PPCMacroFusion.def +++ b/llvm/lib/Target/PowerPC/PPCMacroFusion.def @@ -32,7 +32,7 @@ // {addi} followed by one of these {lxvd2x, lxvw4x, lxvdsx, lvebx, lvehx, // lvewx, lvx, lxsdx} FUSION_FEATURE(AddiLoad, hasAddiLoadFusion, 2, \ - FUSION_OP_SET(ADDI, ADDI8, ADDItocL8), \ + FUSION_OP_SET(ADDI, ADDI8, ADDItocL, ADDItocL8), \ FUSION_OP_SET(LXVD2X, LXVW4X, LXVDSX, LVEBX, LVEHX, LVEWX, \ LVX, LXSDX)) @@ -134,13 +134,13 @@ FUSION_FEATURE(XorisXori, hasWideImmFusion, 1, FUSION_OP_SET(XORIS, XORIS8), // addis rx,ra,si - addi rt,rx,SI, SI >= 0 FUSION_FEATURE(AddisAddi, hasWideImmFusion, 1, - FUSION_OP_SET(ADDIS, ADDIS8, ADDIStocHA8), - FUSION_OP_SET(ADDI, ADDI8, ADDItocL8)) + FUSION_OP_SET(ADDIS, ADDIS8, ADDIStocHA8, ADDIStocHA), + FUSION_OP_SET(ADDI, ADDI8, ADDItocL8, ADDItocL)) // addi rx,ra,si - addis rt,rx,SI, ra > 0, SI >= 2 FUSION_FEATURE(AddiAddis, hasWideImmFusion, 1, - FUSION_OP_SET(ADDI, ADDI8, ADDItocL8), - FUSION_OP_SET(ADDIS, ADDIS8, ADDIStocHA8)) + FUSION_OP_SET(ADDI, ADDI8, ADDItocL8, ADDItocL), + FUSION_OP_SET(ADDIS, ADDIS8, ADDIStocHA8, ADDIStocHA)) // mtctr - { bcctr,bcctrl } FUSION_FEATURE(ZeroMoveCTR, hasZeroMoveFusion, -1, diff --git a/llvm/test/CodeGen/PowerPC/toc-data.ll b/llvm/test/CodeGen/PowerPC/toc-data.ll index cbf3be9fcaad..7f7afe76cfcd 100644 --- a/llvm/test/CodeGen/PowerPC/toc-data.ll +++ b/llvm/test/CodeGen/PowerPC/toc-data.ll @@ -12,6 +12,14 @@ ; RUN: llc -mtriple powerpc-ibm-aix-xcoff -verify-machineinstrs -O0 < %s | FileCheck %s --check-prefix TEST32 ; RUN: llc -mtriple powerpc64-ibm-aix-xcoff -verify-machineinstrs -O0 < %s | FileCheck %s --check-prefix TEST64 +; RUN: llc -mtriple powerpc-ibm-aix-xcoff -code-model=large -verify-machineinstrs < %s \ +; RUN: -stop-before=ppc-vsx-copy | FileCheck %s --check-prefix CHECK32LARGE +; RUN: llc -mtriple powerpc-ibm-aix-xcoff -code-model=large -verify-machineinstrs < %s | FileCheck %s --check-prefix TEST32LARGE + +; Global variables i and f have the toc-data attribute. +; In the following functions, those writing to or reading from +; variables i and f should use the toc-data access pattern. +; All remaining variables should use the regular toc access sequence. @i = dso_local global i32 0, align 4 #0 @d = dso_local local_unnamed_addr global double 3.141590e+00, align 8 @f = dso_local local_unnamed_addr global float 0x4005BE76C0000000, align 4 #0 @@ -44,6 +52,16 @@ define dso_local void @write_int(i32 signext %in) { ; TEST64: la 4, i[TD](2) ; TEST64-NEXT: stw 3, 0(4) +; CHECK32LARGE: name: write_int +; CHECK32LARGE: %[[SCRATCH1:[0-9]+]]:gprc_and_gprc_nor0 = ADDIStocHA $r2, @i +; CHECK32LARGE-NEXT: %[[SCRATCH2:[0-9]+]]:gprc_and_gprc_nor0 = ADDItocL killed %[[SCRATCH1]], @i +; CHECK32LARGE-NEXT: STW %{{[0-9]+}}, 0, killed %[[SCRATCH2]] :: (store (s32) into @i) + +; FIXME: peephole optimization opportunity for lower part relocation @l to the consuming stw +; TEST32LARGE: .write_int: +; TEST32LARGE: addis 4, i[TD]@u(2) +; TEST32LARGE-NEXT: la 4, i[TD]@l(4) +; TEST32LARGE-NEXT: stw 3, 0(4) define dso_local i64 @read_ll() { entry: @@ -70,6 +88,15 @@ define dso_local i64 @read_ll() { ; TEST64: ld 3, L..C0(2) ; TEST64-NEXT: ld 3, 0(3) +; CHECK32LARGE: name: read_ll +; CHECK32LARGE: %[[SCRATCH1:[0-9]+]]:gprc_and_gprc_nor0 = ADDIStocHA $r2, @ll +; CHECK32LARGE: LWZtocL @ll, killed %[[SCRATCH1]] :: (load (s32) from got) + +; TEST32LARGE: .read_ll: +; TEST32LARGE: addis 3, L..C0@u(2) +; TEST32LARGE-NEXT: lwz 4, L..C0@l(3) +; TEST32LARGE-NEXT: lwz 3, 0(4) +; TEST32LARGE-NEXT: lwz 4, 4(4) define dso_local float @read_float() { entry: @@ -96,6 +123,16 @@ define dso_local float @read_float() { ; TEST64: la 3, f[TD](2) ; TEST64-NEXT: lfs 1, 0(3) +; CHECK32LARGE: name: read_float +; CHECK32LARGE: %[[SCRATCH1:[0-9]+]]:gprc_and_gprc_nor0 = ADDIStocHA $r2, @f +; CHECK32LARGE-NEXT: %[[SCRATCH2:[0-9]+]]:gprc_and_gprc_nor0 = ADDItocL killed %[[SCRATCH1]], @f +; CHECK32LARGE-NEXT: LFS 0, killed %[[SCRATCH2]] :: (dereferenceable load (s32) from @f) + +; FIXME: peephole optimization opportunity for lower part relocation @l to the consuming lfs +; TEST32LARGE: .read_float: +; TEST32LARGE: addis 3, f[TD]@u(2) +; TEST32LARGE-NEXT: la 3, f[TD]@l(3) +; TEST32LARGE-NEXT: lfs 1, 0(3) define dso_local void @write_double(double %in) { entry: @@ -121,6 +158,14 @@ define dso_local void @write_double(double %in) { ; TEST64: ld 3, L..C1(2) ; TEST64-NEXT: stfd 1, 0(3) +; CHECK32LARGE: name: write_double +; CHECK32LARGE: %[[SCRATCH1:[0-9]+]]:gprc_and_gprc_nor0 = ADDIStocHA $r2, @d +; CHECK32LARGE: LWZtocL @d, killed %[[SCRATCH1]] :: (load (s32) from got) + +; TEST32LARGE: .write_double: +; TEST32LARGE: addis 3, L..C1@u(2) +; TEST32LARGE-NEXT: lwz 3, L..C1@l(3) +; TEST32LARGE-NEXT: stfd 1, 0(3) define dso_local nonnull ptr @addr() { entry: @@ -144,6 +189,15 @@ define dso_local nonnull ptr @addr() { ; TEST64: .addr ; TEST64: la 3, i[TD](2) +; CHECK32LARGE: name: addr +; CHECK32LARGE: %[[SCRATCH1:[0-9]+]]:gprc_and_gprc_nor0 = ADDIStocHA $r2, @i +; CHECK32LARGE-NEXT: %[[SCRATCH2:[0-9]+]]:gprc = ADDItocL killed %[[SCRATCH1]], @i +; CHECK32LARGE-NEXT: $r3 = COPY %[[SCRATCH2]] + +; TEST32LARGE: .addr: +; TEST32LARGE: addis 3, i[TD]@u(2) +; TEST32LARGE-NEXT: la 3, i[TD]@l(3) + ; TEST32: .toc ; TEST32: .tc ll[TC],ll[RW] ; TEST32-NOT: .csect ll[TD] @@ -170,4 +224,17 @@ define dso_local nonnull ptr @addr() { ; TEST64-NEXT: .globl f[TD] ; TEST64-NOT: .tc f[TD],f[RW] +; TEST32LARGE: .toc +; TEST32LARGE: .tc ll[TE],ll[RW] +; TEST32LARGE-NOT: .csect ll[TD] +; TEST32LARGE: .tc d[TE],d[RW] +; TEST32LARGE-NOT: .csect d[TD],2 +; TEST32LARGE: .csect i[TD],2 +; TEST32LARGE-NEXT: .globl i[TD] +; TEST32LARGE-NEXT: .align 2 +; TEST32LARGE-NOT: .tc i[TE],i[RW] +; TEST32LARGE: .csect f[TD],2 +; TEST32LARGE-NEXT: .globl f[TD] +; TEST32LARGE-NOT: .tc f[TE],f[RW] + attributes #0 = { "toc-data" } -- GitLab From edbeae373489a2e710f328ceba50b4740c738217 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 17 Apr 2024 21:12:08 +0800 Subject: [PATCH 015/714] [RISCV] Explicitly bail if something modifies VL/VTYPE in doLocalPostpass If an instruction between MI and NextMI uses VL or VTYPE we demand the respective fields so as to not clobber them at their uses. But we don't consider if something might modify VL or VTYPE, and will happily coalesce two vsetvlis when we need to preserve them. This fixes this by skipping to the next vsetvli. Demanding the fields isn't enough, as we need to preserve the VL and VTYPE values even if no fields are demanded. In practice this doesn't happen, presumably due to there not being any instructions that write to VL or VTYPE without reading them. But I noticed this whilst working on a separate patch and split it out. --- llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp | 3 +++ llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp index aab91adbb64b..fa37d1ccccd7 100644 --- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp +++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp @@ -1538,6 +1538,9 @@ void RISCVInsertVSETVLI::doLocalPostpass(MachineBasicBlock &MBB) { if (!isVectorConfigInstr(MI)) { doUnion(Used, getDemanded(MI, MRI, ST)); + if (MI.isCall() || MI.isInlineAsm() || MI.modifiesRegister(RISCV::VL) || + MI.modifiesRegister(RISCV::VTYPE)) + NextMI = nullptr; continue; } diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir index d9a87c2cb12a..e8620c848f8d 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir +++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir @@ -516,9 +516,10 @@ body: | ; CHECK-LABEL: name: postpass_modify_vl ; CHECK: liveins: $x1 ; CHECK-NEXT: {{ $}} - ; CHECK-NEXT: $x0 = PseudoVSETIVLI 3, 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype + ; CHECK-NEXT: dead $x0 = PseudoVSETIVLI 3, 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $vtype ; CHECK-NEXT: $vl = COPY $x1 + ; CHECK-NEXT: dead $x0 = PseudoVSETIVLI 3, 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype ; CHECK-NEXT: [[PseudoVADD_VV_M1_:%[0-9]+]]:vr = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, 3, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype ; CHECK-NEXT: PseudoRET dead $x0 = PseudoVSETIVLI 3, 216, implicit-def $vl, implicit-def $vtype -- GitLab From 38205717501237f2b7a57eaabe65a8367e5f91c3 Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Tue, 16 Apr 2024 14:57:27 -0400 Subject: [PATCH 016/714] [C99] Remove WG14 N522 from the C status page This paper is about type compatibility rules that changed in C99, but this is only applicable across translation units and so there's nothing for us to test. The specific change was that C89 allowed different tag types (e.g., struct and union) to be compatible and C99 tightened that restriction. This is a case where the user gets whatever they get if they link two TUs with incompatible tag types. --- clang/www/c_status.html | 5 ----- 1 file changed, 5 deletions(-) diff --git a/clang/www/c_status.html b/clang/www/c_status.html index 9893170ae847..dfc1afefda24 100644 --- a/clang/www/c_status.html +++ b/clang/www/c_status.html @@ -295,11 +295,6 @@ conformance.

N570 Yes - - new structure type compatibility (tag compatibility) - N522 - Unknown - additional predefined macro names Unknown -- GitLab From 41b7341d6b27adf81262a5a0bd4e430675b73bbb Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 17 Apr 2024 14:56:46 +0100 Subject: [PATCH 017/714] [VPlan] Factor out helper to recursively collect all users (NFCI). Factor out logic to collect all users recursively to be re-used in https://github.com/llvm/llvm-project/pull/87816. --- llvm/lib/Transforms/Vectorize/VPlan.h | 5 ++++ .../Transforms/Vectorize/VPlanTransforms.cpp | 30 +++++++++---------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 148227f1f1a5..334b10e2e5d0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1049,6 +1049,11 @@ public: R->getVPDefID() == VPRecipeBase::VPVectorPointerSC; } + static inline bool classof(const VPUser *U) { + auto *R = dyn_cast(U); + return R && classof(R); + } + /// Drop all poison-generating flags. void dropPoisonGeneratingFlags() { // NOTE: This needs to be kept in-sync with diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 382bf5ac1140..78d0b5b95c5e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -852,6 +852,18 @@ bool VPlanTransforms::adjustFixedOrderRecurrences(VPlan &Plan, return true; } +static SmallVector collectUsersRecursively(VPValue *V) { + SetVector Users(V->user_begin(), V->user_end()); + for (unsigned I = 0; I != Users.size(); ++I) { + VPRecipeBase *Cur = dyn_cast(Users[I]); + if (!Cur || isa(Cur)) + continue; + for (VPValue *V : Cur->definedValues()) + Users.insert(V->user_begin(), V->user_end()); + } + return Users.takeVector(); +} + void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) { for (VPRecipeBase &R : Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) { @@ -863,24 +875,10 @@ void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) { if (RK != RecurKind::Add && RK != RecurKind::Mul) continue; - SmallSetVector Worklist; - Worklist.insert(PhiR); - - for (unsigned I = 0; I != Worklist.size(); ++I) { - VPValue *Cur = Worklist[I]; - if (auto *RecWithFlags = - dyn_cast(Cur->getDefiningRecipe())) { + for (VPUser *U : collectUsersRecursively(PhiR)) + if (auto *RecWithFlags = dyn_cast(U)) { RecWithFlags->dropPoisonGeneratingFlags(); } - - for (VPUser *U : Cur->users()) { - auto *UserRecipe = dyn_cast(U); - if (!UserRecipe) - continue; - for (VPValue *V : UserRecipe->definedValues()) - Worklist.insert(V); - } - } } } -- GitLab From 856d1c44103f09f2ed0448001de9dcda63055733 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Wed, 17 Apr 2024 14:58:13 +0100 Subject: [PATCH 018/714] [AMDGPU] Fix predicates for BUFFER_ATOMIC_FMIN/FMAX patterns (#89066) Use OtherPredicates to avoid interfering with other uses of SubtargetPredicate for GFX12. --- llvm/lib/Target/AMDGPU/BUFInstructions.td | 2 +- .../AMDGPU/fp-min-max-buffer-atomics.ll | 72 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index 273f92abf354..8053d89aeb0a 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -1726,7 +1726,7 @@ let SubtargetPredicate = isGFX12Plus in { defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_cond_sub_u32", i32, "BUFFER_ATOMIC_COND_SUB_U32_VBUFFER", ["noret"]>; } -let SubtargetPredicate = isGFX6GFX7GFX10Plus in { +let OtherPredicates = [isGFX6GFX7GFX10Plus] in { defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f32, "BUFFER_ATOMIC_FMIN">; defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f32, "BUFFER_ATOMIC_FMAX">; } diff --git a/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll index 0c62b52eb92a..587340c7aa34 100644 --- a/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll +++ b/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll @@ -4,12 +4,14 @@ ; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs | FileCheck %s -check-prefix=GFX10 ; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs | FileCheck %s -check-prefix=GFX1030 ; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs | FileCheck %s -check-prefix=GFX1100 +; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs | FileCheck %s -check-prefix=GFX12 ; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck %s -check-prefix=G_SI ; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=hawaii -verify-machineinstrs | FileCheck %s -check-prefix=G_GFX7 ; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs | FileCheck %s -check-prefix=G_GFX10 ; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs | FileCheck %s -check-prefix=G_GFX1030 ; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs | FileCheck %s -check-prefix=G_GFX1100 +; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs | FileCheck %s -check-prefix=GFX12 declare float @llvm.amdgcn.raw.buffer.atomic.fmin.f32(float, <4 x i32>, i32, i32, i32 immarg) declare float @llvm.amdgcn.raw.buffer.atomic.fmax.f32(float, <4 x i32>, i32, i32, i32 immarg) @@ -70,6 +72,18 @@ define amdgpu_kernel void @raw_buffer_atomic_min_noret_f32(<4 x i32> inreg %rsrc ; GFX1100-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX1100-NEXT: s_endpgm ; +; GFX12-LABEL: raw_buffer_atomic_min_noret_f32: +; GFX12: ; %bb.0: ; %main_body +; GFX12-NEXT: s_clause 0x1 +; GFX12-NEXT: s_load_b64 s[4:5], s[0:1], 0x34 +; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x24 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5 +; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen +; GFX12-NEXT: s_nop 0 +; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX12-NEXT: s_endpgm +; ; G_SI-LABEL: raw_buffer_atomic_min_noret_f32: ; G_SI: ; %bb.0: ; %main_body ; G_SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd @@ -170,6 +184,15 @@ define amdgpu_ps void @raw_buffer_atomic_min_rtn_f32(<4 x i32> inreg %rsrc, floa ; GFX1100-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX1100-NEXT: s_endpgm ; +; GFX12-LABEL: raw_buffer_atomic_min_rtn_f32: +; GFX12: ; %bb.0: ; %main_body +; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], null offen th:TH_ATOMIC_RETURN +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_store_b32 v[0:1], v0, off +; GFX12-NEXT: s_nop 0 +; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX12-NEXT: s_endpgm +; ; G_SI-LABEL: raw_buffer_atomic_min_rtn_f32: ; G_SI: ; %bb.0: ; %main_body ; G_SI-NEXT: buffer_atomic_fmin v0, v1, s[0:3], 0 offen glc @@ -292,6 +315,20 @@ define amdgpu_kernel void @raw_buffer_atomic_min_rtn_f32_off4_slc(<4 x i32> inre ; GFX1100-NEXT: ds_store_b32 v1, v0 ; GFX1100-NEXT: s_endpgm ; +; GFX12-LABEL: raw_buffer_atomic_min_rtn_f32_off4_slc: +; GFX12: ; %bb.0: ; %main_body +; GFX12-NEXT: s_clause 0x1 +; GFX12-NEXT: s_load_b96 s[4:6], s[0:1], 0x34 +; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x24 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5 +; GFX12-NEXT: s_mov_b32 s4, 4 +; GFX12-NEXT: buffer_atomic_min_num_f32 v0, v1, s[0:3], s4 offen th:TH_ATOMIC_NT_RETURN +; GFX12-NEXT: v_mov_b32_e32 v1, s6 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: ds_store_b32 v1, v0 +; GFX12-NEXT: s_endpgm +; ; G_SI-LABEL: raw_buffer_atomic_min_rtn_f32_off4_slc: ; G_SI: ; %bb.0: ; %main_body ; G_SI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xd @@ -427,6 +464,18 @@ define amdgpu_kernel void @raw_buffer_atomic_max_noret_f32(<4 x i32> inreg %rsrc ; GFX1100-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX1100-NEXT: s_endpgm ; +; GFX12-LABEL: raw_buffer_atomic_max_noret_f32: +; GFX12: ; %bb.0: ; %main_body +; GFX12-NEXT: s_clause 0x1 +; GFX12-NEXT: s_load_b64 s[4:5], s[0:1], 0x34 +; GFX12-NEXT: s_load_b128 s[0:3], s[0:1], 0x24 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5 +; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen +; GFX12-NEXT: s_nop 0 +; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX12-NEXT: s_endpgm +; ; G_SI-LABEL: raw_buffer_atomic_max_noret_f32: ; G_SI: ; %bb.0: ; %main_body ; G_SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd @@ -527,6 +576,15 @@ define amdgpu_ps void @raw_buffer_atomic_max_rtn_f32(<4 x i32> inreg %rsrc, floa ; GFX1100-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX1100-NEXT: s_endpgm ; +; GFX12-LABEL: raw_buffer_atomic_max_rtn_f32: +; GFX12: ; %bb.0: ; %main_body +; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], null offen th:TH_ATOMIC_RETURN +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_store_b32 v[0:1], v0, off +; GFX12-NEXT: s_nop 0 +; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX12-NEXT: s_endpgm +; ; G_SI-LABEL: raw_buffer_atomic_max_rtn_f32: ; G_SI: ; %bb.0: ; %main_body ; G_SI-NEXT: buffer_atomic_fmax v0, v1, s[0:3], 0 offen glc @@ -641,6 +699,20 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f32_off4_slc(<4 x i32> inre ; GFX1100-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX1100-NEXT: s_endpgm ; +; GFX12-LABEL: raw_buffer_atomic_max_rtn_f32_off4_slc: +; GFX12: ; %bb.0: ; %main_body +; GFX12-NEXT: s_load_b256 s[0:7], s[0:1], 0x24 +; GFX12-NEXT: s_wait_kmcnt 0x0 +; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5 +; GFX12-NEXT: s_mov_b32 s4, 4 +; GFX12-NEXT: buffer_atomic_max_num_f32 v0, v1, s[0:3], s4 offen th:TH_ATOMIC_NT_RETURN +; GFX12-NEXT: v_mov_b32_e32 v1, 0 +; GFX12-NEXT: s_wait_loadcnt 0x0 +; GFX12-NEXT: global_store_b32 v1, v0, s[6:7] +; GFX12-NEXT: s_nop 0 +; GFX12-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) +; GFX12-NEXT: s_endpgm +; ; G_SI-LABEL: raw_buffer_atomic_max_rtn_f32_off4_slc: ; G_SI: ; %bb.0: ; %main_body ; G_SI-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x9 -- GitLab From 4f88c2311130791cf69da34b743b1b3ba7584a7b Mon Sep 17 00:00:00 2001 From: Guray Ozen Date: Wed, 17 Apr 2024 15:59:18 +0200 Subject: [PATCH 019/714] [mlir][py] Add NVGPU's `TensorMapDescriptorType` in py bindings (#88855) This PR adds NVGPU dialects' TensorMapDescriptorType in the py bindings. This is a follow-up issue from [this PR](https://github.com/llvm/llvm-project/pull/87153#discussion_r1546193095) --- mlir/include/mlir-c/Dialect/NVGPU.h | 11 ++++++ mlir/lib/Bindings/Python/DialectNVGPU.cpp | 41 +++++++++++++++++++++++ mlir/lib/CAPI/Dialect/NVGPU.cpp | 18 ++++++++++ mlir/python/CMakeLists.txt | 13 +++++++ mlir/python/mlir/dialects/nvgpu.py | 1 + mlir/test/python/dialects/nvgpu.py | 17 ++++++++++ 6 files changed, 101 insertions(+) create mode 100644 mlir/lib/Bindings/Python/DialectNVGPU.cpp diff --git a/mlir/include/mlir-c/Dialect/NVGPU.h b/mlir/include/mlir-c/Dialect/NVGPU.h index 580d566794c0..e58015a4a342 100644 --- a/mlir/include/mlir-c/Dialect/NVGPU.h +++ b/mlir/include/mlir-c/Dialect/NVGPU.h @@ -11,6 +11,7 @@ #define MLIR_C_DIALECT_NVGPU_H #include "mlir-c/IR.h" +#include "mlir-c/Support.h" #ifdef __cplusplus extern "C" { @@ -18,6 +19,16 @@ extern "C" { MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(NVGPU, nvgpu); +//===---------------------------------------------------------------------===// +// TensorMapDescriptorType +//===---------------------------------------------------------------------===// + +MLIR_CAPI_EXPORTED bool mlirTypeIsANVGPUTensorMapDescriptorType(MlirType type); + +MLIR_CAPI_EXPORTED MlirType mlirNVGPUTensorMapDescriptorTypeGet( + MlirContext ctx, MlirType tensorMemrefType, int swizzle, int l2promo, + int oobFill, int interleave); + #ifdef __cplusplus } #endif diff --git a/mlir/lib/Bindings/Python/DialectNVGPU.cpp b/mlir/lib/Bindings/Python/DialectNVGPU.cpp new file mode 100644 index 000000000000..341e4d55bcf2 --- /dev/null +++ b/mlir/lib/Bindings/Python/DialectNVGPU.cpp @@ -0,0 +1,41 @@ +//===--- DialectNvgpu.cpp - Pybind module for Nvgpu dialect API support ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir-c/Dialect/NVGPU.h" +#include "mlir-c/IR.h" +#include "mlir/Bindings/Python/PybindAdaptors.h" +#include + +namespace py = pybind11; +using namespace llvm; +using namespace mlir; +using namespace mlir::python; +using namespace mlir::python::adaptors; + +static void populateDialectNvgpuSubmodule(const pybind11::module &m) { + auto nvgpuTensorMapDescriptorType = mlir_type_subclass( + m, "TensorMapDescriptorType", mlirTypeIsANVGPUTensorMapDescriptorType); + + nvgpuTensorMapDescriptorType.def_classmethod( + "get", + [](py::object cls, MlirType tensorMemrefType, int swizzle, int l2promo, + int oobFill, int interleave, MlirContext ctx) { + return cls(mlirNVGPUTensorMapDescriptorTypeGet( + ctx, tensorMemrefType, swizzle, l2promo, oobFill, interleave)); + }, + "Gets an instance of TensorMapDescriptorType in the same context", + py::arg("cls"), py::arg("tensor_type"), py::arg("swizzle"), + py::arg("l2promo"), py::arg("oob_fill"), py::arg("interleave"), + py::arg("ctx") = py::none()); +} + +PYBIND11_MODULE(_mlirDialectsNvgpu, m) { + m.doc() = "MLIR NVGPU dialect."; + + populateDialectNvgpuSubmodule(m); +} diff --git a/mlir/lib/CAPI/Dialect/NVGPU.cpp b/mlir/lib/CAPI/Dialect/NVGPU.cpp index 02d10954a037..e6da529e1b6b 100644 --- a/mlir/lib/CAPI/Dialect/NVGPU.cpp +++ b/mlir/lib/CAPI/Dialect/NVGPU.cpp @@ -9,5 +9,23 @@ #include "mlir-c/Dialect/NVGPU.h" #include "mlir/CAPI/Registration.h" #include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h" +#include "mlir/IR/BuiltinTypes.h" + +using namespace mlir; +using namespace mlir::nvgpu; MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(NVGPU, nvgpu, mlir::nvgpu::NVGPUDialect) + +bool mlirTypeIsANVGPUTensorMapDescriptorType(MlirType type) { + return isa(unwrap(type)); +} + +MlirType mlirNVGPUTensorMapDescriptorTypeGet(MlirContext ctx, + MlirType tensorMemrefType, + int swizzle, int l2promo, + int oobFill, int interleave) { + return wrap(nvgpu::TensorMapDescriptorType::get( + unwrap(ctx), cast(unwrap(tensorMemrefType)), + TensorMapSwizzleKind(swizzle), TensorMapL2PromoKind(l2promo), + TensorMapOOBKind(oobFill), TensorMapInterleaveKind(interleave))); +} diff --git a/mlir/python/CMakeLists.txt b/mlir/python/CMakeLists.txt index c27ee688a040..0a2dc0754c09 100644 --- a/mlir/python/CMakeLists.txt +++ b/mlir/python/CMakeLists.txt @@ -524,6 +524,19 @@ declare_mlir_python_extension(MLIRPythonExtension.Dialects.Quant.Pybind MLIRCAPIQuant ) +declare_mlir_python_extension(MLIRPythonExtension.Dialects.NVGPU.Pybind + MODULE_NAME _mlirDialectsNvgpu + ADD_TO_PARENT MLIRPythonSources.Dialects.nvgpu + ROOT_DIR "${PYTHON_SOURCE_DIR}" + SOURCES + DialectNVGPU.cpp + PRIVATE_LINK_LIBS + LLVMSupport + EMBED_CAPI_LINK_LIBS + MLIRCAPIIR + MLIRCAPINVGPU +) + declare_mlir_python_extension(MLIRPythonExtension.Dialects.PDL.Pybind MODULE_NAME _mlirDialectsPDL ADD_TO_PARENT MLIRPythonSources.Dialects.pdl diff --git a/mlir/python/mlir/dialects/nvgpu.py b/mlir/python/mlir/dialects/nvgpu.py index 2f6993b768ca..e19bf610ea33 100644 --- a/mlir/python/mlir/dialects/nvgpu.py +++ b/mlir/python/mlir/dialects/nvgpu.py @@ -4,3 +4,4 @@ from ._nvgpu_ops_gen import * from ._nvgpu_enum_gen import * +from .._mlir_libs._mlirDialectsNvgpu import * diff --git a/mlir/test/python/dialects/nvgpu.py b/mlir/test/python/dialects/nvgpu.py index 3158388f0e68..6df32bdd3c27 100644 --- a/mlir/test/python/dialects/nvgpu.py +++ b/mlir/test/python/dialects/nvgpu.py @@ -15,6 +15,23 @@ def constructAndPrintInModule(f): return f +# CHECK-LABEL: testTypes +@constructAndPrintInModule +def testTypes(): + tensorMemrefType = MemRefType.get( + (128, 64), F16Type.get(), memory_space=Attribute.parse("3") + ) + # CHECK: !nvgpu.tensormap.descriptor, swizzle = swizzle_128b, l2promo = l2promo_256b, oob = nan, interleave = none> + tma_desc = nvgpu.TensorMapDescriptorType.get( + tensorMemrefType, + nvgpu.TensorMapSwizzleKind.SWIZZLE_128B, + nvgpu.TensorMapL2PromoKind.L2PROMO_256B, + nvgpu.TensorMapOOBKind.OOB_NAN, + nvgpu.TensorMapInterleaveKind.INTERLEAVE_NONE, + ) + print(tma_desc) + + # CHECK-LABEL: testSmoke @constructAndPrintInModule def testSmoke(): -- GitLab From fda04b1caaf1a61b208f23e717a2db6d9b861f5a Mon Sep 17 00:00:00 2001 From: Rajveer Singh Bharadwaj Date: Wed, 17 Apr 2024 19:37:18 +0530 Subject: [PATCH 020/714] [libc] Replace mentions of `LIBC_FULLBUILD` with `LLVM_LIBC_FULL_BUILD` in 'examples/' (#88657) Resolves #88328 --- libc/examples/README.md | 2 +- libc/examples/examples.cmake | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libc/examples/README.md b/libc/examples/README.md index 36b886090c6c..1bc4a67294f2 100644 --- a/libc/examples/README.md +++ b/libc/examples/README.md @@ -59,7 +59,7 @@ have installed them, you have to inform CMake that we are linking against the full libc as follows: ```bash -cmake ../ -G -DLIBC_FULLBUILD=ON \ +cmake ../ -G -DLLVM_LIBC_FULL_BUILD=ON \ -DCMAKE_SYSROOT= \ -DCMAKE_C_COMPILER=/bin/clang \ -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY diff --git a/libc/examples/examples.cmake b/libc/examples/examples.cmake index 81e99e3cbede..6bb6b41c252f 100644 --- a/libc/examples/examples.cmake +++ b/libc/examples/examples.cmake @@ -4,13 +4,13 @@ function(add_example name) ${ARGN} ) - if(LIBC_FULLBUILD) + if(LLVM_LIBC_FULL_BUILD) target_link_options(${name} PRIVATE -static -rtlib=compiler-rt -fuse-ld=lld) elseif(LIBC_OVERLAY_ARCHIVE_DIR) target_link_directories(${name} PRIVATE ${LIBC_OVERLAY_ARCHIVE_DIR}) target_link_options(${name} PRIVATE -l:libllvmlibc.a) else() - message(FATAL_ERROR "Either LIBC_FULLBUILD should be on or " + message(FATAL_ERROR "Either LLVM_LIBC_FULL_BUILD should be on or " "LIBC_OVERLAY_ARCHIVE_DIR should be set.") endif() endfunction() -- GitLab From d558c090fc78beb6737098f058a084635b893567 Mon Sep 17 00:00:00 2001 From: yronglin Date: Wed, 17 Apr 2024 22:11:04 +0800 Subject: [PATCH 021/714] [NFC] Clean dead code in ParsedAttr.h (#89064) Signed-off-by: yronglin --- clang/include/clang/Sema/ParsedAttr.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/clang/include/clang/Sema/ParsedAttr.h b/clang/include/clang/Sema/ParsedAttr.h index e3857b2f07d9..25a5fa05b21c 100644 --- a/clang/include/clang/Sema/ParsedAttr.h +++ b/clang/include/clang/Sema/ParsedAttr.h @@ -94,7 +94,7 @@ struct PropertyData { : GetterId(getterId), SetterId(setterId) {} }; -} // namespace +} // namespace detail /// Wraps an identifier and optional source location for the identifier. struct IdentifierLoc { @@ -743,11 +743,6 @@ public: IdentifierInfo *scopeName, SourceLocation scopeLoc, ArgsUnion *args, unsigned numArgs, ParsedAttr::Form form, SourceLocation ellipsisLoc = SourceLocation()) { - size_t temp = - ParsedAttr::totalSizeToAlloc(numArgs, 0, 0, 0, 0); - (void)temp; void *memory = allocate( ParsedAttr::totalSizeToAlloc Date: Wed, 17 Apr 2024 14:29:09 +0000 Subject: [PATCH 022/714] [lldb] XFAIL TestDetachResumes on windows --- .../API/commands/process/detach-resumes/TestDetachResumes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lldb/test/API/commands/process/detach-resumes/TestDetachResumes.py b/lldb/test/API/commands/process/detach-resumes/TestDetachResumes.py index 57727294ddc3..797db55a45b9 100644 --- a/lldb/test/API/commands/process/detach-resumes/TestDetachResumes.py +++ b/lldb/test/API/commands/process/detach-resumes/TestDetachResumes.py @@ -12,6 +12,7 @@ from lldbsuite.test import lldbutil class DetachResumesTestCase(TestBase): NO_DEBUG_INFO_TESTCASE = True + @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr89077") def test_detach_resumes(self): self.build() exe = self.getBuildArtifact() -- GitLab From e49043512dbdc68319093da46e95a1e331ef837e Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 17 Apr 2024 14:55:53 +0100 Subject: [PATCH 023/714] [CostModel][X86] Update BITREVERSE costs for GFNI targets Inspired by the recent patches by @shamithoke - we have real scheduler model numbers for GFNI instructions now, allowing us to calculate an upper bounds costs table instead of performing it analytically. --- .../lib/Target/X86/X86TargetTransformInfo.cpp | 41 +++-- .../CostModel/X86/bitreverse-codesize.ll | 64 +++---- .../CostModel/X86/bitreverse-latency.ll | 160 +++++++++--------- .../CostModel/X86/bitreverse-sizelatency.ll | 160 +++++++++--------- .../test/Analysis/CostModel/X86/bitreverse.ll | 136 +++++++-------- 5 files changed, 284 insertions(+), 277 deletions(-) diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index b466624e1334..38064f979269 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -3820,6 +3820,24 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, { ISD::FSQRT, MVT::v2f64, { 27, 27, 1, 1 } }, // vsqrtpd { ISD::FSQRT, MVT::v4f64, { 54, 54, 1, 3 } }, // vsqrtpd }; + static const CostKindTblEntry GFNICostTbl[] = { + { ISD::BITREVERSE, MVT::i8, { 3, 3, 3, 4 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::i16, { 3, 3, 4, 6 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::i32, { 3, 3, 4, 5 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::i64, { 3, 3, 4, 6 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v16i8, { 1, 6, 1, 2 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v32i8, { 1, 6, 1, 2 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v64i8, { 1, 6, 1, 2 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v8i16, { 1, 8, 2, 4 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v16i16, { 1, 9, 2, 4 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v32i16, { 1, 9, 2, 4 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v4i32, { 1, 8, 2, 4 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v8i32, { 1, 9, 2, 4 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v16i32, { 1, 9, 2, 4 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v2i64, { 1, 8, 2, 4 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v4i64, { 1, 9, 2, 4 } }, // gf2p8affineqb + { ISD::BITREVERSE, MVT::v8i64, { 1, 9, 2, 4 } }, // gf2p8affineqb + }; static const CostKindTblEntry GLMCostTbl[] = { { ISD::FSQRT, MVT::f32, { 19, 20, 1, 1 } }, // sqrtss { ISD::FSQRT, MVT::v4f32, { 37, 41, 1, 5 } }, // sqrtps @@ -4156,23 +4174,6 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, std::pair LT = getTypeLegalizationCost(OpTy); MVT MTy = LT.second; - // Attempt to lookup cost. - if (ISD == ISD::BITREVERSE && ST->hasGFNI() && ST->hasSSSE3() && - MTy.isVector()) { - // With PSHUFB the code is very similar for all types. If we have integer - // byte operations, we just need a GF2P8AFFINEQB for vXi8. For other types - // we also need a PSHUFB. - unsigned Cost = MTy.getVectorElementType() == MVT::i8 ? 1 : 2; - - // Without byte operations, we need twice as many GF2P8AFFINEQB and PSHUFB - // instructions. We also need an extract and an insert. - if (!(MTy.is128BitVector() || (ST->hasAVX2() && MTy.is256BitVector()) || - (ST->hasBWI() && MTy.is512BitVector()))) - Cost = Cost * 2 + 2; - - return LT.first * Cost; - } - // Without BMI/LZCNT see if we're only looking for a *_ZERO_UNDEF cost. if (((ISD == ISD::CTTZ && !ST->hasBMI()) || (ISD == ISD::CTLZ && !ST->hasLZCNT())) && @@ -4230,6 +4231,12 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, return adjustTableCost(Entry->ISD, *KindCost, LT.first, ICA.getFlags()); + if (ST->hasGFNI()) + if (const auto *Entry = CostTableLookup(GFNICostTbl, ISD, MTy)) + if (auto KindCost = Entry->Cost[CostKind]) + return adjustTableCost(Entry->ISD, *KindCost, LT.first, + ICA.getFlags()); + if (ST->hasCDI()) if (const auto *Entry = CostTableLookup(AVX512CDCostTbl, ISD, MTy)) if (auto KindCost = Entry->Cost[CostKind]) diff --git a/llvm/test/Analysis/CostModel/X86/bitreverse-codesize.ll b/llvm/test/Analysis/CostModel/X86/bitreverse-codesize.ll index 90e49e6ccb81..e02ba761aa8d 100644 --- a/llvm/test/Analysis/CostModel/X86/bitreverse-codesize.ll +++ b/llvm/test/Analysis/CostModel/X86/bitreverse-codesize.ll @@ -40,23 +40,23 @@ define i64 @var_bitreverse_i64(i64 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) @@ -77,23 +77,23 @@ define i32 @var_bitreverse_i32(i32 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) @@ -114,23 +114,23 @@ define i16 @var_bitreverse_i16(i16 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) @@ -151,23 +151,23 @@ define i8 @var_bitreverse_i8(i8 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) @@ -270,7 +270,7 @@ define <4 x i64> @var_bitreverse_v4i64(<4 x i64> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v4i64' @@ -323,7 +323,7 @@ define <8 x i64> @var_bitreverse_v8i64(<8 x i64> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i64' @@ -331,7 +331,7 @@ define <8 x i64> @var_bitreverse_v8i64(<8 x i64> %a) { ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i64' @@ -421,7 +421,7 @@ define <8 x i32> @var_bitreverse_v8i32(<8 x i32> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i32' @@ -474,7 +474,7 @@ define <16 x i32> @var_bitreverse_v16i32(<16 x i32> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i32' @@ -482,7 +482,7 @@ define <16 x i32> @var_bitreverse_v16i32(<16 x i32> %a) { ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v16i32' @@ -572,7 +572,7 @@ define <16 x i16> @var_bitreverse_v16i16(<16 x i16> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i16' @@ -625,7 +625,7 @@ define <32 x i16> @var_bitreverse_v32i16(<32 x i16> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v32i16' @@ -633,7 +633,7 @@ define <32 x i16> @var_bitreverse_v32i16(<32 x i16> %a) { ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v32i16' @@ -723,7 +723,7 @@ define <32 x i8> @var_bitreverse_v32i8(<32 x i8> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v32i8' @@ -776,7 +776,7 @@ define <64 x i8> @var_bitreverse_v64i8(<64 x i8> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v64i8' @@ -784,7 +784,7 @@ define <64 x i8> @var_bitreverse_v64i8(<64 x i8> %a) { ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v64i8' diff --git a/llvm/test/Analysis/CostModel/X86/bitreverse-latency.ll b/llvm/test/Analysis/CostModel/X86/bitreverse-latency.ll index e7d0d05f8242..ba231b985c26 100644 --- a/llvm/test/Analysis/CostModel/X86/bitreverse-latency.ll +++ b/llvm/test/Analysis/CostModel/X86/bitreverse-latency.ll @@ -40,23 +40,23 @@ define i64 @var_bitreverse_i64(i64 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) @@ -77,23 +77,23 @@ define i32 @var_bitreverse_i32(i32 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) @@ -114,23 +114,23 @@ define i16 @var_bitreverse_i16(i16 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) @@ -151,23 +151,23 @@ define i8 @var_bitreverse_i8(i8 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) @@ -221,23 +221,23 @@ define <2 x i64> @var_bitreverse_v2i64(<2 x i64> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v2i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) @@ -270,23 +270,23 @@ define <4 x i64> @var_bitreverse_v4i64(<4 x i64> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v4i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) @@ -323,23 +323,23 @@ define <8 x i64> @var_bitreverse_v8i64(<8 x i64> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v8i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) @@ -376,23 +376,23 @@ define <4 x i32> @var_bitreverse_v4i32(<4 x i32> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v4i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) @@ -425,23 +425,23 @@ define <8 x i32> @var_bitreverse_v8i32(<8 x i32> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v8i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) @@ -478,23 +478,23 @@ define <16 x i32> @var_bitreverse_v16i32(<16 x i32> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v16i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) @@ -531,23 +531,23 @@ define <8 x i16> @var_bitreverse_v8i16(<8 x i16> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v8i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) @@ -580,23 +580,23 @@ define <16 x i16> @var_bitreverse_v16i16(<16 x i16> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v16i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) @@ -633,23 +633,23 @@ define <32 x i16> @var_bitreverse_v32i16(<32 x i16> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v32i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) @@ -686,23 +686,23 @@ define <16 x i8> @var_bitreverse_v16i8(<16 x i8> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v16i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v16i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v16i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) @@ -735,23 +735,23 @@ define <32 x i8> @var_bitreverse_v32i8(<32 x i8> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v32i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) @@ -788,23 +788,23 @@ define <64 x i8> @var_bitreverse_v64i8(<64 x i8> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v64i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) diff --git a/llvm/test/Analysis/CostModel/X86/bitreverse-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/bitreverse-sizelatency.ll index 1ba93af46ebd..d60fac228fc0 100644 --- a/llvm/test/Analysis/CostModel/X86/bitreverse-sizelatency.ll +++ b/llvm/test/Analysis/CostModel/X86/bitreverse-sizelatency.ll @@ -40,23 +40,23 @@ define i64 @var_bitreverse_i64(i64 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %bitreverse ; %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) @@ -77,23 +77,23 @@ define i32 @var_bitreverse_i32(i32 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i32 %bitreverse ; %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) @@ -114,23 +114,23 @@ define i16 @var_bitreverse_i16(i16 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i16 %bitreverse ; %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) @@ -151,23 +151,23 @@ define i8 @var_bitreverse_i8(i8 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i8 %bitreverse ; %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) @@ -217,23 +217,23 @@ define <2 x i64> @var_bitreverse_v2i64(<2 x i64> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v2i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %bitreverse ; %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) @@ -270,23 +270,23 @@ define <4 x i64> @var_bitreverse_v4i64(<4 x i64> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v4i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %bitreverse ; %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) @@ -323,23 +323,23 @@ define <8 x i64> @var_bitreverse_v8i64(<8 x i64> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v8i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %bitreverse ; %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) @@ -372,23 +372,23 @@ define <4 x i32> @var_bitreverse_v4i32(<4 x i32> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v4i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %bitreverse ; %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) @@ -425,23 +425,23 @@ define <8 x i32> @var_bitreverse_v8i32(<8 x i32> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v8i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %bitreverse ; %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) @@ -478,23 +478,23 @@ define <16 x i32> @var_bitreverse_v16i32(<16 x i32> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v16i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %bitreverse ; %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) @@ -527,23 +527,23 @@ define <8 x i16> @var_bitreverse_v8i16(<8 x i16> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v8i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %bitreverse ; %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) @@ -580,23 +580,23 @@ define <16 x i16> @var_bitreverse_v16i16(<16 x i16> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v16i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %bitreverse ; %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) @@ -633,23 +633,23 @@ define <32 x i16> @var_bitreverse_v32i16(<32 x i16> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v32i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %bitreverse ; %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) @@ -682,23 +682,23 @@ define <16 x i8> @var_bitreverse_v16i8(<16 x i8> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v16i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v16i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v16i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %bitreverse ; %bitreverse = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %a) @@ -735,23 +735,23 @@ define <32 x i8> @var_bitreverse_v32i8(<32 x i8> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v32i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %bitreverse ; %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) @@ -788,23 +788,23 @@ define <64 x i8> @var_bitreverse_v64i8(<64 x i8> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v64i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %bitreverse ; %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) diff --git a/llvm/test/Analysis/CostModel/X86/bitreverse.ll b/llvm/test/Analysis/CostModel/X86/bitreverse.ll index 0b76b0d527ba..a890147fee46 100644 --- a/llvm/test/Analysis/CostModel/X86/bitreverse.ll +++ b/llvm/test/Analysis/CostModel/X86/bitreverse.ll @@ -40,23 +40,23 @@ define i64 @var_bitreverse_i64(i64 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %bitreverse ; %bitreverse = call i64 @llvm.bitreverse.i64(i64 %a) @@ -77,23 +77,23 @@ define i32 @var_bitreverse_i32(i32 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %bitreverse ; %bitreverse = call i32 @llvm.bitreverse.i32(i32 %a) @@ -114,23 +114,23 @@ define i16 @var_bitreverse_i16(i16 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %bitreverse ; %bitreverse = call i16 @llvm.bitreverse.i16(i16 %a) @@ -151,23 +151,23 @@ define i8 @var_bitreverse_i8(i8 %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_i8' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_i8' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_i8' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %bitreverse ; %bitreverse = call i8 @llvm.bitreverse.i8(i8 %a) @@ -217,23 +217,23 @@ define <2 x i64> @var_bitreverse_v2i64(<2 x i64> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v2i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v2i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %bitreverse ; %bitreverse = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %a) @@ -270,23 +270,23 @@ define <4 x i64> @var_bitreverse_v4i64(<4 x i64> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v4i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v4i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %bitreverse ; %bitreverse = call <4 x i64> @llvm.bitreverse.v4i64(<4 x i64> %a) @@ -323,23 +323,23 @@ define <8 x i64> @var_bitreverse_v8i64(<8 x i64> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v8i64' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i64' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %bitreverse ; %bitreverse = call <8 x i64> @llvm.bitreverse.v8i64(<8 x i64> %a) @@ -372,23 +372,23 @@ define <4 x i32> @var_bitreverse_v4i32(<4 x i32> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v4i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v4i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %bitreverse ; %bitreverse = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %a) @@ -425,23 +425,23 @@ define <8 x i32> @var_bitreverse_v8i32(<8 x i32> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v8i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %bitreverse ; %bitreverse = call <8 x i32> @llvm.bitreverse.v8i32(<8 x i32> %a) @@ -478,23 +478,23 @@ define <16 x i32> @var_bitreverse_v16i32(<16 x i32> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v16i32' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v16i32' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i32> %bitreverse ; %bitreverse = call <16 x i32> @llvm.bitreverse.v16i32(<16 x i32> %a) @@ -527,23 +527,23 @@ define <8 x i16> @var_bitreverse_v8i16(<8 x i16> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v8i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v8i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %bitreverse ; %bitreverse = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %a) @@ -580,23 +580,23 @@ define <16 x i16> @var_bitreverse_v16i16(<16 x i16> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v16i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v16i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %bitreverse ; %bitreverse = call <16 x i16> @llvm.bitreverse.v16i16(<16 x i16> %a) @@ -633,23 +633,23 @@ define <32 x i16> @var_bitreverse_v32i16(<32 x i16> %a) { ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %bitreverse ; ; GFNISSE-LABEL: 'var_bitreverse_v32i16' -; GFNISSE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNISSE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v32i16' -; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) +; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) ; GFNIAVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %bitreverse ; %bitreverse = call <32 x i16> @llvm.bitreverse.v32i16(<32 x i16> %a) @@ -743,7 +743,7 @@ define <32 x i8> @var_bitreverse_v32i8(<32 x i8> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v32i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <32 x i8> @llvm.bitreverse.v32i8(<32 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v32i8' @@ -796,7 +796,7 @@ define <64 x i8> @var_bitreverse_v64i8(<64 x i8> %a) { ; GFNISSE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX2-LABEL: 'var_bitreverse_v64i8' @@ -804,7 +804,7 @@ define <64 x i8> @var_bitreverse_v64i8(<64 x i8> %a) { ; GFNIAVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX512F-LABEL: 'var_bitreverse_v64i8' -; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) +; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %bitreverse = call <64 x i8> @llvm.bitreverse.v64i8(<64 x i8> %a) ; GFNIAVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %bitreverse ; ; GFNIAVX512BW-LABEL: 'var_bitreverse_v64i8' -- GitLab From 4a5ab13bf5a94ec7f0eabaf24dfe1a5ee720b860 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 17 Apr 2024 15:46:24 +0100 Subject: [PATCH 024/714] [VectorCombine] Remove single quotes from "-passes=vector-combine" These confuse the update_test_checks.py script when run by DOS cmd.exe --- llvm/test/Transforms/VectorCombine/AArch64/select-shuffle.ll | 2 +- llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll | 2 +- llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/test/Transforms/VectorCombine/AArch64/select-shuffle.ll b/llvm/test/Transforms/VectorCombine/AArch64/select-shuffle.ll index eae087900483..b27c026dbccf 100644 --- a/llvm/test/Transforms/VectorCombine/AArch64/select-shuffle.ll +++ b/llvm/test/Transforms/VectorCombine/AArch64/select-shuffle.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='vector-combine' -S %s | FileCheck %s +; RUN: opt -passes=vector-combine -S %s | FileCheck %s target triple = "aarch64" diff --git a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll index d96dfec84916..a1d4ca1a740e 100644 --- a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll +++ b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='vector-combine' -S %s | FileCheck %s +; RUN: opt -passes=vector-combine -S %s | FileCheck %s target triple = "aarch64" diff --git a/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll b/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll index d69cb75664a8..a22575ccb1ca 100644 --- a/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll +++ b/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes='vector-combine' -S %s | FileCheck %s +; RUN: opt -passes=vector-combine -S %s | FileCheck %s target triple = "aarch64" -- GitLab From 5d314353fbec1a15cd8900f466dcdcf2af40e8c9 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 17 Apr 2024 15:53:15 +0100 Subject: [PATCH 025/714] [VPlan] Check for VPWidenLoadRecipe directly in truncateToMinBW. (NFCI). Since ne After a separate recipe has been introduced for wide loads in a9bafe91dd0, we can directly check for load recipes in the early bail-out and remove the redundant bail out for stores. --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 78d0b5b95c5e..901ecd10c69d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -975,9 +975,7 @@ void VPlanTransforms::truncateToMinimalBitwidths( vp_depth_first_deep(Plan.getVectorLoopRegion()))) { for (VPRecipeBase &R : make_early_inc_range(*VPBB)) { if (!isa(&R)) - continue; - if (isa(&R)) + VPWidenSelectRecipe, VPWidenLoadRecipe>(&R)) continue; VPValue *ResultVPV = R.getVPSingleValue(); -- GitLab From 812963f6aa2adb5e990f273b8ce1a0eabcdefd7f Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Wed, 17 Apr 2024 17:02:07 +0200 Subject: [PATCH 026/714] [libc++][chrono] Improves date formatting. (#86127) The formatting of years has been done manually since the results of %Y outside the "typical" range may produce unexpected values. The same applies to %F which is identical to %Y-%m-%d. None of these conversion specifiers is affected by the locale used. So it's trivial to manually handle this case. This removes several platform specific ifdefs from the tests. --- libcxx/include/__chrono/formatter.h | 16 ++++----- .../time.cal.ymd.nonmembers/ostream.pass.cpp | 34 ------------------- .../sys_date.ostream.pass.cpp | 34 ------------------- .../formatter.year_month_day.pass.cpp | 24 ------------- 4 files changed, 7 insertions(+), 101 deletions(-) diff --git a/libcxx/include/__chrono/formatter.h b/libcxx/include/__chrono/formatter.h index d932a99f4b99..f76e7b2ea0e8 100644 --- a/libcxx/include/__chrono/formatter.h +++ b/libcxx/include/__chrono/formatter.h @@ -322,15 +322,13 @@ _LIBCPP_HIDE_FROM_ABI void __format_chrono_using_chrono_specs( __formatter::__format_year(__sstr, __t.tm_year + 1900); break; - case _CharT('F'): { - int __year = __t.tm_year + 1900; - if (__year < 1000) { - __formatter::__format_year(__sstr, __year); - __sstr << std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "-{:02}-{:02}"), __t.tm_mon + 1, __t.tm_mday); - } else - __facet.put( - {__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1)); - } break; + case _CharT('F'): + // Depending on the platform's libc the range of supported years is + // limited. Instead of testing all conditions use the internal + // implementation unconditionally. + __formatter::__format_year(__sstr, __t.tm_year + 1900); + __sstr << std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "-{:02}-{:02}"), __t.tm_mon + 1, __t.tm_mday); + break; case _CharT('z'): __formatter::__format_zone_offset(__sstr, __z.__offset, false); diff --git a/libcxx/test/std/time/time.cal/time.cal.ymd/time.cal.ymd.nonmembers/ostream.pass.cpp b/libcxx/test/std/time/time.cal/time.cal.ymd/time.cal.ymd.nonmembers/ostream.pass.cpp index ffc737fcad5d..20ffb165558e 100644 --- a/libcxx/test/std/time/time.cal/time.cal.ymd/time.cal.ymd.nonmembers/ostream.pass.cpp +++ b/libcxx/test/std/time/time.cal/time.cal.ymd/time.cal.ymd.nonmembers/ostream.pass.cpp @@ -13,9 +13,6 @@ // TODO FMT This test should not require std::to_chars(floating-point) // XFAIL: availability-fp_to_chars-missing -// TODO FMT Investigate Windows issues. -// XFAIL: msvc - // REQUIRES: locale.fr_FR.UTF-8 // REQUIRES: locale.ja_JP.UTF-8 @@ -89,20 +86,9 @@ static void test() { TEST_EQUAL(stream_c_locale( std::chrono::year_month_day{std::chrono::year{2000}, std::chrono::month{2}, std::chrono::day{29}}), SV("2000-02-29")); - -#if defined(_AIX) - TEST_EQUAL(stream_c_locale( - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}), - SV("+32767-12-31")); -#elif defined(_WIN32) // defined(_AIX) - TEST_EQUAL(stream_c_locale( - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}), - SV("")); -#else // defined(_AIX) TEST_EQUAL(stream_c_locale( std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}), SV("32767-12-31")); -#endif // defined(_AIX) TEST_EQUAL(stream_fr_FR_locale( std::chrono::year_month_day{std::chrono::year{-32'768}, std::chrono::month{1}, std::chrono::day{1}}), @@ -122,19 +108,9 @@ static void test() { TEST_EQUAL(stream_fr_FR_locale( std::chrono::year_month_day{std::chrono::year{2000}, std::chrono::month{2}, std::chrono::day{29}}), SV("2000-02-29")); -#if defined(_AIX) - TEST_EQUAL(stream_fr_FR_locale( - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}), - SV("+32767-12-31")); -#elif defined(_WIN32) // defined(_AIX) - TEST_EQUAL(stream_fr_FR_locale( - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}), - SV("")); -#else // defined(_AIX) TEST_EQUAL(stream_fr_FR_locale( std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}), SV("32767-12-31")); -#endif // defined(_AIX) TEST_EQUAL(stream_ja_JP_locale( std::chrono::year_month_day{std::chrono::year{-32'768}, std::chrono::month{1}, std::chrono::day{1}}), @@ -154,19 +130,9 @@ static void test() { TEST_EQUAL(stream_ja_JP_locale( std::chrono::year_month_day{std::chrono::year{2000}, std::chrono::month{2}, std::chrono::day{29}}), SV("2000-02-29")); -#if defined(_AIX) - TEST_EQUAL(stream_ja_JP_locale( - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}), - SV("+32767-12-31")); -#elif defined(_WIN32) // defined(_AIX) - TEST_EQUAL(stream_ja_JP_locale( - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}), - SV("")); -#else // defined(_AIX) TEST_EQUAL(stream_ja_JP_locale( std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}), SV("32767-12-31")); -#endif // defined(_AIX) } int main(int, char**) { diff --git a/libcxx/test/std/time/time.clock/time.clock.system/sys_date.ostream.pass.cpp b/libcxx/test/std/time/time.clock/time.clock.system/sys_date.ostream.pass.cpp index 7af3ebf77680..c5645884cfed 100644 --- a/libcxx/test/std/time/time.clock/time.clock.system/sys_date.ostream.pass.cpp +++ b/libcxx/test/std/time/time.clock/time.clock.system/sys_date.ostream.pass.cpp @@ -13,9 +13,6 @@ // TODO FMT This test should not require std::to_chars(floating-point) // XFAIL: availability-fp_to_chars-missing -// TODO FMT Investigate Windows issues. -// XFAIL: msvc - // REQUIRES: locale.fr_FR.UTF-8 // REQUIRES: locale.ja_JP.UTF-8 @@ -81,20 +78,9 @@ static void test() { TEST_EQUAL(stream_c_locale(std::chrono::sys_days{ std::chrono::year_month_day{std::chrono::year{2000}, std::chrono::month{2}, std::chrono::day{29}}}), SV("2000-02-29")); - -#if defined(_AIX) - TEST_EQUAL(stream_c_locale(std::chrono::sys_days{ - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}}), - SV("+32767-12-31")); -#elif defined(_WIN32) // defined(_AIX) - TEST_EQUAL(stream_c_locale(std::chrono::sys_days{ - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}}), - SV("")); -#else // defined(_AIX) TEST_EQUAL(stream_c_locale(std::chrono::sys_days{ std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}}), SV("32767-12-31")); -#endif // defined(_AIX) // multiples of days are considered days. TEST_EQUAL(stream_c_locale(std::chrono::sys_time{std::chrono::weeks{3}}), @@ -112,19 +98,9 @@ static void test() { TEST_EQUAL(stream_fr_FR_locale(std::chrono::sys_days{ std::chrono::year_month_day{std::chrono::year{2000}, std::chrono::month{2}, std::chrono::day{29}}}), SV("2000-02-29")); -#if defined(_AIX) - TEST_EQUAL(stream_fr_FR_locale(std::chrono::sys_days{ - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}}), - SV("+32767-12-31")); -#elif defined(_WIN32) // defined(_AIX) - TEST_EQUAL(stream_fr_FR_locale(std::chrono::sys_days{ - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}}), - SV("")); -#else // defined(_AIX) TEST_EQUAL(stream_fr_FR_locale(std::chrono::sys_days{ std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}}), SV("32767-12-31")); -#endif // defined(_AIX) // multiples of days are considered days. TEST_EQUAL(stream_fr_FR_locale(std::chrono::sys_time{std::chrono::weeks{3}}), @@ -142,19 +118,9 @@ static void test() { TEST_EQUAL(stream_ja_JP_locale(std::chrono::sys_days{ std::chrono::year_month_day{std::chrono::year{2000}, std::chrono::month{2}, std::chrono::day{29}}}), SV("2000-02-29")); -#if defined(_AIX) - TEST_EQUAL(stream_ja_JP_locale(std::chrono::sys_days{ - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}}), - SV("+32767-12-31")); -#elif defined(_WIN32) // defined(_AIX) - TEST_EQUAL(stream_ja_JP_locale(std::chrono::sys_days{ - std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}}), - SV("")); -#else // defined(_AIX) TEST_EQUAL(stream_ja_JP_locale(std::chrono::sys_days{ std::chrono::year_month_day{std::chrono::year{32'767}, std::chrono::month{12}, std::chrono::day{31}}}), SV("32767-12-31")); -#endif // defined(_AIX) // multiples of days are considered days. TEST_EQUAL(stream_ja_JP_locale(std::chrono::sys_time{std::chrono::weeks{3}}), diff --git a/libcxx/test/std/time/time.syn/formatter.year_month_day.pass.cpp b/libcxx/test/std/time/time.syn/formatter.year_month_day.pass.cpp index 5a2b7afa37a8..1f2af1cb0530 100644 --- a/libcxx/test/std/time/time.syn/formatter.year_month_day.pass.cpp +++ b/libcxx/test/std/time/time.syn/formatter.year_month_day.pass.cpp @@ -62,17 +62,6 @@ static void test_no_chrono_specs() { std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{2}, std::chrono::day{31}}); // Valid year, invalid month, valid day -#ifdef _WIN32 - check(SV(" is not a valid date"), - SV("{}"), - std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{31}}); - check(SV("****** is not a valid date******"), - SV("{:*^32}"), - std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{31}}); - check(SV("*********** is not a valid date"), - SV("{:*>31}"), - std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{31}}); -#else // _WIN32 check(SV("1970-00-31 is not a valid date"), SV("{}"), std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{31}}); @@ -82,20 +71,8 @@ static void test_no_chrono_specs() { check(SV("*1970-00-31 is not a valid date"), SV("{:*>31}"), std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{31}}); -#endif // _WIN32 // Valid year, invalid month, invalid day -#ifdef _WIN32 - check(SV(" is not a valid date"), - SV("{}"), - std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{32}}); - check(SV("****** is not a valid date******"), - SV("{:*^32}"), - std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{32}}); - check(SV("*********** is not a valid date"), - SV("{:*>31}"), - std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{32}}); -#else // _WIN32 check(SV("1970-00-32 is not a valid date"), SV("{}"), std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{32}}); @@ -105,7 +82,6 @@ static void test_no_chrono_specs() { check(SV("*1970-00-32 is not a valid date"), SV("{:*>31}"), std::chrono::year_month_day{std::chrono::year{1970}, std::chrono::month{0}, std::chrono::day{32}}); -#endif // _WIN32 // Invalid year, valid month, valid day check(SV("-32768-01-31 is not a valid date"), -- GitLab From 458328ae23d318a5055d5bac66426b8551bce01f Mon Sep 17 00:00:00 2001 From: Vlad Serebrennikov Date: Wed, 17 Apr 2024 18:01:55 +0300 Subject: [PATCH 027/714] [clang][NFC] Refactor `Sema::RedeclarationKind` This patch converts the enum into scoped enum, and moves it into its own header for the time being. It's definition is needed in `Sema.h`, and is going to be needed in upcoming `SemaObjC.h`. `Lookup.h` can't hold it, because it includes `Sema.h`. --- clang/include/clang/Sema/Lookup.h | 43 ++++++++++--------- clang/include/clang/Sema/Redeclaration.h | 31 +++++++++++++ clang/include/clang/Sema/Sema.h | 40 ++++------------- clang/lib/Interpreter/Interpreter.cpp | 3 +- clang/lib/Interpreter/InterpreterUtils.cpp | 2 +- clang/lib/Sema/SemaDecl.cpp | 24 ++++++----- clang/lib/Sema/SemaDeclCXX.cpp | 28 ++++++------ clang/lib/Sema/SemaExprCXX.cpp | 2 +- clang/lib/Sema/SemaExprMember.cpp | 2 +- clang/lib/Sema/SemaLookup.cpp | 16 ++++++- clang/lib/Sema/SemaOpenMP.cpp | 2 +- clang/lib/Sema/SemaTemplate.cpp | 5 ++- .../lib/Sema/SemaTemplateInstantiateDecl.cpp | 8 ++-- 13 files changed, 118 insertions(+), 88 deletions(-) create mode 100644 clang/include/clang/Sema/Redeclaration.h diff --git a/clang/include/clang/Sema/Lookup.h b/clang/include/clang/Sema/Lookup.h index 2f2f2607a937..0db5b847038f 100644 --- a/clang/include/clang/Sema/Lookup.h +++ b/clang/include/clang/Sema/Lookup.h @@ -153,28 +153,30 @@ public: using iterator = UnresolvedSetImpl::iterator; - LookupResult(Sema &SemaRef, const DeclarationNameInfo &NameInfo, - Sema::LookupNameKind LookupKind, - Sema::RedeclarationKind Redecl = Sema::NotForRedeclaration) + LookupResult( + Sema &SemaRef, const DeclarationNameInfo &NameInfo, + Sema::LookupNameKind LookupKind, + RedeclarationKind Redecl = RedeclarationKind::NotForRedeclaration) : SemaPtr(&SemaRef), NameInfo(NameInfo), LookupKind(LookupKind), - Redecl(Redecl != Sema::NotForRedeclaration), - ExternalRedecl(Redecl == Sema::ForExternalRedeclaration), - DiagnoseAccess(Redecl == Sema::NotForRedeclaration), - DiagnoseAmbiguous(Redecl == Sema::NotForRedeclaration) { + Redecl(Redecl != RedeclarationKind::NotForRedeclaration), + ExternalRedecl(Redecl == RedeclarationKind::ForExternalRedeclaration), + DiagnoseAccess(Redecl == RedeclarationKind::NotForRedeclaration), + DiagnoseAmbiguous(Redecl == RedeclarationKind::NotForRedeclaration) { configure(); } // TODO: consider whether this constructor should be restricted to take // as input a const IdentifierInfo* (instead of Name), // forcing other cases towards the constructor taking a DNInfo. - LookupResult(Sema &SemaRef, DeclarationName Name, SourceLocation NameLoc, - Sema::LookupNameKind LookupKind, - Sema::RedeclarationKind Redecl = Sema::NotForRedeclaration) + LookupResult( + Sema &SemaRef, DeclarationName Name, SourceLocation NameLoc, + Sema::LookupNameKind LookupKind, + RedeclarationKind Redecl = RedeclarationKind::NotForRedeclaration) : SemaPtr(&SemaRef), NameInfo(Name, NameLoc), LookupKind(LookupKind), - Redecl(Redecl != Sema::NotForRedeclaration), - ExternalRedecl(Redecl == Sema::ForExternalRedeclaration), - DiagnoseAccess(Redecl == Sema::NotForRedeclaration), - DiagnoseAmbiguous(Redecl == Sema::NotForRedeclaration) { + Redecl(Redecl != RedeclarationKind::NotForRedeclaration), + ExternalRedecl(Redecl == RedeclarationKind::ForExternalRedeclaration), + DiagnoseAccess(Redecl == RedeclarationKind::NotForRedeclaration), + DiagnoseAmbiguous(Redecl == RedeclarationKind::NotForRedeclaration) { configure(); } @@ -285,9 +287,10 @@ public: return ExternalRedecl; } - Sema::RedeclarationKind redeclarationKind() const { - return ExternalRedecl ? Sema::ForExternalRedeclaration : - Redecl ? Sema::ForVisibleRedeclaration : Sema::NotForRedeclaration; + RedeclarationKind redeclarationKind() const { + return ExternalRedecl ? RedeclarationKind::ForExternalRedeclaration + : Redecl ? RedeclarationKind::ForVisibleRedeclaration + : RedeclarationKind::NotForRedeclaration; } /// Specify whether hidden declarations are visible, e.g., @@ -615,9 +618,9 @@ public: } /// Change this lookup's redeclaration kind. - void setRedeclarationKind(Sema::RedeclarationKind RK) { - Redecl = (RK != Sema::NotForRedeclaration); - ExternalRedecl = (RK == Sema::ForExternalRedeclaration); + void setRedeclarationKind(RedeclarationKind RK) { + Redecl = (RK != RedeclarationKind::NotForRedeclaration); + ExternalRedecl = (RK == RedeclarationKind::ForExternalRedeclaration); configure(); } diff --git a/clang/include/clang/Sema/Redeclaration.h b/clang/include/clang/Sema/Redeclaration.h new file mode 100644 index 000000000000..ae18b922f5cd --- /dev/null +++ b/clang/include/clang/Sema/Redeclaration.h @@ -0,0 +1,31 @@ +//===- Redeclaration.h - Redeclarations--------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines RedeclarationKind enum. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_SEMA_REDECLARATION_H +#define LLVM_CLANG_SEMA_REDECLARATION_H + +/// Specifies whether (or how) name lookup is being performed for a +/// redeclaration (vs. a reference). +enum class RedeclarationKind { + /// The lookup is a reference to this name that is not for the + /// purpose of redeclaring the name. + NotForRedeclaration = 0, + /// The lookup results will be used for redeclaration of a name, + /// if an entity by that name already exists and is visible. + ForVisibleRedeclaration, + /// The lookup results will be used for redeclaration of a name + /// with external linkage; non-visible lookup results with external linkage + /// may also be found. + ForExternalRedeclaration +}; + +#endif // LLVM_CLANG_SEMA_REDECLARATION_H \ No newline at end of file diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 281e3b91de1d..1e89dfc58d92 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -52,6 +52,7 @@ #include "clang/Sema/IdentifierResolver.h" #include "clang/Sema/ObjCMethodList.h" #include "clang/Sema/Ownership.h" +#include "clang/Sema/Redeclaration.h" #include "clang/Sema/Scope.h" #include "clang/Sema/SemaBase.h" #include "clang/Sema/SemaConcept.h" @@ -7443,40 +7444,17 @@ public: typedef std::function TypoRecoveryCallback; - /// Specifies whether (or how) name lookup is being performed for a - /// redeclaration (vs. a reference). - enum RedeclarationKind { - /// The lookup is a reference to this name that is not for the - /// purpose of redeclaring the name. - NotForRedeclaration = 0, - /// The lookup results will be used for redeclaration of a name, - /// if an entity by that name already exists and is visible. - ForVisibleRedeclaration, - /// The lookup results will be used for redeclaration of a name - /// with external linkage; non-visible lookup results with external linkage - /// may also be found. - ForExternalRedeclaration - }; - - RedeclarationKind forRedeclarationInCurContext() const { - // A declaration with an owning module for linkage can never link against - // anything that is not visible. We don't need to check linkage here; if - // the context has internal linkage, redeclaration lookup won't find things - // from other TUs, and we can't safely compute linkage yet in general. - if (cast(CurContext) - ->getOwningModuleForLinkage(/*IgnoreLinkage*/ true)) - return ForVisibleRedeclaration; - return ForExternalRedeclaration; - } + RedeclarationKind forRedeclarationInCurContext() const; /// Look up a name, looking for a single declaration. Return /// null if the results were absent, ambiguous, or overloaded. /// /// It is preferable to use the elaborated form and explicitly handle /// ambiguity and overloaded. - NamedDecl *LookupSingleName(Scope *S, DeclarationName Name, - SourceLocation Loc, LookupNameKind NameKind, - RedeclarationKind Redecl = NotForRedeclaration); + NamedDecl *LookupSingleName( + Scope *S, DeclarationName Name, SourceLocation Loc, + LookupNameKind NameKind, + RedeclarationKind Redecl = RedeclarationKind::NotForRedeclaration); bool LookupBuiltin(LookupResult &R); void LookupNecessaryTypesForBuiltin(Scope *S, unsigned ID); bool LookupName(LookupResult &R, Scope *S, bool AllowBuiltinCreation = false, @@ -7488,9 +7466,9 @@ public: bool LookupParsedName(LookupResult &R, Scope *S, CXXScopeSpec *SS, bool AllowBuiltinCreation = false, bool EnteringContext = false); - ObjCProtocolDecl * - LookupProtocol(IdentifierInfo *II, SourceLocation IdLoc, - RedeclarationKind Redecl = NotForRedeclaration); + ObjCProtocolDecl *LookupProtocol( + IdentifierInfo *II, SourceLocation IdLoc, + RedeclarationKind Redecl = RedeclarationKind::NotForRedeclaration); bool LookupInSuper(LookupResult &R, CXXRecordDecl *Class); void LookupOverloadedOperatorName(OverloadedOperatorKind Op, Scope *S, diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp index cf31456b6950..b20e6efcebfd 100644 --- a/clang/lib/Interpreter/Interpreter.cpp +++ b/clang/lib/Interpreter/Interpreter.cpp @@ -550,7 +550,8 @@ std::unique_ptr Interpreter::FindRuntimeInterface() { auto LookupInterface = [&](Expr *&Interface, llvm::StringRef Name) { LookupResult R(S, &Ctx.Idents.get(Name), SourceLocation(), - Sema::LookupOrdinaryName, Sema::ForVisibleRedeclaration); + Sema::LookupOrdinaryName, + RedeclarationKind::ForVisibleRedeclaration); S.LookupQualifiedName(R, Ctx.getTranslationUnitDecl()); if (R.empty()) return false; diff --git a/clang/lib/Interpreter/InterpreterUtils.cpp b/clang/lib/Interpreter/InterpreterUtils.cpp index c19cf6aa3156..45f6322b8461 100644 --- a/clang/lib/Interpreter/InterpreterUtils.cpp +++ b/clang/lib/Interpreter/InterpreterUtils.cpp @@ -72,7 +72,7 @@ NamedDecl *LookupNamed(Sema &S, llvm::StringRef Name, const DeclContext *Within) { DeclarationName DName = &S.Context.Idents.get(Name); LookupResult R(S, DName, SourceLocation(), Sema::LookupOrdinaryName, - Sema::ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); R.suppressDiagnostics(); diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 19abd5327b73..455ccb45b406 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -5374,7 +5374,7 @@ static bool CheckAnonMemberRedeclaration(Sema &SemaRef, Scope *S, LookupResult R(SemaRef, Name, NameLoc, Owner->isRecord() ? Sema::LookupMemberName : Sema::LookupOrdinaryName, - Sema::ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); if (!SemaRef.LookupName(R, S)) return false; // Pick a representative declaration. @@ -6470,7 +6470,8 @@ NamedDecl *Sema::HandleDeclarator(Scope *S, Declarator &D, if (IsLinkageLookup) { Previous.clear(LookupRedeclarationWithLinkage); - Previous.setRedeclarationKind(ForExternalRedeclaration); + Previous.setRedeclarationKind( + RedeclarationKind::ForExternalRedeclaration); } LookupName(Previous, S, CreateBuiltins); @@ -8521,7 +8522,8 @@ void Sema::CheckShadow(Scope *S, VarDecl *D) { return; LookupResult R(*this, D->getDeclName(), D->getLocation(), - Sema::LookupOrdinaryName, Sema::ForVisibleRedeclaration); + Sema::LookupOrdinaryName, + RedeclarationKind::ForVisibleRedeclaration); LookupName(R, S); if (NamedDecl *ShadowedDecl = getShadowedDeclaration(D, R)) CheckShadow(D, ShadowedDecl, R); @@ -9161,7 +9163,7 @@ static NamedDecl *DiagnoseInvalidRedeclaration( LookupResult Prev(SemaRef, Name, NewFD->getLocation(), IsLocalFriend ? Sema::LookupLocalFriendName : Sema::LookupOrdinaryName, - Sema::ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); NewFD->setInvalidDecl(); if (IsLocalFriend) @@ -15196,7 +15198,7 @@ Decl *Sema::ActOnParamDeclarator(Scope *S, Declarator &D, const IdentifierInfo *II = D.getIdentifier(); if (II) { LookupResult R(*this, II, D.getIdentifierLoc(), LookupOrdinaryName, - ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); LookupName(R, S); if (!R.empty()) { NamedDecl *PrevDecl = *R.begin(); @@ -17428,7 +17430,7 @@ Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc, RedeclarationKind Redecl = forRedeclarationInCurContext(); if (TUK == TUK_Friend || TUK == TUK_Reference) - Redecl = NotForRedeclaration; + Redecl = RedeclarationKind::NotForRedeclaration; /// Create a new tag decl in C/ObjC. Since the ODR-like semantics for ObjC/C /// implemented asks for structural equivalence checking, the returned decl @@ -18589,7 +18591,7 @@ FieldDecl *Sema::HandleField(Scope *S, RecordDecl *Record, // Check to see if this name was declared as a member previously NamedDecl *PrevDecl = nullptr; LookupResult Previous(*this, II, Loc, LookupMemberName, - ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); LookupName(Previous, S); switch (Previous.getResultKind()) { case LookupResult::Found: @@ -18993,8 +18995,9 @@ Decl *Sema::ActOnIvar(Scope *S, SourceLocation DeclStart, Declarator &D, NewID->setInvalidDecl(); if (II) { - NamedDecl *PrevDecl = LookupSingleName(S, II, Loc, LookupMemberName, - ForVisibleRedeclaration); + NamedDecl *PrevDecl = + LookupSingleName(S, II, Loc, LookupMemberName, + RedeclarationKind::ForVisibleRedeclaration); if (PrevDecl && isDeclInScope(PrevDecl, EnclosingContext, S) && !isa(PrevDecl)) { Diag(Loc, diag::err_duplicate_member) << II; @@ -20039,7 +20042,8 @@ Decl *Sema::ActOnEnumConstant(Scope *S, Decl *theEnumDecl, Decl *lastEnumConst, // Verify that there isn't already something declared with this name in this // scope. - LookupResult R(*this, Id, IdLoc, LookupOrdinaryName, ForVisibleRedeclaration); + LookupResult R(*this, Id, IdLoc, LookupOrdinaryName, + RedeclarationKind::ForVisibleRedeclaration); LookupName(R, S); NamedDecl *PrevDecl = R.getAsSingle(); diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp index 8c6bae545bfd..591016243b0a 100644 --- a/clang/lib/Sema/SemaDeclCXX.cpp +++ b/clang/lib/Sema/SemaDeclCXX.cpp @@ -896,7 +896,7 @@ Sema::ActOnDecompositionDeclarator(Scope *S, Declarator &D, assert(VarName && "Cannot have an unnamed binding declaration"); LookupResult Previous(*this, NameInfo, LookupOrdinaryName, - ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); LookupName(Previous, S, /*CreateBuiltins*/DC->getRedeclContext()->isTranslationUnit()); @@ -951,7 +951,7 @@ Sema::ActOnDecompositionDeclarator(Scope *S, Declarator &D, DeclarationNameInfo NameInfo((IdentifierInfo *)nullptr, Decomp.getLSquareLoc()); LookupResult Previous(*this, NameInfo, LookupOrdinaryName, - ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); // Build the variable that holds the non-decomposed object. bool AddToScope = true; @@ -11715,7 +11715,7 @@ Decl *Sema::ActOnStartNamespaceDef(Scope *NamespcScope, // look through using directives, just look for any ordinary names // as if by qualified name lookup. LookupResult R(*this, II, IdentLoc, LookupOrdinaryName, - ForExternalRedeclaration); + RedeclarationKind::ForExternalRedeclaration); LookupQualifiedName(R, CurContext->getRedeclContext()); NamedDecl *PrevDecl = R.isSingleResult() ? R.getRepresentativeDecl() : nullptr; @@ -12916,7 +12916,7 @@ NamedDecl *Sema::BuildUsingDeclaration( // Do the redeclaration lookup in the current scope. LookupResult Previous(*this, UsingName, LookupUsingDeclName, - ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); Previous.setHideTags(false); if (S) { LookupName(Previous, S); @@ -13159,7 +13159,7 @@ NamedDecl *Sema::BuildUsingEnumDeclaration(Scope *S, AccessSpecifier AS, /// In class scope, check if this is a duplicate, for better a diagnostic. DeclarationNameInfo UsingEnumName(ED->getDeclName(), NameLoc); LookupResult Previous(*this, UsingEnumName, LookupUsingDeclName, - ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); LookupName(Previous, S); @@ -13192,7 +13192,7 @@ NamedDecl *Sema::BuildUsingEnumDeclaration(Scope *S, AccessSpecifier AS, UsingShadowDecl *PrevDecl = nullptr; DeclarationNameInfo DNI(EC->getDeclName(), EC->getLocation()); LookupResult Previous(*this, DNI, LookupOrdinaryName, - ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); LookupName(Previous, S); FilterUsingLookup(S, Previous); @@ -13587,7 +13587,7 @@ Decl *Sema::ActOnAliasDeclaration(Scope *S, AccessSpecifier AS, LookupResult Previous(*this, NameInfo, LookupOrdinaryName, TemplateParamLists.size() ? forRedeclarationInCurContext() - : ForVisibleRedeclaration); + : RedeclarationKind::ForVisibleRedeclaration); LookupName(Previous, S); // Warn about shadowing the name of a template parameter. @@ -13737,7 +13737,7 @@ Decl *Sema::ActOnNamespaceAliasDef(Scope *S, SourceLocation NamespaceLoc, // Check if we have a previous declaration with the same name. LookupResult PrevR(*this, Alias, AliasLoc, LookupOrdinaryName, - ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); LookupName(PrevR, S); // Check we're not shadowing a template parameter. @@ -13983,7 +13983,7 @@ void Sema::CheckImplicitSpecialMemberDeclaration(Scope *S, FunctionDecl *FD) { // implicit special members with this name. DeclarationName Name = FD->getDeclName(); LookupResult R(*this, Name, SourceLocation(), LookupOrdinaryName, - ForExternalRedeclaration); + RedeclarationKind::ForExternalRedeclaration); for (auto *D : FD->getParent()->lookup(Name)) if (auto *Acceptable = R.getAcceptableDecl(D)) R.addDecl(Acceptable); @@ -17113,9 +17113,9 @@ Decl *Sema::ActOnExceptionDeclarator(Scope *S, Declarator &D) { } const IdentifierInfo *II = D.getIdentifier(); - if (NamedDecl *PrevDecl = LookupSingleName(S, II, D.getIdentifierLoc(), - LookupOrdinaryName, - ForVisibleRedeclaration)) { + if (NamedDecl *PrevDecl = + LookupSingleName(S, II, D.getIdentifierLoc(), LookupOrdinaryName, + RedeclarationKind::ForVisibleRedeclaration)) { // The scope should be freshly made just for us. There is just no way // it contains any previous declaration, except for function parameters in // a function-try-block's catch statement. @@ -17906,7 +17906,7 @@ NamedDecl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, DeclContext *DC; Scope *DCScope = S; LookupResult Previous(*this, NameInfo, LookupOrdinaryName, - ForExternalRedeclaration); + RedeclarationKind::ForExternalRedeclaration); bool isTemplateId = D.getName().getKind() == UnqualifiedIdKind::IK_TemplateId; @@ -19242,7 +19242,7 @@ MSPropertyDecl *Sema::HandleMSProperty(Scope *S, RecordDecl *Record, // Check to see if this name was declared as a member previously NamedDecl *PrevDecl = nullptr; LookupResult Previous(*this, II, Loc, LookupMemberName, - ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); LookupName(Previous, S); switch (Previous.getResultKind()) { case LookupResult::Found: diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp index f4a91ececfbb..7582cbd75fec 100644 --- a/clang/lib/Sema/SemaExprCXX.cpp +++ b/clang/lib/Sema/SemaExprCXX.cpp @@ -9153,7 +9153,7 @@ Sema::CheckMicrosoftIfExistsSymbol(Scope *S, // Do the redeclaration lookup in the current scope. LookupResult R(*this, TargetNameInfo, Sema::LookupAnyName, - Sema::NotForRedeclaration); + RedeclarationKind::NotForRedeclaration); LookupParsedName(R, S, &SS); R.suppressDiagnostics(); diff --git a/clang/lib/Sema/SemaExprMember.cpp b/clang/lib/Sema/SemaExprMember.cpp index 7ea6d733fe5a..c79128bc8f39 100644 --- a/clang/lib/Sema/SemaExprMember.cpp +++ b/clang/lib/Sema/SemaExprMember.cpp @@ -728,7 +728,7 @@ static bool LookupMemberExprInRecord(Sema &SemaRef, LookupResult &R, Sema &SemaRef; DeclarationNameInfo NameInfo; Sema::LookupNameKind LookupKind; - Sema::RedeclarationKind Redecl; + RedeclarationKind Redecl; }; QueryState Q = {R.getSema(), R.getLookupNameInfo(), R.getLookupKind(), R.redeclarationKind()}; diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp index d65f52b8efe8..55af414df39f 100644 --- a/clang/lib/Sema/SemaLookup.cpp +++ b/clang/lib/Sema/SemaLookup.cpp @@ -4449,7 +4449,8 @@ LabelDecl *Sema::LookupOrCreateLabel(IdentifierInfo *II, SourceLocation Loc, } // Not a GNU local label. - Res = LookupSingleName(CurScope, II, Loc, LookupLabel, NotForRedeclaration); + Res = LookupSingleName(CurScope, II, Loc, LookupLabel, + RedeclarationKind::NotForRedeclaration); // If we found a label, check to see if it is in the same context as us. // When in a Block, we don't want to reuse a label in an enclosing function. if (Res && Res->getDeclContext() != CurContext) @@ -5889,7 +5890,8 @@ void Sema::clearDelayedTypo(TypoExpr *TE) { void Sema::ActOnPragmaDump(Scope *S, SourceLocation IILoc, IdentifierInfo *II) { DeclarationNameInfo Name(II, IILoc); - LookupResult R(*this, Name, LookupAnyName, Sema::NotForRedeclaration); + LookupResult R(*this, Name, LookupAnyName, + RedeclarationKind::NotForRedeclaration); R.suppressDiagnostics(); R.setHideTags(false); LookupName(R, S); @@ -5899,3 +5901,13 @@ void Sema::ActOnPragmaDump(Scope *S, SourceLocation IILoc, IdentifierInfo *II) { void Sema::ActOnPragmaDump(Expr *E) { E->dump(); } + +RedeclarationKind Sema::forRedeclarationInCurContext() const { + // A declaration with an owning module for linkage can never link against + // anything that is not visible. We don't need to check linkage here; if + // the context has internal linkage, redeclaration lookup won't find things + // from other TUs, and we can't safely compute linkage yet in general. + if (cast(CurContext)->getOwningModuleForLinkage(/*IgnoreLinkage*/ true)) + return RedeclarationKind::ForVisibleRedeclaration; + return RedeclarationKind::ForExternalRedeclaration; +} diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp index d229ef650bcc..3e9f6cba2507 100644 --- a/clang/lib/Sema/SemaOpenMP.cpp +++ b/clang/lib/Sema/SemaOpenMP.cpp @@ -24944,7 +24944,7 @@ ExprResult SemaOpenMP::ActOnOMPIteratorExpr(Scope *S, // Check for conflicting previous declaration. DeclarationNameInfo NameInfo(VD->getDeclName(), D.DeclIdentLoc); LookupResult Previous(SemaRef, NameInfo, Sema::LookupOrdinaryName, - Sema::ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); Previous.suppressDiagnostics(); SemaRef.LookupName(Previous, S); diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp index 95171359f0ab..f4b6e1ceb6f0 100644 --- a/clang/lib/Sema/SemaTemplate.cpp +++ b/clang/lib/Sema/SemaTemplate.cpp @@ -972,8 +972,9 @@ void Sema::translateTemplateArguments(const ASTTemplateArgsPtr &TemplateArgsIn, static void maybeDiagnoseTemplateParameterShadow(Sema &SemaRef, Scope *S, SourceLocation Loc, const IdentifierInfo *Name) { - NamedDecl *PrevDecl = SemaRef.LookupSingleName( - S, Name, Loc, Sema::LookupOrdinaryName, Sema::ForVisibleRedeclaration); + NamedDecl *PrevDecl = + SemaRef.LookupSingleName(S, Name, Loc, Sema::LookupOrdinaryName, + RedeclarationKind::ForVisibleRedeclaration); if (PrevDecl && PrevDecl->isTemplateParameter()) SemaRef.DiagnoseTemplateParameterShadow(Loc, PrevDecl); } diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp index 6d359c5a9a02..caa07abb61fe 100644 --- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp +++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp @@ -2296,7 +2296,7 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl( SemaRef, Function->getDeclName(), SourceLocation(), D->isLocalExternDecl() ? Sema::LookupRedeclarationWithLinkage : Sema::LookupOrdinaryName, - D->isLocalExternDecl() ? Sema::ForExternalRedeclaration + D->isLocalExternDecl() ? RedeclarationKind::ForExternalRedeclaration : SemaRef.forRedeclarationInCurContext()); if (DependentFunctionTemplateSpecializationInfo *DFTSI = @@ -2697,7 +2697,7 @@ Decl *TemplateDeclInstantiator::VisitCXXMethodDecl( Method->setInvalidDecl(); LookupResult Previous(SemaRef, NameInfo, Sema::LookupOrdinaryName, - Sema::ForExternalRedeclaration); + RedeclarationKind::ForExternalRedeclaration); bool IsExplicitSpecialization = false; @@ -3365,7 +3365,7 @@ Decl *TemplateDeclInstantiator::VisitUsingDecl(UsingDecl *D) { // fact, it's not really even possible in non-class scopes). bool CheckRedeclaration = Owner->isRecord(); LookupResult Prev(SemaRef, NameInfo, Sema::LookupUsingDeclName, - Sema::ForVisibleRedeclaration); + RedeclarationKind::ForVisibleRedeclaration); UsingDecl *NewUD = UsingDecl::Create(SemaRef.Context, Owner, D->getUsingLoc(), @@ -5388,7 +5388,7 @@ void Sema::BuildVariableInstantiation( *this, NewVar->getDeclName(), NewVar->getLocation(), NewVar->isLocalExternDecl() ? Sema::LookupRedeclarationWithLinkage : Sema::LookupOrdinaryName, - NewVar->isLocalExternDecl() ? Sema::ForExternalRedeclaration + NewVar->isLocalExternDecl() ? RedeclarationKind::ForExternalRedeclaration : forRedeclarationInCurContext()); if (NewVar->isLocalExternDecl() && OldVar->getPreviousDecl() && -- GitLab From 950bb097e11d6ee26533c00519c62df994322228 Mon Sep 17 00:00:00 2001 From: Dinar Temirbulatov Date: Wed, 17 Apr 2024 15:30:40 +0000 Subject: [PATCH 028/714] Revert "[Clang][AArch64] Warn when calling non/streaming about vector size difference (#79842)" This reverts commit 4e85e1ffcaf161736e27a24c291c1177be865976 --- clang/include/clang/Basic/DiagnosticGroups.td | 3 - .../clang/Basic/DiagnosticSemaKinds.td | 10 -- clang/lib/Sema/SemaChecking.cpp | 22 +-- clang/lib/Sema/SemaDecl.cpp | 12 +- .../Sema/aarch64-incompat-sm-builtin-calls.c | 6 +- clang/test/Sema/aarch64-sme-func-attrs.c | 136 +----------------- 6 files changed, 5 insertions(+), 184 deletions(-) diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td index 47747d8704b6..5251774ff4ef 100644 --- a/clang/include/clang/Basic/DiagnosticGroups.td +++ b/clang/include/clang/Basic/DiagnosticGroups.td @@ -1412,9 +1412,6 @@ def MultiGPU: DiagGroup<"multi-gpu">; // libc and the CRT to be skipped. def AVRRtlibLinkingQuirks : DiagGroup<"avr-rtlib-linking-quirks">; -// A warning group related to AArch64 SME function attribues. -def AArch64SMEAttributes : DiagGroup<"aarch64-sme-attributes">; - // A warning group for things that will change semantics in the future. def FutureCompat : DiagGroup<"future-compat">; diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 44f802c0c28e..30a8543489f4 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -3751,16 +3751,6 @@ def err_sme_definition_using_za_in_non_sme_target : Error< "function using ZA state requires 'sme'">; def err_sme_definition_using_zt0_in_non_sme2_target : Error< "function using ZT0 state requires 'sme2'">; -def warn_sme_streaming_pass_return_vl_to_non_streaming : Warning< - "passing a VL-dependent argument to/from a function that has a different" - " streaming-mode. The streaming and non-streaming vector lengths may be" - " different">, - InGroup, DefaultIgnore; -def warn_sme_locally_streaming_has_vl_args_returns : Warning< - "passing/returning a VL-dependent argument to/from a __arm_locally_streaming" - " function. The streaming and non-streaming vector" - " lengths may be different">, - InGroup, DefaultIgnore; def err_conflicting_attributes_arm_state : Error< "conflicting attributes for state '%0'">; def err_sme_streaming_cannot_be_multiversioned : Error< diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 99b0a0008353..d814327d2347 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -7949,7 +7949,6 @@ void Sema::checkCall(NamedDecl *FDecl, const FunctionProtoType *Proto, // For variadic functions, we may have more args than parameters. // For some K&R functions, we may have less args than parameters. const auto N = std::min(Proto->getNumParams(), Args.size()); - bool AnyScalableArgsOrRet = Proto->getReturnType()->isSizelessVectorType(); for (unsigned ArgIdx = 0; ArgIdx < N; ++ArgIdx) { // Args[ArgIdx] can be null in malformed code. if (const Expr *Arg = Args[ArgIdx]) { @@ -7963,8 +7962,6 @@ void Sema::checkCall(NamedDecl *FDecl, const FunctionProtoType *Proto, checkAIXMemberAlignment((Arg->getExprLoc()), Arg); QualType ParamTy = Proto->getParamType(ArgIdx); - if (ParamTy->isSizelessVectorType()) - AnyScalableArgsOrRet = true; QualType ArgTy = Arg->getType(); CheckArgAlignment(Arg->getExprLoc(), FDecl, std::to_string(ArgIdx + 1), ArgTy, ParamTy); @@ -7985,23 +7982,6 @@ void Sema::checkCall(NamedDecl *FDecl, const FunctionProtoType *Proto, } } - // If the call requires a streaming-mode change and has scalable vector - // arguments or return values, then warn the user that the streaming and - // non-streaming vector lengths may be different. - const auto *CallerFD = dyn_cast(CurContext); - if (CallerFD && (!FD || !FD->getBuiltinID()) && AnyScalableArgsOrRet) { - bool IsCalleeStreaming = - ExtInfo.AArch64SMEAttributes & FunctionType::SME_PStateSMEnabledMask; - bool IsCalleeStreamingCompatible = - ExtInfo.AArch64SMEAttributes & - FunctionType::SME_PStateSMCompatibleMask; - ArmStreamingType CallerFnType = getArmStreamingFnType(CallerFD); - if (!IsCalleeStreamingCompatible && - (CallerFnType == ArmStreamingCompatible || - ((CallerFnType == ArmStreaming) ^ IsCalleeStreaming))) - Diag(Loc, diag::warn_sme_streaming_pass_return_vl_to_non_streaming); - } - FunctionType::ArmStateValue CalleeArmZAState = FunctionType::getArmZAState(ExtInfo.AArch64SMEAttributes); FunctionType::ArmStateValue CalleeArmZT0State = @@ -8010,7 +7990,7 @@ void Sema::checkCall(NamedDecl *FDecl, const FunctionProtoType *Proto, CalleeArmZT0State != FunctionType::ARM_None) { bool CallerHasZAState = false; bool CallerHasZT0State = false; - if (CallerFD) { + if (const auto *CallerFD = dyn_cast(CurContext)) { auto *Attr = CallerFD->getAttr(); if (Attr && Attr->isNewZA()) CallerHasZAState = true; diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 455ccb45b406..1bde99d6fce7 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -12408,22 +12408,12 @@ bool Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD, } // Check if the function definition uses any AArch64 SME features without - // having the '+sme' feature enabled and warn user if sme locally streaming - // function returns or uses arguments with VL-based types. + // having the '+sme' feature enabled. if (DeclIsDefn) { const auto *Attr = NewFD->getAttr(); bool UsesSM = NewFD->hasAttr(); bool UsesZA = Attr && Attr->isNewZA(); bool UsesZT0 = Attr && Attr->isNewZT0(); - - if (NewFD->hasAttr()) { - if (NewFD->getReturnType()->isSizelessVectorType() || - llvm::any_of(NewFD->parameters(), [](ParmVarDecl *P) { - return P->getOriginalType()->isSizelessVectorType(); - })) - Diag(NewFD->getLocation(), - diag::warn_sme_locally_streaming_has_vl_args_returns); - } if (const auto *FPT = NewFD->getType()->getAs()) { FunctionProtoType::ExtProtoInfo EPI = FPT->getExtProtoInfo(); UsesSM |= diff --git a/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c b/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c index 6a1feeb9bf53..55c97c73e8b6 100644 --- a/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c +++ b/clang/test/Sema/aarch64-incompat-sm-builtin-calls.c @@ -1,6 +1,6 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve \ -// RUN: -target-feature +sme2 -target-feature +sve2 -target-feature +neon -Waarch64-sme-attributes -fsyntax-only -verify %s +// RUN: -target-feature +sme2 -target-feature +sve2 -target-feature +neon -fsyntax-only -verify %s // REQUIRES: aarch64-registered-target @@ -33,7 +33,6 @@ svuint32_t incompat_sve_sm(svbool_t pg, svuint32_t a, int16_t b) __arm_streaming return __builtin_sve_svld1_gather_u32base_index_u32(pg, a, b); } -// expected-warning@+1 {{passing/returning a VL-dependent argument to/from a __arm_locally_streaming function. The streaming and non-streaming vector lengths may be different}} __arm_locally_streaming svuint32_t incompat_sve_ls(svbool_t pg, svuint32_t a, int64_t b) { // expected-warning@+1 {{builtin call has undefined behaviour when called from a streaming function}} return __builtin_sve_svld1_gather_u32base_index_u32(pg, a, b); @@ -49,7 +48,6 @@ svuint32_t incompat_sve2_sm(svbool_t pg, svuint32_t a, int64_t b) __arm_streamin return __builtin_sve_svldnt1_gather_u32base_index_u32(pg, a, b); } -// expected-warning@+1 {{passing/returning a VL-dependent argument to/from a __arm_locally_streaming function. The streaming and non-streaming vector lengths may be different}} __arm_locally_streaming svuint32_t incompat_sve2_ls(svbool_t pg, svuint32_t a, int64_t b) { // expected-warning@+1 {{builtin call has undefined behaviour when called from a streaming function}} return __builtin_sve_svldnt1_gather_u32base_index_u32(pg, a, b); @@ -70,7 +68,6 @@ svfloat64_t streaming_caller_sve(svbool_t pg, svfloat64_t a, float64_t b) __arm_ return svadd_n_f64_m(pg, a, b); } -// expected-warning@+1 {{passing/returning a VL-dependent argument to/from a __arm_locally_streaming function. The streaming and non-streaming vector lengths may be different}} __arm_locally_streaming svfloat64_t locally_streaming_caller_sve(svbool_t pg, svfloat64_t a, float64_t b) { // expected-no-warning return svadd_n_f64_m(pg, a, b); @@ -86,7 +83,6 @@ svint16_t streaming_caller_sve2(svint16_t op1, svint16_t op2) __arm_streaming { return svmul_lane_s16(op1, op2, 0); } -// expected-warning@+1 {{passing/returning a VL-dependent argument to/from a __arm_locally_streaming function. The streaming and non-streaming vector lengths may be different}} __arm_locally_streaming svint16_t locally_streaming_caller_sve2(svint16_t op1, svint16_t op2) { // expected-no-warning return svmul_lane_s16(op1, op2, 0); diff --git a/clang/test/Sema/aarch64-sme-func-attrs.c b/clang/test/Sema/aarch64-sme-func-attrs.c index 12de16509ccb..bfc8768c3f36 100644 --- a/clang/test/Sema/aarch64-sme-func-attrs.c +++ b/clang/test/Sema/aarch64-sme-func-attrs.c @@ -1,5 +1,5 @@ -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve -Waarch64-sme-attributes -fsyntax-only -verify %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -target-feature +sve -Waarch64-sme-attributes -fsyntax-only -verify=expected-cpp -x c++ %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -fsyntax-only -verify %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -fsyntax-only -verify=expected-cpp -x c++ %s // Valid attributes @@ -496,135 +496,3 @@ void fmv_caller() { just_fine(); incompatible_locally_streaming(); } - -void sme_streaming_with_vl_arg(__SVInt8_t a) __arm_streaming { } - -__SVInt8_t sme_streaming_returns_vl(void) __arm_streaming { __SVInt8_t r; return r; } - -void sme_streaming_compatible_with_vl_arg(__SVInt8_t a) __arm_streaming_compatible { } - -__SVInt8_t sme_streaming_compatible_returns_vl(void) __arm_streaming_compatible { __SVInt8_t r; return r; } - -void sme_no_streaming_with_vl_arg(__SVInt8_t a) { } - -__SVInt8_t sme_no_streaming_returns_vl(void) { __SVInt8_t r; return r; } - -// expected-warning@+2 {{passing/returning a VL-dependent argument to/from a __arm_locally_streaming function. The streaming and non-streaming vector lengths may be different}} -// expected-cpp-warning@+1 {{passing/returning a VL-dependent argument to/from a __arm_locally_streaming function. The streaming and non-streaming vector lengths may be different}} -__arm_locally_streaming void sme_locally_streaming_with_vl_arg(__SVInt8_t a) { } - -// expected-warning@+2 {{passing/returning a VL-dependent argument to/from a __arm_locally_streaming function. The streaming and non-streaming vector lengths may be different}} -// expected-cpp-warning@+1 {{passing/returning a VL-dependent argument to/from a __arm_locally_streaming function. The streaming and non-streaming vector lengths may be different}} -__arm_locally_streaming __SVInt8_t sme_locally_streaming_returns_vl(void) { __SVInt8_t r; return r; } - -void sme_no_streaming_calling_streaming_with_vl_args() { - __SVInt8_t a; - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - sme_streaming_with_vl_arg(a); -} - -void sme_no_streaming_calling_streaming_with_return_vl() { - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - __SVInt8_t r = sme_streaming_returns_vl(); -} - -void sme_streaming_calling_non_streaming_with_vl_args(void) __arm_streaming { - __SVInt8_t a; - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - sme_no_streaming_with_vl_arg(a); -} - -void sme_streaming_calling_non_streaming_with_return_vl(void) __arm_streaming { - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - __SVInt8_t r = sme_no_streaming_returns_vl(); -} - -void sme_no_streaming_calling_streaming_with_vl_args_param(__SVInt8_t arg, void (*sc)( __SVInt8_t arg) __arm_streaming) { - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - sc(arg); -} - -__SVInt8_t sme_no_streaming_calling_streaming_return_vl_param(__SVInt8_t (*s)(void) __arm_streaming) { - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - return s(); -} - -void sme_streaming_compatible_calling_streaming_with_vl_args(__SVInt8_t arg) __arm_streaming_compatible { - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - sme_streaming_with_vl_arg(arg); -} - -void sme_streaming_compatible_calling_sme_streaming_return_vl(void) __arm_streaming_compatible { - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - __SVInt8_t r = sme_streaming_returns_vl(); -} - -void sme_streaming_compatible_calling_no_streaming_with_vl_args(__SVInt8_t arg) __arm_streaming_compatible { - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - sme_no_streaming_with_vl_arg(arg); -} - -void sme_streaming_compatible_calling_no_sme_streaming_return_vl(void) __arm_streaming_compatible { - // expected-warning@+2 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - // expected-cpp-warning@+1 {{passing a VL-dependent argument to/from a function that has a different streaming-mode. The streaming and non-streaming vector lengths may be different}} - __SVInt8_t r = sme_no_streaming_returns_vl(); -} - -void sme_streaming_calling_streaming(__SVInt8_t arg, void (*s)( __SVInt8_t arg) __arm_streaming) __arm_streaming { - s(arg); -} - -__SVInt8_t sme_streaming_calling_streaming_return_vl(__SVInt8_t (*s)(void) __arm_streaming) __arm_streaming { - return s(); -} - -void sme_streaming_calling_streaming_with_vl_args(__SVInt8_t a) __arm_streaming { - sme_streaming_with_vl_arg(a); -} - -void sme_streaming_calling_streaming_with_return_vl(void) __arm_streaming { - __SVInt8_t r = sme_streaming_returns_vl(); -} - -void sme_streaming_calling_streaming_compatible_with_vl_args(__SVInt8_t a) __arm_streaming { - sme_streaming_compatible_with_vl_arg(a); -} - -void sme_streaming_calling_streaming_compatible_with_return_vl(void) __arm_streaming { - __SVInt8_t r = sme_streaming_compatible_returns_vl(); -} - -void sme_no_streaming_calling_streaming_compatible_with_vl_args() { - __SVInt8_t a; - sme_streaming_compatible_with_vl_arg(a); -} - -void sme_no_streaming_calling_streaming_compatible_with_return_vl() { - __SVInt8_t r = sme_streaming_compatible_returns_vl(); -} - -void sme_no_streaming_calling_non_streaming_compatible_with_vl_args() { - __SVInt8_t a; - sme_no_streaming_with_vl_arg(a); -} - -void sme_no_streaming_calling_non_streaming_compatible_with_return_vl() { - __SVInt8_t r = sme_no_streaming_returns_vl(); -} - -void sme_streaming_compatible_calling_streaming_compatible_with_vl_args(__SVInt8_t arg) __arm_streaming_compatible { - sme_streaming_compatible_with_vl_arg(arg); -} - -void sme_streaming_compatible_calling_streaming_compatible_with_return_vl(void) __arm_streaming_compatible { - __SVInt8_t r = sme_streaming_compatible_returns_vl(); -} -- GitLab From b854a2323337be2633b1135f590678a17e9d1ade Mon Sep 17 00:00:00 2001 From: Robin Caloudis Date: Wed, 17 Apr 2024 17:38:47 +0200 Subject: [PATCH 029/714] [libc][c23][fenv] Implement fetestexceptflag (#87828) Provide C23 `fetestexceptflag` function according to 7.6.4.6 in the latest [revision of the C standard](https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3096.pdf) from 2023-04-02. Closes https://github.com/llvm/llvm-project/issues/87565. --- libc/config/baremetal/arm/entrypoints.txt | 1 + libc/config/baremetal/riscv/entrypoints.txt | 1 + libc/config/darwin/arm/entrypoints.txt | 1 + libc/config/darwin/x86_64/entrypoints.txt | 1 + libc/config/linux/aarch64/entrypoints.txt | 1 + libc/config/linux/arm/entrypoints.txt | 1 + libc/config/linux/riscv/entrypoints.txt | 1 + libc/config/linux/x86_64/entrypoints.txt | 1 + libc/config/windows/entrypoints.txt | 1 + libc/docs/c23.rst | 2 +- libc/docs/fenv.rst | 4 +-- libc/spec/stdc.td | 5 +++ libc/src/fenv/CMakeLists.txt | 13 ++++++++ libc/src/fenv/fetestexceptflag.cpp | 23 +++++++++++++ libc/src/fenv/fetestexceptflag.h | 20 ++++++++++++ libc/test/src/fenv/CMakeLists.txt | 1 + libc/test/src/fenv/exception_flags_test.cpp | 32 +++++++++++++++---- .../llvm-project-overlay/libc/BUILD.bazel | 10 ++++++ .../libc/test/src/fenv/BUILD.bazel | 1 + 19 files changed, 110 insertions(+), 10 deletions(-) create mode 100644 libc/src/fenv/fetestexceptflag.cpp create mode 100644 libc/src/fenv/fetestexceptflag.h diff --git a/libc/config/baremetal/arm/entrypoints.txt b/libc/config/baremetal/arm/entrypoints.txt index f33f9430c792..4e3d1cb9f533 100644 --- a/libc/config/baremetal/arm/entrypoints.txt +++ b/libc/config/baremetal/arm/entrypoints.txt @@ -201,6 +201,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.fesetround libc.src.fenv.feraiseexcept libc.src.fenv.fetestexcept + libc.src.fenv.fetestexceptflag libc.src.fenv.feupdateenv # math.h entrypoints diff --git a/libc/config/baremetal/riscv/entrypoints.txt b/libc/config/baremetal/riscv/entrypoints.txt index dad187fa0496..7efd9bcd5b3c 100644 --- a/libc/config/baremetal/riscv/entrypoints.txt +++ b/libc/config/baremetal/riscv/entrypoints.txt @@ -201,6 +201,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.fesetround libc.src.fenv.feraiseexcept libc.src.fenv.fetestexcept + libc.src.fenv.fetestexceptflag libc.src.fenv.feupdateenv # math.h entrypoints diff --git a/libc/config/darwin/arm/entrypoints.txt b/libc/config/darwin/arm/entrypoints.txt index aea2f6d5771e..e1303265b9ac 100644 --- a/libc/config/darwin/arm/entrypoints.txt +++ b/libc/config/darwin/arm/entrypoints.txt @@ -112,6 +112,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.fesetround libc.src.fenv.feraiseexcept libc.src.fenv.fetestexcept + libc.src.fenv.fetestexceptflag libc.src.fenv.feupdateenv # math.h entrypoints diff --git a/libc/config/darwin/x86_64/entrypoints.txt b/libc/config/darwin/x86_64/entrypoints.txt index 09fe3d7b4768..02912decadcf 100644 --- a/libc/config/darwin/x86_64/entrypoints.txt +++ b/libc/config/darwin/x86_64/entrypoints.txt @@ -106,6 +106,7 @@ set(TARGET_LIBM_ENTRYPOINTS # libc.src.fenv.fesetround # libc.src.fenv.feraiseexcept # libc.src.fenv.fetestexcept + # libc.src.fenv.fetestexceptflag # libc.src.fenv.feupdateenv ## Currently disabled for failing tests. diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt index 2952baacdd67..1ac6bd930000 100644 --- a/libc/config/linux/aarch64/entrypoints.txt +++ b/libc/config/linux/aarch64/entrypoints.txt @@ -324,6 +324,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.fesetround libc.src.fenv.feraiseexcept libc.src.fenv.fetestexcept + libc.src.fenv.fetestexceptflag libc.src.fenv.feupdateenv # math.h entrypoints diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt index 35fd588a9a6c..335981ff7dc7 100644 --- a/libc/config/linux/arm/entrypoints.txt +++ b/libc/config/linux/arm/entrypoints.txt @@ -192,6 +192,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.fesetround libc.src.fenv.feraiseexcept libc.src.fenv.fetestexcept + libc.src.fenv.fetestexceptflag libc.src.fenv.feupdateenv # math.h entrypoints diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt index 47c03a61c45a..87e82e5eb9a0 100644 --- a/libc/config/linux/riscv/entrypoints.txt +++ b/libc/config/linux/riscv/entrypoints.txt @@ -332,6 +332,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.fesetround libc.src.fenv.feraiseexcept libc.src.fenv.fetestexcept + libc.src.fenv.fetestexceptflag libc.src.fenv.feupdateenv # math.h entrypoints diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index 8fdd4575e27e..70f130a4399a 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -346,6 +346,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.fesetround libc.src.fenv.feraiseexcept libc.src.fenv.fetestexcept + libc.src.fenv.fetestexceptflag libc.src.fenv.feupdateenv # math.h entrypoints diff --git a/libc/config/windows/entrypoints.txt b/libc/config/windows/entrypoints.txt index c46c947bf313..71216530c404 100644 --- a/libc/config/windows/entrypoints.txt +++ b/libc/config/windows/entrypoints.txt @@ -110,6 +110,7 @@ set(TARGET_LIBM_ENTRYPOINTS libc.src.fenv.fesetround libc.src.fenv.feraiseexcept libc.src.fenv.fetestexcept + libc.src.fenv.fetestexceptflag libc.src.fenv.feupdateenv # math.h entrypoints diff --git a/libc/docs/c23.rst b/libc/docs/c23.rst index 4138c9d7104f..44724fe1660c 100644 --- a/libc/docs/c23.rst +++ b/libc/docs/c23.rst @@ -21,7 +21,7 @@ Additions: * fenv.h * fesetexcept |check| - * fetestexceptflag + * fetestexceptflag |check| * fegetmode * fesetmode * math.h diff --git a/libc/docs/fenv.rst b/libc/docs/fenv.rst index 6574fb7246dd..1dee5515e117 100644 --- a/libc/docs/fenv.rst +++ b/libc/docs/fenv.rst @@ -42,7 +42,7 @@ fenv.h Functions - |check| - 7.6.6.3 * - fesetexcept - - + - |check| - 7.6.4.4 * - fesetexceptflag - |check| @@ -57,7 +57,7 @@ fenv.h Functions - |check| - 7.6.4.7 * - fetestexceptflag - - + - |check| - 7.6.4.6 * - feupdateenv - |check| diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index 63d044986711..01aa7c70b3b9 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -149,6 +149,11 @@ def StdC : StandardSpec<"stdc"> { RetValSpec, [ArgSpec] >, + FunctionSpec< + "fetestexceptflag", + RetValSpec, + [ArgSpec, ArgSpec] + >, FunctionSpec< "feraiseexcept", RetValSpec, diff --git a/libc/src/fenv/CMakeLists.txt b/libc/src/fenv/CMakeLists.txt index 17e994741206..c5431b1b9d55 100644 --- a/libc/src/fenv/CMakeLists.txt +++ b/libc/src/fenv/CMakeLists.txt @@ -58,6 +58,19 @@ add_entrypoint_object( -O2 ) +add_entrypoint_object( + fetestexceptflag + SRCS + fetestexceptflag.cpp + HDRS + fetestexceptflag.h + DEPENDS + libc.hdr.types.fexcept_t + libc.src.__support.FPUtil.fenv_impl + COMPILE_OPTIONS + -O2 +) + add_entrypoint_object( fegetenv SRCS diff --git a/libc/src/fenv/fetestexceptflag.cpp b/libc/src/fenv/fetestexceptflag.cpp new file mode 100644 index 000000000000..63453350a199 --- /dev/null +++ b/libc/src/fenv/fetestexceptflag.cpp @@ -0,0 +1,23 @@ +//===-- Implementation of fetestexceptflag function -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/fenv/fetestexceptflag.h" +#include "hdr/types/fexcept_t.h" +#include "src/__support/FPUtil/FEnvImpl.h" +#include "src/__support/common.h" + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, fetestexceptflag, + (const fexcept_t *flagp, int excepts)) { + static_assert(sizeof(int) >= sizeof(fexcept_t), + "fexcept_t value cannot fit in an int value."); + return *flagp | fputil::test_except(excepts); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/fenv/fetestexceptflag.h b/libc/src/fenv/fetestexceptflag.h new file mode 100644 index 000000000000..1c8b0b843f54 --- /dev/null +++ b/libc/src/fenv/fetestexceptflag.h @@ -0,0 +1,20 @@ +//===-- Implementation header for fetestexceptflag --------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_FENV_FETESTEXCEPTFLAG_H +#define LLVM_LIBC_SRC_FENV_FETESTEXCEPTFLAG_H + +#include "hdr/types/fexcept_t.h" + +namespace LIBC_NAMESPACE { + +int fetestexceptflag(const fexcept_t *, int excepts); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_FENV_FETESTEXCEPTFLAG_H diff --git a/libc/test/src/fenv/CMakeLists.txt b/libc/test/src/fenv/CMakeLists.txt index 577735599dc0..f277b65e2d42 100644 --- a/libc/test/src/fenv/CMakeLists.txt +++ b/libc/test/src/fenv/CMakeLists.txt @@ -48,6 +48,7 @@ add_libc_unittest( DEPENDS libc.src.fenv.fegetexceptflag libc.src.fenv.fesetexceptflag + libc.src.fenv.fetestexceptflag libc.src.__support.FPUtil.fenv_impl ) diff --git a/libc/test/src/fenv/exception_flags_test.cpp b/libc/test/src/fenv/exception_flags_test.cpp index d1d8bfcc53db..9d2be6426a6d 100644 --- a/libc/test/src/fenv/exception_flags_test.cpp +++ b/libc/test/src/fenv/exception_flags_test.cpp @@ -1,4 +1,4 @@ -//===-- Unittests for fegetexceptflag and fesetexceptflag -----------------===// +//===-- Unittests for fe{get|set|test}exceptflag --------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -9,11 +9,12 @@ #include "hdr/types/fexcept_t.h" #include "src/fenv/fegetexceptflag.h" #include "src/fenv/fesetexceptflag.h" +#include "src/fenv/fetestexceptflag.h" #include "src/__support/FPUtil/FEnvImpl.h" #include "test/UnitTest/Test.h" -TEST(LlvmLibcFenvTest, GetExceptFlagAndSetExceptFlag) { +TEST(LlvmLibcFenvTest, GetSetTestExceptFlag) { // We will disable all exceptions to prevent invocation of the exception // handler. LIBC_NAMESPACE::fputil::disable_except(FE_ALL_EXCEPT); @@ -39,19 +40,36 @@ TEST(LlvmLibcFenvTest, GetExceptFlagAndSetExceptFlag) { ASSERT_EQ(LIBC_NAMESPACE::fesetexceptflag(&eflags, FE_ALL_EXCEPT), 0); ASSERT_NE(LIBC_NAMESPACE::fputil::test_except(FE_ALL_EXCEPT) & e, 0); + // Exception flags are exactly the flags corresponding to the previously + // raised exception. + ASSERT_EQ(LIBC_NAMESPACE::fetestexceptflag(&eflags, FE_ALL_EXCEPT), + LIBC_NAMESPACE::fputil::test_except(FE_ALL_EXCEPT)); + // Cleanup. We clear all excepts as raising excepts like FE_OVERFLOW // can also raise FE_INEXACT. LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT); } - // Next, we will raise one exception and save the flags. + // Next, we will raise one exception, save the flag and clear all exceptions. LIBC_NAMESPACE::fputil::raise_except(FE_INVALID); - fexcept_t eflags; - LIBC_NAMESPACE::fegetexceptflag(&eflags, FE_ALL_EXCEPT); - // Clear all exceptions and raise two other exceptions. + fexcept_t invalid_flag; + LIBC_NAMESPACE::fegetexceptflag(&invalid_flag, FE_ALL_EXCEPT); + ASSERT_EQ(LIBC_NAMESPACE::fetestexceptflag(&invalid_flag, FE_ALL_EXCEPT), + FE_INVALID); LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT); + + // Raise two other exceptions and verify that they are set. LIBC_NAMESPACE::fputil::raise_except(FE_OVERFLOW | FE_INEXACT); + fexcept_t overflow_and_inexact_flag; + LIBC_NAMESPACE::fegetexceptflag(&overflow_and_inexact_flag, FE_ALL_EXCEPT); + ASSERT_EQ(LIBC_NAMESPACE::fetestexceptflag(&overflow_and_inexact_flag, + FE_ALL_EXCEPT), + FE_OVERFLOW | FE_INEXACT); + ASSERT_EQ(LIBC_NAMESPACE::fetestexceptflag(&overflow_and_inexact_flag, + FE_OVERFLOW | FE_INEXACT), + FE_OVERFLOW | FE_INEXACT); + // When we set the flags and test, we should only see FE_INVALID. - LIBC_NAMESPACE::fesetexceptflag(&eflags, FE_ALL_EXCEPT); + LIBC_NAMESPACE::fesetexceptflag(&invalid_flag, FE_ALL_EXCEPT); EXPECT_EQ(LIBC_NAMESPACE::fputil::test_except(FE_ALL_EXCEPT), FE_INVALID); } diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index fb37f113b310..6029cc3fee61 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -1145,6 +1145,16 @@ libc_function( ], ) +libc_function( + name = "fetestexceptflag", + srcs = ["src/fenv/fetestexceptflag.cpp"], + hdrs = ["src/fenv/fetestexceptflag.h"], + deps = [ + ":__support_common", + ":__support_fputil_fenv_impl", + ], +) + libc_function( name = "feclearexcept", srcs = ["src/fenv/feclearexcept.cpp"], diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/fenv/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/fenv/BUILD.bazel index 359db0723dfd..fc3ab3da3587 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/src/fenv/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/test/src/fenv/BUILD.bazel @@ -75,6 +75,7 @@ libc_test( libc_function_deps = [ "//libc:fegetexceptflag", "//libc:fesetexceptflag", + "//libc:fetestexceptflag", ], deps = [ "//libc:__support_fputil_fenv_impl", -- GitLab From 8656d4c6a7a742c6fa6ee02c2ace7415163e65e4 Mon Sep 17 00:00:00 2001 From: Krystian Stasiowski Date: Wed, 17 Apr 2024 11:41:03 -0400 Subject: [PATCH 030/714] [Clang][Parse] Diagnose requires expressions with explicit object parameters (#88974) Clang currently allows the following: ``` auto x = requires (this int) { true; }; ``` This patch addresses that. --- clang/docs/ReleaseNotes.rst | 2 ++ clang/include/clang/Basic/DiagnosticParseKinds.td | 2 ++ clang/lib/Parse/ParseDecl.cpp | 15 ++++++++++++++- .../CXX/dcl.decl/dcl.meaning/dcl.fct/p6-cxx23.cpp | 7 +++++++ 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 clang/test/CXX/dcl.decl/dcl.meaning/dcl.fct/p6-cxx23.cpp diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 96ad92b540b4..c19ad9fba58f 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -364,6 +364,8 @@ Improvements to Clang's diagnostics - Clang now uses the correct type-parameter-key (``class`` or ``typename``) when printing template template parameter declarations. +- Clang now diagnoses requires expressions with explicit object parameters. + Improvements to Clang's time-trace ---------------------------------- diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td index bb9ca2a50cc0..66405095d51d 100644 --- a/clang/include/clang/Basic/DiagnosticParseKinds.td +++ b/clang/include/clang/Basic/DiagnosticParseKinds.td @@ -863,6 +863,8 @@ def err_empty_requires_expr : Error< "a requires expression must contain at least one requirement">; def err_requires_expr_parameter_list_ellipsis : Error< "varargs not allowed in requires expression">; +def err_requires_expr_explicit_object_parameter: Error< + "a requires expression cannot have an explicit object parameter">; def err_expected_semi_requirement : Error< "expected ';' at end of requirement">; def err_requires_expr_missing_arrow : Error< diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp index 274ee7b10c17..5f26b5a9e46b 100644 --- a/clang/lib/Parse/ParseDecl.cpp +++ b/clang/lib/Parse/ParseDecl.cpp @@ -7660,8 +7660,21 @@ void Parser::ParseParameterDeclarationClause( // Parse a C++23 Explicit Object Parameter // We do that in all language modes to produce a better diagnostic. SourceLocation ThisLoc; - if (getLangOpts().CPlusPlus && Tok.is(tok::kw_this)) + if (getLangOpts().CPlusPlus && Tok.is(tok::kw_this)) { ThisLoc = ConsumeToken(); + // C++23 [dcl.fct]p6: + // An explicit-object-parameter-declaration is a parameter-declaration + // with a this specifier. An explicit-object-parameter-declaration + // shall appear only as the first parameter-declaration of a + // parameter-declaration-list of either: + // - a member-declarator that declares a member function, or + // - a lambda-declarator. + // + // The parameter-declaration-list of a requires-expression is not such + // a context. + if (DeclaratorCtx == DeclaratorContext::RequiresExpr) + Diag(ThisLoc, diag::err_requires_expr_explicit_object_parameter); + } ParseDeclarationSpecifiers(DS, /*TemplateInfo=*/ParsedTemplateInfo(), AS_none, DeclSpecContext::DSC_normal, diff --git a/clang/test/CXX/dcl.decl/dcl.meaning/dcl.fct/p6-cxx23.cpp b/clang/test/CXX/dcl.decl/dcl.meaning/dcl.fct/p6-cxx23.cpp new file mode 100644 index 000000000000..9c1f30f81a01 --- /dev/null +++ b/clang/test/CXX/dcl.decl/dcl.meaning/dcl.fct/p6-cxx23.cpp @@ -0,0 +1,7 @@ +// RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify %s + +auto x0 = requires (this int) { true; }; // expected-error {{a requires expression cannot have an explicit object parameter}} +auto x1 = requires (int, this int) { true; }; // expected-error {{a requires expression cannot have an explicit object parameter}} + +template // expected-error {{expected template parameter}} +void f(); // expected-error {{no function template matches function template specialization 'f'}} -- GitLab From abd5e45a96954d80f6ffe6d8676c0059fae8573b Mon Sep 17 00:00:00 2001 From: Alexander Richardson Date: Wed, 17 Apr 2024 08:42:41 -0700 Subject: [PATCH 031/714] [compiler-rt] Use __atomic builtins whenever possible The code in this file dates back to 2012 when Clang's support for atomic builtins was still quite limited. The bugs referenced in the comment at the top of the file have long been fixed and using the compiler builtins directly should now generate slightly better code. Additionally, this allows using the atomic builtin header for platforms where the __sync_builtins are lacking (e.g. Arm Morello). This change does not introduce any code generation changes for __tsan_read*/__tsan_write* or __tsan_func_{entry,exit} on x86, which indicates the previously noted compiler issues have been fixed. We also have to touch the non-clang codepaths here since the only way we can make this work easily is by making the memory_order enum match the compiler-provided macros, so we have to update the debug checks that assumed the enum was always a bitflag. The one downside of this change is that 32-bit MIPS now definitely requires libatomic (but that may already have been needed for RMW ops). Reviewed By: dvyukov Pull Request: https://github.com/llvm/llvm-project/pull/84439 --- .../lib/sanitizer_common/CMakeLists.txt | 3 - .../lib/sanitizer_common/sanitizer_atomic.h | 12 ++ .../sanitizer_common/sanitizer_atomic_clang.h | 85 ++++++------- .../sanitizer_atomic_clang_mips.h | 117 ------------------ .../sanitizer_atomic_clang_other.h | 85 ------------- .../sanitizer_atomic_clang_x86.h | 113 ----------------- .../sanitizer_common/sanitizer_atomic_msvc.h | 8 +- .../compiler-rt/lib/sanitizer_common/BUILD.gn | 3 - 8 files changed, 56 insertions(+), 370 deletions(-) delete mode 100644 compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_mips.h delete mode 100644 compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_other.h delete mode 100644 compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h diff --git a/compiler-rt/lib/sanitizer_common/CMakeLists.txt b/compiler-rt/lib/sanitizer_common/CMakeLists.txt index f2b4ac72ae15..66f2d259aa5f 100644 --- a/compiler-rt/lib/sanitizer_common/CMakeLists.txt +++ b/compiler-rt/lib/sanitizer_common/CMakeLists.txt @@ -122,9 +122,6 @@ set(SANITIZER_IMPL_HEADERS sanitizer_asm.h sanitizer_atomic.h sanitizer_atomic_clang.h - sanitizer_atomic_clang_mips.h - sanitizer_atomic_clang_other.h - sanitizer_atomic_clang_x86.h sanitizer_atomic_msvc.h sanitizer_bitvector.h sanitizer_bvgraph.h diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_atomic.h b/compiler-rt/lib/sanitizer_common/sanitizer_atomic.h index 46f06957228c..0609a11ffdeb 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_atomic.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_atomic.h @@ -18,12 +18,24 @@ namespace __sanitizer { enum memory_order { +// If the __atomic atomic builtins are supported (Clang/GCC), use the +// compiler provided macro values so that we can map the atomic operations +// to __atomic_* directly. +#ifdef __ATOMIC_SEQ_CST + memory_order_relaxed = __ATOMIC_RELAXED, + memory_order_consume = __ATOMIC_CONSUME, + memory_order_acquire = __ATOMIC_ACQUIRE, + memory_order_release = __ATOMIC_RELEASE, + memory_order_acq_rel = __ATOMIC_ACQ_REL, + memory_order_seq_cst = __ATOMIC_SEQ_CST +#else memory_order_relaxed = 1 << 0, memory_order_consume = 1 << 1, memory_order_acquire = 1 << 2, memory_order_release = 1 << 3, memory_order_acq_rel = 1 << 4, memory_order_seq_cst = 1 << 5 +#endif }; struct atomic_uint8_t { diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang.h b/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang.h index 4318d64d16cf..1414092e38d7 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang.h @@ -14,60 +14,63 @@ #ifndef SANITIZER_ATOMIC_CLANG_H #define SANITIZER_ATOMIC_CLANG_H -#if defined(__i386__) || defined(__x86_64__) -# include "sanitizer_atomic_clang_x86.h" -#else -# include "sanitizer_atomic_clang_other.h" -#endif - namespace __sanitizer { -// We would like to just use compiler builtin atomic operations -// for loads and stores, but they are mostly broken in clang: -// - they lead to vastly inefficient code generation -// (http://llvm.org/bugs/show_bug.cgi?id=17281) -// - 64-bit atomic operations are not implemented on x86_32 -// (http://llvm.org/bugs/show_bug.cgi?id=15034) -// - they are not implemented on ARM -// error: undefined reference to '__atomic_load_4' +// We use the compiler builtin atomic operations for loads and stores, which +// generates correct code for all architectures, but may require libatomic +// on platforms where e.g. 64-bit atomics are not supported natively. // See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html // for mappings of the memory model to different processors. -inline void atomic_signal_fence(memory_order) { +inline void atomic_signal_fence(memory_order mo) { __atomic_signal_fence(mo); } + +inline void atomic_thread_fence(memory_order mo) { __atomic_thread_fence(mo); } + +inline void proc_yield(int cnt) { + __asm__ __volatile__("" ::: "memory"); +#if defined(__i386__) || defined(__x86_64__) + for (int i = 0; i < cnt; i++) __asm__ __volatile__("pause"); __asm__ __volatile__("" ::: "memory"); +#endif } -inline void atomic_thread_fence(memory_order) { - __sync_synchronize(); +template +inline typename T::Type atomic_load(const volatile T *a, memory_order mo) { + DCHECK(mo == memory_order_relaxed || mo == memory_order_consume || + mo == memory_order_acquire || mo == memory_order_seq_cst); + DCHECK(!((uptr)a % sizeof(*a))); + return __atomic_load_n(&a->val_dont_use, mo); } -template -inline typename T::Type atomic_fetch_add(volatile T *a, - typename T::Type v, memory_order mo) { - (void)mo; +template +inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { + DCHECK(mo == memory_order_relaxed || mo == memory_order_release || + mo == memory_order_seq_cst); DCHECK(!((uptr)a % sizeof(*a))); - return __sync_fetch_and_add(&a->val_dont_use, v); + __atomic_store_n(&a->val_dont_use, v, mo); } -template -inline typename T::Type atomic_fetch_sub(volatile T *a, - typename T::Type v, memory_order mo) { +template +inline typename T::Type atomic_fetch_add(volatile T *a, typename T::Type v, + memory_order mo) { + DCHECK(!((uptr)a % sizeof(*a))); + return __atomic_fetch_add(&a->val_dont_use, v, mo); +} + +template +inline typename T::Type atomic_fetch_sub(volatile T *a, typename T::Type v, + memory_order mo) { (void)mo; DCHECK(!((uptr)a % sizeof(*a))); - return __sync_fetch_and_add(&a->val_dont_use, -v); + return __atomic_fetch_sub(&a->val_dont_use, v, mo); } -template -inline typename T::Type atomic_exchange(volatile T *a, - typename T::Type v, memory_order mo) { +template +inline typename T::Type atomic_exchange(volatile T *a, typename T::Type v, + memory_order mo) { DCHECK(!((uptr)a % sizeof(*a))); - if (mo & (memory_order_release | memory_order_acq_rel | memory_order_seq_cst)) - __sync_synchronize(); - v = __sync_lock_test_and_set(&a->val_dont_use, v); - if (mo == memory_order_seq_cst) - __sync_synchronize(); - return v; + return __atomic_exchange_n(&a->val_dont_use, v, mo); } template @@ -82,9 +85,8 @@ inline bool atomic_compare_exchange_strong(volatile T *a, typename T::Type *cmp, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } -template -inline bool atomic_compare_exchange_weak(volatile T *a, - typename T::Type *cmp, +template +inline bool atomic_compare_exchange_weak(volatile T *a, typename T::Type *cmp, typename T::Type xchg, memory_order mo) { return atomic_compare_exchange_strong(a, cmp, xchg, mo); @@ -92,13 +94,6 @@ inline bool atomic_compare_exchange_weak(volatile T *a, } // namespace __sanitizer -// This include provides explicit template instantiations for atomic_uint64_t -// on MIPS32, which does not directly support 8 byte atomics. It has to -// proceed the template definitions above. -#if defined(_MIPS_SIM) && defined(_ABIO32) && _MIPS_SIM == _ABIO32 -# include "sanitizer_atomic_clang_mips.h" -#endif - #undef ATOMIC_ORDER #endif // SANITIZER_ATOMIC_CLANG_H diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_mips.h b/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_mips.h deleted file mode 100644 index f3d3052e5b7c..000000000000 --- a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_mips.h +++ /dev/null @@ -1,117 +0,0 @@ -//===-- sanitizer_atomic_clang_mips.h ---------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file is a part of ThreadSanitizer/AddressSanitizer runtime. -// Not intended for direct inclusion. Include sanitizer_atomic.h. -// -//===----------------------------------------------------------------------===// - -#ifndef SANITIZER_ATOMIC_CLANG_MIPS_H -#define SANITIZER_ATOMIC_CLANG_MIPS_H - -namespace __sanitizer { - -// MIPS32 does not support atomics > 4 bytes. To address this lack of -// functionality, the sanitizer library provides helper methods which use an -// internal spin lock mechanism to emulate atomic operations when the size is -// 8 bytes. -static void __spin_lock(volatile int *lock) { - while (__sync_lock_test_and_set(lock, 1)) - while (*lock) { - } -} - -static void __spin_unlock(volatile int *lock) { __sync_lock_release(lock); } - -// Make sure the lock is on its own cache line to prevent false sharing. -// Put it inside a struct that is aligned and padded to the typical MIPS -// cacheline which is 32 bytes. -static struct { - int lock; - char pad[32 - sizeof(int)]; -} __attribute__((aligned(32))) lock = {0, {0}}; - -template <> -inline atomic_uint64_t::Type atomic_fetch_add(volatile atomic_uint64_t *ptr, - atomic_uint64_t::Type val, - memory_order mo) { - DCHECK(mo & - (memory_order_relaxed | memory_order_release | memory_order_seq_cst)); - DCHECK(!((uptr)ptr % sizeof(*ptr))); - - atomic_uint64_t::Type ret; - - __spin_lock(&lock.lock); - ret = *(const_cast(&ptr->val_dont_use)); - ptr->val_dont_use = ret + val; - __spin_unlock(&lock.lock); - - return ret; -} - -template <> -inline atomic_uint64_t::Type atomic_fetch_sub(volatile atomic_uint64_t *ptr, - atomic_uint64_t::Type val, - memory_order mo) { - return atomic_fetch_add(ptr, -val, mo); -} - -template <> -inline bool atomic_compare_exchange_strong(volatile atomic_uint64_t *ptr, - atomic_uint64_t::Type *cmp, - atomic_uint64_t::Type xchg, - memory_order mo) { - DCHECK(mo & - (memory_order_relaxed | memory_order_release | memory_order_seq_cst)); - DCHECK(!((uptr)ptr % sizeof(*ptr))); - - typedef atomic_uint64_t::Type Type; - Type cmpv = *cmp; - Type prev; - bool ret = false; - - __spin_lock(&lock.lock); - prev = *(const_cast(&ptr->val_dont_use)); - if (prev == cmpv) { - ret = true; - ptr->val_dont_use = xchg; - } - __spin_unlock(&lock.lock); - - return ret; -} - -template <> -inline atomic_uint64_t::Type atomic_load(const volatile atomic_uint64_t *ptr, - memory_order mo) { - DCHECK(mo & - (memory_order_relaxed | memory_order_release | memory_order_seq_cst)); - DCHECK(!((uptr)ptr % sizeof(*ptr))); - - atomic_uint64_t::Type zero = 0; - volatile atomic_uint64_t *Newptr = - const_cast(ptr); - return atomic_fetch_add(Newptr, zero, mo); -} - -template <> -inline void atomic_store(volatile atomic_uint64_t *ptr, atomic_uint64_t::Type v, - memory_order mo) { - DCHECK(mo & - (memory_order_relaxed | memory_order_release | memory_order_seq_cst)); - DCHECK(!((uptr)ptr % sizeof(*ptr))); - - __spin_lock(&lock.lock); - ptr->val_dont_use = v; - __spin_unlock(&lock.lock); -} - -} // namespace __sanitizer - -#endif // SANITIZER_ATOMIC_CLANG_MIPS_H - diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_other.h b/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_other.h deleted file mode 100644 index 557082a636b8..000000000000 --- a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_other.h +++ /dev/null @@ -1,85 +0,0 @@ -//===-- sanitizer_atomic_clang_other.h --------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file is a part of ThreadSanitizer/AddressSanitizer runtime. -// Not intended for direct inclusion. Include sanitizer_atomic.h. -// -//===----------------------------------------------------------------------===// - -#ifndef SANITIZER_ATOMIC_CLANG_OTHER_H -#define SANITIZER_ATOMIC_CLANG_OTHER_H - -namespace __sanitizer { - - -inline void proc_yield(int cnt) { - __asm__ __volatile__("" ::: "memory"); -} - -template -inline typename T::Type atomic_load( - const volatile T *a, memory_order mo) { - DCHECK(mo & (memory_order_relaxed | memory_order_consume - | memory_order_acquire | memory_order_seq_cst)); - DCHECK(!((uptr)a % sizeof(*a))); - typename T::Type v; - - if (sizeof(*a) < 8 || sizeof(void*) == 8) { - // Assume that aligned loads are atomic. - if (mo == memory_order_relaxed) { - v = a->val_dont_use; - } else if (mo == memory_order_consume) { - // Assume that processor respects data dependencies - // (and that compiler won't break them). - __asm__ __volatile__("" ::: "memory"); - v = a->val_dont_use; - __asm__ __volatile__("" ::: "memory"); - } else if (mo == memory_order_acquire) { - __asm__ __volatile__("" ::: "memory"); - v = a->val_dont_use; - __sync_synchronize(); - } else { // seq_cst - // E.g. on POWER we need a hw fence even before the store. - __sync_synchronize(); - v = a->val_dont_use; - __sync_synchronize(); - } - } else { - __atomic_load(const_cast(&a->val_dont_use), &v, - __ATOMIC_SEQ_CST); - } - return v; -} - -template -inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { - DCHECK(mo & (memory_order_relaxed | memory_order_release - | memory_order_seq_cst)); - DCHECK(!((uptr)a % sizeof(*a))); - - if (sizeof(*a) < 8 || sizeof(void*) == 8) { - // Assume that aligned stores are atomic. - if (mo == memory_order_relaxed) { - a->val_dont_use = v; - } else if (mo == memory_order_release) { - __sync_synchronize(); - a->val_dont_use = v; - __asm__ __volatile__("" ::: "memory"); - } else { // seq_cst - __sync_synchronize(); - a->val_dont_use = v; - __sync_synchronize(); - } - } else { - __atomic_store(&a->val_dont_use, &v, __ATOMIC_SEQ_CST); - } -} - -} // namespace __sanitizer - -#endif // #ifndef SANITIZER_ATOMIC_CLANG_OTHER_H diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h b/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h deleted file mode 100644 index b81a354d2098..000000000000 --- a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_clang_x86.h +++ /dev/null @@ -1,113 +0,0 @@ -//===-- sanitizer_atomic_clang_x86.h ----------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file is a part of ThreadSanitizer/AddressSanitizer runtime. -// Not intended for direct inclusion. Include sanitizer_atomic.h. -// -//===----------------------------------------------------------------------===// - -#ifndef SANITIZER_ATOMIC_CLANG_X86_H -#define SANITIZER_ATOMIC_CLANG_X86_H - -namespace __sanitizer { - -inline void proc_yield(int cnt) { - __asm__ __volatile__("" ::: "memory"); - for (int i = 0; i < cnt; i++) - __asm__ __volatile__("pause"); - __asm__ __volatile__("" ::: "memory"); -} - -template -inline typename T::Type atomic_load( - const volatile T *a, memory_order mo) { - DCHECK(mo & (memory_order_relaxed | memory_order_consume - | memory_order_acquire | memory_order_seq_cst)); - DCHECK(!((uptr)a % sizeof(*a))); - typename T::Type v; - - if (sizeof(*a) < 8 || sizeof(void*) == 8) { - // Assume that aligned loads are atomic. - if (mo == memory_order_relaxed) { - v = a->val_dont_use; - } else if (mo == memory_order_consume) { - // Assume that processor respects data dependencies - // (and that compiler won't break them). - __asm__ __volatile__("" ::: "memory"); - v = a->val_dont_use; - __asm__ __volatile__("" ::: "memory"); - } else if (mo == memory_order_acquire) { - __asm__ __volatile__("" ::: "memory"); - v = a->val_dont_use; - // On x86 loads are implicitly acquire. - __asm__ __volatile__("" ::: "memory"); - } else { // seq_cst - // On x86 plain MOV is enough for seq_cst store. - __asm__ __volatile__("" ::: "memory"); - v = a->val_dont_use; - __asm__ __volatile__("" ::: "memory"); - } - } else { - // 64-bit load on 32-bit platform. - __asm__ __volatile__( - "movq %1, %%mm0;" // Use mmx reg for 64-bit atomic moves - "movq %%mm0, %0;" // (ptr could be read-only) - "emms;" // Empty mmx state/Reset FP regs - : "=m" (v) - : "m" (a->val_dont_use) - : // mark the mmx registers as clobbered -#ifdef __MMX__ - "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7", -#endif // #ifdef __MMX__ - "memory"); - } - return v; -} - -template -inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { - DCHECK(mo & (memory_order_relaxed | memory_order_release - | memory_order_seq_cst)); - DCHECK(!((uptr)a % sizeof(*a))); - - if (sizeof(*a) < 8 || sizeof(void*) == 8) { - // Assume that aligned stores are atomic. - if (mo == memory_order_relaxed) { - a->val_dont_use = v; - } else if (mo == memory_order_release) { - // On x86 stores are implicitly release. - __asm__ __volatile__("" ::: "memory"); - a->val_dont_use = v; - __asm__ __volatile__("" ::: "memory"); - } else { // seq_cst - // On x86 stores are implicitly release. - __asm__ __volatile__("" ::: "memory"); - a->val_dont_use = v; - __sync_synchronize(); - } - } else { - // 64-bit store on 32-bit platform. - __asm__ __volatile__( - "movq %1, %%mm0;" // Use mmx reg for 64-bit atomic moves - "movq %%mm0, %0;" - "emms;" // Empty mmx state/Reset FP regs - : "=m" (a->val_dont_use) - : "m" (v) - : // mark the mmx registers as clobbered -#ifdef __MMX__ - "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7", -#endif // #ifdef __MMX__ - "memory"); - if (mo == memory_order_seq_cst) - __sync_synchronize(); - } -} - -} // namespace __sanitizer - -#endif // #ifndef SANITIZER_ATOMIC_CLANG_X86_H diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_msvc.h b/compiler-rt/lib/sanitizer_common/sanitizer_atomic_msvc.h index 31317adcdfc9..d80bfdbf6a08 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_atomic_msvc.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_atomic_msvc.h @@ -70,8 +70,8 @@ inline void proc_yield(int cnt) { template inline typename T::Type atomic_load( const volatile T *a, memory_order mo) { - DCHECK(mo & (memory_order_relaxed | memory_order_consume - | memory_order_acquire | memory_order_seq_cst)); + DCHECK(mo == memory_order_relaxed || mo == memory_order_consume || + mo == memory_order_acquire || mo == memory_order_seq_cst); DCHECK(!((uptr)a % sizeof(*a))); typename T::Type v; // FIXME(dvyukov): 64-bit load is not atomic on 32-bits. @@ -87,8 +87,8 @@ inline typename T::Type atomic_load( template inline void atomic_store(volatile T *a, typename T::Type v, memory_order mo) { - DCHECK(mo & (memory_order_relaxed | memory_order_release - | memory_order_seq_cst)); + DCHECK(mo == memory_order_relaxed || mo == memory_order_release || + mo == memory_order_seq_cst); DCHECK(!((uptr)a % sizeof(*a))); // FIXME(dvyukov): 64-bit store is not atomic on 32-bits. if (mo == memory_order_relaxed) { diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/sanitizer_common/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/sanitizer_common/BUILD.gn index 051907323943..f7f1fce10bf5 100644 --- a/llvm/utils/gn/secondary/compiler-rt/lib/sanitizer_common/BUILD.gn +++ b/llvm/utils/gn/secondary/compiler-rt/lib/sanitizer_common/BUILD.gn @@ -33,9 +33,6 @@ source_set("sources") { "sanitizer_asm.h", "sanitizer_atomic.h", "sanitizer_atomic_clang.h", - "sanitizer_atomic_clang_mips.h", - "sanitizer_atomic_clang_other.h", - "sanitizer_atomic_clang_x86.h", "sanitizer_atomic_msvc.h", "sanitizer_bitvector.h", "sanitizer_bvgraph.h", -- GitLab From a88ea8fbb3953c2fe2887438baf342e381a79d8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Wed, 17 Apr 2024 08:43:11 -0700 Subject: [PATCH 032/714] [flang][cuda] Update memory effect on fir.cuda_allocate op (#88930) Add MemRead effect on the box operand as the descriptor might be read when performing the allocation of the data. Also update the expected type of the box operand to be a reference. Check in the verifier that this is a reference to a box or class type. This addresses the comment made post commit on #88586 --- flang/include/flang/Optimizer/Dialect/FIROps.td | 2 +- flang/lib/Optimizer/Dialect/FIROps.cpp | 2 +- flang/test/Fir/cuf-invalid.fir | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td index 580e840587ab..92790a691e47 100644 --- a/flang/include/flang/Optimizer/Dialect/FIROps.td +++ b/flang/include/flang/Optimizer/Dialect/FIROps.td @@ -3200,7 +3200,7 @@ def fir_CUDAAllocateOp : fir_Op<"cuda_allocate", [AttrSizedOperandSegments, is initialized before with the standard flang runtime calls. }]; - let arguments = (ins Arg:$box, + let arguments = (ins Arg:$box, Arg, "", [MemWrite]>:$errmsg, Optional:$stream, Arg, "", [MemWrite]>:$pinned, diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp index be27256d911b..5c24c95db427 100644 --- a/flang/lib/Optimizer/Dialect/FIROps.cpp +++ b/flang/lib/Optimizer/Dialect/FIROps.cpp @@ -3998,7 +3998,7 @@ mlir::LogicalResult fir::CUDAAllocateOp::verify() { return emitOpError("pinned and stream cannot appears at the same time"); if (!fir::unwrapRefType(getBox().getType()).isa()) return emitOpError( - "expect box to be a reference to/or a class or box type value"); + "expect box to be a reference to a class or box type value"); if (getSource() && !fir::unwrapRefType(getSource().getType()).isa()) return emitOpError( diff --git a/flang/test/Fir/cuf-invalid.fir b/flang/test/Fir/cuf-invalid.fir index 5d3aa55cf346..6c533a32ccf9 100644 --- a/flang/test/Fir/cuf-invalid.fir +++ b/flang/test/Fir/cuf-invalid.fir @@ -16,7 +16,7 @@ func.func @_QPsub1() { func.func @_QPsub1() { %1 = fir.alloca i32 - // expected-error@+1{{'fir.cuda_allocate' op expect box to be a reference to/or a class or box type value}} + // expected-error@+1{{'fir.cuda_allocate' op expect box to be a reference to a class or box type value}} %2 = fir.cuda_allocate %1 : !fir.ref {cuda_attr = #fir.cuda} -> i32 return } -- GitLab From da70f2cdcde8cb96e75ce0236db1fb5353407a69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Wed, 17 Apr 2024 08:43:25 -0700 Subject: [PATCH 033/714] [flang][cuda] Lower ALLOCATE for device variable (#88980) Replace the runtime call to `AllocatableAllocate` for CUDA device variable to the newly added `fir.cuda_allocate` operation. --- flang/lib/Lower/Allocatable.cpp | 57 +++++++++-- flang/test/Lower/CUDA/cuda-allocatable.cuf | 107 +++++++++++++++++++++ 2 files changed, 154 insertions(+), 10 deletions(-) create mode 100644 flang/test/Lower/CUDA/cuda-allocatable.cuf diff --git a/flang/lib/Lower/Allocatable.cpp b/flang/lib/Lower/Allocatable.cpp index 42e78fc96e44..1d434d512d0c 100644 --- a/flang/lib/Lower/Allocatable.cpp +++ b/flang/lib/Lower/Allocatable.cpp @@ -14,6 +14,7 @@ #include "flang/Evaluate/tools.h" #include "flang/Lower/AbstractConverter.h" #include "flang/Lower/ConvertType.h" +#include "flang/Lower/ConvertVariable.h" #include "flang/Lower/IterationSpace.h" #include "flang/Lower/Mangler.h" #include "flang/Lower/OpenACC.h" @@ -368,20 +369,17 @@ private: [&](const Fortran::parser::AllocOpt::Mold &mold) { moldExpr = Fortran::semantics::GetExpr(mold.v.value()); }, - [&](const Fortran::parser::AllocOpt::Stream &) { - TODO(loc, "CUDA ALLOCATE(STREAM=)"); + [&](const Fortran::parser::AllocOpt::Stream &stream) { + streamExpr = Fortran::semantics::GetExpr(stream.v.value()); }, - [&](const Fortran::parser::AllocOpt::Pinned &) { - TODO(loc, "CUDA ALLOCATE(PINNED=)"); + [&](const Fortran::parser::AllocOpt::Pinned &pinned) { + pinnedExpr = Fortran::semantics::GetExpr(pinned.v.value()); }, }, allocOption.u); } void lowerAllocation(const Allocation &alloc) { - if (Fortran::semantics::HasCUDAAttr(alloc.getSymbol())) - TODO(loc, "Allocation of variable with CUDA attributes"); - fir::MutableBoxValue boxAddr = genMutableBoxValue(converter, loc, alloc.getAllocObj()); @@ -456,7 +454,8 @@ private: const fir::MutableBoxValue &box) { if (!box.isDerived() && !errorManager.hasStatSpec() && !alloc.type.IsPolymorphic() && !alloc.hasCoarraySpec() && - !useAllocateRuntime && !box.isPointer()) { + !useAllocateRuntime && !box.isPointer() && + !Fortran::semantics::HasCUDAAttr(alloc.getSymbol())) { // Pointers must use PointerAllocate so that their deallocations // can be validated. genInlinedAllocation(alloc, box); @@ -472,7 +471,12 @@ private: genSetType(alloc, box, loc); genSetDeferredLengthParameters(alloc, box); genAllocateObjectBounds(alloc, box); - mlir::Value stat = genRuntimeAllocate(builder, loc, box, errorManager); + mlir::Value stat; + if (!Fortran::semantics::HasCUDAAttr(alloc.getSymbol())) + stat = genRuntimeAllocate(builder, loc, box, errorManager); + else + stat = + genCudaAllocate(builder, loc, box, errorManager, alloc.getSymbol()); fir::factory::syncMutableBoxFromIRBox(builder, loc, box); postAllocationAction(alloc); errorManager.assignStat(builder, loc, stat); @@ -602,7 +606,10 @@ private: genSetDeferredLengthParameters(alloc, box); genAllocateObjectBounds(alloc, box); mlir::Value stat; - if (isSource) + if (Fortran::semantics::HasCUDAAttr(alloc.getSymbol())) + stat = + genCudaAllocate(builder, loc, box, errorManager, alloc.getSymbol()); + else if (isSource) stat = genRuntimeAllocateSource(builder, loc, box, exv, errorManager); else stat = genRuntimeAllocate(builder, loc, box, errorManager); @@ -717,6 +724,34 @@ private: return nullptr; } + mlir::Value genCudaAllocate(fir::FirOpBuilder &builder, mlir::Location loc, + const fir::MutableBoxValue &box, + ErrorManager &errorManager, + const Fortran::semantics::Symbol &sym) { + Fortran::lower::StatementContext stmtCtx; + fir::CUDADataAttributeAttr cudaAttr = + Fortran::lower::translateSymbolCUDADataAttribute(builder.getContext(), + sym); + mlir::Value errmsg = errMsgExpr ? errorManager.errMsgAddr : nullptr; + mlir::Value stream = + streamExpr + ? fir::getBase(converter.genExprValue(loc, *streamExpr, stmtCtx)) + : nullptr; + mlir::Value pinned = + pinnedExpr + ? fir::getBase(converter.genExprAddr(loc, *pinnedExpr, stmtCtx)) + : nullptr; + mlir::Value source = sourceExpr ? fir::getBase(sourceExv) : nullptr; + + // Keep return type the same as a standard AllocatableAllocate call. + mlir::Type retTy = fir::runtime::getModel()(builder.getContext()); + return builder + .create( + loc, retTy, box.getAddr(), errmsg, stream, pinned, source, cudaAttr, + errorManager.hasStatSpec() ? builder.getUnitAttr() : nullptr) + .getResult(); + } + Fortran::lower::AbstractConverter &converter; fir::FirOpBuilder &builder; const Fortran::parser::AllocateStmt &stmt; @@ -724,6 +759,8 @@ private: const Fortran::lower::SomeExpr *moldExpr{nullptr}; const Fortran::lower::SomeExpr *statExpr{nullptr}; const Fortran::lower::SomeExpr *errMsgExpr{nullptr}; + const Fortran::lower::SomeExpr *pinnedExpr{nullptr}; + const Fortran::lower::SomeExpr *streamExpr{nullptr}; // If the allocate has a type spec, lenParams contains the // value of the length parameters that were specified inside. llvm::SmallVector lenParams; diff --git a/flang/test/Lower/CUDA/cuda-allocatable.cuf b/flang/test/Lower/CUDA/cuda-allocatable.cuf new file mode 100644 index 000000000000..55223011e8d9 --- /dev/null +++ b/flang/test/Lower/CUDA/cuda-allocatable.cuf @@ -0,0 +1,107 @@ +! RUN: bbc -emit-hlfir -fcuda %s -o - | FileCheck %s + +! Test lowering of CUDA allocatable allocate/deallocate statements. + +subroutine sub1() + real, allocatable, device :: a(:) + allocate(a(10)) +end subroutine + +! CHECK-LABEL: func.func @_QPsub1() +! CHECK: %[[BOX:.*]] = fir.alloca !fir.box>> {bindc_name = "a", uniq_name = "_QFsub1Ea"} +! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %[[BOX]] {cuda_attr = #fir.cuda, fortran_attrs = #fir.var_attrs, uniq_name = "_QFsub1Ea"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: fir.call @_FortranAAllocatableSetBounds +! CHECK: %{{.*}} = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref>>> {cuda_attr = #fir.cuda} -> i32 + +subroutine sub2() + real, allocatable, managed :: a(:) + integer :: istat + allocate(a(10), stat=istat) +end subroutine + +! CHECK-LABEL: func.func @_QPsub2() +! CHECK: %[[BOX:.*]] = fir.alloca !fir.box>> {bindc_name = "a", uniq_name = "_QFsub2Ea"} +! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %[[BOX]] {cuda_attr = #fir.cuda, fortran_attrs = #fir.var_attrs, uniq_name = "_QFsub2Ea"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[ISTAT:.*]] = fir.alloca i32 {bindc_name = "istat", uniq_name = "_QFsub2Eistat"} +! CHECK: %[[ISTAT_DECL:.*]]:2 = hlfir.declare %[[ISTAT]] {uniq_name = "_QFsub2Eistat"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: fir.call @_FortranAAllocatableSetBounds +! CHECK: %[[STAT:.*]] = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref>>> {cuda_attr = #fir.cuda, hasStat} -> i32 +! CHECK: fir.store %[[STAT]] to %[[ISTAT_DECL]]#1 : !fir.ref + +subroutine sub3() + integer, allocatable, pinned :: a(:,:) + logical :: plog + allocate(a(20,30), pinned = plog) +end subroutine + +! CHECK-LABEL: func.func @_QPsub3() +! CHECK: %[[BOX:.*]] = fir.alloca !fir.box>> {bindc_name = "a", uniq_name = "_QFsub3Ea"} +! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %[[BOX]] {cuda_attr = #fir.cuda, fortran_attrs = #fir.var_attrs, uniq_name = "_QFsub3Ea"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[PLOG:.*]] = fir.alloca !fir.logical<4> {bindc_name = "plog", uniq_name = "_QFsub3Eplog"} +! CHECK: %[[PLOG_DECL:.*]]:2 = hlfir.declare %5 {uniq_name = "_QFsub3Eplog"} : (!fir.ref>) -> (!fir.ref>, !fir.ref>) +! CHECK-2: fir.call @_FortranAAllocatableSetBounds +! CHECK: %{{.*}} = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref>>> pinned(%[[PLOG_DECL]]#1 : !fir.ref>) {cuda_attr = #fir.cuda} -> i32 + +subroutine sub4() + real, allocatable, unified :: a(:) + integer :: istream + allocate(a(10), stream=istream) +end subroutine + +! CHECK-LABEL: func.func @_QPsub4() +! CHECK: %[[BOX:.*]] = fir.alloca !fir.box>> {bindc_name = "a", uniq_name = "_QFsub4Ea"} +! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %0 {cuda_attr = #fir.cuda, fortran_attrs = #fir.var_attrs, uniq_name = "_QFsub4Ea"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[ISTREAM:.*]] = fir.alloca i32 {bindc_name = "istream", uniq_name = "_QFsub4Eistream"} +! CHECK: %[[ISTREAM_DECL:.*]]:2 = hlfir.declare %[[ISTREAM]] {uniq_name = "_QFsub4Eistream"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: fir.call @_FortranAAllocatableSetBounds +! CHECK: %[[STREAM:.*]] = fir.load %[[ISTREAM_DECL]]#0 : !fir.ref +! CHECK: %{{.*}} = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref>>> stream(%[[STREAM]] : i32) {cuda_attr = #fir.cuda} -> i32 + +subroutine sub5() + real, allocatable, device :: a(:) + real, allocatable :: b(:) + allocate(a, source=b) +end subroutine + +! CHECK-LABEL: func.func @_QPsub5() +! CHECK: %[[BOX_A:.*]] = fir.alloca !fir.box>> {bindc_name = "a", uniq_name = "_QFsub5Ea"} +! CHECK: %[[BOX_A_DECL:.*]]:2 = hlfir.declare %[[BOX]] {cuda_attr = #fir.cuda, fortran_attrs = #fir.var_attrs, uniq_name = "_QFsub5Ea"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[BOX_B:.*]] = fir.alloca !fir.box>> {bindc_name = "b", uniq_name = "_QFsub5Eb"} +! CHECK: %[[BOX_B_DECL:.*]]:2 = hlfir.declare %[[BOX_B]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFsub5Eb"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[LOAD_B:.*]] = fir.load %[[BOX_B_DECL]]#1 : !fir.ref>>> +! CHECK: fir.call @_FortranAAllocatableSetBounds +! CHECK: %{{.*}} = fir.cuda_allocate %[[BOX_A_DECL]]#1 : !fir.ref>>> source(%[[LOAD_B]] : !fir.box>>) {cuda_attr = #fir.cuda} -> i32 + +subroutine sub6() + real, allocatable, device :: a(:) + real, allocatable :: b(:) + allocate(a, mold=b) +end subroutine + +! CHECK-LABEL: func.func @_QPsub6() +! CHECK: %[[BOX_A:.*]] = fir.alloca !fir.box>> {bindc_name = "a", uniq_name = "_QFsub6Ea"} +! CHECK: %[[BOX_A_DECL:.*]]:2 = hlfir.declare %[[BOX]] {cuda_attr = #fir.cuda, fortran_attrs = #fir.var_attrs, uniq_name = "_QFsub6Ea"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[BOX_B:.*]] = fir.alloca !fir.box>> {bindc_name = "b", uniq_name = "_QFsub6Eb"} +! CHECK: %[[BOX_B_DECL:.*]]:2 = hlfir.declare %[[BOX_B]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFsub6Eb"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[LOAD_B:.*]] = fir.load %[[BOX_B_DECL]]#1 : !fir.ref>>> +! CHECK: fir.call @_FortranAAllocatableApplyMold +! CHECK: %{{.*}} = fir.cuda_allocate %[[BOX_A_DECL]]#1 : !fir.ref>>> {cuda_attr = #fir.cuda} -> i32 + +subroutine sub7() + real, allocatable, device :: a(:) + integer :: istat + character(50) :: err + allocate(a(100), stat=istat, errmsg=err) +end subroutine + +! CHECK-LABEL: func.func @_QPsub7() +! CHECK: %[[BOX:.*]] = fir.alloca !fir.box>> {bindc_name = "a", uniq_name = "_QFsub7Ea"} +! CHECK: %[[BOX_DECL:.*]]:2 = hlfir.declare %[[BOX]] {cuda_attr = #fir.cuda, fortran_attrs = #fir.var_attrs, uniq_name = "_QFsub7Ea"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[ERR:.*]] = fir.alloca !fir.char<1,50> {bindc_name = "err", uniq_name = "_QFsub7Eerr"} +! CHECK: %[[ERR_DECL:.*]]:2 = hlfir.declare %[[ERR]] typeparams %{{.*}} {uniq_name = "_QFsub7Eerr"} : (!fir.ref>, index) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[ISTAT:.*]] = fir.alloca i32 {bindc_name = "istat", uniq_name = "_QFsub7Eistat"} +! CHECK: %[[ISTAT_DECL:.*]]:2 = hlfir.declare %[[ISTAT]] {uniq_name = "_QFsub7Eistat"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[ERR_BOX:.*]] = fir.embox %[[ERR_DECL]]#1 : (!fir.ref>) -> !fir.box> +! CHECK: fir.call @_FortranAAllocatableSetBounds +! CHECK: %[[STAT:.*]] = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref>>> errmsg(%[[ERR_BOX]] : !fir.box>) {cuda_attr = #fir.cuda, hasStat} -> i32 +! CHECK: fir.store %[[STAT]] to %[[ISTAT_DECL]]#1 : !fir.ref -- GitLab From 19c6a7feca6e1558ef7cbe18efd2477c1126899d Mon Sep 17 00:00:00 2001 From: Alexandros Lamprineas Date: Wed, 17 Apr 2024 17:16:58 +0100 Subject: [PATCH 034/714] [FMV] Remove useless features according the latest ACLE spec. (#88965) As explained in https://github.com/ARM-software/acle/pull/315 we are deprecating features which aren't adding any value. These are: sha1, pmull, dit, dgh, ebf16, sve-bf16, sve-ebf16, sve-i8mm, sve2-pmull128, memtag2, memtag3, ssbs2, bti, ls64_v, ls64_accdata --- clang/test/CodeGen/aarch64-cpu-supports.c | 24 +- .../CodeGen/aarch64-mixed-target-attributes.c | 12 +- .../test/CodeGen/attr-target-clones-aarch64.c | 77 ++--- clang/test/CodeGen/attr-target-version.c | 326 +++++++++--------- .../CodeGenCXX/attr-target-clones-aarch64.cpp | 41 +-- clang/test/CodeGenCXX/attr-target-version.cpp | 22 +- clang/test/Sema/aarch64-cpu-supports.c | 2 +- clang/test/Sema/attr-target-clones-aarch64.c | 21 +- clang/test/Sema/attr-target-version.c | 18 +- clang/test/SemaCXX/attr-target-version.cpp | 16 +- compiler-rt/lib/builtins/cpu_model/aarch64.c | 17 +- .../builtins/cpu_model/aarch64/fmv/apple.inc | 6 +- .../cpu_model/aarch64/fmv/fuchsia.inc | 4 - .../builtins/cpu_model/aarch64/fmv/mrs.inc | 42 +-- .../llvm/TargetParser/AArch64TargetParser.h | 36 +- 15 files changed, 287 insertions(+), 377 deletions(-) diff --git a/clang/test/CodeGen/aarch64-cpu-supports.c b/clang/test/CodeGen/aarch64-cpu-supports.c index c54b7475a3fd..7fad9724dfb6 100644 --- a/clang/test/CodeGen/aarch64-cpu-supports.c +++ b/clang/test/CodeGen/aarch64-cpu-supports.c @@ -1,15 +1,17 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals --version 2 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -emit-llvm -o - %s | FileCheck %s +//. // CHECK: @__aarch64_cpu_features = external dso_local global { i64 } +//. // CHECK-LABEL: define dso_local i32 @main // CHECK-SAME: () #[[ATTR0:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[RETVAL:%.*]] = alloca i32, align 4 // CHECK-NEXT: store i32 0, ptr [[RETVAL]], align 4 // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 70368744177664 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 70368744177664 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 34359738368 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 34359738368 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] // CHECK: if.then: @@ -17,8 +19,8 @@ // CHECK-NEXT: br label [[RETURN:%.*]] // CHECK: if.end: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 9070970929152 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 9070970929152 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 17716740096 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 17716740096 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[IF_THEN1:%.*]], label [[IF_END2:%.*]] // CHECK: if.then1: @@ -26,8 +28,8 @@ // CHECK-NEXT: br label [[RETURN]] // CHECK: if.end2: // CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 166633186212708352 -// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 166633186212708352 +// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 5222680231936 +// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 5222680231936 // CHECK-NEXT: [[TMP11:%.*]] = and i1 true, [[TMP10]] // CHECK-NEXT: br i1 [[TMP11]], label [[IF_THEN3:%.*]], label [[IF_END4:%.*]] // CHECK: if.then3: @@ -49,10 +51,10 @@ int main(void) { if (__builtin_cpu_supports("sb")) return 1; - if (__builtin_cpu_supports("sve2-pmull128+memtag")) + if (__builtin_cpu_supports("sve2-aes+memtag")) return 2; - if (__builtin_cpu_supports("sme2+ls64_v+wfxt")) + if (__builtin_cpu_supports("sme2+ls64+wfxt")) return 3; if (__builtin_cpu_supports("avx2")) @@ -60,3 +62,9 @@ int main(void) { return 0; } +//. +// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +//. +// CHECK: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4} +// CHECK: [[META1:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"} +//. diff --git a/clang/test/CodeGen/aarch64-mixed-target-attributes.c b/clang/test/CodeGen/aarch64-mixed-target-attributes.c index aef6ce36ab1c..be290ff9ecee 100644 --- a/clang/test/CodeGen/aarch64-mixed-target-attributes.c +++ b/clang/test/CodeGen/aarch64-mixed-target-attributes.c @@ -69,8 +69,8 @@ __attribute__((target_version("jscvt"))) int default_def_with_version_decls(void // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 131072 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 131072 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -143,8 +143,8 @@ __attribute__((target_version("jscvt"))) int default_def_with_version_decls(void // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 131072 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 131072 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -210,8 +210,8 @@ __attribute__((target_version("jscvt"))) int default_def_with_version_decls(void // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 131072 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 131072 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: diff --git a/clang/test/CodeGen/attr-target-clones-aarch64.c b/clang/test/CodeGen/attr-target-clones-aarch64.c index 8c8b951e9118..c715001f6a72 100644 --- a/clang/test/CodeGen/attr-target-clones-aarch64.c +++ b/clang/test/CodeGen/attr-target-clones-aarch64.c @@ -3,7 +3,7 @@ // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature -fmv -S -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK-NOFMV int __attribute__((target_clones("lse+aes", "sve2"))) ftc(void) { return 0; } -int __attribute__((target_clones("sha2", "sha2+memtag2", " default "))) ftc_def(void) { return 1; } +int __attribute__((target_clones("sha2", "sha2+memtag", " default "))) ftc_def(void) { return 1; } int __attribute__((target_clones("sha2", "default"))) ftc_dup1(void) { return 2; } int __attribute__((target_clones("fp", "crc+dotprod"))) ftc_dup2(void) { return 3; } int foo() { @@ -12,7 +12,7 @@ int foo() { inline int __attribute__((target_clones("rng+simd", "rcpc+predres", "sve2-aes+wfxt"))) ftc_inline1(void) { return 1; } inline int __attribute__((target_clones("fp16", "fcma+sve2-bitperm", "default"))) ftc_inline2(void); -inline int __attribute__((target_clones("bti", "sve+sb"))) ftc_inline3(void) { return 3; } +inline int __attribute__((target_clones("mops", "sve+sb"))) ftc_inline3(void) { return 3; } int __attribute__((target_clones("default"))) ftc_direct(void) { return 4; } @@ -56,16 +56,16 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default")) // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 16512 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 16512 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 8320 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 8320 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: // CHECK-NEXT: ret ptr @ftc._MaesMlse // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 68719476736 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 68719476736 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 268435456 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 268435456 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: @@ -81,7 +81,7 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default")) // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: @ftc_def._Mmemtag2Msha2( +// CHECK-LABEL: @ftc_def._MmemtagMsha2( // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // @@ -90,16 +90,16 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default")) // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 17592186048512 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 17592186048512 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 17179871232 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 17179871232 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: -// CHECK-NEXT: ret ptr @ftc_def._Mmemtag2Msha2 +// CHECK-NEXT: ret ptr @ftc_def._MmemtagMsha2 // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 4096 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 4096 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 2048 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 2048 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: @@ -118,8 +118,8 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default")) // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 4096 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 4096 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 2048 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 2048 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -198,16 +198,16 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default")) // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 18014535948435456 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 18014535948435456 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 550292684800 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 550292684800 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: // CHECK-NEXT: ret ptr @ftc_inline1._Msve2-aesMwfxt // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 140737492549632 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 140737492549632 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 68720001024 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 68720001024 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: @@ -228,16 +228,16 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default")) // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 549757911040 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 549757911040 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1074003968 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1074003968 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: // CHECK-NEXT: ret ptr @ftc_inline2._MfcmaMsve2-bitperm // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 65536 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 65536 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 16384 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 16384 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: @@ -250,20 +250,20 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default")) // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 70369817919488 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 70369817919488 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 34393292800 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 34393292800 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: // CHECK-NEXT: ret ptr @ftc_inline3._MsbMsve // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 1125899906842624 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 1125899906842624 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 17592186044416 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 17592186044416 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: -// CHECK-NEXT: ret ptr @ftc_inline3._Mbti +// CHECK-NEXT: ret ptr @ftc_inline3._Mmops // CHECK: resolver_else2: // CHECK-NEXT: ret ptr @ftc_inline3.default // @@ -329,7 +329,7 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default")) // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: @ftc_inline3._Mbti( +// CHECK-LABEL: @ftc_inline3._Mmops( // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // @@ -407,17 +407,16 @@ inline int __attribute__((target_clones("fp16", "sve2-bitperm+fcma", "default")) // CHECK: attributes #[[ATTR0:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+lse,+neon" } // CHECK: attributes #[[ATTR1:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2" } // CHECK: attributes #[[ATTR2:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+sha2" } -// CHECK: attributes #[[ATTR3:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+mte,+neon,+sha2" } -// CHECK: attributes #[[ATTR4:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" } -// CHECK: attributes #[[ATTR5:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,+dotprod,+fp-armv8,+neon" } -// CHECK: attributes #[[ATTR6:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } -// CHECK: attributes #[[ATTR7:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" } -// CHECK: attributes #[[ATTR8:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+complxnum,+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-bitperm" } -// CHECK: attributes #[[ATTR9:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rand" } -// CHECK: attributes #[[ATTR10:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+predres,+rcpc" } -// CHECK: attributes #[[ATTR11:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-aes,+wfxt" } -// CHECK: attributes #[[ATTR12:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti" } -// CHECK: attributes #[[ATTR13:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sb,+sve" } +// CHECK: attributes #[[ATTR3:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" } +// CHECK: attributes #[[ATTR4:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,+dotprod,+fp-armv8,+neon" } +// CHECK: attributes #[[ATTR5:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +// CHECK: attributes #[[ATTR6:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" } +// CHECK: attributes #[[ATTR7:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+complxnum,+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-bitperm" } +// CHECK: attributes #[[ATTR8:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rand" } +// CHECK: attributes #[[ATTR9:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+predres,+rcpc" } +// CHECK: attributes #[[ATTR10:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve,+sve2,+sve2-aes,+wfxt" } +// CHECK: attributes #[[ATTR11:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops" } +// CHECK: attributes #[[ATTR12:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sb,+sve" } //. // CHECK-NOFMV: attributes #[[ATTR0:[0-9]+]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" } // CHECK-NOFMV: attributes #[[ATTR1:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" } diff --git a/clang/test/CodeGen/attr-target-version.c b/clang/test/CodeGen/attr-target-version.c index dd4cbbf5a898..e71370e6d91d 100644 --- a/clang/test/CodeGen/attr-target-version.c +++ b/clang/test/CodeGen/attr-target-version.c @@ -5,11 +5,11 @@ int __attribute__((target_version("rng+flagm+fp16fml"))) fmv(void) { return 1; } int __attribute__((target_version("flagm2+sme-i16i64"))) fmv(void) { return 2; } int __attribute__((target_version("lse+sha2"))) fmv(void) { return 3; } -int __attribute__((target_version("dotprod+ls64_accdata"))) fmv(void) { return 4; } +int __attribute__((target_version("dotprod+ls64"))) fmv(void) { return 4; } int __attribute__((target_version("fp16fml+memtag"))) fmv(void) { return 5; } int __attribute__((target_version("fp+aes"))) fmv(void) { return 6; } -int __attribute__((target_version("crc+ls64_v"))) fmv(void) { return 7; } -int __attribute__((target_version("bti"))) fmv(void) { return 8; } +int __attribute__((target_version("crc+ls64"))) fmv(void) { return 7; } +int __attribute__((target_version("mops"))) fmv(void) { return 8; } int __attribute__((target_version("sme2"))) fmv(void) { return 9; } int __attribute__((target_version("default"))) fmv(void); int __attribute__((target_version("ls64+simd"))) fmv_one(void) { return 1; } @@ -17,25 +17,25 @@ int __attribute__((target_version("dpb"))) fmv_one(void) { return 2; } int __attribute__((target_version("default"))) fmv_one(void); int __attribute__((target_version("fp"))) fmv_two(void) { return 1; } int __attribute__((target_version("simd"))) fmv_two(void) { return 2; } -int __attribute__((target_version("dgh"))) fmv_two(void) { return 3; } +int __attribute__((target_version("frintts"))) fmv_two(void) { return 3; } int __attribute__((target_version("fp16+simd"))) fmv_two(void) { return 4; } int __attribute__((target_version("default"))) fmv_two(void); int foo() { return fmv()+fmv_one()+fmv_two(); } -inline int __attribute__((target_version("sha1+pmull+f64mm"))) fmv_inline(void) { return 1; } +inline int __attribute__((target_version("crypto+f64mm"))) fmv_inline(void) { return 1; } inline int __attribute__((target_version("fp16+fcma+rdma+sme+ fp16 "))) fmv_inline(void) { return 2; } inline int __attribute__((target_version("sha3+i8mm+f32mm"))) fmv_inline(void) { return 12; } -inline int __attribute__((target_version("dit+sve-ebf16"))) fmv_inline(void) { return 8; } +inline int __attribute__((target_version("sme2+ssbs"))) fmv_inline(void) { return 8; } inline int __attribute__((target_version("dpb+rcpc2 "))) fmv_inline(void) { return 6; } inline int __attribute__((target_version(" dpb2 + jscvt"))) fmv_inline(void) { return 7; } inline int __attribute__((target_version("rcpc+frintts"))) fmv_inline(void) { return 3; } -inline int __attribute__((target_version("sve+sve-bf16"))) fmv_inline(void) { return 4; } +inline int __attribute__((target_version("sve+sme"))) fmv_inline(void) { return 4; } inline int __attribute__((target_version("sve2-aes+sve2-sha3"))) fmv_inline(void) { return 5; } -inline int __attribute__((target_version("sve2+sve2-pmull128+sve2-bitperm"))) fmv_inline(void) { return 9; } -inline int __attribute__((target_version("sve2-sm4+memtag2"))) fmv_inline(void) { return 10; } -inline int __attribute__((target_version("memtag3+rcpc3+mops"))) fmv_inline(void) { return 11; } +inline int __attribute__((target_version("sve2+sve2-aes+sve2-bitperm"))) fmv_inline(void) { return 9; } +inline int __attribute__((target_version("sve2-sm4+memtag"))) fmv_inline(void) { return 10; } +inline int __attribute__((target_version("memtag+rcpc3+mops"))) fmv_inline(void) { return 11; } inline int __attribute__((target_version("aes+dotprod"))) fmv_inline(void) { return 13; } inline int __attribute__((target_version("simd+fp16fml"))) fmv_inline(void) { return 14; } inline int __attribute__((target_version("fp+sm4"))) fmv_inline(void) { return 15; } @@ -186,7 +186,7 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._MdotprodMls64_accdata +// CHECK-LABEL: define {{[^@]+}}@fmv._MdotprodMls64 // CHECK-SAME: () #[[ATTR3:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 @@ -207,14 +207,14 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._McrcMls64_v +// CHECK-LABEL: define {{[^@]+}}@fmv._McrcMls64 // CHECK-SAME: () #[[ATTR6:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 7 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._Mbti +// CHECK-LABEL: define {{[^@]+}}@fmv._Mmops // CHECK-SAME: () #[[ATTR7:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 8 @@ -256,7 +256,7 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mdgh +// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfrintts // CHECK-SAME: () #[[ATTR11:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 @@ -271,7 +271,7 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@foo -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[CALL:%.*]] = call i32 @fmv() // CHECK-NEXT: [[CALL1:%.*]] = call i32 @fmv_one() @@ -293,68 +293,68 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: ret ptr @fmv._MflagmMfp16fmlMrng // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 72057594037927940 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 72057594037927940 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 2199023255556 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 2199023255556 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: // CHECK-NEXT: ret ptr @fmv._Mflagm2Msme-i16i64 // CHECK: resolver_else2: // CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 9007199254741008 -// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 9007199254741008 +// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 274877906960 +// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 274877906960 // CHECK-NEXT: [[TMP11:%.*]] = and i1 true, [[TMP10]] // CHECK-NEXT: br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]] // CHECK: resolver_return3: -// CHECK-NEXT: ret ptr @fmv._MdotprodMls64_accdata +// CHECK-NEXT: ret ptr @fmv._MdotprodMls64 // CHECK: resolver_else4: // CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP13:%.*]] = and i64 [[TMP12]], 4503599627371520 -// CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 4503599627371520 +// CHECK-NEXT: [[TMP13:%.*]] = and i64 [[TMP12]], 274877907968 +// CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 274877907968 // CHECK-NEXT: [[TMP15:%.*]] = and i1 true, [[TMP14]] // CHECK-NEXT: br i1 [[TMP15]], label [[RESOLVER_RETURN5:%.*]], label [[RESOLVER_ELSE6:%.*]] // CHECK: resolver_return5: -// CHECK-NEXT: ret ptr @fmv._McrcMls64_v +// CHECK-NEXT: ret ptr @fmv._McrcMls64 // CHECK: resolver_else6: // CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP17:%.*]] = and i64 [[TMP16]], 8796093022216 -// CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 8796093022216 +// CHECK-NEXT: [[TMP17:%.*]] = and i64 [[TMP16]], 17179869192 +// CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 17179869192 // CHECK-NEXT: [[TMP19:%.*]] = and i1 true, [[TMP18]] // CHECK-NEXT: br i1 [[TMP19]], label [[RESOLVER_RETURN7:%.*]], label [[RESOLVER_ELSE8:%.*]] // CHECK: resolver_return7: // CHECK-NEXT: ret ptr @fmv._Mfp16fmlMmemtag // CHECK: resolver_else8: // CHECK-NEXT: [[TMP20:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP21:%.*]] = and i64 [[TMP20]], 16640 -// CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[TMP21]], 16640 +// CHECK-NEXT: [[TMP21:%.*]] = and i64 [[TMP20]], 8448 +// CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[TMP21]], 8448 // CHECK-NEXT: [[TMP23:%.*]] = and i1 true, [[TMP22]] // CHECK-NEXT: br i1 [[TMP23]], label [[RESOLVER_RETURN9:%.*]], label [[RESOLVER_ELSE10:%.*]] // CHECK: resolver_return9: // CHECK-NEXT: ret ptr @fmv._MaesMfp // CHECK: resolver_else10: // CHECK-NEXT: [[TMP24:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP25:%.*]] = and i64 [[TMP24]], 4224 -// CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 4224 +// CHECK-NEXT: [[TMP25:%.*]] = and i64 [[TMP24]], 2176 +// CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 2176 // CHECK-NEXT: [[TMP27:%.*]] = and i1 true, [[TMP26]] // CHECK-NEXT: br i1 [[TMP27]], label [[RESOLVER_RETURN11:%.*]], label [[RESOLVER_ELSE12:%.*]] // CHECK: resolver_return11: // CHECK-NEXT: ret ptr @fmv._MlseMsha2 // CHECK: resolver_else12: // CHECK-NEXT: [[TMP28:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP29:%.*]] = and i64 [[TMP28]], 144115188075855872 -// CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[TMP29]], 144115188075855872 +// CHECK-NEXT: [[TMP29:%.*]] = and i64 [[TMP28]], 17592186044416 +// CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[TMP29]], 17592186044416 // CHECK-NEXT: [[TMP31:%.*]] = and i1 true, [[TMP30]] // CHECK-NEXT: br i1 [[TMP31]], label [[RESOLVER_RETURN13:%.*]], label [[RESOLVER_ELSE14:%.*]] // CHECK: resolver_return13: -// CHECK-NEXT: ret ptr @fmv._Msme2 +// CHECK-NEXT: ret ptr @fmv._Mmops // CHECK: resolver_else14: // CHECK-NEXT: [[TMP32:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP33:%.*]] = and i64 [[TMP32]], 1125899906842624 -// CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[TMP33]], 1125899906842624 +// CHECK-NEXT: [[TMP33:%.*]] = and i64 [[TMP32]], 4398046511104 +// CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[TMP33]], 4398046511104 // CHECK-NEXT: [[TMP35:%.*]] = and i1 true, [[TMP34]] // CHECK-NEXT: br i1 [[TMP35]], label [[RESOLVER_RETURN15:%.*]], label [[RESOLVER_ELSE16:%.*]] // CHECK: resolver_return15: -// CHECK-NEXT: ret ptr @fmv._Mbti +// CHECK-NEXT: ret ptr @fmv._Msme2 // CHECK: resolver_else16: // CHECK-NEXT: ret ptr @fmv.default // @@ -363,16 +363,16 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 2251799813685760 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 2251799813685760 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 274877907456 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 274877907456 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: // CHECK-NEXT: ret ptr @fmv_one._Mls64Msimd // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 262144 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 262144 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 32768 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 32768 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: @@ -385,20 +385,20 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 66048 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 66048 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 16896 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 16896 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: // CHECK-NEXT: ret ptr @fmv_two._Mfp16Msimd // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 33554432 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 33554432 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 2097152 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 2097152 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: -// CHECK-NEXT: ret ptr @fmv_two._Mdgh +// CHECK-NEXT: ret ptr @fmv_two._Mfrintts // CHECK: resolver_else2: // CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 // CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 512 @@ -421,49 +421,49 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_e.default -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 20 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_d._Msb -// CHECK-SAME: () #[[ATTR13:[0-9]+]] { +// CHECK-SAME: () #[[ATTR14:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_d.default -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_default -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 111 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_c._Mssbs -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret void // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_c.default -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret void // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@goo -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[CALL:%.*]] = call i32 @fmv_inline() // CHECK-NEXT: [[CALL1:%.*]] = call i32 @fmv_e() @@ -477,96 +477,96 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 4398048673856 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 4398048673856 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 8590213184 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 8590213184 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: // CHECK-NEXT: ret ptr @fmv_inline._MfcmaMfp16MrdmMsme // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 864726312827224064 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 864726312827224064 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 26405458935808 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 26405458935808 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: -// CHECK-NEXT: ret ptr @fmv_inline._Mmemtag3MmopsMrcpc3 +// CHECK-NEXT: ret ptr @fmv_inline._MmemtagMmopsMrcpc3 // CHECK: resolver_else2: // CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 893353197568 -// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 893353197568 +// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 1879048192 +// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 1879048192 // CHECK-NEXT: [[TMP11:%.*]] = and i1 true, [[TMP10]] // CHECK-NEXT: br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]] // CHECK: resolver_return3: -// CHECK-NEXT: ret ptr @fmv_inline._Msve2Msve2-bitpermMsve2-pmull128 +// CHECK-NEXT: ret ptr @fmv_inline._Msve2Msve2-aesMsve2-bitperm // CHECK: resolver_else4: // CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP13:%.*]] = and i64 [[TMP12]], 34359773184 -// CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 34359773184 +// CHECK-NEXT: [[TMP13:%.*]] = and i64 [[TMP12]], 71307264 +// CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 71307264 // CHECK-NEXT: [[TMP15:%.*]] = and i1 true, [[TMP14]] // CHECK-NEXT: br i1 [[TMP15]], label [[RESOLVER_RETURN5:%.*]], label [[RESOLVER_ELSE6:%.*]] // CHECK: resolver_return5: -// CHECK-NEXT: ret ptr @fmv_inline._Mf64mmMpmullMsha1 +// CHECK-NEXT: ret ptr @fmv_inline._Mf32mmMi8mmMsha3 // CHECK: resolver_else6: // CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP17:%.*]] = and i64 [[TMP16]], 17246986240 -// CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 17246986240 +// CHECK-NEXT: [[TMP17:%.*]] = and i64 [[TMP16]], 4535485464576 +// CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 4535485464576 // CHECK-NEXT: [[TMP19:%.*]] = and i1 true, [[TMP18]] // CHECK-NEXT: br i1 [[TMP19]], label [[RESOLVER_RETURN7:%.*]], label [[RESOLVER_ELSE8:%.*]] // CHECK: resolver_return7: -// CHECK-NEXT: ret ptr @fmv_inline._Mf32mmMi8mmMsha3 +// CHECK-NEXT: ret ptr @fmv_inline._Msme2Mssbs // CHECK: resolver_else8: // CHECK-NEXT: [[TMP20:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP21:%.*]] = and i64 [[TMP20]], 19791209299968 -// CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[TMP21]], 19791209299968 +// CHECK-NEXT: [[TMP21:%.*]] = and i64 [[TMP20]], 21474836480 +// CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[TMP21]], 21474836480 // CHECK-NEXT: [[TMP23:%.*]] = and i1 true, [[TMP22]] // CHECK-NEXT: br i1 [[TMP23]], label [[RESOLVER_RETURN9:%.*]], label [[RESOLVER_ELSE10:%.*]] // CHECK: resolver_return9: -// CHECK-NEXT: ret ptr @fmv_inline._Mmemtag2Msve2-sm4 +// CHECK-NEXT: ret ptr @fmv_inline._MmemtagMsve2-sm4 // CHECK: resolver_else10: // CHECK-NEXT: [[TMP24:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP25:%.*]] = and i64 [[TMP24]], 1236950581248 -// CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 1236950581248 +// CHECK-NEXT: [[TMP25:%.*]] = and i64 [[TMP24]], 8623489024 +// CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 8623489024 // CHECK-NEXT: [[TMP27:%.*]] = and i1 true, [[TMP26]] // CHECK-NEXT: br i1 [[TMP27]], label [[RESOLVER_RETURN11:%.*]], label [[RESOLVER_ELSE12:%.*]] // CHECK: resolver_return11: -// CHECK-NEXT: ret ptr @fmv_inline._Msve2-aesMsve2-sha3 +// CHECK-NEXT: ret ptr @fmv_inline._MsmeMsve // CHECK: resolver_else12: // CHECK-NEXT: [[TMP28:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP29:%.*]] = and i64 [[TMP28]], 4295098368 -// CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[TMP29]], 4295098368 +// CHECK-NEXT: [[TMP29:%.*]] = and i64 [[TMP28]], 2684354560 +// CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[TMP29]], 2684354560 // CHECK-NEXT: [[TMP31:%.*]] = and i1 true, [[TMP30]] // CHECK-NEXT: br i1 [[TMP31]], label [[RESOLVER_RETURN13:%.*]], label [[RESOLVER_ELSE14:%.*]] // CHECK: resolver_return13: -// CHECK-NEXT: ret ptr @fmv_inline._MditMsve-ebf16 +// CHECK-NEXT: ret ptr @fmv_inline._Msve2-aesMsve2-sha3 // CHECK: resolver_else14: // CHECK-NEXT: [[TMP32:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP33:%.*]] = and i64 [[TMP32]], 3221225472 -// CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[TMP33]], 3221225472 +// CHECK-NEXT: [[TMP33:%.*]] = and i64 [[TMP32]], 281475110928384 +// CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[TMP33]], 281475110928384 // CHECK-NEXT: [[TMP35:%.*]] = and i1 true, [[TMP34]] // CHECK-NEXT: br i1 [[TMP35]], label [[RESOLVER_RETURN15:%.*]], label [[RESOLVER_ELSE16:%.*]] // CHECK: resolver_return15: -// CHECK-NEXT: ret ptr @fmv_inline._MsveMsve-bf16 +// CHECK-NEXT: ret ptr @fmv_inline._McryptoMf64mm // CHECK: resolver_else16: // CHECK-NEXT: [[TMP36:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP37:%.*]] = and i64 [[TMP36]], 20971520 -// CHECK-NEXT: [[TMP38:%.*]] = icmp eq i64 [[TMP37]], 20971520 +// CHECK-NEXT: [[TMP37:%.*]] = and i64 [[TMP36]], 2621440 +// CHECK-NEXT: [[TMP38:%.*]] = icmp eq i64 [[TMP37]], 2621440 // CHECK-NEXT: [[TMP39:%.*]] = and i1 true, [[TMP38]] // CHECK-NEXT: br i1 [[TMP39]], label [[RESOLVER_RETURN17:%.*]], label [[RESOLVER_ELSE18:%.*]] // CHECK: resolver_return17: // CHECK-NEXT: ret ptr @fmv_inline._MfrinttsMrcpc // CHECK: resolver_else18: // CHECK-NEXT: [[TMP40:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP41:%.*]] = and i64 [[TMP40]], 8650752 -// CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[TMP41]], 8650752 +// CHECK-NEXT: [[TMP41:%.*]] = and i64 [[TMP40]], 1081344 +// CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[TMP41]], 1081344 // CHECK-NEXT: [[TMP43:%.*]] = and i1 true, [[TMP42]] // CHECK-NEXT: br i1 [[TMP43]], label [[RESOLVER_RETURN19:%.*]], label [[RESOLVER_ELSE20:%.*]] // CHECK: resolver_return19: // CHECK-NEXT: ret ptr @fmv_inline._MdpbMrcpc2 // CHECK: resolver_else20: // CHECK-NEXT: [[TMP44:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP45:%.*]] = and i64 [[TMP44]], 1572864 -// CHECK-NEXT: [[TMP46:%.*]] = icmp eq i64 [[TMP45]], 1572864 +// CHECK-NEXT: [[TMP45:%.*]] = and i64 [[TMP44]], 196608 +// CHECK-NEXT: [[TMP46:%.*]] = icmp eq i64 [[TMP45]], 196608 // CHECK-NEXT: [[TMP47:%.*]] = and i1 true, [[TMP46]] // CHECK-NEXT: br i1 [[TMP47]], label [[RESOLVER_RETURN21:%.*]], label [[RESOLVER_ELSE22:%.*]] // CHECK: resolver_return21: @@ -581,8 +581,8 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: ret ptr @fmv_inline._Mfp16fmlMsimd // CHECK: resolver_else24: // CHECK-NEXT: [[TMP52:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP53:%.*]] = and i64 [[TMP52]], 16400 -// CHECK-NEXT: [[TMP54:%.*]] = icmp eq i64 [[TMP53]], 16400 +// CHECK-NEXT: [[TMP53:%.*]] = and i64 [[TMP52]], 8208 +// CHECK-NEXT: [[TMP54:%.*]] = icmp eq i64 [[TMP53]], 8208 // CHECK-NEXT: [[TMP55:%.*]] = and i1 true, [[TMP54]] // CHECK-NEXT: br i1 [[TMP55]], label [[RESOLVER_RETURN25:%.*]], label [[RESOLVER_ELSE26:%.*]] // CHECK: resolver_return25: @@ -611,8 +611,8 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 2251799813685248 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 2251799813685248 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 274877906944 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 274877906944 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -625,8 +625,8 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 70368744177664 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 70368744177664 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 34359738368 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 34359738368 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -639,8 +639,8 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 281474976710656 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 281474976710656 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 137438953472 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 137438953472 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -651,7 +651,7 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@recur -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: call void @reca() // CHECK-NEXT: ret void @@ -659,7 +659,7 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@main -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[RETVAL:%.*]] = alloca i32, align 4 // CHECK-NEXT: store i32 0, ptr [[RETVAL]], align 4 @@ -670,7 +670,7 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@hoo -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[FP1:%.*]] = alloca ptr, align 8 // CHECK-NEXT: [[FP2:%.*]] = alloca ptr, align 8 @@ -687,14 +687,14 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_forward_default_decl._Mmops -// CHECK-SAME: () #[[ATTR14:[0-9]+]] { +// CHECK-SAME: () #[[ATTR7]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_extern_forward_default_decl._Mdotprod -// CHECK-SAME: () #[[ATTR15:[0-9]+]] { +// CHECK-SAME: () #[[ATTR3]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // @@ -708,14 +708,14 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_default_def._Msve -// CHECK-SAME: () #[[ATTR16:[0-9]+]] { +// CHECK-SAME: () #[[ATTR15:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.default -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // @@ -729,56 +729,56 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def.default -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def.default -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def._Mlse -// CHECK-SAME: () #[[ATTR17:[0-9]+]] { +// CHECK-SAME: () #[[ATTR16:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@unused_without_default._Mrdm -// CHECK-SAME: () #[[ATTR18:[0-9]+]] { +// CHECK-SAME: () #[[ATTR17:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@default_def_with_version_decls.default -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl._Mjscvt -// CHECK-SAME: () #[[ATTR21:[0-9]+]] { +// CHECK-SAME: () #[[ATTR20:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@used_def_without_default_decl._Mrdm -// CHECK-SAME: () #[[ATTR18]] { +// CHECK-SAME: () #[[ATTR17]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@caller -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[CALL:%.*]] = call i32 @used_def_without_default_decl() // CHECK-NEXT: [[CALL1:%.*]] = call i32 @used_decl_without_default_decl() @@ -790,8 +790,8 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 131072 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 131072 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -812,8 +812,8 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 131072 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 131072 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -831,92 +831,92 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf64mmMpmullMsha1 -// CHECK-SAME: () #[[ATTR22:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@fmv_inline._McryptoMf64mm +// CHECK-SAME: () #[[ATTR21:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfcmaMfp16MrdmMsme -// CHECK-SAME: () #[[ATTR23:[0-9]+]] { +// CHECK-SAME: () #[[ATTR22:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf32mmMi8mmMsha3 -// CHECK-SAME: () #[[ATTR24:[0-9]+]] { +// CHECK-SAME: () #[[ATTR23:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 12 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MditMsve-ebf16 -// CHECK-SAME: () #[[ATTR25:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msme2Mssbs +// CHECK-SAME: () #[[ATTR8]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 8 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MdpbMrcpc2 -// CHECK-SAME: () #[[ATTR26:[0-9]+]] { +// CHECK-SAME: () #[[ATTR24:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 6 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mdpb2Mjscvt -// CHECK-SAME: () #[[ATTR27:[0-9]+]] { +// CHECK-SAME: () #[[ATTR25:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 7 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfrinttsMrcpc -// CHECK-SAME: () #[[ATTR28:[0-9]+]] { +// CHECK-SAME: () #[[ATTR26:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MsveMsve-bf16 -// CHECK-SAME: () #[[ATTR29:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MsmeMsve +// CHECK-SAME: () #[[ATTR27:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2-aesMsve2-sha3 -// CHECK-SAME: () #[[ATTR30:[0-9]+]] { +// CHECK-SAME: () #[[ATTR28:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 5 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2Msve2-bitpermMsve2-pmull128 -// CHECK-SAME: () #[[ATTR31:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2Msve2-aesMsve2-bitperm +// CHECK-SAME: () #[[ATTR29:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 9 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag2Msve2-sm4 -// CHECK-SAME: () #[[ATTR32:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MmemtagMsve2-sm4 +// CHECK-SAME: () #[[ATTR30:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 10 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag3MmopsMrcpc3 -// CHECK-SAME: () #[[ATTR33:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@fmv_inline._MmemtagMmopsMrcpc3 +// CHECK-SAME: () #[[ATTR31:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 11 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MaesMdotprod -// CHECK-SAME: () #[[ATTR15]] { +// CHECK-SAME: () #[[ATTR3]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 13 // @@ -930,21 +930,21 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfpMsm4 -// CHECK-SAME: () #[[ATTR34:[0-9]+]] { +// CHECK-SAME: () #[[ATTR32:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 15 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MlseMrdm -// CHECK-SAME: () #[[ATTR35:[0-9]+]] { +// CHECK-SAME: () #[[ATTR33:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 16 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline.default -// CHECK-SAME: () #[[ATTR11]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // @@ -953,8 +953,8 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1073741824 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1073741824 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 33554432 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 33554432 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -967,8 +967,8 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 65536 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 65536 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 16384 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 16384 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -995,8 +995,8 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1048576 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1048576 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 131072 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 131072 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -1132,39 +1132,37 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de // CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+flagm,+fp16fml,+fullfp16,+neon,+rand,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+altnzcv,+bf16,+flagm,+sme,+sme-i16i64,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+sha2,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+ls64,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+neon,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp16fml,+fullfp16,+neon,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme2,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR9:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,-fp-armv8,-v9.5a" } // CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR18]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR19:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR20:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fullfp16,+neon,+rdm,+sme,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+jsconv,+neon,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR33]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+mte,+rcpc,+rcpc3,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR34]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+sm4,-fp-armv8,-v9.5a" } -// CHECK: attributes #[[ATTR35]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+rdm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR18:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR19:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR20]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fullfp16,+neon,+sha2,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fullfp16,+neon,+rdm,+sme,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fullfp16,+neon,+sme,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-sm4,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+rcpc,+rcpc3,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+sm4,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR33]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+rdm,-fp-armv8,-v9.5a" } //. // CHECK-NOFMV: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" } // CHECK-NOFMV: attributes #[[ATTR1:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" } diff --git a/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp b/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp index 7953f902bf09..d439f96982eb 100644 --- a/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp +++ b/clang/test/CodeGenCXX/attr-target-clones-aarch64.cpp @@ -1,8 +1,8 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals --include-generated-funcs // RUN: %clang_cc1 -std=c++11 -triple aarch64-linux-gnu -emit-llvm %s -o - | FileCheck %s -int __attribute__((target_clones("ls64_v+fp16", "default"))) foo_ovl(int) { return 1; } -int __attribute__((target_clones("ls64_accdata+ls64"))) foo_ovl(void) { return 2; } +int __attribute__((target_clones("ls64+fp16", "default"))) foo_ovl(int) { return 1; } +int __attribute__((target_clones("sme+ls64"))) foo_ovl(void) { return 2; } int bar() { return foo_ovl(1) + foo_ovl(); @@ -35,9 +35,6 @@ void run_foo_tml() { Mc4.foo_tml(); } - - - //. // CHECK: @__aarch64_cpu_features = external dso_local global { i64 } // CHECK: @_Z7foo_ovli.ifunc = weak_odr alias i32 (i32), ptr @_Z7foo_ovli @@ -49,7 +46,7 @@ void run_foo_tml() { // CHECK: @_ZN7MyClassIssE7foo_tmlEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClassIssE7foo_tmlEv.resolver // CHECK: @_ZN7MyClassIisE7foo_tmlEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClassIisE7foo_tmlEv.resolver //. -// CHECK-LABEL: @_Z7foo_ovli._Mfp16Mls64_v( +// CHECK-LABEL: @_Z7foo_ovli._Mfp16Mls64( // CHECK-NEXT: entry: // CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 // CHECK-NEXT: store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4 @@ -60,17 +57,17 @@ void run_foo_tml() { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 4503599627436032 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 4503599627436032 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 274877923328 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 274877923328 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: -// CHECK-NEXT: ret ptr @_Z7foo_ovli._Mfp16Mls64_v +// CHECK-NEXT: ret ptr @_Z7foo_ovli._Mfp16Mls64 // CHECK: resolver_else: // CHECK-NEXT: ret ptr @_Z7foo_ovli.default // // -// CHECK-LABEL: @_Z7foo_ovlv._Mls64Mls64_accdata( +// CHECK-LABEL: @_Z7foo_ovlv._Mls64Msme( // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // @@ -79,12 +76,12 @@ void run_foo_tml() { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 11258999068426240 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 11258999068426240 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 283467841536 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 283467841536 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: -// CHECK-NEXT: ret ptr @_Z7foo_ovlv._Mls64Mls64_accdata +// CHECK-NEXT: ret ptr @_Z7foo_ovlv._Mls64Msme // CHECK: resolver_else: // CHECK-NEXT: ret ptr @_Z7foo_ovlv.default // @@ -114,16 +111,16 @@ void run_foo_tml() { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 36310271995674624 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 36310271995674624 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1236950581248 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1236950581248 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: // CHECK-NEXT: ret ptr @_ZN7MyClassIssE7foo_tmlEv._Msme-f64f64Mssbs // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 16777216 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 16777216 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 2097152 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 2097152 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: @@ -136,16 +133,16 @@ void run_foo_tml() { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 36310271995674624 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 36310271995674624 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1236950581248 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1236950581248 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: // CHECK-NEXT: ret ptr @_ZN7MyClassIisE7foo_tmlEv._Msme-f64f64Mssbs // CHECK: resolver_else: // CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 16777216 -// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 16777216 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 2097152 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 2097152 // CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] // CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] // CHECK: resolver_return1: @@ -231,7 +228,7 @@ void run_foo_tml() { // //. // CHECK: attributes #[[ATTR0:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" } -// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ls64" } +// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme" } // CHECK: attributes #[[ATTR2:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } // CHECK: attributes #[[ATTR3:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint" } // CHECK: attributes #[[ATTR4:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" } diff --git a/clang/test/CodeGenCXX/attr-target-version.cpp b/clang/test/CodeGenCXX/attr-target-version.cpp index 8b7273fe3bb5..24433b8f20b7 100644 --- a/clang/test/CodeGenCXX/attr-target-version.cpp +++ b/clang/test/CodeGenCXX/attr-target-version.cpp @@ -3,7 +3,7 @@ int __attribute__((target_version("sme-f64f64+bf16"))) foo(int) { return 1; } int __attribute__((target_version("default"))) foo(int) { return 2; } -int __attribute__((target_version("sm4+ebf16"))) foo(void) { return 3; } +int __attribute__((target_version("sm4+bf16"))) foo(void) { return 3; } int __attribute__((target_version("default"))) foo(void) { return 4; } struct MyClass { @@ -89,7 +89,7 @@ int bar() { // CHECK-NEXT: ret i32 2 // // -// CHECK-LABEL: @_Z3foov._Mebf16Msm4( +// CHECK-LABEL: @_Z3foov._Mbf16Msm4( // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // @@ -246,8 +246,8 @@ int bar() { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 36028797153181696 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 36028797153181696 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1099520016384 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1099520016384 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -260,12 +260,12 @@ int bar() { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 268435488 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 268435488 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 8388640 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 8388640 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: -// CHECK-NEXT: ret ptr @_Z3foov._Mebf16Msm4 +// CHECK-NEXT: ret ptr @_Z3foov._Mbf16Msm4 // CHECK: resolver_else: // CHECK-NEXT: ret ptr @_Z3foov.default // @@ -274,8 +274,8 @@ int bar() { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1073741824 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1073741824 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 33554432 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 33554432 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -288,8 +288,8 @@ int bar() { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 65536 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 65536 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 16384 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 16384 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: diff --git a/clang/test/Sema/aarch64-cpu-supports.c b/clang/test/Sema/aarch64-cpu-supports.c index ddeed7c5bc9e..cf8e043dfb1e 100644 --- a/clang/test/Sema/aarch64-cpu-supports.c +++ b/clang/test/Sema/aarch64-cpu-supports.c @@ -20,7 +20,7 @@ int test_aarch64_features(void) { // expected-warning@+1 {{invalid cpu feature string}} if (__builtin_cpu_supports("default")) return 6; - if (__builtin_cpu_supports(" ssbs + bti ")) + if (__builtin_cpu_supports(" ssbs + crc ")) return 7; return 0; } diff --git a/clang/test/Sema/attr-target-clones-aarch64.c b/clang/test/Sema/attr-target-clones-aarch64.c index bc3fceab8282..511e8a941180 100644 --- a/clang/test/Sema/attr-target-clones-aarch64.c +++ b/clang/test/Sema/attr-target-clones-aarch64.c @@ -9,7 +9,7 @@ void __attribute__((target_clones("ssbs+ls64"))) warn2(void); // expected-error@+2 {{'target_clones' and 'target_version' attributes are not compatible}} // expected-note@+1 {{conflicting attribute is here}} -void __attribute__((target_version("sve-bf16"), target_clones("sme+memtag"))) not_compat(void); +void __attribute__((target_version("bf16"), target_clones("sme+memtag"))) not_compat(void); int redecl(void); int __attribute__((target_clones("frintts", "simd+fp", "default"))) redecl(void) { return 1; } @@ -21,26 +21,25 @@ int __attribute__((target_clones("sve+dotprod"))) redecl3(void); int redecl3(void); int __attribute__((target_clones("rng", "fp16fml+fp", "default"))) redecl4(void); -// expected-error@+3 {{'target_clones' attribute does not match previous declaration}} +// expected-error@+2 {{'target_clones' attribute does not match previous declaration}} // expected-note@-2 {{previous declaration is here}} -// expected-warning@+1 {{version list contains entries that don't impact code generation}} -int __attribute__((target_clones("dgh+memtag+rpres+ls64_v", "ebf16+dpb+sha1", "default"))) redecl4(void) { return 1; } +int __attribute__((target_clones("dpb2+memtag+rpres+ls64", "bf16+dpb+sha2", "default"))) redecl4(void) { return 1; } int __attribute__((target_version("flagm2"))) redef2(void) { return 1; } // expected-error@+2 {{multiversioned function redeclarations require identical target attributes}} // expected-note@-2 {{previous declaration is here}} int __attribute__((target_clones("flagm2", "default"))) redef2(void) { return 1; } -int __attribute__((target_clones("f32mm", "f64mm", "sha1+fp"))) redef3(void) { return 1; } +int __attribute__((target_clones("f32mm", "f64mm", "sha3+fp"))) redef3(void) { return 1; } // expected-error@+2 {{'target_clones' attribute does not match previous declaration}} // expected-note@-2 {{previous declaration is here}} -int __attribute__((target_clones("f32mm", "sha1+fp", "f64mm"))) redef3(void) { return 1; } +int __attribute__((target_clones("f32mm", "sha3+fp", "f64mm"))) redef3(void) { return 1; } int __attribute__((target_clones("rdm+lse+rdm", "lse+rdm"))) dup1(void) { return 1; } // expected-warning@+1 {{version list contains duplicate entries}} int __attribute__((target_clones("rdm+lse+rdm", "rdm+lse+rdm"))) dup2(void) { return 2; } // expected-warning@+1 {{version list contains duplicate entries}} -int __attribute__((target_clones("rcpc2+sve2-pmull128", "rcpc2+sve2-pmull128"))) dup3(void) { return 3; } +int __attribute__((target_clones("rcpc2+sve2-aes", "rcpc2+sve2-aes"))) dup3(void) { return 3; } // expected-warning@+1 {{version list contains duplicate entries}} void __attribute__((target_clones("sha3", "default", "default"))) dup4(void); // expected-warning@+2 {{version list contains duplicate entries}} @@ -49,7 +48,7 @@ int __attribute__((target_clones("fp", "fp", "crc+dotprod", "dotprod+crc"))) dup // expected-warning@+1 {{version list contains duplicate entries}} int __attribute__((target_clones("fp16+memtag", "memtag+fp16"))) dup6(void) { return 6; } -int __attribute__((target_clones("simd+ssbs2", "simd+dpb2"))) dup7(void) { return 7; } +int __attribute__((target_clones("simd+ssbs", "simd+dpb2"))) dup7(void) { return 7; } // expected-warning@+1 {{unsupported '' in the 'target_clones' attribute string;}} void __attribute__((target_clones(""))) empty_target_1(void); @@ -72,13 +71,13 @@ empty_target_5(void); void __attribute__((target_clones("sve2-bitperm", "sve2-bitperm"))) dupe_normal(void); -void __attribute__((target_clones("default"), target_clones("memtag3+bti"))) dupe_normal2(void); +void __attribute__((target_clones("default"), target_clones("memtag+mops"))) dupe_normal2(void); int mv_after_use(void); int useage(void) { return mv_after_use(); } // expected-error@+1 {{function declaration cannot become a multiversioned function after first usage}} -int __attribute__((target_clones("sve2-sha3+ssbs2", "sm4"))) mv_after_use(void) { return 1; } +int __attribute__((target_clones("sve2-sha3+ssbs", "sm4"))) mv_after_use(void) { return 1; } // expected-error@+1 {{'main' cannot be a multiversioned function}} -int __attribute__((target_clones("sve-i8mm"))) main() { return 1; } +int __attribute__((target_clones("sve2-aes"))) main() { return 1; } diff --git a/clang/test/Sema/attr-target-version.c b/clang/test/Sema/attr-target-version.c index cd5be459456e..4ff8e1eee619 100644 --- a/clang/test/Sema/attr-target-version.c +++ b/clang/test/Sema/attr-target-version.c @@ -16,7 +16,7 @@ int __attribute__((target_version("aes"))) foo(void) { return 1; } int __attribute__((target_version("default"))) foo(void) { return 2; } //expected-note@+1 {{previous definition is here}} -int __attribute__((target_version("sha3 + pmull "))) foo(void) { return 1; } +int __attribute__((target_version("sha3 + aes "))) foo(void) { return 1; } //expected-note@-1 {{previous definition is here}} //expected-error@+1 {{redefinition of 'foo'}} @@ -32,11 +32,11 @@ __attribute__ ((target("bf16,sve,sve2,dotprod"))) int func(void) { return 1; } __attribute__ ((target("default"))) int func(void) { return 0; } //expected-note@+1 {{previous declaration is here}} -void __attribute__((target_version("bti+flagm2"))) one(void) {} +void __attribute__((target_version("mops+flagm2"))) one(void) {} //expected-error@+1 {{multiversioned function redeclarations require identical target attributes}} -void __attribute__((target_version("flagm2+bti"))) one(void) {} +void __attribute__((target_version("flagm2+mops"))) one(void) {} -void __attribute__((target_version("ssbs+sha1"))) two(void) {} +void __attribute__((target_version("ssbs+sha2"))) two(void) {} void __attribute__((target_version("ssbs+fp16fml"))) two(void) {} //expected-error@+1 {{'main' cannot be a multiversioned function}} @@ -44,7 +44,7 @@ int __attribute__((target_version("lse"))) main(void) { return 1; } // It is ok for the default version to appear first. int default_first(void) { return 1; } -int __attribute__((target_version("dit"))) default_first(void) { return 2; } +int __attribute__((target_version("crc"))) default_first(void) { return 2; } int __attribute__((target_version("mops"))) default_first(void) { return 3; } // It is ok if the default version is between other versions. @@ -77,7 +77,7 @@ void __attribute__((target_version("rdm+rng+crc"))) redef(void) {} void __attribute__((target_version("rdm+rng+crc"))) redef(void) {} int def(void); -void __attribute__((target_version("dit"))) nodef(void); +void __attribute__((target_version("crc"))) nodef(void); void __attribute__((target_version("ls64"))) nodef(void); void __attribute__((target_version("aes"))) ovl(void); void __attribute__((target_version("default"))) ovl(void); @@ -89,12 +89,12 @@ int bar() { return def(); } // expected-error@+1 {{function declaration cannot become a multiversioned function after first usage}} -int __attribute__((target_version("sha1"))) def(void) { return 1; } +int __attribute__((target_version("sha3"))) def(void) { return 1; } int __attribute__((target_version("sve"))) prot(); // expected-error@-1 {{multiversioned function must have a prototype}} -int __attribute__((target_version("pmull"))) rtype(int); +int __attribute__((target_version("aes"))) rtype(int); // expected-error@+1 {{multiversioned function declaration has a different return type}} float __attribute__((target_version("rdm"))) rtype(int); @@ -102,7 +102,7 @@ int __attribute__((target_version("sha2"))) combine(void) { return 1; } // expected-error@+1 {{multiversioned function declaration has a different calling convention}} int __attribute__((aarch64_vector_pcs, target_version("sha3"))) combine(void) { return 2; } -int __attribute__((target_version("fp+aes+pmull+rcpc"))) unspec_args() { return -1; } +int __attribute__((target_version("fp+aes+sha3+rcpc"))) unspec_args() { return -1; } // expected-error@-1 {{multiversioned function must have a prototype}} // expected-error@+1 {{multiversioned function must have a prototype}} int __attribute__((target_version("default"))) unspec_args() { return 0; } diff --git a/clang/test/SemaCXX/attr-target-version.cpp b/clang/test/SemaCXX/attr-target-version.cpp index b3385f043590..ae2923cc303c 100644 --- a/clang/test/SemaCXX/attr-target-version.cpp +++ b/clang/test/SemaCXX/attr-target-version.cpp @@ -25,13 +25,13 @@ int __attribute__((target_version("dpb"))) diff_link(void); int __attribute__((target_version("memtag"))) diff_link1(void) { return 1; } //expected-error@+1 {{multiversioned function declaration has a different linkage}} -static int __attribute__((target_version("bti"))) diff_link1(void); +static int __attribute__((target_version("mops"))) diff_link1(void); int __attribute__((target_version("flagm2"))) diff_link2(void) { return 1; } extern int __attribute__((target_version("flagm"))) diff_link2(void); namespace { -static int __attribute__((target_version("memtag3"))) diff_link2(void) { return 2; } +static int __attribute__((target_version("memtag"))) diff_link2(void) { return 2; } int __attribute__((target_version("sve2-bitperm"))) diff_link2(void) { return 1; } } // namespace @@ -49,7 +49,7 @@ double __attribute__((target_version("rcpc"))) diff_type1(void); auto __attribute__((target_version("rcpc2"))) diff_type2(void) -> int { return 1; } //expected-error@+1 {{multiversioned function declaration has a different return type}} -auto __attribute__((target_version("sve-bf16"))) diff_type2(void) -> long { return (long)1; } +auto __attribute__((target_version("sve2-aes"))) diff_type2(void) -> long { return (long)1; } int __attribute__((target_version("fp16fml"))) diff_type3(void) noexcept(false) { return 1; } //expected-error@+2 {{exception specification in declaration does not match previous declaration}} @@ -75,7 +75,7 @@ auto __attribute__((target_version("dpb2"))) ret3(void) -> int { return 1; } class Cls { __attribute__((target_version("rng"))) Cls(); // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support constructors}} - __attribute__((target_version("sve-i8mm"))) ~Cls(); + __attribute__((target_version("sve2-aes"))) ~Cls(); // expected-error@-1 {{attribute 'target_version' multiversioned functions do not yet support destructors}} Cls &__attribute__((target_version("f32mm"))) operator=(const Cls &) = default; @@ -98,11 +98,11 @@ __attribute__((target_version("jscvt"))) void Decl(); } // namespace Nms class Out { - int __attribute__((target_version("bti"))) func(void); - int __attribute__((target_version("ssbs2"))) func(void); + int __attribute__((target_version("mops"))) func(void); + int __attribute__((target_version("ssbs"))) func(void); }; -int __attribute__((target_version("bti"))) Out::func(void) { return 1; } -int __attribute__((target_version("ssbs2"))) Out::func(void) { return 2; } +int __attribute__((target_version("mops"))) Out::func(void) { return 1; } +int __attribute__((target_version("ssbs"))) Out::func(void) { return 2; } // expected-error@+3 {{out-of-line definition of 'func' does not match any declaration in 'Out'}} // expected-note@-3 {{member declaration nearly matches}} // expected-note@-3 {{member declaration nearly matches}} diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64.c b/compiler-rt/lib/builtins/cpu_model/aarch64.c index 17bddfca46f0..1ac4d85a0c13 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64.c +++ b/compiler-rt/lib/builtins/cpu_model/aarch64.c @@ -67,13 +67,10 @@ enum CPUFeatures { FEAT_FP, FEAT_SIMD, FEAT_CRC, - FEAT_SHA1, FEAT_SHA2, FEAT_SHA3, FEAT_AES, - FEAT_PMULL, FEAT_FP16, - FEAT_DIT, FEAT_DPB, FEAT_DPB2, FEAT_JSCVT, @@ -81,35 +78,23 @@ enum CPUFeatures { FEAT_RCPC, FEAT_RCPC2, FEAT_FRINTTS, - FEAT_DGH, FEAT_I8MM, FEAT_BF16, - FEAT_EBF16, FEAT_RPRES, FEAT_SVE, - FEAT_SVE_BF16, - FEAT_SVE_EBF16, - FEAT_SVE_I8MM, FEAT_SVE_F32MM, FEAT_SVE_F64MM, FEAT_SVE2, FEAT_SVE_AES, - FEAT_SVE_PMULL128, FEAT_SVE_BITPERM, FEAT_SVE_SHA3, FEAT_SVE_SM4, FEAT_SME, FEAT_MEMTAG, - FEAT_MEMTAG2, - FEAT_MEMTAG3, FEAT_SB, FEAT_PREDRES, FEAT_SSBS, - FEAT_SSBS2, - FEAT_BTI, FEAT_LS64, - FEAT_LS64_V, - FEAT_LS64_ACCDATA, FEAT_WFXT, FEAT_SME_F64, FEAT_SME_I64, @@ -117,7 +102,7 @@ enum CPUFeatures { FEAT_RCPC3, FEAT_MOPS, FEAT_MAX, - FEAT_EXT = 62, // Reserved to indicate presence of additional features field + FEAT_EXT = 47, // Reserved to indicate presence of additional features field // in __aarch64_cpu_features FEAT_INIT // Used as flag of features initialization completion }; diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc index 6fef109567b6..19114c4abdfc 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc +++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc @@ -35,13 +35,10 @@ void __init_cpu_features_resolver(void) { {"hw.optional.floatingpoint", FEAT_FP}, {"hw.optional.AdvSIMD", FEAT_SIMD}, {"hw.optional.armv8_crc32", FEAT_CRC}, - {"hw.optional.arm.FEAT_SHA1", FEAT_SHA1}, {"hw.optional.arm.FEAT_SHA256", FEAT_SHA2}, {"hw.optional.arm.FEAT_SHA3", FEAT_SHA3}, {"hw.optional.arm.FEAT_AES", FEAT_AES}, - {"hw.optional.arm.FEAT_PMULL", FEAT_PMULL}, {"hw.optional.arm.FEAT_FP16", FEAT_FP16}, - {"hw.optional.arm.FEAT_DIT", FEAT_DIT}, {"hw.optional.arm.FEAT_DPB", FEAT_DPB}, {"hw.optional.arm.FEAT_DPB2", FEAT_DPB2}, {"hw.optional.arm.FEAT_JSCVT", FEAT_JSCVT}, @@ -53,8 +50,7 @@ void __init_cpu_features_resolver(void) { {"hw.optional.arm.FEAT_BF16", FEAT_BF16}, {"hw.optional.arm.FEAT_SB", FEAT_SB}, {"hw.optional.arm.FEAT_SPECRES", FEAT_PREDRES}, - {"hw.optional.arm.FEAT_SSBS", FEAT_SSBS2}, - {"hw.optional.arm.FEAT_BTI", FEAT_BTI}, + {"hw.optional.arm.FEAT_SSBS", FEAT_SSBS}, }; for (size_t I = 0, E = sizeof(feature_checks) / sizeof(feature_checks[0]); diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/fuchsia.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/fuchsia.inc index d8e0280f4041..579aa00dc5c7 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/fuchsia.inc +++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/fuchsia.inc @@ -22,10 +22,6 @@ void __init_cpu_features_resolver() { setCPUFeature(FEAT_SIMD); if (features & ZX_ARM64_FEATURE_ISA_AES) setCPUFeature(FEAT_AES); - if (features & ZX_ARM64_FEATURE_ISA_PMULL) - setCPUFeature(FEAT_PMULL); - if (features & ZX_ARM64_FEATURE_ISA_SHA1) - setCPUFeature(FEAT_SHA1); if (features & ZX_ARM64_FEATURE_ISA_SHA256) setCPUFeature(FEAT_SHA2); if (features & ZX_ARM64_FEATURE_ISA_CRC32) diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc index 32a21a2fba9a..54797852e4aa 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc +++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc @@ -16,8 +16,6 @@ static void __init_cpu_features_constructor(unsigned long hwcap, hwcap2 = arg->_hwcap2; if (hwcap & HWCAP_CRC32) setCPUFeature(FEAT_CRC); - if (hwcap & HWCAP_PMULL) - setCPUFeature(FEAT_PMULL); if (hwcap & HWCAP_FLAGM) setCPUFeature(FEAT_FLAGM); if (hwcap2 & HWCAP2_FLAGM2) { @@ -34,16 +32,12 @@ static void __init_cpu_features_constructor(unsigned long hwcap, setCPUFeature(FEAT_FP16); setCPUFeature(FEAT_FP); } - if (hwcap & HWCAP_DIT) - setCPUFeature(FEAT_DIT); if (hwcap & HWCAP_ASIMDRDM) setCPUFeature(FEAT_RDM); if (hwcap & HWCAP_ILRCPC) setCPUFeature(FEAT_RCPC2); if (hwcap & HWCAP_AES) setCPUFeature(FEAT_AES); - if (hwcap & HWCAP_SHA1) - setCPUFeature(FEAT_SHA1); if (hwcap & HWCAP_SHA2) setCPUFeature(FEAT_SHA2); if (hwcap & HWCAP_JSCVT) @@ -53,22 +47,11 @@ static void __init_cpu_features_constructor(unsigned long hwcap, if (hwcap & HWCAP_SB) setCPUFeature(FEAT_SB); if (hwcap & HWCAP_SSBS) - setCPUFeature(FEAT_SSBS2); - if (hwcap2 & HWCAP2_MTE) { + setCPUFeature(FEAT_SSBS); + if (hwcap2 & HWCAP2_MTE) setCPUFeature(FEAT_MEMTAG); - setCPUFeature(FEAT_MEMTAG2); - } - if (hwcap2 & HWCAP2_MTE3) { - setCPUFeature(FEAT_MEMTAG); - setCPUFeature(FEAT_MEMTAG2); - setCPUFeature(FEAT_MEMTAG3); - } if (hwcap2 & HWCAP2_SVEAES) setCPUFeature(FEAT_SVE_AES); - if (hwcap2 & HWCAP2_SVEPMULL) { - setCPUFeature(FEAT_SVE_AES); - setCPUFeature(FEAT_SVE_PMULL128); - } if (hwcap2 & HWCAP2_SVEBITPERM) setCPUFeature(FEAT_SVE_BITPERM); if (hwcap2 & HWCAP2_SVESHA3) @@ -83,22 +66,12 @@ static void __init_cpu_features_constructor(unsigned long hwcap, setCPUFeature(FEAT_RNG); if (hwcap2 & HWCAP2_I8MM) setCPUFeature(FEAT_I8MM); - if (hwcap2 & HWCAP2_EBF16) - setCPUFeature(FEAT_EBF16); - if (hwcap2 & HWCAP2_SVE_EBF16) - setCPUFeature(FEAT_SVE_EBF16); - if (hwcap2 & HWCAP2_DGH) - setCPUFeature(FEAT_DGH); if (hwcap2 & HWCAP2_FRINT) setCPUFeature(FEAT_FRINTTS); - if (hwcap2 & HWCAP2_SVEI8MM) - setCPUFeature(FEAT_SVE_I8MM); if (hwcap2 & HWCAP2_SVEF32MM) setCPUFeature(FEAT_SVE_F32MM); if (hwcap2 & HWCAP2_SVEF64MM) setCPUFeature(FEAT_SVE_F64MM); - if (hwcap2 & HWCAP2_BTI) - setCPUFeature(FEAT_BTI); if (hwcap2 & HWCAP2_RPRES) setCPUFeature(FEAT_RPRES); if (hwcap2 & HWCAP2_WFXT) @@ -141,9 +114,6 @@ static void __init_cpu_features_constructor(unsigned long hwcap, // ID_AA64ZFR0_EL1.SVEver == 0b0001 if (extractBits(ftr, 0, 4) == 0x1) setCPUFeature(FEAT_SVE2); - // ID_AA64ZFR0_EL1.BF16 != 0b0000 - if (extractBits(ftr, 20, 4) != 0x0) - setCPUFeature(FEAT_SVE_BF16); } getCPUFeature(ID_AA64ISAR0_EL1, ftr); // ID_AA64ISAR0_EL1.SHA3 != 0b0000 @@ -168,12 +138,6 @@ static void __init_cpu_features_constructor(unsigned long hwcap, // ID_AA64ISAR1_EL1.LS64 >= 0b0001 if (extractBits(ftr, 60, 4) >= 0x1) setCPUFeature(FEAT_LS64); - // ID_AA64ISAR1_EL1.LS64 >= 0b0010 - if (extractBits(ftr, 60, 4) >= 0x2) - setCPUFeature(FEAT_LS64_V); - // ID_AA64ISAR1_EL1.LS64 >= 0b0011 - if (extractBits(ftr, 60, 4) >= 0x3) - setCPUFeature(FEAT_LS64_ACCDATA); } else { // Set some features in case of no CPUID support if (hwcap & (HWCAP_FP | HWCAP_FPHP)) { @@ -187,8 +151,6 @@ static void __init_cpu_features_constructor(unsigned long hwcap, setCPUFeature(FEAT_RCPC); if (hwcap2 & HWCAP2_BF16 || hwcap2 & HWCAP2_EBF16) setCPUFeature(FEAT_BF16); - if (hwcap2 & HWCAP2_SVEBF16) - setCPUFeature(FEAT_SVE_BF16); if (hwcap2 & HWCAP2_SVE2 && hwcap & HWCAP_SVE) setCPUFeature(FEAT_SVE2); if (hwcap & HWCAP_SHA3) diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h index 805b963a7a13..bcb36beb3cc0 100644 --- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h +++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h @@ -46,13 +46,10 @@ enum CPUFeatures { FEAT_FP, FEAT_SIMD, FEAT_CRC, - FEAT_SHA1, FEAT_SHA2, FEAT_SHA3, FEAT_AES, - FEAT_PMULL, FEAT_FP16, - FEAT_DIT, FEAT_DPB, FEAT_DPB2, FEAT_JSCVT, @@ -60,35 +57,23 @@ enum CPUFeatures { FEAT_RCPC, FEAT_RCPC2, FEAT_FRINTTS, - FEAT_DGH, FEAT_I8MM, FEAT_BF16, - FEAT_EBF16, FEAT_RPRES, FEAT_SVE, - FEAT_SVE_BF16, - FEAT_SVE_EBF16, - FEAT_SVE_I8MM, FEAT_SVE_F32MM, FEAT_SVE_F64MM, FEAT_SVE2, FEAT_SVE_AES, - FEAT_SVE_PMULL128, FEAT_SVE_BITPERM, FEAT_SVE_SHA3, FEAT_SVE_SM4, FEAT_SME, FEAT_MEMTAG, - FEAT_MEMTAG2, - FEAT_MEMTAG3, FEAT_SB, FEAT_PREDRES, FEAT_SSBS, - FEAT_SSBS2, - FEAT_BTI, FEAT_LS64, - FEAT_LS64_V, - FEAT_LS64_ACCDATA, FEAT_WFXT, FEAT_SME_F64, FEAT_SME_I64, @@ -96,12 +81,12 @@ enum CPUFeatures { FEAT_RCPC3, FEAT_MOPS, FEAT_MAX, - FEAT_EXT = 62, + FEAT_EXT = 47, FEAT_INIT }; -static_assert(FEAT_MAX < 62, - "Number of features in CPUFeatures are limited to 62 entries"); +static_assert(FEAT_MAX < 47, + "Number of features in CPUFeatures are limited to 47 entries"); // Arch extension modifiers for CPUs. These are labelled with their Arm ARM // feature name (though the canonical reference for those is AArch64.td) @@ -215,17 +200,13 @@ inline constexpr ExtensionInfo Extensions[] = { {"b16b16", AArch64::AEK_B16B16, "+b16b16", "-b16b16", FEAT_INIT, "", 0}, {"bf16", AArch64::AEK_BF16, "+bf16", "-bf16", FEAT_BF16, "+bf16", 280}, {"brbe", AArch64::AEK_BRBE, "+brbe", "-brbe", FEAT_INIT, "", 0}, - {"bti", AArch64::AEK_NONE, {}, {}, FEAT_BTI, "+bti", 510}, {"crc", AArch64::AEK_CRC, "+crc", "-crc", FEAT_CRC, "+crc", 110}, {"crypto", AArch64::AEK_CRYPTO, "+crypto", "-crypto", FEAT_INIT, "+aes,+sha2", 0}, {"cssc", AArch64::AEK_CSSC, "+cssc", "-cssc", FEAT_INIT, "", 0}, {"d128", AArch64::AEK_D128, "+d128", "-d128", FEAT_INIT, "", 0}, - {"dgh", AArch64::AEK_NONE, {}, {}, FEAT_DGH, "", 260}, - {"dit", AArch64::AEK_NONE, {}, {}, FEAT_DIT, "+dit", 180}, {"dotprod", AArch64::AEK_DOTPROD, "+dotprod", "-dotprod", FEAT_DOTPROD, "+dotprod,+fp-armv8,+neon", 104}, {"dpb", AArch64::AEK_NONE, {}, {}, FEAT_DPB, "+ccpp", 190}, {"dpb2", AArch64::AEK_NONE, {}, {}, FEAT_DPB2, "+ccpp,+ccdp", 200}, - {"ebf16", AArch64::AEK_NONE, {}, {}, FEAT_EBF16, "+bf16", 290}, {"f32mm", AArch64::AEK_F32MM, "+f32mm", "-f32mm", FEAT_SVE_F32MM, "+sve,+f32mm,+fullfp16,+fp-armv8,+neon", 350}, {"f64mm", AArch64::AEK_F64MM, "+f64mm", "-f64mm", FEAT_SVE_F64MM, "+sve,+f64mm,+fullfp16,+fp-armv8,+neon", 360}, {"fcma", AArch64::AEK_FCMA, "+complxnum", "-complxnum", FEAT_FCMA, "+fp-armv8,+neon,+complxnum", 220}, @@ -239,17 +220,12 @@ inline constexpr ExtensionInfo Extensions[] = { {"i8mm", AArch64::AEK_I8MM, "+i8mm", "-i8mm", FEAT_I8MM, "+i8mm", 270}, {"ite", AArch64::AEK_ITE, "+ite", "-ite", FEAT_INIT, "", 0}, {"jscvt", AArch64::AEK_JSCVT, "+jsconv", "-jsconv", FEAT_JSCVT, "+fp-armv8,+neon,+jsconv", 210}, - {"ls64_accdata", AArch64::AEK_NONE, {}, {}, FEAT_LS64_ACCDATA, "+ls64", 540}, - {"ls64_v", AArch64::AEK_NONE, {}, {}, FEAT_LS64_V, "", 530}, {"ls64", AArch64::AEK_LS64, "+ls64", "-ls64", FEAT_LS64, "", 520}, {"lse", AArch64::AEK_LSE, "+lse", "-lse", FEAT_LSE, "+lse", 80}, {"lse128", AArch64::AEK_LSE128, "+lse128", "-lse128", FEAT_INIT, "", 0}, {"memtag", AArch64::AEK_MTE, "+mte", "-mte", FEAT_MEMTAG, "", 440}, - {"memtag2", AArch64::AEK_NONE, {}, {}, FEAT_MEMTAG2, "+mte", 450}, - {"memtag3", AArch64::AEK_NONE, {}, {}, FEAT_MEMTAG3, "+mte", 460}, {"mops", AArch64::AEK_MOPS, "+mops", "-mops", FEAT_MOPS, "+mops", 650}, {"pauth", AArch64::AEK_PAUTH, "+pauth", "-pauth", FEAT_INIT, "", 0}, - {"pmull", AArch64::AEK_NONE, {}, {}, FEAT_PMULL, "+aes,+fp-armv8,+neon", 160}, {"pmuv3", AArch64::AEK_PERFMON, "+perfmon", "-perfmon", FEAT_INIT, "", 0}, {"predres", AArch64::AEK_PREDRES, "+predres", "-predres", FEAT_PREDRES, "+predres", 480}, {"predres2", AArch64::AEK_SPECRES2, "+specres2", "-specres2", FEAT_INIT, "", 0}, @@ -263,7 +239,6 @@ inline constexpr ExtensionInfo Extensions[] = { {"rng", AArch64::AEK_RAND, "+rand", "-rand", FEAT_RNG, "+rand", 10}, {"rpres", AArch64::AEK_NONE, {}, {}, FEAT_RPRES, "", 300}, {"sb", AArch64::AEK_SB, "+sb", "-sb", FEAT_SB, "+sb", 470}, - {"sha1", AArch64::AEK_NONE, {}, {}, FEAT_SHA1, "+fp-armv8,+neon", 120}, {"sha2", AArch64::AEK_SHA2, "+sha2", "-sha2", FEAT_SHA2, "+sha2,+fp-armv8,+neon", 130}, {"sha3", AArch64::AEK_SHA3, "+sha3", "-sha3", FEAT_SHA3, "+sha3,+sha2,+fp-armv8,+neon", 140}, {"simd", AArch64::AEK_SIMD, "+neon", "-neon", FEAT_SIMD, "+fp-armv8,+neon", 100}, @@ -275,14 +250,9 @@ inline constexpr ExtensionInfo Extensions[] = { {"sme2", AArch64::AEK_SME2, "+sme2", "-sme2", FEAT_SME2, "+sme2,+sme,+bf16", 580}, {"sme2p1", AArch64::AEK_SME2p1, "+sme2p1", "-sme2p1", FEAT_INIT, "+sme2p1,+sme2,+sme,+bf16", 0}, {"ssbs", AArch64::AEK_SSBS, "+ssbs", "-ssbs", FEAT_SSBS, "", 490}, - {"ssbs2", AArch64::AEK_NONE, {}, {}, FEAT_SSBS2, "+ssbs", 500}, - {"sve-bf16", AArch64::AEK_NONE, {}, {}, FEAT_SVE_BF16, "+sve,+bf16,+fullfp16,+fp-armv8,+neon", 320}, - {"sve-ebf16", AArch64::AEK_NONE, {}, {}, FEAT_SVE_EBF16, "+sve,+bf16,+fullfp16,+fp-armv8,+neon", 330}, - {"sve-i8mm", AArch64::AEK_NONE, {}, {}, FEAT_SVE_I8MM, "+sve,+i8mm,+fullfp16,+fp-armv8,+neon", 340}, {"sve", AArch64::AEK_SVE, "+sve", "-sve", FEAT_SVE, "+sve,+fullfp16,+fp-armv8,+neon", 310}, {"sve2-aes", AArch64::AEK_SVE2AES, "+sve2-aes", "-sve2-aes", FEAT_SVE_AES, "+sve2,+sve,+sve2-aes,+fullfp16,+fp-armv8,+neon", 380}, {"sve2-bitperm", AArch64::AEK_SVE2BITPERM, "+sve2-bitperm", "-sve2-bitperm", FEAT_SVE_BITPERM, "+sve2,+sve,+sve2-bitperm,+fullfp16,+fp-armv8,+neon", 400}, - {"sve2-pmull128", AArch64::AEK_NONE, {}, {}, FEAT_SVE_PMULL128, "+sve2,+sve,+sve2-aes,+fullfp16,+fp-armv8,+neon", 390}, {"sve2-sha3", AArch64::AEK_SVE2SHA3, "+sve2-sha3", "-sve2-sha3", FEAT_SVE_SHA3, "+sve2,+sve,+sve2-sha3,+fullfp16,+fp-armv8,+neon", 410}, {"sve2-sm4", AArch64::AEK_SVE2SM4, "+sve2-sm4", "-sve2-sm4", FEAT_SVE_SM4, "+sve2,+sve,+sve2-sm4,+fullfp16,+fp-armv8,+neon", 420}, {"sve2", AArch64::AEK_SVE2, "+sve2", "-sve2", FEAT_SVE2, "+sve2,+sve,+fullfp16,+fp-armv8,+neon", 370}, -- GitLab From 06947b9f8d258fe66fc69f1e7c0197cb621da3a5 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Wed, 17 Apr 2024 09:31:29 -0700 Subject: [PATCH 035/714] [libc][POSIX][pthreads] implement pthread_condattr_t functions (#88987) Implement: - pthread_condattr_destroy - pthread_condattr_getclock - pthread_condattr_getpshared - pthread_condattr_init - pthread_condattr_setclock - pthread_condattr_setpshared Fixes: #88581 --- libc/config/linux/api.td | 29 ++++++-- libc/config/linux/x86_64/entrypoints.txt | 6 ++ libc/include/CMakeLists.txt | 1 + libc/include/llvm-libc-types/CMakeLists.txt | 3 +- .../llvm-libc-types/pthread_condattr_t.h | 18 +++++ libc/include/pthread.h.def | 6 ++ libc/spec/posix.td | 64 ++++++++++++++++- libc/src/pthread/CMakeLists.txt | 65 +++++++++++++++++ libc/src/pthread/pthread_condattr_destroy.cpp | 23 ++++++ libc/src/pthread/pthread_condattr_destroy.h | 20 ++++++ .../src/pthread/pthread_condattr_getclock.cpp | 25 +++++++ libc/src/pthread/pthread_condattr_getclock.h | 22 ++++++ .../pthread/pthread_condattr_getpshared.cpp | 24 +++++++ .../src/pthread/pthread_condattr_getpshared.h | 21 ++++++ libc/src/pthread/pthread_condattr_init.cpp | 24 +++++++ libc/src/pthread/pthread_condattr_init.h | 20 ++++++ .../src/pthread/pthread_condattr_setclock.cpp | 30 ++++++++ libc/src/pthread/pthread_condattr_setclock.h | 21 ++++++ .../pthread/pthread_condattr_setpshared.cpp | 28 ++++++++ .../src/pthread/pthread_condattr_setpshared.h | 20 ++++++ libc/test/src/pthread/CMakeLists.txt | 12 ++++ .../src/pthread/pthread_condattr_test.cpp | 71 +++++++++++++++++++ 22 files changed, 545 insertions(+), 8 deletions(-) create mode 100644 libc/include/llvm-libc-types/pthread_condattr_t.h create mode 100644 libc/src/pthread/pthread_condattr_destroy.cpp create mode 100644 libc/src/pthread/pthread_condattr_destroy.h create mode 100644 libc/src/pthread/pthread_condattr_getclock.cpp create mode 100644 libc/src/pthread/pthread_condattr_getclock.h create mode 100644 libc/src/pthread/pthread_condattr_getpshared.cpp create mode 100644 libc/src/pthread/pthread_condattr_getpshared.h create mode 100644 libc/src/pthread/pthread_condattr_init.cpp create mode 100644 libc/src/pthread/pthread_condattr_init.h create mode 100644 libc/src/pthread/pthread_condattr_setclock.cpp create mode 100644 libc/src/pthread/pthread_condattr_setclock.h create mode 100644 libc/src/pthread/pthread_condattr_setpshared.cpp create mode 100644 libc/src/pthread/pthread_condattr_setpshared.h create mode 100644 libc/test/src/pthread/pthread_condattr_test.cpp diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td index 9964971f191b..5fb92a9c299c 100644 --- a/libc/config/linux/api.td +++ b/libc/config/linux/api.td @@ -175,6 +175,7 @@ def PThreadAPI : PublicAPI<"pthread.h"> { "__pthread_start_t", "__pthread_tss_dtor_t", "pthread_attr_t", + "pthread_condattr_t", "pthread_mutex_t", "pthread_mutexattr_t", "pthread_t", @@ -241,10 +242,30 @@ def SysSendfileAPI : PublicAPI<"sys/sendfile.h"> { } def SysTypesAPI : PublicAPI<"sys/types.h"> { - let Types = ["blkcnt_t", "blksize_t", "clockid_t", "dev_t", "gid_t", "ino_t", - "mode_t", "nlink_t", "off_t", "pid_t", "pthread_attr_t", "pthread_key_t", - "pthread_mutex_t", "pthread_mutexattr_t", "pthread_once_t", "pthread_t", - "size_t", "ssize_t", "suseconds_t", "time_t", "uid_t"]; + let Types = [ + "blkcnt_t", + "blksize_t", + "clockid_t", + "dev_t", + "gid_t", + "ino_t", + "mode_t", + "nlink_t", + "off_t", + "pid_t", + "pthread_attr_t", + "pthread_condattr_t", + "pthread_key_t", + "pthread_mutex_t", + "pthread_mutexattr_t", + "pthread_once_t", + "pthread_t", + "size_t", + "ssize_t", + "suseconds_t", + "time_t", + "uid_t" + ]; } def SysUtsNameAPI : PublicAPI<"sys/utsname.h"> { diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index 70f130a4399a..2d8136536b21 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -639,6 +639,12 @@ if(LLVM_LIBC_FULL_BUILD) libc.src.pthread.pthread_attr_setguardsize libc.src.pthread.pthread_attr_setstack libc.src.pthread.pthread_attr_setstacksize + libc.src.pthread.pthread_condattr_destroy + libc.src.pthread.pthread_condattr_getclock + libc.src.pthread.pthread_condattr_getpshared + libc.src.pthread.pthread_condattr_init + libc.src.pthread.pthread_condattr_setclock + libc.src.pthread.pthread_condattr_setpshared libc.src.pthread.pthread_create libc.src.pthread.pthread_detach libc.src.pthread.pthread_equal diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt index b85366c8deaf..f5ba2791af3f 100644 --- a/libc/include/CMakeLists.txt +++ b/libc/include/CMakeLists.txt @@ -321,6 +321,7 @@ add_gen_header( .llvm-libc-types.__pthread_start_t .llvm-libc-types.__pthread_tss_dtor_t .llvm-libc-types.pthread_attr_t + .llvm-libc-types.pthread_condattr_t .llvm-libc-types.pthread_mutex_t .llvm-libc-types.pthread_mutexattr_t .llvm-libc-types.pthread_t diff --git a/libc/include/llvm-libc-types/CMakeLists.txt b/libc/include/llvm-libc-types/CMakeLists.txt index 93a79e1477b3..f26fc0729dc9 100644 --- a/libc/include/llvm-libc-types/CMakeLists.txt +++ b/libc/include/llvm-libc-types/CMakeLists.txt @@ -49,11 +49,12 @@ add_header(pid_t HDR pid_t.h) add_header(posix_spawn_file_actions_t HDR posix_spawn_file_actions_t.h) add_header(posix_spawnattr_t HDR posix_spawnattr_t.h) add_header(pthread_attr_t HDR pthread_attr_t.h DEPENDS .size_t) +add_header(pthread_condattr_t HDR pthread_condattr_t.h DEPENDS .clockid_t) add_header(pthread_key_t HDR pthread_key_t.h) add_header(pthread_mutex_t HDR pthread_mutex_t.h DEPENDS .__futex_word .__mutex_type) -add_header(pthread_t HDR pthread_t.h DEPENDS .__thread_type) add_header(pthread_mutexattr_t HDR pthread_mutexattr_t.h) add_header(pthread_once_t HDR pthread_once_t.h DEPENDS .__futex_word) +add_header(pthread_t HDR pthread_t.h DEPENDS .__thread_type) add_header(rlim_t HDR rlim_t.h) add_header(time_t HDR time_t.h) add_header(stack_t HDR stack_t.h) diff --git a/libc/include/llvm-libc-types/pthread_condattr_t.h b/libc/include/llvm-libc-types/pthread_condattr_t.h new file mode 100644 index 000000000000..b91fc2950aa3 --- /dev/null +++ b/libc/include/llvm-libc-types/pthread_condattr_t.h @@ -0,0 +1,18 @@ +//===-- Definition of pthread_condattr_t type -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_LIBC_TYPES_PTHREAD_CONDATTR_T_H +#define LLVM_LIBC_TYPES_PTHREAD_CONDATTR_T_H + +#include "clockid_t.h" + +typedef struct { + clockid_t clock; + int pshared; +} pthread_condattr_t; + +#endif // LLVM_LIBC_TYPES_PTHREAD_CONDATTR_T_H diff --git a/libc/include/pthread.h.def b/libc/include/pthread.h.def index abeb839ee83d..a94d770657e1 100644 --- a/libc/include/pthread.h.def +++ b/libc/include/pthread.h.def @@ -11,6 +11,9 @@ #include "__llvm-libc-common.h" +// TODO: move to a pthreads-macros.h file: +// https://github.com/llvm/llvm-project/issues/88997 + #define PTHREAD_STACK_MIN (1 << 14) // 16KB #define PTHREAD_MUTEX_INITIALIZER {0} @@ -32,6 +35,9 @@ enum { PTHREAD_MUTEX_ROBUST = 0x1, }; +#define PTHREAD_PROCESS_PRIVATE 0 +#define PTHREAD_PROCESS_SHARED 1 + %%public_api() #endif // LLVM_LIBC_PTHREAD_H diff --git a/libc/spec/posix.td b/libc/spec/posix.td index 7095a3964ee3..0c88dbd848a3 100644 --- a/libc/spec/posix.td +++ b/libc/spec/posix.td @@ -26,6 +26,7 @@ def UidT : NamedType<"uid_t">; def GidT : NamedType<"gid_t">; def DevT : NamedType<"dev_t">; def ClockIdT : NamedType<"clockid_t">; +def RestrictedClockIdTPtr : RestrictedPtrType; def BlkSizeT : NamedType<"blksize_t">; def BlkCntT : NamedType<"blkcnt_t">; def NLinkT : NamedType<"nlink_t">; @@ -105,6 +106,10 @@ def POSIX : StandardSpec<"POSIX"> { ConstType ConstPThreadAttrTPtr = ConstType; ConstType ConstRestrictedPThreadAttrTPtr = ConstType; + NamedType PThreadCondAttrTType = NamedType<"pthread_condattr_t">; + PtrType PThreadCondAttrTPtr = PtrType; + ConstType ConstRestrictedPThreadCondAttrTPtr = ConstType>; + NamedType PThreadMutexAttrTType = NamedType<"pthread_mutexattr_t">; PtrType PThreadMutexAttrTPtr = PtrType; RestrictedPtrType RestrictedPThreadMutexAttrTPtr = RestrictedPtrType; @@ -980,7 +985,9 @@ def POSIX : StandardSpec<"POSIX"> { [], // Macros [ AtForkCallbackT, + ClockIdT, PThreadAttrTType, + PThreadCondAttrTType, PThreadKeyT, PThreadMutexAttrTType, PThreadMutexTType, @@ -1047,6 +1054,36 @@ def POSIX : StandardSpec<"POSIX"> { RetValSpec, [ArgSpec, ArgSpec, ArgSpec] >, + FunctionSpec< + "pthread_condattr_destroy", + RetValSpec, + [ArgSpec] + >, + FunctionSpec< + "pthread_condattr_getclock", + RetValSpec, + [ArgSpec, ArgSpec] + >, + FunctionSpec< + "pthread_condattr_getpshared", + RetValSpec, + [ArgSpec, ArgSpec] + >, + FunctionSpec< + "pthread_condattr_init", + RetValSpec, + [ArgSpec] + >, + FunctionSpec< + "pthread_condattr_setclock", + RetValSpec, + [ArgSpec, ArgSpec] + >, + FunctionSpec< + "pthread_condattr_setpshared", + RetValSpec, + [ArgSpec, ArgSpec] + >, FunctionSpec< "pthread_create", RetValSpec, @@ -1522,9 +1559,30 @@ def POSIX : StandardSpec<"POSIX"> { HeaderSpec SysTypes = HeaderSpec< "sys/types.h", [], // Macros - [BlkCntT, BlkSizeT, ClockIdT, DevT, GidT, InoT, ModeTType, NLinkT, OffTType, PidT, - PThreadAttrTType, PThreadKeyT, PThreadMutexTType, PThreadMutexAttrTType, PThreadOnceT, PThreadTType, - SizeTType, SSizeTType, SuSecondsT, TimeTType, UidT], + [ + BlkCntT, + BlkSizeT, + ClockIdT, + DevT, + GidT, + InoT, + ModeTType, + NLinkT, + OffTType, + PThreadAttrTType, + PThreadCondAttrTType, + PThreadKeyT, + PThreadMutexAttrTType, + PThreadMutexTType, + PThreadOnceT, + PThreadTType, + PidT, + SSizeTType, + SizeTType, + SuSecondsT, + TimeTType, + UidT + ], // Types [], // Enumerations [] // Functions >; diff --git a/libc/src/pthread/CMakeLists.txt b/libc/src/pthread/CMakeLists.txt index d5e6c802a845..3d6cf6dde010 100644 --- a/libc/src/pthread/CMakeLists.txt +++ b/libc/src/pthread/CMakeLists.txt @@ -100,6 +100,71 @@ add_entrypoint_object( libc.src.pthread.pthread_attr_setstacksize ) +add_entrypoint_object( + pthread_condattr_destroy + SRCS + pthread_condattr_destroy.cpp + HDRS + pthread_condattr_destroy.h + DEPENDS + libc.include.pthread +) + +add_entrypoint_object( + pthread_condattr_getclock + SRCS + pthread_condattr_getclock.cpp + HDRS + pthread_condattr_getclock.h + DEPENDS + libc.include.pthread + libc.include.sys_types +) + +add_entrypoint_object( + pthread_condattr_getpshared + SRCS + pthread_condattr_getpshared.cpp + HDRS + pthread_condattr_getpshared.h + DEPENDS + libc.include.pthread +) + +add_entrypoint_object( + pthread_condattr_init + SRCS + pthread_condattr_init.cpp + HDRS + pthread_condattr_init.h + DEPENDS + libc.include.pthread + libc.include.time +) + +add_entrypoint_object( + pthread_condattr_setclock + SRCS + pthread_condattr_setclock.cpp + HDRS + pthread_condattr_setclock.h + DEPENDS + libc.include.errno + libc.include.pthread + libc.include.sys_types + libc.include.time +) + +add_entrypoint_object( + pthread_condattr_setpshared + SRCS + pthread_condattr_setpshared.cpp + HDRS + pthread_condattr_setpshared.h + DEPENDS + libc.include.pthread +) + add_header_library( pthread_mutexattr HDRS diff --git a/libc/src/pthread/pthread_condattr_destroy.cpp b/libc/src/pthread/pthread_condattr_destroy.cpp new file mode 100644 index 000000000000..45cc011a4a92 --- /dev/null +++ b/libc/src/pthread/pthread_condattr_destroy.cpp @@ -0,0 +1,23 @@ +//===-- Implementation of the pthread_condattr_destroy --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "pthread_condattr_destroy.h" + +#include "src/__support/common.h" + +#include + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, pthread_condattr_destroy, (pthread_condattr_t * attr)) { + // Initializing a pthread_condattr_t acquires no resources, so this is a + // no-op. + return 0; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/pthread/pthread_condattr_destroy.h b/libc/src/pthread/pthread_condattr_destroy.h new file mode 100644 index 000000000000..2910fa9f9616 --- /dev/null +++ b/libc/src/pthread/pthread_condattr_destroy.h @@ -0,0 +1,20 @@ +//===-- Implementation header for pthread_condattr_destroy ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_DESTROY_H +#define LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_DESTROY_H + +#include + +namespace LIBC_NAMESPACE { + +int pthread_condattr_destroy(pthread_condattr_t *attr); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_DESTROY_H diff --git a/libc/src/pthread/pthread_condattr_getclock.cpp b/libc/src/pthread/pthread_condattr_getclock.cpp new file mode 100644 index 000000000000..a3a3963f4f42 --- /dev/null +++ b/libc/src/pthread/pthread_condattr_getclock.cpp @@ -0,0 +1,25 @@ +//===-- Implementation of the pthread_condattr_getclock -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "pthread_condattr_getclock.h" + +#include "src/__support/common.h" + +#include // pthread_condattr_t +#include // clockid_t + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, pthread_condattr_getclock, + (const pthread_condattr_t *__restrict attr, + clockid_t *__restrict clock_id)) { + *clock_id = attr->clock; + return 0; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/pthread/pthread_condattr_getclock.h b/libc/src/pthread/pthread_condattr_getclock.h new file mode 100644 index 000000000000..d5878c4f45b5 --- /dev/null +++ b/libc/src/pthread/pthread_condattr_getclock.h @@ -0,0 +1,22 @@ +//===-- Implementation header for pthread_condattr_getclock -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_GETCLOCK_H +#define LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_GETCLOCK_H + +#include // pthread_condattr_t +#include // clockid_t + +namespace LIBC_NAMESPACE { + +int pthread_condattr_getclock(const pthread_condattr_t *__restrict attr, + clockid_t *__restrict clock_id); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_GETCLOCK_H diff --git a/libc/src/pthread/pthread_condattr_getpshared.cpp b/libc/src/pthread/pthread_condattr_getpshared.cpp new file mode 100644 index 000000000000..0c5fdc115c25 --- /dev/null +++ b/libc/src/pthread/pthread_condattr_getpshared.cpp @@ -0,0 +1,24 @@ +//===-- Implementation of the pthread_condattr_getpshared -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "pthread_condattr_getpshared.h" + +#include "src/__support/common.h" + +#include + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, pthread_condattr_getpshared, + (const pthread_condattr_t *__restrict attr, + int *__restrict pshared)) { + *pshared = attr->pshared; + return 0; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/pthread/pthread_condattr_getpshared.h b/libc/src/pthread/pthread_condattr_getpshared.h new file mode 100644 index 000000000000..3d7a0c1d357c --- /dev/null +++ b/libc/src/pthread/pthread_condattr_getpshared.h @@ -0,0 +1,21 @@ +//===-- Implementation header for pthread_condattr_getpshared ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_PSHARED_H +#define LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_PSHARED_H + +#include + +namespace LIBC_NAMESPACE { + +int pthread_condattr_getpshared(const pthread_condattr_t *__restrict attr, + int *__restrict pshared); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_PSHARED_H diff --git a/libc/src/pthread/pthread_condattr_init.cpp b/libc/src/pthread/pthread_condattr_init.cpp new file mode 100644 index 000000000000..54633b2e3a5e --- /dev/null +++ b/libc/src/pthread/pthread_condattr_init.cpp @@ -0,0 +1,24 @@ +//===-- Implementation of the pthread_condattr_init -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "pthread_condattr_init.h" + +#include "src/__support/common.h" + +#include // pthread_condattr_t, PTHREAD_PROCESS_PRIVATE +#include // CLOCK_REALTIME + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, pthread_condattr_init, (pthread_condattr_t * attr)) { + attr->clock = CLOCK_REALTIME; + attr->pshared = PTHREAD_PROCESS_PRIVATE; + return 0; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/pthread/pthread_condattr_init.h b/libc/src/pthread/pthread_condattr_init.h new file mode 100644 index 000000000000..9f3c06bb6f4a --- /dev/null +++ b/libc/src/pthread/pthread_condattr_init.h @@ -0,0 +1,20 @@ +//===-- Implementation header for pthread_condattr_init ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_INIT_H +#define LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_INIT_H + +#include + +namespace LIBC_NAMESPACE { + +int pthread_condattr_init(pthread_condattr_t *attr); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_INIT_H diff --git a/libc/src/pthread/pthread_condattr_setclock.cpp b/libc/src/pthread/pthread_condattr_setclock.cpp new file mode 100644 index 000000000000..6eca8b30ef7f --- /dev/null +++ b/libc/src/pthread/pthread_condattr_setclock.cpp @@ -0,0 +1,30 @@ +//===-- Implementation of the pthread_condattr_setclock -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "pthread_condattr_setclock.h" + +#include "src/__support/common.h" + +#include // EINVAL +#include // pthread_condattr_t +#include // clockid_t +#include // CLOCK_MONOTONIC, CLOCK_REALTIME + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, pthread_condattr_setclock, + (pthread_condattr_t * attr, clockid_t clock)) { + + if (clock != CLOCK_MONOTONIC && clock != CLOCK_REALTIME) + return EINVAL; + + attr->clock = clock; + return 0; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/pthread/pthread_condattr_setclock.h b/libc/src/pthread/pthread_condattr_setclock.h new file mode 100644 index 000000000000..328766fe7883 --- /dev/null +++ b/libc/src/pthread/pthread_condattr_setclock.h @@ -0,0 +1,21 @@ +//===-- Implementation header for pthread_condattr_setclock -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_SETCLOCK_H +#define LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_SETCLOCK_H + +#include +#include // clockid_t + +namespace LIBC_NAMESPACE { + +int pthread_condattr_setclock(pthread_condattr_t *attr, clockid_t clock); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_SETCLOCK_H diff --git a/libc/src/pthread/pthread_condattr_setpshared.cpp b/libc/src/pthread/pthread_condattr_setpshared.cpp new file mode 100644 index 000000000000..7f1560acad84 --- /dev/null +++ b/libc/src/pthread/pthread_condattr_setpshared.cpp @@ -0,0 +1,28 @@ +//===-- Implementation of the pthread_condattr_setpshared -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "pthread_condattr_setpshared.h" + +#include "src/__support/common.h" + +#include // EINVAL +#include // pthread_condattr_t, PTHREAD_PROCESS_SHARED, PTHREAD_PROCESS_PRIVATE + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, pthread_condattr_setpshared, + (pthread_condattr_t * attr, int pshared)) { + + if (pshared != PTHREAD_PROCESS_SHARED && pshared != PTHREAD_PROCESS_PRIVATE) + return EINVAL; + + attr->pshared = pshared; + return 0; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/pthread/pthread_condattr_setpshared.h b/libc/src/pthread/pthread_condattr_setpshared.h new file mode 100644 index 000000000000..8083bdec78cc --- /dev/null +++ b/libc/src/pthread/pthread_condattr_setpshared.h @@ -0,0 +1,20 @@ +//===-- Implementation header for pthread_condattr_setpshared ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_SETPSHARED_H +#define LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_SETPSHARED_H + +#include + +namespace LIBC_NAMESPACE { + +int pthread_condattr_setpshared(pthread_condattr_t *attr, int pshared); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_CONDATTR_SETPSHARED_H diff --git a/libc/test/src/pthread/CMakeLists.txt b/libc/test/src/pthread/CMakeLists.txt index fb0d22ab9d2a..46f38422a537 100644 --- a/libc/test/src/pthread/CMakeLists.txt +++ b/libc/test/src/pthread/CMakeLists.txt @@ -39,3 +39,15 @@ add_libc_unittest( libc.src.pthread.pthread_mutexattr_setrobust libc.src.pthread.pthread_mutexattr_settype ) + +add_libc_unittest( + pthread_condattr_test + SUITE + libc_pthread_unittests + SRCS + pthread_condattr_test.cpp + DEPENDS + libc.include.errno + libc.include.pthread + libc.include.time + ) diff --git a/libc/test/src/pthread/pthread_condattr_test.cpp b/libc/test/src/pthread/pthread_condattr_test.cpp new file mode 100644 index 000000000000..accb62de92e4 --- /dev/null +++ b/libc/test/src/pthread/pthread_condattr_test.cpp @@ -0,0 +1,71 @@ +//===-- Unittests for pthread_condattr_t ----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "test/UnitTest/Test.h" + +#include +#include +#include + +TEST(LlvmLibcPThreadCondAttrTest, InitAndDestroy) { + pthread_condattr_t cond; + ASSERT_EQ(pthread_condattr_init(&cond), 0); + ASSERT_EQ(pthread_condattr_destroy(&cond), 0); +} + +TEST(LlvmLibcPThreadCondAttrTest, GetDefaultValues) { + pthread_condattr_t cond; + + // Invalid clock id. + clockid_t clock = 7; + // Invalid value. + int pshared = 42; + + ASSERT_EQ(pthread_condattr_init(&cond), 0); + ASSERT_EQ(pthread_condattr_getclock(&cond, &clock), 0); + ASSERT_EQ(clock, CLOCK_REALTIME); + ASSERT_EQ(pthread_condattr_getpshared(&cond, &pshared), 0); + ASSERT_EQ(pshared, PTHREAD_PROCESS_PRIVATE); + ASSERT_EQ(pthread_condattr_destroy(&cond), 0); +} + +TEST(LlvmLibcPThreadCondAttrTest, SetGoodValues) { + pthread_condattr_t cond; + + // Invalid clock id. + clockid_t clock = 7; + // Invalid value. + int pshared = 42; + + ASSERT_EQ(pthread_condattr_init(&cond), 0); + ASSERT_EQ(pthread_condattr_setclock(&cond, CLOCK_MONOTONIC), 0); + ASSERT_EQ(pthread_condattr_getclock(&cond, &clock), 0); + ASSERT_EQ(clock, CLOCK_MONOTONIC); + ASSERT_EQ(pthread_condattr_setpshared(&cond, PTHREAD_PROCESS_SHARED), 0); + ASSERT_EQ(pthread_condattr_getpshared(&cond, &pshared), 0); + ASSERT_EQ(pshared, PTHREAD_PROCESS_SHARED); + ASSERT_EQ(pthread_condattr_destroy(&cond), 0); +} + +TEST(LlvmLibcPThreadCondAttrTest, SetBadValues) { + pthread_condattr_t cond; + + // Invalid clock id. + clockid_t clock = 7; + // Invalid value. + int pshared = 42; + + ASSERT_EQ(pthread_condattr_init(&cond), 0); + ASSERT_EQ(pthread_condattr_setclock(&cond, clock), EINVAL); + ASSERT_EQ(pthread_condattr_getclock(&cond, &clock), 0); + ASSERT_EQ(clock, CLOCK_REALTIME); + ASSERT_EQ(pthread_condattr_setpshared(&cond, pshared), EINVAL); + ASSERT_EQ(pthread_condattr_getpshared(&cond, &pshared), 0); + ASSERT_EQ(pshared, PTHREAD_PROCESS_PRIVATE); + ASSERT_EQ(pthread_condattr_destroy(&cond), 0); +} -- GitLab From 4edeaffbf255137861f5153eb1a6183d956efede Mon Sep 17 00:00:00 2001 From: fabrizio-indirli Date: Wed, 17 Apr 2024 17:43:22 +0100 Subject: [PATCH 036/714] [mlir][tosa] Fix tosa.Resize-to-linalg lowering (#88514) --- .../Conversion/TosaToLinalg/TosaToLinalg.cpp | 21 ++-- .../TosaToLinalg/tosa-to-linalg-resize.mlir | 112 ++++++++---------- 2 files changed, 60 insertions(+), 73 deletions(-) diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp index d8dd1c93722b..af19ebaea937 100644 --- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp +++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp @@ -1582,17 +1582,16 @@ public: } // x = x * scale_d + offset; // ix = floor(x / scale_n) - // dx = x / scale_n - ix - Value val = b.create(floatTy, in); - scaleN = b.create(floatTy, scaleN); - scaleD = b.create(floatTy, scaleD); - offset = b.create(floatTy, offset); - val = b.create(val, scaleD); - val = b.create(val, offset); - val = b.create(val, scaleN); - index = b.create(val); - delta = b.create(val, index); - index = b.create(b.getI32Type(), index); + Value val = b.create(in, scaleD); + val = b.create(val, offset); + index = b.create(val, scaleN); + + // rx = x % scale_n + // dx = rx / scale_n + Value r = b.create(val, scaleN); + Value rFp = b.create(floatTy, r); + Value scaleNfp = b.create(floatTy, scaleN); + delta = b.create(rFp, scaleNfp); }; // Compute the ix and dx values for the X and Y dimensions - int case. diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-resize.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-resize.mlir index 468e92e2a266..d42d0a46692d 100644 --- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-resize.mlir +++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-resize.mlir @@ -304,42 +304,36 @@ func.func @resize_nearest_fp32(%input: tensor<1x50x48x1xf32>) -> () { // CHECK-DAG: %[[XMAX:.*]] = arith.constant 47 // CHECK: %[[Y:.+]] = arith.index_cast %[[IDX1]] // CHECK: %[[X:.+]] = arith.index_cast %[[IDX2]] - // CHECK-DAG: %[[ISCALE_Y_N:.*]] = arith.constant 64 - // CHECK-DAG: %[[ISCALE_Y_D:.*]] = arith.constant 2 - // CHECK-DAG: %[[ISCALE_X_N:.*]] = arith.constant 64 - // CHECK-DAG: %[[ISCALE_X_D:.*]] = arith.constant 2 - // CHECK-DAG: %[[IOFFSET_Y:.*]] = arith.constant -31 - // CHECK-DAG: %[[IOFFSET_X:.*]] = arith.constant -31 - // CHECK-DAG: %[[IBORDER_Y:.*]] = arith.constant 31 - // CHECK-DAG: %[[IBORDER_X:.*]] = arith.constant 31 - - // CHECK: %[[Y0:.+]] = arith.uitofp %[[Y]] - // CHECK: %[[SCALE_Y_N:.*]] = arith.uitofp %[[ISCALE_Y_N]] - // CHECK: %[[SCALE_Y_D:.*]] = arith.uitofp %[[ISCALE_Y_D]] - // CHECK: %[[OFFSET_Y:.*]] = arith.sitofp %[[IOFFSET_Y]] - // CHECK: %[[VAL_29:.*]] = arith.mulf %[[Y0]], %[[SCALE_Y_D]] - // CHECK: %[[VAL_31:.*]] = arith.addf %[[VAL_29]], %[[OFFSET_Y]] - // CHECK: %[[VAL_33:.*]] = arith.divf %[[VAL_31]], %[[SCALE_Y_N]] - // CHECK: %[[VAL_35:.*]] = math.floor %[[VAL_33]] - // CHECK: %[[D_Y:.*]] = arith.subf %[[VAL_33]], %[[VAL_35]] - // CHECK: %[[VAL_39:.*]] = arith.fptosi %[[VAL_35]] - - // CHECK: %[[X0:.+]] = arith.uitofp %[[X]] - // CHECK: %[[SCALE_X_N:.*]] = arith.uitofp %[[ISCALE_X_N]] - // CHECK: %[[SCALE_X_D:.*]] = arith.uitofp %[[ISCALE_X_D]] - // CHECK: %[[OFFSET_X:.*]] = arith.sitofp %[[IOFFSET_X]] - // CHECK: %[[VAL_30:.*]] = arith.mulf %[[X0]], %[[SCALE_X_D]] - // CHECK: %[[VAL_32:.*]] = arith.addf %[[VAL_30]], %[[OFFSET_X]] - // CHECK: %[[VAL_34:.*]] = arith.divf %[[VAL_32]], %[[SCALE_X_N]] - // CHECK: %[[VAL_36:.*]] = math.floor %[[VAL_34]] - // CHECK: %[[D_X:.*]] = arith.subf %[[VAL_34]], %[[VAL_36]] - // CHECK: %[[VAL_40:.*]] = arith.fptosi %[[VAL_36]] + // CHECK-DAG: %[[SCALE_Y_N:.*]] = arith.constant 64 + // CHECK-DAG: %[[SCALE_Y_D:.*]] = arith.constant 2 + // CHECK-DAG: %[[SCALE_X_N:.*]] = arith.constant 64 + // CHECK-DAG: %[[SCALE_X_D:.*]] = arith.constant 2 + // CHECK-DAG: %[[OFFSET_Y:.*]] = arith.constant -31 + // CHECK-DAG: %[[OFFSET_X:.*]] = arith.constant -31 + // CHECK-DAG: %[[BORDER_Y:.*]] = arith.constant 31 + // CHECK-DAG: %[[BORDER_X:.*]] = arith.constant 31 + + // CHECK: %[[VAL_29:.*]] = arith.muli %[[Y]], %[[SCALE_Y_D]] + // CHECK: %[[Y_TEMP:.*]] = arith.addi %[[VAL_29]], %[[OFFSET_Y]] + // CHECK: %[[IY_TEMP:.*]] = arith.floordivsi %[[Y_TEMP]], %[[SCALE_Y_N]] + // CHECK: %[[RY:.*]] = arith.remsi %[[Y_TEMP]], %[[SCALE_Y_N]] + // CHECK: %[[RY_FP:.*]] = arith.sitofp %[[RY]] + // CHECK: %[[SCALE_Y_N_FP:.*]] = arith.uitofp %[[SCALE_Y_N]] + // CHECK: %[[D_Y:.*]] = arith.divf %[[RY_FP]], %[[SCALE_Y_N_FP]] + + // CHECK: %[[VAL_30:.*]] = arith.muli %[[X]], %[[SCALE_X_D]] + // CHECK: %[[X_TEMP:.*]] = arith.addi %[[VAL_30]], %[[OFFSET_X]] + // CHECK: %[[IX_TEMP:.*]] = arith.floordivsi %[[X_TEMP]], %[[SCALE_X_N]] + // CHECK: %[[RX:.*]] = arith.remsi %[[X_TEMP]], %[[SCALE_X_N]] + // CHECK: %[[RX_FP:.*]] = arith.sitofp %[[RX]] + // CHECK: %[[SCALE_X_N_FP:.*]] = arith.uitofp %[[SCALE_X_N]] + // CHECK: %[[D_X:.*]] = arith.divf %[[RX_FP]], %[[SCALE_X_N_FP]] // CHECK-DAG: %[[ONE:.*]] = arith.constant 1 // CHECK-DAG: %[[HALF:.*]] = arith.constant 5.000000e-01 // CHECK: %[[PRED_Y:.*]] = arith.cmpf oge, %[[D_Y]], %[[HALF]] // CHECK: %[[ROUND_Y:.*]] = arith.select %[[PRED_Y]], %[[ONE]], %[[ZERO]] - // CHECK: %[[VAL_48:.*]] = arith.addi %[[VAL_39]], %[[ROUND_Y]] + // CHECK: %[[VAL_48:.*]] = arith.addi %[[IY_TEMP]], %[[ROUND_Y]] // CHECK: %[[LOWER:.*]] = arith.maxsi %[[ZERO]], %[[VAL_48]] // CHECK: %[[CLAMPED:.*]] = arith.minsi %[[YMAX]], %[[LOWER]] // CHECK: %[[IDY:.*]] = arith.index_cast %[[CLAMPED]] @@ -347,7 +341,7 @@ func.func @resize_nearest_fp32(%input: tensor<1x50x48x1xf32>) -> () { // CHECK-DAG: %[[HALF:.*]] = arith.constant 5.000000e-01 // CHECK: %[[PRED_X:.*]] = arith.cmpf oge, %[[D_X]], %[[HALF]] // CHECK: %[[ROUND_X:.*]] = arith.select %[[PRED_X]], %[[ONE]], %[[ZERO]] - // CHECK: %[[VAL_49:.*]] = arith.addi %[[VAL_40]], %[[ROUND_X]] + // CHECK: %[[VAL_49:.*]] = arith.addi %[[IX_TEMP]], %[[ROUND_X]] // CHECK: %[[LOWER:.*]] = arith.maxsi %[[ZERO]], %[[VAL_49]] // CHECK: %[[CLAMPED:.*]] = arith.minsi %[[XMAX]], %[[LOWER]] // CHECK: %[[IDX:.*]] = arith.index_cast %[[CLAMPED]] @@ -374,36 +368,30 @@ func.func @resize_bilinear_fp(%input: tensor<1x23x24x1xf32>) -> () { // CHECK-DAG: %[[X_MAX:.*]] = arith.constant 23 // CHECK: %[[Y:.+]] = arith.index_cast %[[IDX_1]] // CHECK: %[[X:.+]] = arith.index_cast %[[IDX_2]] - // CHECK-DAG: %[[ISCALE_Y_N:.*]] = arith.constant 4 - // CHECK-DAG: %[[ISCALE_Y_D:.*]] = arith.constant 1 - // CHECK-DAG: %[[ISCALE_X_N:.*]] = arith.constant 4 - // CHECK-DAG: %[[ISCALE_X_D:.*]] = arith.constant 1 - // CHECK-DAG: %[[IOFFSET_Y:.*]] = arith.constant 0 - // CHECK-DAG: %[[IOFFSET_X:.*]] = arith.constant 0 - // CHECK-DAG: %[[IBORDER_Y:.*]] = arith.constant 0 - // CHECK-DAG: %[[IBORDER_X:.*]] = arith.constant 0 - - // CHECK: %[[Y0:.+]] = arith.uitofp %[[Y]] - // CHECK: %[[SCALE_Y_N:.*]] = arith.uitofp %[[ISCALE_Y_N]] - // CHECK: %[[SCALE_Y_D:.*]] = arith.uitofp %[[ISCALE_Y_D]] - // CHECK: %[[OFFSET_Y:.*]] = arith.sitofp %[[IOFFSET_Y]] - // CHECK: %[[VAL_29:.*]] = arith.mulf %[[Y0]], %[[SCALE_Y_D]] - // CHECK: %[[VAL_31:.*]] = arith.addf %[[VAL_29]], %[[OFFSET_Y]] - // CHECK: %[[VAL_33:.*]] = arith.divf %[[VAL_31]], %[[SCALE_Y_N]] - // CHECK: %[[VAL_35:.*]] = math.floor %[[VAL_33]] - // CHECK: %[[D_Y:.*]] = arith.subf %[[VAL_33]], %[[VAL_35]] - // CHECK: %[[I_Y:.*]] = arith.fptosi %[[VAL_35]] - - // CHECK: %[[X0:.+]] = arith.uitofp %[[X]] - // CHECK: %[[SCALE_X_N:.*]] = arith.uitofp %[[ISCALE_X_N]] - // CHECK: %[[SCALE_X_D:.*]] = arith.uitofp %[[ISCALE_X_D]] - // CHECK: %[[OFFSET_X:.*]] = arith.sitofp %[[IOFFSET_X]] - // CHECK: %[[VAL_30:.*]] = arith.mulf %[[X0]], %[[SCALE_X_D]] - // CHECK: %[[VAL_32:.*]] = arith.addf %[[VAL_30]], %[[OFFSET_X]] - // CHECK: %[[VAL_34:.*]] = arith.divf %[[VAL_32]], %[[SCALE_X_N]] - // CHECK: %[[VAL_36:.*]] = math.floor %[[VAL_34]] - // CHECK: %[[D_X:.*]] = arith.subf %[[VAL_34]], %[[VAL_36]] - // CHECK: %[[I_X:.*]] = arith.fptosi %[[VAL_36]] + // CHECK-DAG: %[[SCALE_Y_N:.*]] = arith.constant 4 + // CHECK-DAG: %[[SCALE_Y_D:.*]] = arith.constant 1 + // CHECK-DAG: %[[SCALE_X_N:.*]] = arith.constant 4 + // CHECK-DAG: %[[SCALE_X_D:.*]] = arith.constant 1 + // CHECK-DAG: %[[OFFSET_Y:.*]] = arith.constant 0 + // CHECK-DAG: %[[OFFSET_X:.*]] = arith.constant 0 + // CHECK-DAG: %[[BORDER_Y:.*]] = arith.constant 0 + // CHECK-DAG: %[[BORDER_X:.*]] = arith.constant 0 + + // CHECK: %[[VAL_29:.*]] = arith.muli %[[Y]], %[[SCALE_Y_D]] + // CHECK: %[[Y_TEMP:.*]] = arith.addi %[[VAL_29]], %[[OFFSET_Y]] + // CHECK: %[[I_Y:.*]] = arith.floordivsi %[[Y_TEMP]], %[[SCALE_Y_N]] + // CHECK: %[[RY:.*]] = arith.remsi %[[Y_TEMP]], %[[SCALE_Y_N]] + // CHECK: %[[RY_FP:.*]] = arith.sitofp %[[RY]] + // CHECK: %[[SCALE_Y_N_FP:.*]] = arith.uitofp %[[SCALE_Y_N]] + // CHECK: %[[D_Y:.*]] = arith.divf %[[RY_FP]], %[[SCALE_Y_N_FP]] + + // CHECK: %[[VAL_30:.*]] = arith.muli %[[X]], %[[SCALE_X_D]] + // CHECK: %[[X_TEMP:.*]] = arith.addi %[[VAL_30]], %[[OFFSET_X]] + // CHECK: %[[I_X:.*]] = arith.floordivsi %[[X_TEMP]], %[[SCALE_X_N]] + // CHECK: %[[RX:.*]] = arith.remsi %[[X_TEMP]], %[[SCALE_X_N]] + // CHECK: %[[RX_FP:.*]] = arith.sitofp %[[RX]] + // CHECK: %[[SCALE_X_N_FP:.*]] = arith.uitofp %[[SCALE_X_N]] + // CHECK: %[[D_X:.*]] = arith.divf %[[RX_FP]], %[[SCALE_X_N_FP]] // Compute the left, right, and top indices for the bilinear interpolation. -- GitLab From 564f9abfcc3b99a01843f88b5a2c7309bfab5a33 Mon Sep 17 00:00:00 2001 From: Jorge Gorbe Moya Date: Wed, 17 Apr 2024 10:11:03 -0700 Subject: [PATCH 037/714] [bazel][mlir] Add missing dep after 4f88c2311130791cf69da34b743b1b3ba7584a7b --- utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 03386549a011..58538b66c5e0 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -577,6 +577,7 @@ mlir_c_api_cc_library( ], includes = ["include"], deps = [ + ":IR", ":NVGPUDialect", ], ) -- GitLab From e59632bdfd4c70caed437216af17d335858686fd Mon Sep 17 00:00:00 2001 From: Michael Maitland Date: Wed, 17 Apr 2024 06:19:11 -0700 Subject: [PATCH 038/714] [RISCV] Fix typo in RISCVScheduleV.td that was introduced in 60a1158 --- llvm/lib/Target/RISCV/RISCVScheduleV.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td index 5993884bc2c1..5be06d4c3f7e 100644 --- a/llvm/lib/Target/RISCV/RISCVScheduleV.td +++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td @@ -1079,7 +1079,7 @@ defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>; defm "" : LMULSEWReadAdvanceW<"ReadVFWCvtIToFV", 0>; defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>; defm "" : LMULSEWReadAdvanceFW<"ReadVFWCvtFToFV", 0>; -defm "" : LMULSEWReadAdvanceFW<"SEWReadVFNCvtIToFV", 0>; +defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtIToFV", 0>; defm "" : LMULReadAdvanceW<"ReadVFNCvtFToIV", 0>; defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtFToFV", 0>; -- GitLab From 676d3bafc09d0c331a04b813804407334de12917 Mon Sep 17 00:00:00 2001 From: Jorge Gorbe Moya Date: Wed, 17 Apr 2024 10:20:07 -0700 Subject: [PATCH 039/714] [bazel][libc] Add missing dep after b854a2323337be2633b1135f590678a17e9d1ade --- utils/bazel/llvm-project-overlay/libc/BUILD.bazel | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 6029cc3fee61..be02c2270432 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -786,7 +786,7 @@ libc_support_library( ":errno", ":hdr_fenv_macros", ":hdr_math_macros", - ":types_fenv_t" + ":types_fenv_t", ], ) @@ -1152,6 +1152,7 @@ libc_function( deps = [ ":__support_common", ":__support_fputil_fenv_impl", + ":types_fexcept_t", ], ) @@ -1272,7 +1273,7 @@ libc_function( deps = [ ":__support_common", ":__support_fputil_fenv_impl", - ":types_fexcept_t" + ":types_fexcept_t", ], ) @@ -1283,7 +1284,7 @@ libc_function( deps = [ ":__support_common", ":__support_fputil_fenv_impl", - ":types_fexcept_t", + ":types_fexcept_t", ], ) -- GitLab From 693a458287d019c5c6a66fe3019d099df2978cdb Mon Sep 17 00:00:00 2001 From: Abdul Raheem <55028856+abdulraheembeigh@users.noreply.github.com> Date: Wed, 17 Apr 2024 18:24:04 +0100 Subject: [PATCH 040/714] [MLIR] Update doc comment in ViewLikeInterface.td (NFC) (#89074) Signed-off: Abdul Raheem Beigh --- mlir/include/mlir/Interfaces/ViewLikeInterface.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlir/include/mlir/Interfaces/ViewLikeInterface.td b/mlir/include/mlir/Interfaces/ViewLikeInterface.td index ea5bb1b5ac48..9397f271e1bc 100644 --- a/mlir/include/mlir/Interfaces/ViewLikeInterface.td +++ b/mlir/include/mlir/Interfaces/ViewLikeInterface.td @@ -158,7 +158,7 @@ def OffsetSizeAndStrideOpInterface : OpInterface<"OffsetSizeAndStrideOpInterface >, InterfaceMethod< /*desc=*/[{ - Return a vector of all the static or dynamic sizes of the op. + Return a vector of all the static or dynamic offsets of the op. }], /*retTy=*/"::llvm::SmallVector<::mlir::OpFoldResult, 4>", /*methodName=*/"getMixedOffsets", -- GitLab From 6f7160eedb2db02f37d4ffd52fff7b0cf88b3fdc Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Tue, 16 Apr 2024 14:55:41 -0400 Subject: [PATCH 041/714] [SLP]Attempt to vectorize long stores, if short one failed. We can try to vectorize long store sequences, if short ones were unsuccessful because of the non-profitable vectorization. It should not increase compile time significantly (stores are sorted already, complexity is n x log n), but vectorize extra code. Metric: size..text Program size..text results results0 diff test-suite :: External/SPEC/CINT2006/400.perlbench/400.perlbench.test 1088012.00 1088236.00 0.0% test-suite :: SingleSource/UnitTests/matrix-types-spec.test 480396.00 480476.00 0.0% test-suite :: External/SPEC/CINT2017rate/525.x264_r/525.x264_r.test 664613.00 664661.00 0.0% test-suite :: External/SPEC/CINT2017speed/625.x264_s/625.x264_s.test 664613.00 664661.00 0.0% test-suite :: External/SPEC/CFP2017rate/510.parest_r/510.parest_r.test 2041105.00 2040961.00 -0.0% test-suite :: MultiSource/Applications/JM/lencod/lencod.test 836563.00 836387.00 -0.0% test-suite :: MultiSource/Benchmarks/7zip/7zip-benchmark.test 1035100.00 1032140.00 -0.3% In all benchmarks extra code gets vectorized Reviewers: RKSimon Reviewed By: RKSimon Pull Request: https://github.com/llvm/llvm-project/pull/88563 --- .../Transforms/Vectorize/SLPVectorizer.cpp | 106 ++++++++++++------ .../Transforms/SLPVectorizer/X86/pr46983.ll | 46 ++------ 2 files changed, 80 insertions(+), 72 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 7694627c3b04..806e8085038b 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -15164,10 +15164,6 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef Stores, BoUpSLP::ValueSet VectorizedStores; bool Changed = false; - // Stores the pair of stores (first_store, last_store) in a range, that were - // already tried to be vectorized. Allows to skip the store ranges that were - // already tried to be vectorized but the attempts were unsuccessful. - DenseSet> TriedSequences; struct StoreDistCompare { bool operator()(const std::pair &Op1, const std::pair &Op2) const { @@ -15209,8 +15205,10 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef Stores, Type *ValueTy = StoreTy; if (auto *Trunc = dyn_cast(Store->getValueOperand())) ValueTy = Trunc->getSrcTy(); - unsigned MinVF = PowerOf2Ceil(TTI->getStoreMinimumVF( - R.getMinVF(DL->getTypeStoreSizeInBits(StoreTy)), StoreTy, ValueTy)); + unsigned MinVF = std::max( + 2, PowerOf2Ceil(TTI->getStoreMinimumVF( + R.getMinVF(DL->getTypeStoreSizeInBits(StoreTy)), StoreTy, + ValueTy))); if (MaxVF < MinVF) { LLVM_DEBUG(dbgs() << "SLP: Vectorization infeasible as MaxVF (" << MaxVF @@ -15236,40 +15234,74 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef Stores, VF = Size > MaxVF ? NonPowerOf2VF : Size; Size *= 2; }); - unsigned StartIdx = 0; - for (unsigned Size : CandidateVFs) { - for (unsigned Cnt = StartIdx, E = Operands.size(); Cnt + Size <= E;) { - ArrayRef Slice = ArrayRef(Operands).slice(Cnt, Size); - assert( - all_of( - Slice, - [&](Value *V) { - return cast(V)->getValueOperand()->getType() == - cast(Slice.front()) - ->getValueOperand() - ->getType(); - }) && - "Expected all operands of same type."); - if (!VectorizedStores.count(Slice.front()) && - !VectorizedStores.count(Slice.back()) && - TriedSequences.insert(std::make_pair(Slice.front(), Slice.back())) - .second && - vectorizeStoreChain(Slice, R, Cnt, MinVF)) { - // Mark the vectorized stores so that we don't vectorize them again. - VectorizedStores.insert(Slice.begin(), Slice.end()); - Changed = true; - // If we vectorized initial block, no need to try to vectorize it - // again. - if (Cnt == StartIdx) - StartIdx += Size; - Cnt += Size; - continue; + unsigned End = Operands.size(); + unsigned Repeat = 0; + constexpr unsigned MaxAttempts = 2; + SmallBitVector Range(Operands.size()); + while (true) { + ++Repeat; + for (unsigned Size : CandidateVFs) { + int StartIdx = Range.find_first_unset(); + while (StartIdx != -1) { + int EndIdx = Range.find_next(StartIdx); + unsigned Sz = EndIdx == -1 ? End : EndIdx; + for (unsigned Cnt = StartIdx; Cnt + Size <= Sz;) { + ArrayRef Slice = ArrayRef(Operands).slice(Cnt, Size); + assert(all_of(Slice, + [&](Value *V) { + return cast(V) + ->getValueOperand() + ->getType() == + cast(Slice.front()) + ->getValueOperand() + ->getType(); + }) && + "Expected all operands of same type."); + if (vectorizeStoreChain(Slice, R, Cnt, MinVF)) { + // Mark the vectorized stores so that we don't vectorize them + // again. + VectorizedStores.insert(Slice.begin(), Slice.end()); + // Mark the vectorized stores so that we don't vectorize them + // again. + Changed = true; + // If we vectorized initial block, no need to try to vectorize + // it again. + Range.set(Cnt, Cnt + Size); + if (Cnt < StartIdx + MinVF) + Range.set(StartIdx, Cnt); + if (Cnt > EndIdx - Size - MinVF) { + Range.set(Cnt + Size, EndIdx); + End = Cnt; + } + Cnt += Size; + continue; + } + ++Cnt; + } + if (Sz >= End) + break; + StartIdx = Range.find_next_unset(EndIdx); } - ++Cnt; } - // Check if the whole array was vectorized already - exit. - if (StartIdx >= Operands.size()) + // All values vectorize - exit. + if (Range.all()) + break; + // Check if tried all attempts or no need for the last attempts at all. + if (Repeat >= MaxAttempts) + break; + constexpr unsigned MaxVFScale = 4; + constexpr unsigned StoresLimit = 16; + const unsigned MaxTotalNum = std::min( + std::max(StoresLimit, MaxVFScale * MaxVF), + bit_floor(static_cast(Range.find_last_unset() - + Range.find_first_unset() + 1))); + if (MaxVF >= MaxTotalNum) break; + // Last attempt to vectorize max number of elements, if all previous + // attempts were unsuccessful because of the cost issues. + CandidateVFs.clear(); + for (unsigned Size = MaxTotalNum; Size > MaxVF; Size /= 2) + CandidateVFs.push_back(Size); } } }; diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll index 75505f632a43..3deab0975ce7 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll @@ -100,41 +100,17 @@ define void @store_i8(ptr nocapture %0, i32 %1, i32 %2) { define void @store_i64(ptr nocapture %0, i32 %1, i32 %2) { ; SSE-LABEL: @store_i64( ; SSE-NEXT: [[TMP4:%.*]] = zext i32 [[TMP1:%.*]] to i64 -; SSE-NEXT: [[TMP5:%.*]] = load i64, ptr [[TMP0:%.*]], align 8, !tbaa [[TBAA5:![0-9]+]] -; SSE-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], [[TMP4]] -; SSE-NEXT: [[TMP7:%.*]] = lshr i64 [[TMP6]], 15 -; SSE-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32 -; SSE-NEXT: [[TMP9:%.*]] = icmp ult i32 [[TMP8]], 255 -; SSE-NEXT: [[TMP10:%.*]] = and i64 [[TMP7]], 4294967295 -; SSE-NEXT: [[TMP11:%.*]] = select i1 [[TMP9]], i64 [[TMP10]], i64 255 -; SSE-NEXT: store i64 [[TMP11]], ptr [[TMP0]], align 8, !tbaa [[TBAA5]] -; SSE-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8 -; SSE-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP12]], align 8, !tbaa [[TBAA5]] -; SSE-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], [[TMP4]] -; SSE-NEXT: [[TMP15:%.*]] = lshr i64 [[TMP14]], 15 -; SSE-NEXT: [[TMP16:%.*]] = trunc i64 [[TMP15]] to i32 -; SSE-NEXT: [[TMP17:%.*]] = icmp ult i32 [[TMP16]], 255 -; SSE-NEXT: [[TMP18:%.*]] = and i64 [[TMP15]], 4294967295 -; SSE-NEXT: [[TMP19:%.*]] = select i1 [[TMP17]], i64 [[TMP18]], i64 255 -; SSE-NEXT: store i64 [[TMP19]], ptr [[TMP12]], align 8, !tbaa [[TBAA5]] -; SSE-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16 -; SSE-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP20]], align 8, !tbaa [[TBAA5]] -; SSE-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], [[TMP4]] -; SSE-NEXT: [[TMP23:%.*]] = lshr i64 [[TMP22]], 15 -; SSE-NEXT: [[TMP24:%.*]] = trunc i64 [[TMP23]] to i32 -; SSE-NEXT: [[TMP25:%.*]] = icmp ult i32 [[TMP24]], 255 -; SSE-NEXT: [[TMP26:%.*]] = and i64 [[TMP23]], 4294967295 -; SSE-NEXT: [[TMP27:%.*]] = select i1 [[TMP25]], i64 [[TMP26]], i64 255 -; SSE-NEXT: store i64 [[TMP27]], ptr [[TMP20]], align 8, !tbaa [[TBAA5]] -; SSE-NEXT: [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 24 -; SSE-NEXT: [[TMP29:%.*]] = load i64, ptr [[TMP28]], align 8, !tbaa [[TBAA5]] -; SSE-NEXT: [[TMP30:%.*]] = mul i64 [[TMP29]], [[TMP4]] -; SSE-NEXT: [[TMP31:%.*]] = lshr i64 [[TMP30]], 15 -; SSE-NEXT: [[TMP32:%.*]] = trunc i64 [[TMP31]] to i32 -; SSE-NEXT: [[TMP33:%.*]] = icmp ult i32 [[TMP32]], 255 -; SSE-NEXT: [[TMP34:%.*]] = and i64 [[TMP31]], 4294967295 -; SSE-NEXT: [[TMP35:%.*]] = select i1 [[TMP33]], i64 [[TMP34]], i64 255 -; SSE-NEXT: store i64 [[TMP35]], ptr [[TMP28]], align 8, !tbaa [[TBAA5]] +; SSE-NEXT: [[TMP5:%.*]] = load <4 x i64>, ptr [[TMP0:%.*]], align 8, !tbaa [[TBAA5:![0-9]+]] +; SSE-NEXT: [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0 +; SSE-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> zeroinitializer +; SSE-NEXT: [[TMP8:%.*]] = mul <4 x i64> [[TMP5]], [[TMP7]] +; SSE-NEXT: [[TMP9:%.*]] = lshr <4 x i64> [[TMP8]], +; SSE-NEXT: [[TMP10:%.*]] = trunc <4 x i64> [[TMP9]] to <4 x i32> +; SSE-NEXT: [[TMP11:%.*]] = icmp ult <4 x i32> [[TMP10]], +; SSE-NEXT: [[TMP12:%.*]] = trunc <4 x i64> [[TMP9]] to <4 x i32> +; SSE-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP11]], <4 x i32> [[TMP12]], <4 x i32> +; SSE-NEXT: [[TMP14:%.*]] = zext <4 x i32> [[TMP13]] to <4 x i64> +; SSE-NEXT: store <4 x i64> [[TMP14]], ptr [[TMP0]], align 8, !tbaa [[TBAA5]] ; SSE-NEXT: ret void ; ; AVX-LABEL: @store_i64( -- GitLab From eefee382186005d3662958e076c8e61e286ea1ab Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Wed, 17 Apr 2024 10:34:23 -0700 Subject: [PATCH 042/714] [libc] set cmake dependencies for condattr test (#89103) The entrypoints are not yet exposed on non-x86. Express this dependency to unbreak post submit. Fixes #88987 --- libc/src/pthread/pthread_condattr_destroy.cpp | 3 ++- libc/test/src/pthread/CMakeLists.txt | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/libc/src/pthread/pthread_condattr_destroy.cpp b/libc/src/pthread/pthread_condattr_destroy.cpp index 45cc011a4a92..41994c6941ff 100644 --- a/libc/src/pthread/pthread_condattr_destroy.cpp +++ b/libc/src/pthread/pthread_condattr_destroy.cpp @@ -14,7 +14,8 @@ namespace LIBC_NAMESPACE { -LLVM_LIBC_FUNCTION(int, pthread_condattr_destroy, (pthread_condattr_t * attr)) { +LLVM_LIBC_FUNCTION(int, pthread_condattr_destroy, + (pthread_condattr_t * attr [[gnu::unused]])) { // Initializing a pthread_condattr_t acquires no resources, so this is a // no-op. return 0; diff --git a/libc/test/src/pthread/CMakeLists.txt b/libc/test/src/pthread/CMakeLists.txt index 46f38422a537..51954a5babd2 100644 --- a/libc/test/src/pthread/CMakeLists.txt +++ b/libc/test/src/pthread/CMakeLists.txt @@ -50,4 +50,10 @@ add_libc_unittest( libc.include.errno libc.include.pthread libc.include.time + libc.src.pthread.pthread_condattr_destroy + libc.src.pthread.pthread_condattr_getclock + libc.src.pthread.pthread_condattr_getpshared + libc.src.pthread.pthread_condattr_init + libc.src.pthread.pthread_condattr_setclock + libc.src.pthread.pthread_condattr_setpshared ) -- GitLab From 825536039d667eeb933c590fe40c358fdea03a8d Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 17 Apr 2024 16:38:31 +0100 Subject: [PATCH 043/714] [CostModel][X86] Add basic GFNI target test coverage for shift/rotate costs --- .../Analysis/CostModel/X86/fshl-codesize.ll | 181 +++++++++++++ .../Analysis/CostModel/X86/fshl-latency.ll | 181 +++++++++++++ .../CostModel/X86/fshl-sizelatency.ll | 241 ++++++++++++++++++ llvm/test/Analysis/CostModel/X86/fshl.ll | 181 +++++++++++++ .../Analysis/CostModel/X86/fshr-codesize.ll | 181 +++++++++++++ .../Analysis/CostModel/X86/fshr-latency.ll | 181 +++++++++++++ .../CostModel/X86/fshr-sizelatency.ll | 241 ++++++++++++++++++ llvm/test/Analysis/CostModel/X86/fshr.ll | 181 +++++++++++++ .../CostModel/X86/vshift-ashr-codesize.ll | 85 ++++++ .../X86/vshift-ashr-cost-inseltpoison.ll | 87 +++++++ .../CostModel/X86/vshift-ashr-cost.ll | 87 +++++++ .../CostModel/X86/vshift-ashr-latency.ll | 93 +++++++ .../CostModel/X86/vshift-ashr-sizelatency.ll | 93 +++++++ .../CostModel/X86/vshift-lshr-codesize.ll | 65 +++++ .../X86/vshift-lshr-cost-inseltpoison.ll | 77 ++++++ .../CostModel/X86/vshift-lshr-cost.ll | 77 ++++++ .../CostModel/X86/vshift-lshr-latency.ll | 75 ++++++ .../CostModel/X86/vshift-lshr-sizelatency.ll | 77 ++++++ .../CostModel/X86/vshift-shl-codesize.ll | 57 +++++ .../X86/vshift-shl-cost-inseltpoison.ll | 81 ++++++ .../Analysis/CostModel/X86/vshift-shl-cost.ll | 81 ++++++ .../CostModel/X86/vshift-shl-latency.ll | 61 +++++ .../CostModel/X86/vshift-shl-sizelatency.ll | 73 ++++++ 23 files changed, 2737 insertions(+) diff --git a/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll b/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll index c212ecfc3eb9..c46e32ffb4ad 100644 --- a/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll +++ b/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll @@ -12,6 +12,7 @@ ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=code-size -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=GLM ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=code-size -mtriple=x86_64-apple-macosx10.8.0 -mcpu=bdver2 | FileCheck %s --check-prefixes=XOP ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=code-size -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=code-size -mtriple=x86_64-apple-macosx10.8.0 -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512,AVX512GFNI target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" @@ -97,6 +98,13 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) @@ -182,6 +190,13 @@ define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) @@ -267,6 +282,13 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) @@ -352,6 +374,13 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) @@ -463,6 +492,15 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -572,6 +610,15 @@ define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -681,6 +728,15 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -790,6 +846,15 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -881,6 +946,13 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -966,6 +1038,13 @@ define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 7) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1051,6 +1130,13 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1136,6 +1222,13 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1225,6 +1318,13 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -1310,6 +1410,13 @@ define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1395,6 +1502,13 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1480,6 +1594,13 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1690,6 +1811,13 @@ define void @var_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) @@ -1775,6 +1903,13 @@ define void @var_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) @@ -2023,6 +2158,15 @@ define void @splatvar_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -2123,6 +2267,15 @@ define void @splatvar_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -2335,6 +2488,13 @@ define void @constant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 7) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2420,6 +2580,13 @@ define void @constant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 7) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) @@ -2616,6 +2783,13 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2694,6 +2868,13 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) diff --git a/llvm/test/Analysis/CostModel/X86/fshl-latency.ll b/llvm/test/Analysis/CostModel/X86/fshl-latency.ll index 487adfe79d94..fa32497c63ec 100644 --- a/llvm/test/Analysis/CostModel/X86/fshl-latency.ll +++ b/llvm/test/Analysis/CostModel/X86/fshl-latency.ll @@ -12,6 +12,7 @@ ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=GLM ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=bdver2 | FileCheck %s --check-prefixes=XOP ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512,AVX512GFNI target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" @@ -97,6 +98,13 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) @@ -182,6 +190,13 @@ define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) @@ -267,6 +282,13 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) @@ -352,6 +374,13 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 58 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 51 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) @@ -463,6 +492,15 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -563,6 +601,15 @@ define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -663,6 +710,15 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -763,6 +819,15 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -854,6 +919,13 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -939,6 +1011,13 @@ define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 7) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1024,6 +1103,13 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1109,6 +1195,13 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 57 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1198,6 +1291,13 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -1276,6 +1376,13 @@ define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1354,6 +1461,13 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1432,6 +1546,13 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1642,6 +1763,13 @@ define void @var_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) @@ -1727,6 +1855,13 @@ define void @var_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) @@ -1975,6 +2110,15 @@ define void @splatvar_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -2075,6 +2219,15 @@ define void @splatvar_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -2287,6 +2440,13 @@ define void @constant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 7) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2372,6 +2532,13 @@ define void @constant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 7) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) @@ -2568,6 +2735,13 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2646,6 +2820,13 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) diff --git a/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll index 52ad7e13c84c..832a574a9b33 100644 --- a/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll +++ b/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll @@ -12,6 +12,7 @@ ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=GLM ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=bdver2 | FileCheck %s --check-prefixes=XOP ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512GFNI target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" @@ -97,6 +98,13 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) @@ -182,6 +190,13 @@ define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) @@ -267,6 +282,13 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 62 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) @@ -352,6 +374,13 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 62 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 39 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) @@ -463,6 +492,15 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -572,6 +610,15 @@ define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -681,6 +728,15 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -790,6 +846,15 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 62 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -881,6 +946,13 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -966,6 +1038,13 @@ define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 7) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1051,6 +1130,13 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 58 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1136,6 +1222,13 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 58 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 37 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1225,6 +1318,13 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 26 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 52 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -1310,6 +1410,13 @@ define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 26 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 52 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1395,6 +1502,13 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 54 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1480,6 +1594,13 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 58 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1562,6 +1683,13 @@ define void @var_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %a64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %a64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> %c128) @@ -1647,6 +1775,13 @@ define void @var_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %a32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %a32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> %c128) @@ -1732,6 +1867,13 @@ define void @var_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) @@ -1817,6 +1959,13 @@ define void @var_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) @@ -1919,6 +2068,15 @@ define void @splatvar_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -2019,6 +2177,15 @@ define void @splatvar_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -2119,6 +2286,15 @@ define void @splatvar_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -2219,6 +2395,15 @@ define void @splatvar_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -2303,6 +2488,13 @@ define void @constant_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %a64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %a64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) @@ -2388,6 +2580,13 @@ define void @constant_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %a32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %a32, i32 7) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) @@ -2473,6 +2672,13 @@ define void @constant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 7) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2558,6 +2764,13 @@ define void @constant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 7) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) @@ -2640,6 +2853,13 @@ define void @splatconstant_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %a64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %a64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) @@ -2718,6 +2938,13 @@ define void @splatconstant_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %a32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %a32, i32 5) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) @@ -2796,6 +3023,13 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2874,6 +3108,13 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) diff --git a/llvm/test/Analysis/CostModel/X86/fshl.ll b/llvm/test/Analysis/CostModel/X86/fshl.ll index 4e688c29c7ea..311d8d5ed7d2 100644 --- a/llvm/test/Analysis/CostModel/X86/fshl.ll +++ b/llvm/test/Analysis/CostModel/X86/fshl.ll @@ -12,6 +12,7 @@ ; RUN: opt < %s -passes="print" 2>&1 -disable-output -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=GLM ; RUN: opt < %s -passes="print" 2>&1 -disable-output -mtriple=x86_64-apple-macosx10.8.0 -mcpu=bdver2 | FileCheck %s --check-prefixes=XOP ; RUN: opt < %s -passes="print" 2>&1 -disable-output -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -mtriple=x86_64-apple-macosx10.8.0 -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512,AVX512GFNI target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" @@ -97,6 +98,13 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) @@ -182,6 +190,13 @@ define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) @@ -267,6 +282,13 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) @@ -352,6 +374,13 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) @@ -463,6 +492,15 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -563,6 +601,15 @@ define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -663,6 +710,15 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -763,6 +819,15 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -854,6 +919,13 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -932,6 +1004,13 @@ define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 7) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1017,6 +1096,13 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1102,6 +1188,13 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1191,6 +1284,13 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -1269,6 +1369,13 @@ define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5) %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1347,6 +1454,13 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1425,6 +1539,13 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1635,6 +1756,13 @@ define void @var_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) @@ -1720,6 +1848,13 @@ define void @var_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) @@ -1968,6 +2103,15 @@ define void @splatvar_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -2068,6 +2212,15 @@ define void @splatvar_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -2273,6 +2426,13 @@ define void @constant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 7) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2358,6 +2518,13 @@ define void @constant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 7) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) @@ -2554,6 +2721,13 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2632,6 +2806,13 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) diff --git a/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll b/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll index 0e76246fad85..f9d30e4ced3e 100644 --- a/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll +++ b/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll @@ -12,6 +12,7 @@ ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=code-size -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=GLM ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=code-size -mtriple=x86_64-apple-macosx10.8.0 -mcpu=bdver2 | FileCheck %s --check-prefixes=XOP ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=code-size -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=code-size -mtriple=x86_64-apple-macosx10.8.0 -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512,AVX512GFNI target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" @@ -97,6 +98,13 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) @@ -182,6 +190,13 @@ define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) @@ -267,6 +282,13 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) @@ -352,6 +374,13 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) @@ -463,6 +492,15 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 42 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -572,6 +610,15 @@ define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -681,6 +728,15 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -790,6 +846,15 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -881,6 +946,13 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -966,6 +1038,13 @@ define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 7) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1051,6 +1130,13 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1136,6 +1222,13 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1225,6 +1318,13 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -1310,6 +1410,13 @@ define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1395,6 +1502,13 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1480,6 +1594,13 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 24 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1690,6 +1811,13 @@ define void @var_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) @@ -1775,6 +1903,13 @@ define void @var_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) @@ -2023,6 +2158,15 @@ define void @splatvar_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -2123,6 +2267,15 @@ define void @splatvar_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -2335,6 +2488,13 @@ define void @constant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 7) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2420,6 +2580,13 @@ define void @constant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 7) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) @@ -2616,6 +2783,13 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2694,6 +2868,13 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) diff --git a/llvm/test/Analysis/CostModel/X86/fshr-latency.ll b/llvm/test/Analysis/CostModel/X86/fshr-latency.ll index 73e32dc92a69..ed2227591847 100644 --- a/llvm/test/Analysis/CostModel/X86/fshr-latency.ll +++ b/llvm/test/Analysis/CostModel/X86/fshr-latency.ll @@ -12,6 +12,7 @@ ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=GLM ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=bdver2 | FileCheck %s --check-prefixes=XOP ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512,AVX512GFNI target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" @@ -97,6 +98,13 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) @@ -182,6 +190,13 @@ define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) @@ -267,6 +282,13 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) @@ -352,6 +374,13 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 21 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 58 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 51 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) @@ -463,6 +492,15 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -563,6 +601,15 @@ define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -663,6 +710,15 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -763,6 +819,15 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -854,6 +919,13 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -939,6 +1011,13 @@ define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 7) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1024,6 +1103,13 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1109,6 +1195,13 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 57 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1198,6 +1291,13 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -1276,6 +1376,13 @@ define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1354,6 +1461,13 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1432,6 +1546,13 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1642,6 +1763,13 @@ define void @var_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) @@ -1727,6 +1855,13 @@ define void @var_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) @@ -1975,6 +2110,15 @@ define void @splatvar_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -2075,6 +2219,15 @@ define void @splatvar_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -2287,6 +2440,13 @@ define void @constant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 7) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2372,6 +2532,13 @@ define void @constant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 7) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) @@ -2568,6 +2735,13 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2646,6 +2820,13 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) diff --git a/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll index 2d106c6dd306..8931781f70bd 100644 --- a/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll +++ b/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll @@ -12,6 +12,7 @@ ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=GLM ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=bdver2 | FileCheck %s --check-prefixes=XOP ; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mtriple=x86_64-apple-macosx10.8.0 -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512GFNI target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" @@ -97,6 +98,13 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) @@ -182,6 +190,13 @@ define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) @@ -267,6 +282,13 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 62 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) @@ -352,6 +374,13 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 62 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 39 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) @@ -463,6 +492,15 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -572,6 +610,15 @@ define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -681,6 +728,15 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 66 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -790,6 +846,15 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 62 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -881,6 +946,13 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -966,6 +1038,13 @@ define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 7) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1051,6 +1130,13 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 58 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1136,6 +1222,13 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 58 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 37 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1225,6 +1318,13 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 26 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 52 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -1310,6 +1410,13 @@ define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 26 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 52 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1395,6 +1502,13 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 54 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1480,6 +1594,13 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 58 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1562,6 +1683,13 @@ define void @var_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %a64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %a64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> %c128) @@ -1647,6 +1775,13 @@ define void @var_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %a32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %a32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> %c128) @@ -1732,6 +1867,13 @@ define void @var_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) @@ -1817,6 +1959,13 @@ define void @var_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) @@ -1919,6 +2068,15 @@ define void @splatvar_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -2019,6 +2177,15 @@ define void @splatvar_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -2119,6 +2286,15 @@ define void @splatvar_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -2219,6 +2395,15 @@ define void @splatvar_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -2303,6 +2488,13 @@ define void @constant_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %a64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %a64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) @@ -2388,6 +2580,13 @@ define void @constant_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %a32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %a32, i32 7) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) @@ -2473,6 +2672,13 @@ define void @constant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 7) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2558,6 +2764,13 @@ define void @constant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 7) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) @@ -2640,6 +2853,13 @@ define void @splatconstant_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %a64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %a64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) @@ -2718,6 +2938,13 @@ define void @splatconstant_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %a32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %a32, i32 5) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) @@ -2796,6 +3023,13 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2874,6 +3108,13 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) diff --git a/llvm/test/Analysis/CostModel/X86/fshr.ll b/llvm/test/Analysis/CostModel/X86/fshr.ll index 9565630677d5..ca9ddcc52938 100644 --- a/llvm/test/Analysis/CostModel/X86/fshr.ll +++ b/llvm/test/Analysis/CostModel/X86/fshr.ll @@ -12,6 +12,7 @@ ; RUN: opt < %s -passes="print" 2>&1 -disable-output -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=GLM ; RUN: opt < %s -passes="print" 2>&1 -disable-output -mtriple=x86_64-apple-macosx10.8.0 -mcpu=bdver2 | FileCheck %s --check-prefixes=XOP ; RUN: opt < %s -passes="print" 2>&1 -disable-output -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -passes="print" 2>&1 -disable-output -mtriple=x86_64-apple-macosx10.8.0 -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512,AVX512GFNI target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" @@ -97,6 +98,13 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128) @@ -182,6 +190,13 @@ define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3 ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 %c32) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 %c32) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128) @@ -267,6 +282,13 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128) @@ -352,6 +374,13 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128) @@ -463,6 +492,15 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer @@ -563,6 +601,15 @@ define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer @@ -663,6 +710,15 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 19 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 38 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -763,6 +819,15 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -854,6 +919,13 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -932,6 +1004,13 @@ define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 7) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1017,6 +1096,13 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1102,6 +1188,13 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1191,6 +1284,13 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i64' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7) %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> ) @@ -1269,6 +1369,13 @@ define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i32' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5) %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> ) @@ -1347,6 +1454,13 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> ) @@ -1425,6 +1539,13 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_funnel_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> ) @@ -1635,6 +1756,13 @@ define void @var_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 %c16) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 %c16) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %c128) @@ -1720,6 +1848,13 @@ define void @var_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> % ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'var_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 %c8) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %c256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %c512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 %c8) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %c128) @@ -1968,6 +2103,15 @@ define void @splatvar_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer @@ -2068,6 +2212,15 @@ define void @splatvar_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatvar_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> %u128) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> %u256) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> %u512) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer @@ -2273,6 +2426,13 @@ define void @constant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3 ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 7) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2358,6 +2518,13 @@ define void @constant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'constant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 7) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 7) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) @@ -2554,6 +2721,13 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) @@ -2632,6 +2806,13 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; XOP-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) diff --git a/llvm/test/Analysis/CostModel/X86/vshift-ashr-codesize.ll b/llvm/test/Analysis/CostModel/X86/vshift-ashr-codesize.ll index 8e3f38c796e6..a3c24bdd1a88 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-ashr-codesize.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-ashr-codesize.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -232,6 +233,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, %b ret <8 x i16> %shift @@ -265,6 +270,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, %b ret <16 x i16> %shift @@ -298,6 +307,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, %b ret <32 x i16> %shift @@ -327,6 +340,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, %b ret <16 x i8> %shift @@ -389,6 +406,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <64 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, %b ret <64 x i8> %shift @@ -778,6 +799,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = ashr <32 x i16> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = ashr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -827,6 +854,12 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = ashr <16 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <16 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = ashr <16 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %insert = insertelement <16 x i8> undef, i8 %b, i32 0 %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer @@ -882,6 +915,12 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = ashr <32 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = ashr <32 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %insert = insertelement <32 x i8> undef, i8 %b, i32 0 %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer @@ -937,6 +976,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = ashr <64 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = ashr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1158,6 +1203,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, ret <8 x i16> %shift @@ -1191,6 +1240,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1224,6 +1277,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1253,6 +1310,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1315,6 +1376,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift @@ -1531,6 +1596,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1568,6 +1637,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1601,6 +1674,10 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1634,6 +1711,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <32 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, ret <32 x i8> %shift @@ -1667,6 +1748,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-ashr-cost-inseltpoison.ll b/llvm/test/Analysis/CostModel/X86/vshift-ashr-cost-inseltpoison.ll index d4ece3b2a113..67679f2cb856 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-ashr-cost-inseltpoison.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-ashr-cost-inseltpoison.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector arithmetic shift right instructions. @@ -228,6 +229,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v8i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, %b ret <8 x i16> %shift @@ -269,6 +274,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, %b ret <16 x i16> %shift @@ -310,6 +319,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, %b ret <32 x i16> %shift @@ -351,6 +364,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, %b ret <16 x i8> %shift @@ -392,6 +409,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <32 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, %b ret <32 x i8> %shift @@ -433,6 +454,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, %b ret <64 x i8> %shift @@ -834,6 +859,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> poison, <32 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = ashr <32 x i16> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> poison, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> poison, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = ashr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> poison, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> poison, <32 x i32> zeroinitializer @@ -950,6 +981,12 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, i8 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> poison, <32 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <32 x i8> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i8> poison, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> poison, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <32 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %insert = insertelement <32 x i8> poison, i8 %b, i32 0 %splat = shufflevector <32 x i8> %insert, <32 x i8> poison, <32 x i32> zeroinitializer @@ -1017,6 +1054,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> poison, <64 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <64 x i8> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> poison, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> poison, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> poison, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> poison, <64 x i32> zeroinitializer @@ -1226,6 +1269,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v8i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, ret <8 x i16> %shift @@ -1267,6 +1314,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1308,6 +1359,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1349,6 +1404,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1390,6 +1449,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, ret <32 x i8> %shift @@ -1431,6 +1494,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift @@ -1631,6 +1698,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1672,6 +1743,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1713,6 +1788,10 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1754,6 +1833,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, ret <32 x i8> %shift @@ -1795,6 +1878,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-ashr-cost.ll b/llvm/test/Analysis/CostModel/X86/vshift-ashr-cost.ll index f8fb2d76a778..efd378ee0b5a 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-ashr-cost.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-ashr-cost.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector arithmetic shift right instructions. @@ -228,6 +229,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v8i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, %b ret <8 x i16> %shift @@ -269,6 +274,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, %b ret <16 x i16> %shift @@ -310,6 +319,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, %b ret <32 x i16> %shift @@ -351,6 +364,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, %b ret <16 x i8> %shift @@ -392,6 +409,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <32 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, %b ret <32 x i8> %shift @@ -433,6 +454,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, %b ret <64 x i8> %shift @@ -834,6 +859,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = ashr <32 x i16> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = ashr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -950,6 +981,12 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, i8 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <32 x i8> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <32 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %insert = insertelement <32 x i8> undef, i8 %b, i32 0 %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer @@ -1017,6 +1054,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <64 x i8> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1226,6 +1269,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v8i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, ret <8 x i16> %shift @@ -1267,6 +1314,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1308,6 +1359,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1349,6 +1404,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1390,6 +1449,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, ret <32 x i8> %shift @@ -1431,6 +1494,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift @@ -1631,6 +1698,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1672,6 +1743,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1713,6 +1788,10 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1754,6 +1833,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, ret <32 x i8> %shift @@ -1795,6 +1878,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-ashr-latency.ll b/llvm/test/Analysis/CostModel/X86/vshift-ashr-latency.ll index af5278e355cf..cd4189d4a7f8 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-ashr-latency.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-ashr-latency.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=slm | FileCheck %s --check-prefixes=SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=goldmont | FileCheck %s --check-prefixes=SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=btver2 | FileCheck %s --check-prefixes=AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -232,6 +233,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, %b ret <8 x i16> %shift @@ -265,6 +270,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, %b ret <16 x i16> %shift @@ -298,6 +307,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, %b ret <32 x i16> %shift @@ -331,6 +344,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = ashr <16 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = ashr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, %b ret <16 x i8> %shift @@ -364,6 +381,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = ashr <32 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = ashr <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, %b ret <32 x i8> %shift @@ -397,6 +418,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, %b ret <64 x i8> %shift @@ -834,6 +859,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <32 x i16> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = ashr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -889,6 +920,12 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %shift = ashr <16 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <16 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %shift = ashr <16 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %insert = insertelement <16 x i8> undef, i8 %b, i32 0 %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer @@ -944,6 +981,12 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = ashr <32 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = ashr <32 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %insert = insertelement <32 x i8> undef, i8 %b, i32 0 %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer @@ -999,6 +1042,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = ashr <64 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = ashr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1220,6 +1269,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, ret <8 x i16> %shift @@ -1253,6 +1306,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1286,6 +1343,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1319,6 +1380,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1352,6 +1417,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = ashr <32 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = ashr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, ret <32 x i8> %shift @@ -1385,6 +1454,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift @@ -1649,6 +1722,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1686,6 +1763,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1723,6 +1804,10 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1760,6 +1845,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %shift = ashr <32 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %shift = ashr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, ret <32 x i8> %shift @@ -1797,6 +1886,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-ashr-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/vshift-ashr-sizelatency.ll index 8286b3967af6..84ccad029415 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-ashr-sizelatency.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-ashr-sizelatency.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -232,6 +233,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, %b ret <8 x i16> %shift @@ -265,6 +270,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, %b ret <16 x i16> %shift @@ -298,6 +307,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, %b ret <32 x i16> %shift @@ -331,6 +344,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <16 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, %b ret <16 x i8> %shift @@ -364,6 +381,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <32 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, %b ret <32 x i8> %shift @@ -397,6 +418,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <64 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, %b ret <64 x i8> %shift @@ -786,6 +811,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = ashr <32 x i16> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = ashr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -835,6 +866,12 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %shift = ashr <16 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <16 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %shift = ashr <16 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %insert = insertelement <16 x i8> undef, i8 %b, i32 0 %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer @@ -890,6 +927,12 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = ashr <32 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = ashr <32 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %insert = insertelement <32 x i8> undef, i8 %b, i32 0 %splat = shufflevector <32 x i8> %insert, <32 x i8> undef, <32 x i32> zeroinitializer @@ -945,6 +988,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = ashr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1166,6 +1215,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = ashr <8 x i16> %a, ret <8 x i16> %shift @@ -1199,6 +1252,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1232,6 +1289,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1265,6 +1326,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1298,6 +1363,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <32 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, ret <32 x i8> %shift @@ -1331,6 +1400,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift @@ -1547,6 +1620,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = ashr <16 x i16> %a, ret <16 x i16> %shift @@ -1584,6 +1661,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = ashr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = ashr <32 x i16> %a, ret <32 x i16> %shift @@ -1617,6 +1698,10 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = ashr <16 x i8> %a, ret <16 x i8> %shift @@ -1654,6 +1739,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <32 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = ashr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = ashr <32 x i8> %a, ret <32 x i8> %shift @@ -1691,6 +1780,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = ashr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = ashr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-lshr-codesize.ll b/llvm/test/Analysis/CostModel/X86/vshift-lshr-codesize.ll index 7c506af62e81..a0e15bb8ff73 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-lshr-codesize.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-lshr-codesize.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -240,6 +241,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, %b ret <8 x i16> %shift @@ -273,6 +278,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, %b ret <16 x i16> %shift @@ -306,6 +315,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, %b ret <32 x i16> %shift @@ -335,6 +348,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, %b ret <16 x i8> %shift @@ -397,6 +414,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = lshr <64 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = lshr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, %b ret <64 x i8> %shift @@ -762,6 +783,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <32 x i16> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -909,6 +936,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1138,6 +1171,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, ret <8 x i16> %shift @@ -1171,6 +1208,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1204,6 +1245,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1233,6 +1278,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, ret <16 x i8> %shift @@ -1295,6 +1344,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift @@ -1495,6 +1548,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1532,6 +1589,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1631,6 +1692,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-lshr-cost-inseltpoison.ll b/llvm/test/Analysis/CostModel/X86/vshift-lshr-cost-inseltpoison.ll index 855d2b8e88e3..34ea1a644f6c 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-lshr-cost-inseltpoison.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-lshr-cost-inseltpoison.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -232,6 +233,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v8i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, %b ret <8 x i16> %shift @@ -273,6 +278,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, %b ret <16 x i16> %shift @@ -314,6 +323,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, %b ret <32 x i16> %shift @@ -355,6 +368,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, %b ret <16 x i8> %shift @@ -396,6 +413,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = lshr <32 x i8> %a, %b ret <32 x i8> %shift @@ -437,6 +458,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, %b ret <64 x i8> %shift @@ -826,6 +851,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> poison, <32 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <32 x i16> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> poison, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> poison, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> poison, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> poison, <32 x i32> zeroinitializer @@ -991,6 +1022,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> poison, <64 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <64 x i8> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> poison, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> poison, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> poison, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> poison, <64 x i32> zeroinitializer @@ -1200,6 +1237,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v8i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, ret <8 x i16> %shift @@ -1241,6 +1282,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1282,6 +1327,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1323,6 +1372,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, ret <16 x i8> %shift @@ -1364,6 +1417,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = lshr <32 x i8> %a, ret <32 x i8> %shift @@ -1405,6 +1462,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift @@ -1593,6 +1654,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1634,6 +1699,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1700,6 +1769,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = lshr <32 x i8> %a, ret <32 x i8> %shift @@ -1741,6 +1814,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-lshr-cost.ll b/llvm/test/Analysis/CostModel/X86/vshift-lshr-cost.ll index 55894dc93295..93b844eddf18 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-lshr-cost.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-lshr-cost.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -232,6 +233,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v8i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, %b ret <8 x i16> %shift @@ -273,6 +278,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, %b ret <16 x i16> %shift @@ -314,6 +323,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BWVL-LABEL: 'var_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, %b ret <32 x i16> %shift @@ -355,6 +368,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, %b ret <16 x i8> %shift @@ -396,6 +413,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = lshr <32 x i8> %a, %b ret <32 x i8> %shift @@ -437,6 +458,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BWVL-LABEL: 'var_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, %b ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, %b ret <64 x i8> %shift @@ -826,6 +851,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <32 x i16> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = lshr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -991,6 +1022,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <64 x i8> %a, %splat ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1200,6 +1237,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v8i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, ret <8 x i16> %shift @@ -1241,6 +1282,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1282,6 +1327,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1323,6 +1372,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v16i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, ret <16 x i8> %shift @@ -1364,6 +1417,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = lshr <32 x i8> %a, ret <32 x i8> %shift @@ -1405,6 +1462,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'constant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift @@ -1593,6 +1654,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1634,6 +1699,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1700,6 +1769,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = lshr <32 x i8> %a, ret <32 x i8> %shift @@ -1741,6 +1814,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-lshr-latency.ll b/llvm/test/Analysis/CostModel/X86/vshift-lshr-latency.ll index 92fe253d38e7..61620e2cc97e 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-lshr-latency.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-lshr-latency.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=slm | FileCheck %s --check-prefixes=SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=goldmont | FileCheck %s --check-prefixes=SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -240,6 +241,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, %b ret <8 x i16> %shift @@ -277,6 +282,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, %b ret <16 x i16> %shift @@ -314,6 +323,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, %b ret <32 x i16> %shift @@ -351,6 +364,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <16 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, %b ret <16 x i8> %shift @@ -421,6 +438,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %shift = lshr <64 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %shift = lshr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, %b ret <64 x i8> %shift @@ -834,6 +855,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i16> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = lshr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -889,6 +916,12 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = lshr <16 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <16 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = lshr <16 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %insert = insertelement <16 x i8> undef, i8 %b, i32 0 %splat = shufflevector <16 x i8> %insert, <16 x i8> undef, <16 x i32> zeroinitializer @@ -993,6 +1026,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <64 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1222,6 +1261,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, ret <8 x i16> %shift @@ -1259,6 +1302,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1296,6 +1343,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1333,6 +1384,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, ret <16 x i8> %shift @@ -1403,6 +1458,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift @@ -1643,6 +1702,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1676,6 +1739,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1709,6 +1776,10 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = lshr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, ret <16 x i8> %shift @@ -1771,6 +1842,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-lshr-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/vshift-lshr-sizelatency.ll index fe8a7d91cc37..e6b6ac75b65d 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-lshr-sizelatency.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-lshr-sizelatency.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -240,6 +241,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, %b ret <8 x i16> %shift @@ -273,6 +278,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, %b ret <16 x i16> %shift @@ -306,6 +315,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, %b ret <32 x i16> %shift @@ -335,6 +348,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = lshr <16 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = lshr <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, %b ret <16 x i8> %shift @@ -368,6 +385,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %shift = lshr <32 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %shift = lshr <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = lshr <32 x i8> %a, %b ret <32 x i8> %shift @@ -401,6 +422,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %shift = lshr <64 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %shift = lshr <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, %b ret <64 x i8> %shift @@ -766,6 +791,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = lshr <32 x i16> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = lshr <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -913,6 +944,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = lshr <64 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %shift = lshr <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1142,6 +1179,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = lshr <8 x i16> %a, ret <8 x i16> %shift @@ -1175,6 +1216,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1208,6 +1253,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1237,6 +1286,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = lshr <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = lshr <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = lshr <16 x i8> %a, ret <16 x i8> %shift @@ -1270,6 +1323,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %shift = lshr <32 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %shift = lshr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = lshr <32 x i8> %a, ret <32 x i8> %shift @@ -1303,6 +1360,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift @@ -1503,6 +1564,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = lshr <16 x i16> %a, ret <16 x i16> %shift @@ -1540,6 +1605,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = lshr <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = lshr <32 x i16> %a, ret <32 x i16> %shift @@ -1606,6 +1675,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = lshr <32 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = lshr <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = lshr <32 x i8> %a, ret <32 x i8> %shift @@ -1643,6 +1716,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = lshr <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = lshr <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = lshr <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-shl-codesize.ll b/llvm/test/Analysis/CostModel/X86/vshift-shl-codesize.ll index 07229e22873c..265658b1e3a2 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-shl-codesize.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-shl-codesize.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=code-size -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -240,6 +241,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <8 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = shl <8 x i16> %a, %b ret <8 x i16> %shift @@ -273,6 +278,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, %b ret <16 x i16> %shift @@ -306,6 +315,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, %b ret <32 x i16> %shift @@ -335,6 +348,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = shl <16 x i8> %a, %b ret <16 x i8> %shift @@ -397,6 +414,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = shl <64 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = shl <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, %b ret <64 x i8> %shift @@ -762,6 +783,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <32 x i16> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -915,6 +942,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = shl <64 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = shl <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1186,6 +1219,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1215,6 +1252,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = shl <16 x i8> %a, ret <16 x i8> %shift @@ -1277,6 +1318,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = shl <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 13 for instruction: %shift = shl <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, ret <64 x i8> %shift @@ -1477,6 +1522,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, ret <16 x i16> %shift @@ -1514,6 +1563,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1613,6 +1666,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-shl-cost-inseltpoison.ll b/llvm/test/Analysis/CostModel/X86/vshift-shl-cost-inseltpoison.ll index d8737d236564..f093d1c8ca35 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-shl-cost-inseltpoison.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-shl-cost-inseltpoison.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SLM ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector shift left instructions. @@ -236,6 +237,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; SLM-LABEL: 'var_shift_v8i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %shift = shl <8 x i16> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = shl <8 x i16> %a, %b ret <8 x i16> %shift @@ -281,6 +286,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; SLM-LABEL: 'var_shift_v16i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %shift = shl <16 x i16> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, %b ret <16 x i16> %shift @@ -326,6 +335,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; SLM-LABEL: 'var_shift_v32i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %shift = shl <32 x i16> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, %b ret <32 x i16> %shift @@ -371,6 +384,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; SLM-LABEL: 'var_shift_v16i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = shl <16 x i8> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = shl <16 x i8> %a, %b ret <16 x i8> %shift @@ -416,6 +433,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; SLM-LABEL: 'var_shift_v32i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = shl <32 x i8> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = shl <32 x i8> %a, %b ret <32 x i8> %shift @@ -461,6 +482,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; SLM-LABEL: 'var_shift_v64i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %shift = shl <64 x i8> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = shl <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, %b ret <64 x i8> %shift @@ -868,6 +893,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> poison, <32 x i32> zeroinitializer ; SLM-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <32 x i16> %a, %splat ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> poison, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> poison, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> poison, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> poison, <32 x i32> zeroinitializer @@ -1051,6 +1082,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> poison, <64 x i32> zeroinitializer ; SLM-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %shift = shl <64 x i8> %a, %splat ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> poison, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> poison, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> poison, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> poison, <64 x i32> zeroinitializer @@ -1305,6 +1342,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; SLM-LABEL: 'constant_shift_v16i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i16> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, ret <16 x i16> %shift @@ -1354,6 +1395,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; SLM-LABEL: 'constant_shift_v32i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <32 x i16> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1399,6 +1444,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; SLM-LABEL: 'constant_shift_v16i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = shl <16 x i8> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = shl <16 x i8> %a, ret <16 x i8> %shift @@ -1444,6 +1493,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; SLM-LABEL: 'constant_shift_v32i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = shl <32 x i8> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = shl <32 x i8> %a, ret <32 x i8> %shift @@ -1489,6 +1542,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; SLM-LABEL: 'constant_shift_v64i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %shift = shl <64 x i8> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = shl <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, ret <64 x i8> %shift @@ -1677,6 +1734,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, ret <16 x i16> %shift @@ -1718,6 +1779,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1784,6 +1849,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = shl <32 x i8> %a, ret <32 x i8> %shift @@ -1825,6 +1894,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, ret <64 x i8> %shift @@ -2044,6 +2117,10 @@ define <16 x i16> @test6(<16 x i16> %a) { ; SLM-LABEL: 'test6' ; SLM-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shl = shl <16 x i16> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shl +; +; AVX512GFNI-LABEL: 'test6' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shl = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shl ; %shl = shl <16 x i16> %a, ret <16 x i16> %shl @@ -2162,6 +2239,10 @@ define <32 x i16> @test9(<32 x i16> %a) { ; SLM-LABEL: 'test9' ; SLM-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shl = shl <32 x i16> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shl +; +; AVX512GFNI-LABEL: 'test9' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shl = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shl ; %shl = shl <32 x i16> %a, ret <32 x i16> %shl diff --git a/llvm/test/Analysis/CostModel/X86/vshift-shl-cost.ll b/llvm/test/Analysis/CostModel/X86/vshift-shl-cost.ll index b254ff27dacc..09521cfa2d23 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-shl-cost.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-shl-cost.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SLM ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector shift left instructions. @@ -236,6 +237,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; SLM-LABEL: 'var_shift_v8i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %shift = shl <8 x i16> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i16> %shift ; %shift = shl <8 x i16> %a, %b ret <8 x i16> %shift @@ -281,6 +286,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; SLM-LABEL: 'var_shift_v16i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %shift = shl <16 x i16> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, %b ret <16 x i16> %shift @@ -326,6 +335,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; SLM-LABEL: 'var_shift_v32i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %shift = shl <32 x i16> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, %b ret <32 x i16> %shift @@ -371,6 +384,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; SLM-LABEL: 'var_shift_v16i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = shl <16 x i8> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = shl <16 x i8> %a, %b ret <16 x i8> %shift @@ -416,6 +433,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; SLM-LABEL: 'var_shift_v32i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = shl <32 x i8> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = shl <32 x i8> %a, %b ret <32 x i8> %shift @@ -461,6 +482,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; SLM-LABEL: 'var_shift_v64i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %shift = shl <64 x i8> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = shl <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, %b ret <64 x i8> %shift @@ -868,6 +893,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; SLM-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <32 x i16> %a, %splat ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %shift = shl <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -1051,6 +1082,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; SLM-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %shift = shl <64 x i8> %a, %splat ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1305,6 +1342,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; SLM-LABEL: 'constant_shift_v16i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i16> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, ret <16 x i16> %shift @@ -1354,6 +1395,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; SLM-LABEL: 'constant_shift_v32i16' ; SLM-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <32 x i16> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1399,6 +1444,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; SLM-LABEL: 'constant_shift_v16i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = shl <16 x i8> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %shift ; %shift = shl <16 x i8> %a, ret <16 x i8> %shift @@ -1444,6 +1493,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; SLM-LABEL: 'constant_shift_v32i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %shift = shl <32 x i8> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = shl <32 x i8> %a, ret <32 x i8> %shift @@ -1489,6 +1542,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; SLM-LABEL: 'constant_shift_v64i8' ; SLM-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %shift = shl <64 x i8> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %shift = shl <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, ret <64 x i8> %shift @@ -1677,6 +1734,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v16i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, ret <16 x i16> %shift @@ -1718,6 +1779,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i16' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1784,6 +1849,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v32i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %shift ; %shift = shl <32 x i8> %a, ret <32 x i8> %shift @@ -1825,6 +1894,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BWVL-LABEL: 'splatconstant_shift_v64i8' ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <64 x i8> %a, ; AVX512BWVL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, ret <64 x i8> %shift @@ -2044,6 +2117,10 @@ define <16 x i16> @test6(<16 x i16> %a) { ; SLM-LABEL: 'test6' ; SLM-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shl = shl <16 x i16> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shl +; +; AVX512GFNI-LABEL: 'test6' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shl = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %shl ; %shl = shl <16 x i16> %a, ret <16 x i16> %shl @@ -2162,6 +2239,10 @@ define <32 x i16> @test9(<32 x i16> %a) { ; SLM-LABEL: 'test9' ; SLM-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shl = shl <32 x i16> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shl +; +; AVX512GFNI-LABEL: 'test9' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shl = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i16> %shl ; %shl = shl <32 x i16> %a, ret <32 x i16> %shl diff --git a/llvm/test/Analysis/CostModel/X86/vshift-shl-latency.ll b/llvm/test/Analysis/CostModel/X86/vshift-shl-latency.ll index c53dbc5f586a..42c91144ff6f 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-shl-latency.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-shl-latency.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=slm | FileCheck %s --check-prefixes=SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=goldmont | FileCheck %s --check-prefixes=SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=latency -mcpu=tigerlake | FileCheck %s --check-prefixes=AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -240,6 +241,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <8 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = shl <8 x i16> %a, %b ret <8 x i16> %shift @@ -277,6 +282,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, %b ret <16 x i16> %shift @@ -314,6 +323,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; AVX512BW-LABEL: 'var_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, %b ret <32 x i16> %shift @@ -351,6 +364,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; AVX512BW-LABEL: 'var_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <16 x i8> %a, %b ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = shl <16 x i8> %a, %b ret <16 x i8> %shift @@ -830,6 +847,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <32 x i16> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %shift = shl <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -983,6 +1006,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = shl <64 x i8> %a, %splat ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = shl <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1208,6 +1237,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v8i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <8 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <8 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; %shift = shl <8 x i16> %a, ret <8 x i16> %shift @@ -1241,6 +1274,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, ret <16 x i16> %shift @@ -1274,6 +1311,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1311,6 +1352,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = shl <16 x i8> %a, ret <16 x i8> %shift @@ -1617,6 +1662,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, ret <16 x i16> %shift @@ -1650,6 +1699,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1683,6 +1736,10 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = shl <16 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %shift = shl <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; %shift = shl <16 x i8> %a, ret <16 x i8> %shift @@ -1745,6 +1802,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, ret <64 x i8> %shift diff --git a/llvm/test/Analysis/CostModel/X86/vshift-shl-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/vshift-shl-sizelatency.ll index cbf0f24b90fb..47b24df063ef 100644 --- a/llvm/test/Analysis/CostModel/X86/vshift-shl-sizelatency.ll +++ b/llvm/test/Analysis/CostModel/X86/vshift-shl-sizelatency.ll @@ -15,6 +15,7 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SLM ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print" 2>&1 -disable-output -cost-kind=size-latency -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512GFNI ; Verify the cost of vector logical shift right instructions. @@ -263,6 +264,10 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %shift = shl <8 x i16> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift ; +; AVX512GFNI-LABEL: 'var_shift_v8i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <8 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift +; ; BTVER2-LABEL: 'var_shift_v8i16' ; BTVER2-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %shift = shl <8 x i16> %a, %b ; BTVER2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %shift @@ -303,6 +308,10 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %shift = shl <16 x i16> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; +; AVX512GFNI-LABEL: 'var_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; ; BTVER2-LABEL: 'var_shift_v16i16' ; BTVER2-NEXT: Cost Model: Found an estimated cost of 25 for instruction: %shift = shl <16 x i16> %a, %b ; BTVER2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift @@ -343,6 +352,10 @@ define <32 x i16> @var_shift_v32i16(<32 x i16> %a, <32 x i16> %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %shift = shl <32 x i16> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; +; AVX512GFNI-LABEL: 'var_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; ; BTVER2-LABEL: 'var_shift_v32i16' ; BTVER2-NEXT: Cost Model: Found an estimated cost of 50 for instruction: %shift = shl <32 x i16> %a, %b ; BTVER2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift @@ -383,6 +396,10 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %shift = shl <16 x i8> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; +; AVX512GFNI-LABEL: 'var_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = shl <16 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; ; BTVER2-LABEL: 'var_shift_v16i8' ; BTVER2-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %shift = shl <16 x i8> %a, %b ; BTVER2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift @@ -423,6 +440,10 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %shift = shl <32 x i8> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; +; AVX512GFNI-LABEL: 'var_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %shift = shl <32 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; ; BTVER2-LABEL: 'var_shift_v32i8' ; BTVER2-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %shift = shl <32 x i8> %a, %b ; BTVER2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift @@ -463,6 +484,10 @@ define <64 x i8> @var_shift_v64i8(<64 x i8> %a, <64 x i8> %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 88 for instruction: %shift = shl <64 x i8> %a, %b ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; +; AVX512GFNI-LABEL: 'var_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = shl <64 x i8> %a, %b +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; ; BTVER2-LABEL: 'var_shift_v64i8' ; BTVER2-NEXT: Cost Model: Found an estimated cost of 80 for instruction: %shift = shl <64 x i8> %a, %b ; BTVER2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift @@ -824,6 +849,12 @@ define <32 x i16> @splatvar_shift_v32i16(<32 x i16> %a, i16 %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer ; SLM-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <32 x i16> %a, %splat ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <32 x i16> undef, i16 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = shl <32 x i16> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %insert = insertelement <32 x i16> undef, i16 %b, i32 0 %splat = shufflevector <32 x i16> %insert, <32 x i16> undef, <32 x i32> zeroinitializer @@ -995,6 +1026,12 @@ define <64 x i8> @splatvar_shift_v64i8(<64 x i8> %a, i8 %b) { ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer ; SLM-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %shift = shl <64 x i8> %a, %splat ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatvar_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %insert = insertelement <64 x i8> undef, i8 %b, i32 0 +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %shift = shl <64 x i8> %a, %splat +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %insert = insertelement <64 x i8> undef, i8 %b, i32 0 %splat = shufflevector <64 x i8> %insert, <64 x i8> undef, <64 x i32> zeroinitializer @@ -1238,6 +1275,10 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, ret <16 x i16> %shift @@ -1271,6 +1312,10 @@ define <32 x i16> @constant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'constant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'constant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1309,6 +1354,10 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) { ; SLM-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %shift = shl <16 x i8> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift ; +; AVX512GFNI-LABEL: 'constant_shift_v16i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %shift = shl <16 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift +; ; BTVER2-LABEL: 'constant_shift_v16i8' ; BTVER2-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %shift = shl <16 x i8> %a, ; BTVER2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %shift @@ -1349,6 +1398,10 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) { ; SLM-NEXT: Cost Model: Found an estimated cost of 44 for instruction: %shift = shl <32 x i8> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; +; AVX512GFNI-LABEL: 'constant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %shift = shl <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; ; BTVER2-LABEL: 'constant_shift_v32i8' ; BTVER2-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %shift = shl <32 x i8> %a, ; BTVER2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift @@ -1389,6 +1442,10 @@ define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) { ; SLM-NEXT: Cost Model: Found an estimated cost of 88 for instruction: %shift = shl <64 x i8> %a, ; SLM-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; +; AVX512GFNI-LABEL: 'constant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %shift = shl <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; ; BTVER2-LABEL: 'constant_shift_v64i8' ; BTVER2-NEXT: Cost Model: Found an estimated cost of 80 for instruction: %shift = shl <64 x i8> %a, ; BTVER2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift @@ -1571,6 +1628,10 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v16i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v16i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <16 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %shift ; %shift = shl <16 x i16> %a, ret <16 x i16> %shift @@ -1604,6 +1665,10 @@ define <32 x i16> @splatconstant_shift_v32i16(<32 x i16> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i16' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %shift = shl <32 x i16> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %shift ; %shift = shl <32 x i16> %a, ret <32 x i16> %shift @@ -1662,6 +1727,10 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v32i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = shl <32 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v32i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = shl <32 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %shift ; %shift = shl <32 x i8> %a, ret <32 x i8> %shift @@ -1695,6 +1764,10 @@ define <64 x i8> @splatconstant_shift_v64i8(<64 x i8> %a) { ; AVX512BW-LABEL: 'splatconstant_shift_v64i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = shl <64 x i8> %a, ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift +; +; AVX512GFNI-LABEL: 'splatconstant_shift_v64i8' +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %shift = shl <64 x i8> %a, +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %shift ; %shift = shl <64 x i8> %a, ret <64 x i8> %shift -- GitLab From da04e4afd3cae13581cac85688fbf10a5848655f Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Wed, 17 Apr 2024 12:05:26 -0500 Subject: [PATCH 044/714] [InstCombine] Use `auto *` instead of `auto` in `visitSIToFP`; NFC --- llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp index d242d3f443de..4537a47da2ce 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp @@ -1990,7 +1990,7 @@ Instruction *InstCombinerImpl::visitSIToFP(CastInst &CI) { if (Instruction *R = commonCastTransforms(CI)) return R; if (isKnownNonNegative(CI.getOperand(0), SQ)) { - auto UI = + auto *UI = CastInst::Create(Instruction::UIToFP, CI.getOperand(0), CI.getType()); UI->setNonNeg(true); return UI; -- GitLab From d423d80e560d8bf7ca493596d9f34a9e1f0eede7 Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Wed, 17 Apr 2024 13:36:53 -0400 Subject: [PATCH 045/714] [libc++][pstl] Promote CPU backends to top-level backends (#88968) This patch removes the two-level backend dispatching mechanism we had in the PSTL. Instead of selecting both a PSTL backend and a PSTL CPU backend, we now only select a top-level PSTL backend. This greatly simplifies the PSTL configuration layer. While this patch technically removes some flexibility from the PSTL configuration mechanism because CPU backends are not considered separately, it opens the door to a much more powerful configuration mechanism based on chained backends in a follow-up patch. This is a step towards overhauling the PSTL dispatching mechanism. --- libcxx/CMakeLists.txt | 18 +++---- libcxx/cmake/caches/Apple.cmake | 2 +- libcxx/include/CMakeLists.txt | 11 ++--- .../__algorithm/pstl_backends/cpu_backend.h | 23 --------- .../pstl_backends/cpu_backends/any_of.h | 2 +- .../pstl_backends/cpu_backends/backend.h | 45 ------------------ .../pstl_backends/cpu_backends/fill.h | 2 +- .../pstl_backends/cpu_backends/find_if.h | 2 +- .../pstl_backends/cpu_backends/for_each.h | 2 +- .../pstl_backends/cpu_backends/merge.h | 2 +- .../pstl_backends/cpu_backends/stable_sort.h | 2 +- .../pstl_backends/cpu_backends/transform.h | 2 +- .../cpu_backends/transform_reduce.h | 2 +- libcxx/include/__algorithm/pstl_copy.h | 2 +- libcxx/include/__algorithm/pstl_count.h | 2 +- libcxx/include/__algorithm/pstl_find.h | 2 +- libcxx/include/__algorithm/pstl_for_each.h | 2 +- libcxx/include/__algorithm/pstl_generate.h | 2 +- .../include/__algorithm/pstl_is_partitioned.h | 2 +- libcxx/include/__algorithm/pstl_merge.h | 2 +- libcxx/include/__algorithm/pstl_move.h | 2 +- libcxx/include/__algorithm/pstl_replace.h | 2 +- libcxx/include/__algorithm/pstl_rotate_copy.h | 2 +- libcxx/include/__algorithm/pstl_sort.h | 2 +- libcxx/include/__algorithm/pstl_stable_sort.h | 2 +- libcxx/include/__algorithm/pstl_transform.h | 2 +- libcxx/include/__config_site.in | 6 +-- .../include/__numeric/pstl_transform_reduce.h | 2 +- .../backends}/libdispatch.h | 19 ++++++-- .../cpu_backends => __pstl/backends}/serial.h | 19 ++++++-- .../thread.h => __pstl/backends/std_thread.h} | 19 ++++++-- libcxx/include/__pstl/configuration.h | 27 +++++++++++ .../configuration_fwd.h} | 25 +++++++--- libcxx/include/libcxx.imp | 6 --- libcxx/include/module.modulemap | 47 ++++++++----------- libcxx/src/CMakeLists.txt | 2 +- libcxx/src/pstl/libdispatch.cpp | 2 +- ...pstl.libdispatch.chunk_partitions.pass.cpp | 4 +- .../apple/system-install-properties.sh.cpp | 2 +- libcxx/utils/libcxx/test/features.py | 2 +- 40 files changed, 154 insertions(+), 169 deletions(-) delete mode 100644 libcxx/include/__algorithm/pstl_backends/cpu_backend.h delete mode 100644 libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h rename libcxx/include/{__algorithm/pstl_backends/cpu_backends => __pstl/backends}/libdispatch.h (94%) rename libcxx/include/{__algorithm/pstl_backends/cpu_backends => __pstl/backends}/serial.h (78%) rename libcxx/include/{__algorithm/pstl_backends/cpu_backends/thread.h => __pstl/backends/std_thread.h} (79%) create mode 100644 libcxx/include/__pstl/configuration.h rename libcxx/include/{__algorithm/pstl_backend.h => __pstl/configuration_fwd.h} (93%) diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt index 043d5a8295c1..2977c26646cb 100644 --- a/libcxx/CMakeLists.txt +++ b/libcxx/CMakeLists.txt @@ -300,9 +300,9 @@ option(LIBCXX_HAS_EXTERNAL_THREAD_API This option may only be set to ON when LIBCXX_ENABLE_THREADS=ON." OFF) if (LIBCXX_ENABLE_THREADS) - set(LIBCXX_PSTL_CPU_BACKEND "std_thread" CACHE STRING "Which PSTL CPU backend to use") + set(LIBCXX_PSTL_BACKEND "std_thread" CACHE STRING "Which PSTL backend to use") else() - set(LIBCXX_PSTL_CPU_BACKEND "serial" CACHE STRING "Which PSTL CPU backend to use") + set(LIBCXX_PSTL_BACKEND "serial" CACHE STRING "Which PSTL backend to use") endif() # Misc options ---------------------------------------------------------------- @@ -792,14 +792,14 @@ elseif (LIBCXX_HARDENING_MODE STREQUAL "debug") config_define(8 _LIBCPP_HARDENING_MODE_DEFAULT) endif() -if (LIBCXX_PSTL_CPU_BACKEND STREQUAL "serial") - config_define(1 _LIBCPP_PSTL_CPU_BACKEND_SERIAL) -elseif(LIBCXX_PSTL_CPU_BACKEND STREQUAL "std_thread") - config_define(1 _LIBCPP_PSTL_CPU_BACKEND_THREAD) -elseif(LIBCXX_PSTL_CPU_BACKEND STREQUAL "libdispatch") - config_define(1 _LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH) +if (LIBCXX_PSTL_BACKEND STREQUAL "serial") + config_define(1 _LIBCPP_PSTL_BACKEND_SERIAL) +elseif(LIBCXX_PSTL_BACKEND STREQUAL "std_thread") + config_define(1 _LIBCPP_PSTL_BACKEND_STD_THREAD) +elseif(LIBCXX_PSTL_BACKEND STREQUAL "libdispatch") + config_define(1 _LIBCPP_PSTL_BACKEND_LIBDISPATCH) else() - message(FATAL_ERROR "LIBCXX_PSTL_CPU_BACKEND is set to ${LIBCXX_PSTL_CPU_BACKEND}, which is not a valid backend. + message(FATAL_ERROR "LIBCXX_PSTL_BACKEND is set to ${LIBCXX_PSTL_BACKEND}, which is not a valid backend. Valid backends are: serial, std_thread and libdispatch") endif() diff --git a/libcxx/cmake/caches/Apple.cmake b/libcxx/cmake/caches/Apple.cmake index cec13c08acf1..8768653e620a 100644 --- a/libcxx/cmake/caches/Apple.cmake +++ b/libcxx/cmake/caches/Apple.cmake @@ -7,7 +7,7 @@ set(LIBCXX_ENABLE_STATIC ON CACHE BOOL "") set(LIBCXX_ENABLE_SHARED ON CACHE BOOL "") set(LIBCXX_CXX_ABI libcxxabi CACHE STRING "") set(LIBCXX_ENABLE_VENDOR_AVAILABILITY_ANNOTATIONS ON CACHE BOOL "") -set(LIBCXX_PSTL_CPU_BACKEND libdispatch CACHE STRING "") +set(LIBCXX_PSTL_BACKEND libdispatch CACHE STRING "") set(LIBCXX_HERMETIC_STATIC_LIBRARY ON CACHE BOOL "") set(LIBCXXABI_HERMETIC_STATIC_LIBRARY ON CACHE BOOL "") diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt index a2af1d9915be..ee4979bfc6f8 100644 --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -73,18 +73,12 @@ set(files __algorithm/pop_heap.h __algorithm/prev_permutation.h __algorithm/pstl_any_all_none_of.h - __algorithm/pstl_backend.h - __algorithm/pstl_backends/cpu_backend.h __algorithm/pstl_backends/cpu_backends/any_of.h - __algorithm/pstl_backends/cpu_backends/backend.h __algorithm/pstl_backends/cpu_backends/fill.h __algorithm/pstl_backends/cpu_backends/find_if.h __algorithm/pstl_backends/cpu_backends/for_each.h - __algorithm/pstl_backends/cpu_backends/libdispatch.h __algorithm/pstl_backends/cpu_backends/merge.h - __algorithm/pstl_backends/cpu_backends/serial.h __algorithm/pstl_backends/cpu_backends/stable_sort.h - __algorithm/pstl_backends/cpu_backends/thread.h __algorithm/pstl_backends/cpu_backends/transform.h __algorithm/pstl_backends/cpu_backends/transform_reduce.h __algorithm/pstl_copy.h @@ -594,6 +588,11 @@ set(files __numeric/transform_exclusive_scan.h __numeric/transform_inclusive_scan.h __numeric/transform_reduce.h + __pstl/backends/libdispatch.h + __pstl/backends/serial.h + __pstl/backends/std_thread.h + __pstl/configuration.h + __pstl/configuration_fwd.h __pstl/cpu_algos/cpu_traits.h __random/bernoulli_distribution.h __random/binomial_distribution.h diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backend.h b/libcxx/include/__algorithm/pstl_backends/cpu_backend.h deleted file mode 100644 index 53eae58f9609..000000000000 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backend.h +++ /dev/null @@ -1,23 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_H -#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_H - -#include <__algorithm/pstl_backends/cpu_backends/any_of.h> -#include <__algorithm/pstl_backends/cpu_backends/backend.h> -#include <__algorithm/pstl_backends/cpu_backends/fill.h> -#include <__algorithm/pstl_backends/cpu_backends/find_if.h> -#include <__algorithm/pstl_backends/cpu_backends/for_each.h> -#include <__algorithm/pstl_backends/cpu_backends/merge.h> -#include <__algorithm/pstl_backends/cpu_backends/stable_sort.h> -#include <__algorithm/pstl_backends/cpu_backends/transform.h> -#include <__algorithm/pstl_backends/cpu_backends/transform_reduce.h> -#include <__config> - -#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_H diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h index 3755d288047e..3db4765da64b 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h @@ -11,12 +11,12 @@ #include <__algorithm/any_of.h> #include <__algorithm/find_if.h> -#include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__atomic/atomic.h> #include <__atomic/memory_order.h> #include <__config> #include <__functional/operations.h> #include <__iterator/concepts.h> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/move.h> diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h deleted file mode 100644 index cb9425862a2b..000000000000 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h +++ /dev/null @@ -1,45 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_BACKEND_H -#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_BACKEND_H - -#include <__config> -#include - -#if defined(_LIBCPP_PSTL_CPU_BACKEND_SERIAL) -# include <__algorithm/pstl_backends/cpu_backends/serial.h> -#elif defined(_LIBCPP_PSTL_CPU_BACKEND_THREAD) -# include <__algorithm/pstl_backends/cpu_backends/thread.h> -#elif defined(_LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH) -# include <__algorithm/pstl_backends/cpu_backends/libdispatch.h> -#else -# error "Invalid CPU backend choice" -#endif - -#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) -# pragma GCC system_header -#endif - -#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 - -_LIBCPP_BEGIN_NAMESPACE_STD - -# if defined(_LIBCPP_PSTL_CPU_BACKEND_SERIAL) -using __cpu_backend_tag = __pstl::__serial_backend_tag; -# elif defined(_LIBCPP_PSTL_CPU_BACKEND_THREAD) -using __cpu_backend_tag = __pstl::__std_thread_backend_tag; -# elif defined(_LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH) -using __cpu_backend_tag = __pstl::__libdispatch_backend_tag; -# endif - -_LIBCPP_END_NAMESPACE_STD - -#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && && _LIBCPP_STD_VER >= 17 - -#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_BACKEND_H diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h index 0c20bdff6267..b5a49f8417d3 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h @@ -10,9 +10,9 @@ #define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_FILL_H #include <__algorithm/fill.h> -#include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__config> #include <__iterator/concepts.h> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/empty.h> diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h index 626293faef69..2b1754ea3a75 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h @@ -10,12 +10,12 @@ #define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_FIND_IF_H #include <__algorithm/find_if.h> -#include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__atomic/atomic.h> #include <__config> #include <__functional/operations.h> #include <__iterator/concepts.h> #include <__iterator/iterator_traits.h> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/move.h> diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h index d637084e151d..6db212ead8ae 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h @@ -10,9 +10,9 @@ #define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKNEDS_FOR_EACH_H #include <__algorithm/for_each.h> -#include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__config> #include <__iterator/concepts.h> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/empty.h> diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h index c93f4051c9d0..f3e59e8c0285 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h @@ -10,9 +10,9 @@ #define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_MERGE_H #include <__algorithm/merge.h> -#include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__config> #include <__iterator/concepts.h> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/move.h> diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h index 8c60cf897ff8..9ad8cc8fb0f2 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h @@ -9,9 +9,9 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_STABLE_SORT_H #define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_STABLE_SORT_H -#include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__algorithm/stable_sort.h> #include <__config> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/is_execution_policy.h> #include <__utility/empty.h> diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h index 4b9b29686683..65e166d847e1 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h @@ -9,11 +9,11 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_TRANSFORM_H #define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_TRANSFORM_H -#include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__algorithm/transform.h> #include <__config> #include <__iterator/concepts.h> #include <__iterator/iterator_traits.h> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h index c074eea9861c..af481d505bb9 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h +++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h @@ -9,11 +9,11 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_TRANSFORM_REDUCE_H #define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_TRANSFORM_REDUCE_H -#include <__algorithm/pstl_backends/cpu_backends/backend.h> #include <__config> #include <__iterator/concepts.h> #include <__iterator/iterator_traits.h> #include <__numeric/transform_reduce.h> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__type_traits/desugars_to.h> #include <__type_traits/is_arithmetic.h> diff --git a/libcxx/include/__algorithm/pstl_copy.h b/libcxx/include/__algorithm/pstl_copy.h index f35bb9713ef1..0fcea33c3919 100644 --- a/libcxx/include/__algorithm/pstl_copy.h +++ b/libcxx/include/__algorithm/pstl_copy.h @@ -10,13 +10,13 @@ #define _LIBCPP___ALGORITHM_PSTL_COPY_H #include <__algorithm/copy_n.h> -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__algorithm/pstl_transform.h> #include <__config> #include <__functional/identity.h> #include <__iterator/concepts.h> #include <__iterator/cpp17_iterator_concepts.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_constant_evaluated.h> #include <__type_traits/is_execution_policy.h> diff --git a/libcxx/include/__algorithm/pstl_count.h b/libcxx/include/__algorithm/pstl_count.h index 6ff57cac334e..64c84d855e4f 100644 --- a/libcxx/include/__algorithm/pstl_count.h +++ b/libcxx/include/__algorithm/pstl_count.h @@ -11,7 +11,6 @@ #include <__algorithm/count.h> #include <__algorithm/for_each.h> -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_for_each.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__atomic/atomic.h> @@ -20,6 +19,7 @@ #include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> #include <__numeric/pstl_transform_reduce.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> diff --git a/libcxx/include/__algorithm/pstl_find.h b/libcxx/include/__algorithm/pstl_find.h index 3b30a7bc9b45..b4c4dfb2ffb6 100644 --- a/libcxx/include/__algorithm/pstl_find.h +++ b/libcxx/include/__algorithm/pstl_find.h @@ -11,10 +11,10 @@ #include <__algorithm/comp.h> #include <__algorithm/find.h> -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__config> #include <__iterator/cpp17_iterator_concepts.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> diff --git a/libcxx/include/__algorithm/pstl_for_each.h b/libcxx/include/__algorithm/pstl_for_each.h index a9ebed74a62f..a99eb6d97fd2 100644 --- a/libcxx/include/__algorithm/pstl_for_each.h +++ b/libcxx/include/__algorithm/pstl_for_each.h @@ -11,11 +11,11 @@ #include <__algorithm/for_each.h> #include <__algorithm/for_each_n.h> -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__config> #include <__iterator/concepts.h> #include <__iterator/cpp17_iterator_concepts.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> diff --git a/libcxx/include/__algorithm/pstl_generate.h b/libcxx/include/__algorithm/pstl_generate.h index 886af290d7f2..350c0e4798be 100644 --- a/libcxx/include/__algorithm/pstl_generate.h +++ b/libcxx/include/__algorithm/pstl_generate.h @@ -9,12 +9,12 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_GENERATE_H #define _LIBCPP___ALGORITHM_PSTL_GENERATE_H -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_for_each.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__config> #include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> diff --git a/libcxx/include/__algorithm/pstl_is_partitioned.h b/libcxx/include/__algorithm/pstl_is_partitioned.h index 108bb1e43252..c016b388e378 100644 --- a/libcxx/include/__algorithm/pstl_is_partitioned.h +++ b/libcxx/include/__algorithm/pstl_is_partitioned.h @@ -10,11 +10,11 @@ #define _LIBCPP___ALGORITHM_PSTL_IS_PARITTIONED #include <__algorithm/pstl_any_all_none_of.h> -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_find.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__config> #include <__iterator/cpp17_iterator_concepts.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> diff --git a/libcxx/include/__algorithm/pstl_merge.h b/libcxx/include/__algorithm/pstl_merge.h index d03cd8c7fbd5..87f634a67f58 100644 --- a/libcxx/include/__algorithm/pstl_merge.h +++ b/libcxx/include/__algorithm/pstl_merge.h @@ -9,10 +9,10 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_MERGE_H #define _LIBCPP___ALGORITHM_PSTL_MERGE_H -#include <__algorithm/pstl_backend.h> #include <__config> #include <__functional/operations.h> #include <__iterator/cpp17_iterator_concepts.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> diff --git a/libcxx/include/__algorithm/pstl_move.h b/libcxx/include/__algorithm/pstl_move.h index f4c8c1fbb2e8..3155ddedf91b 100644 --- a/libcxx/include/__algorithm/pstl_move.h +++ b/libcxx/include/__algorithm/pstl_move.h @@ -10,13 +10,13 @@ #define _LIBCPP___ALGORITHM_PSTL_MOVE_H #include <__algorithm/copy_n.h> -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__algorithm/pstl_transform.h> #include <__config> #include <__functional/identity.h> #include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_constant_evaluated.h> #include <__type_traits/is_execution_policy.h> diff --git a/libcxx/include/__algorithm/pstl_replace.h b/libcxx/include/__algorithm/pstl_replace.h index 73ac11cda26a..b2ded54dfe25 100644 --- a/libcxx/include/__algorithm/pstl_replace.h +++ b/libcxx/include/__algorithm/pstl_replace.h @@ -9,13 +9,13 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_REPLACE_H #define _LIBCPP___ALGORITHM_PSTL_REPLACE_H -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_for_each.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__algorithm/pstl_transform.h> #include <__config> #include <__iterator/cpp17_iterator_concepts.h> #include <__iterator/iterator_traits.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/remove_cvref.h> #include <__utility/move.h> diff --git a/libcxx/include/__algorithm/pstl_rotate_copy.h b/libcxx/include/__algorithm/pstl_rotate_copy.h index adab3958fe31..1a32b710877c 100644 --- a/libcxx/include/__algorithm/pstl_rotate_copy.h +++ b/libcxx/include/__algorithm/pstl_rotate_copy.h @@ -9,10 +9,10 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_ROTATE_COPY_H #define _LIBCPP___ALGORITHM_PSTL_ROTATE_COPY_H -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_copy.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__iterator/cpp17_iterator_concepts.h> +#include <__pstl/configuration.h> #include <__type_traits/is_execution_policy.h> #include diff --git a/libcxx/include/__algorithm/pstl_sort.h b/libcxx/include/__algorithm/pstl_sort.h index 65bc794ca6f4..769dd81af77e 100644 --- a/libcxx/include/__algorithm/pstl_sort.h +++ b/libcxx/include/__algorithm/pstl_sort.h @@ -9,12 +9,12 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_SORT_H #define _LIBCPP___ALGORITHM_PSTL_SORT_H -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__algorithm/pstl_stable_sort.h> #include <__config> #include <__functional/operations.h> #include <__iterator/cpp17_iterator_concepts.h> +#include <__pstl/configuration.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> #include <__utility/empty.h> diff --git a/libcxx/include/__algorithm/pstl_stable_sort.h b/libcxx/include/__algorithm/pstl_stable_sort.h index 79b94557e3dc..f5e0dd40f72b 100644 --- a/libcxx/include/__algorithm/pstl_stable_sort.h +++ b/libcxx/include/__algorithm/pstl_stable_sort.h @@ -9,10 +9,10 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_STABLE_SORT_H #define _LIBCPP___ALGORITHM_PSTL_STABLE_SORT_H -#include <__algorithm/pstl_backend.h> #include <__config> #include <__functional/operations.h> #include <__iterator/cpp17_iterator_concepts.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> diff --git a/libcxx/include/__algorithm/pstl_transform.h b/libcxx/include/__algorithm/pstl_transform.h index a01a64a43cf1..80e1d6b496f2 100644 --- a/libcxx/include/__algorithm/pstl_transform.h +++ b/libcxx/include/__algorithm/pstl_transform.h @@ -9,9 +9,9 @@ #ifndef _LIBCPP___ALGORITHM_PSTL_TRANSFORM_H #define _LIBCPP___ALGORITHM_PSTL_TRANSFORM_H -#include <__algorithm/pstl_backend.h> #include <__config> #include <__iterator/cpp17_iterator_concepts.h> +#include <__pstl/configuration.h> #include <__type_traits/enable_if.h> #include <__type_traits/is_execution_policy.h> #include <__type_traits/remove_cvref.h> diff --git a/libcxx/include/__config_site.in b/libcxx/include/__config_site.in index 7c002c5bfcf8..89a14609ee3f 100644 --- a/libcxx/include/__config_site.in +++ b/libcxx/include/__config_site.in @@ -32,9 +32,9 @@ #cmakedefine _LIBCPP_INSTRUMENTED_WITH_ASAN // PSTL backends -#cmakedefine _LIBCPP_PSTL_CPU_BACKEND_SERIAL -#cmakedefine _LIBCPP_PSTL_CPU_BACKEND_THREAD -#cmakedefine _LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH +#cmakedefine _LIBCPP_PSTL_BACKEND_SERIAL +#cmakedefine _LIBCPP_PSTL_BACKEND_STD_THREAD +#cmakedefine _LIBCPP_PSTL_BACKEND_LIBDISPATCH // Hardening. #cmakedefine _LIBCPP_HARDENING_MODE_DEFAULT @_LIBCPP_HARDENING_MODE_DEFAULT@ diff --git a/libcxx/include/__numeric/pstl_transform_reduce.h b/libcxx/include/__numeric/pstl_transform_reduce.h index 2d2621dc8dad..fe41b1c86f3b 100644 --- a/libcxx/include/__numeric/pstl_transform_reduce.h +++ b/libcxx/include/__numeric/pstl_transform_reduce.h @@ -9,12 +9,12 @@ #ifndef _LIBCPP___NUMERIC_PSTL_TRANSFORM_REDUCE_H #define _LIBCPP___NUMERIC_PSTL_TRANSFORM_REDUCE_H -#include <__algorithm/pstl_backend.h> #include <__algorithm/pstl_frontend_dispatch.h> #include <__config> #include <__functional/operations.h> #include <__iterator/cpp17_iterator_concepts.h> #include <__numeric/transform_reduce.h> +#include <__pstl/configuration.h> #include <__type_traits/is_execution_policy.h> #include <__utility/move.h> #include diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h b/libcxx/include/__pstl/backends/libdispatch.h similarity index 94% rename from libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h rename to libcxx/include/__pstl/backends/libdispatch.h index 17faadf55dd4..977b06b9a489 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h +++ b/libcxx/include/__pstl/backends/libdispatch.h @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_LIBDISPATCH_H -#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_LIBDISPATCH_H +#ifndef _LIBCPP___PSTL_BACKENDS_LIBDISPATCH_H +#define _LIBCPP___PSTL_BACKENDS_LIBDISPATCH_H #include <__algorithm/inplace_merge.h> #include <__algorithm/lower_bound.h> @@ -23,6 +23,7 @@ #include <__memory/construct_at.h> #include <__memory/unique_ptr.h> #include <__numeric/reduce.h> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__utility/empty.h> #include <__utility/exception_guard.h> @@ -40,8 +41,6 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD namespace __pstl { -struct __libdispatch_backend_tag {}; - namespace __libdispatch { // ::dispatch_apply is marked as __attribute__((nothrow)) because it doesn't let exceptions propagate, and neither do // we. @@ -349,4 +348,14 @@ _LIBCPP_END_NAMESPACE_STD _LIBCPP_POP_MACROS -#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_LIBDISPATCH_H +// Implement PSTL algorithms based on the __cpu_traits specialized above +#include <__algorithm/pstl_backends/cpu_backends/any_of.h> +#include <__algorithm/pstl_backends/cpu_backends/fill.h> +#include <__algorithm/pstl_backends/cpu_backends/find_if.h> +#include <__algorithm/pstl_backends/cpu_backends/for_each.h> +#include <__algorithm/pstl_backends/cpu_backends/merge.h> +#include <__algorithm/pstl_backends/cpu_backends/stable_sort.h> +#include <__algorithm/pstl_backends/cpu_backends/transform.h> +#include <__algorithm/pstl_backends/cpu_backends/transform_reduce.h> + +#endif // _LIBCPP___PSTL_BACKENDS_LIBDISPATCH_H diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h b/libcxx/include/__pstl/backends/serial.h similarity index 78% rename from libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h rename to libcxx/include/__pstl/backends/serial.h index 7544619a8eef..8bb894509309 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h +++ b/libcxx/include/__pstl/backends/serial.h @@ -7,10 +7,11 @@ // //===----------------------------------------------------------------------===// -#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_SERIAL_H -#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_SERIAL_H +#ifndef _LIBCPP___PSTL_BACKENDS_SERIAL_H +#define _LIBCPP___PSTL_BACKENDS_SERIAL_H #include <__config> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__utility/empty.h> #include <__utility/move.h> @@ -29,8 +30,6 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD namespace __pstl { -struct __serial_backend_tag {}; - template <> struct __cpu_traits<__serial_backend_tag> { template @@ -82,4 +81,14 @@ _LIBCPP_POP_MACROS #endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && && _LIBCPP_STD_VER >= 17 -#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_SERIAL_H +// Implement PSTL algorithms based on the __cpu_traits specialized above +#include <__algorithm/pstl_backends/cpu_backends/any_of.h> +#include <__algorithm/pstl_backends/cpu_backends/fill.h> +#include <__algorithm/pstl_backends/cpu_backends/find_if.h> +#include <__algorithm/pstl_backends/cpu_backends/for_each.h> +#include <__algorithm/pstl_backends/cpu_backends/merge.h> +#include <__algorithm/pstl_backends/cpu_backends/stable_sort.h> +#include <__algorithm/pstl_backends/cpu_backends/transform.h> +#include <__algorithm/pstl_backends/cpu_backends/transform_reduce.h> + +#endif // _LIBCPP___PSTL_BACKENDS_SERIAL_H diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h b/libcxx/include/__pstl/backends/std_thread.h similarity index 79% rename from libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h rename to libcxx/include/__pstl/backends/std_thread.h index 2acf912264a0..ab09f42cfdd8 100644 --- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h +++ b/libcxx/include/__pstl/backends/std_thread.h @@ -6,11 +6,12 @@ // //===----------------------------------------------------------------------===// -#ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_THREAD_H -#define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_THREAD_H +#ifndef _LIBCPP___PSTL_BACKENDS_STD_THREAD_H +#define _LIBCPP___PSTL_BACKENDS_STD_THREAD_H #include <__assert> #include <__config> +#include <__pstl/configuration_fwd.h> #include <__pstl/cpu_algos/cpu_traits.h> #include <__utility/empty.h> #include <__utility/move.h> @@ -32,8 +33,6 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD namespace __pstl { -struct __std_thread_backend_tag {}; - template <> struct __cpu_traits<__std_thread_backend_tag> { template @@ -85,4 +84,14 @@ _LIBCPP_END_NAMESPACE_STD _LIBCPP_POP_MACROS -#endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_THREAD_H +// Implement PSTL algorithms based on the __cpu_traits specialized above +#include <__algorithm/pstl_backends/cpu_backends/any_of.h> +#include <__algorithm/pstl_backends/cpu_backends/fill.h> +#include <__algorithm/pstl_backends/cpu_backends/find_if.h> +#include <__algorithm/pstl_backends/cpu_backends/for_each.h> +#include <__algorithm/pstl_backends/cpu_backends/merge.h> +#include <__algorithm/pstl_backends/cpu_backends/stable_sort.h> +#include <__algorithm/pstl_backends/cpu_backends/transform.h> +#include <__algorithm/pstl_backends/cpu_backends/transform_reduce.h> + +#endif // _LIBCPP___PSTL_BACKENDS_STD_THREAD_H diff --git a/libcxx/include/__pstl/configuration.h b/libcxx/include/__pstl/configuration.h new file mode 100644 index 000000000000..d32bd21df1f9 --- /dev/null +++ b/libcxx/include/__pstl/configuration.h @@ -0,0 +1,27 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___PSTL_CONFIGURATION_H +#define _LIBCPP___PSTL_CONFIGURATION_H + +#include <__config> +#include <__pstl/configuration_fwd.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +#if defined(_LIBCPP_PSTL_BACKEND_SERIAL) +# include <__pstl/backends/serial.h> +#elif defined(_LIBCPP_PSTL_BACKEND_STD_THREAD) +# include <__pstl/backends/std_thread.h> +#elif defined(_LIBCPP_PSTL_BACKEND_LIBDISPATCH) +# include <__pstl/backends/libdispatch.h> +#endif + +#endif // _LIBCPP___PSTL_CONFIGURATION_H diff --git a/libcxx/include/__algorithm/pstl_backend.h b/libcxx/include/__pstl/configuration_fwd.h similarity index 93% rename from libcxx/include/__algorithm/pstl_backend.h rename to libcxx/include/__pstl/configuration_fwd.h index 3af03ce2fbc8..995fcfce847c 100644 --- a/libcxx/include/__algorithm/pstl_backend.h +++ b/libcxx/include/__pstl/configuration_fwd.h @@ -6,10 +6,9 @@ // //===----------------------------------------------------------------------===// -#ifndef _LIBCPP___ALGORITHM_PSTL_BACKEND_H -#define _LIBCPP___ALGORITHM_PSTL_BACKEND_H +#ifndef _LIBCPP___PSTL_CONFIGURATION_FWD_H +#define _LIBCPP___PSTL_CONFIGURATION_FWD_H -#include <__algorithm/pstl_backends/cpu_backend.h> #include <__config> #include @@ -191,6 +190,20 @@ into a program termination at the front-end level. When a backend returns a dise frontend will turn that into a call to `std::__throw_bad_alloc();` to report the internal failure to the user. */ +namespace __pstl { +struct __libdispatch_backend_tag {}; +struct __serial_backend_tag {}; +struct __std_thread_backend_tag {}; +} // namespace __pstl + +# if defined(_LIBCPP_PSTL_BACKEND_SERIAL) +using __cpu_backend_tag = __pstl::__serial_backend_tag; +# elif defined(_LIBCPP_PSTL_BACKEND_STD_THREAD) +using __cpu_backend_tag = __pstl::__std_thread_backend_tag; +# elif defined(_LIBCPP_PSTL_BACKEND_LIBDISPATCH) +using __cpu_backend_tag = __pstl::__libdispatch_backend_tag; +# endif + template struct __select_backend; @@ -206,8 +219,8 @@ struct __select_backend { }; # endif -# if defined(_LIBCPP_PSTL_CPU_BACKEND_SERIAL) || defined(_LIBCPP_PSTL_CPU_BACKEND_THREAD) || \ - defined(_LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH) +# if defined(_LIBCPP_PSTL_BACKEND_SERIAL) || defined(_LIBCPP_PSTL_BACKEND_STD_THREAD) || \ + defined(_LIBCPP_PSTL_BACKEND_LIBDISPATCH) template <> struct __select_backend { using type = __cpu_backend_tag; @@ -229,4 +242,4 @@ _LIBCPP_END_NAMESPACE_STD #endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17 -#endif // _LIBCPP___ALGORITHM_PSTL_BACKEND_H +#endif // _LIBCPP___PSTL_CONFIGURATION_FWD_H diff --git a/libcxx/include/libcxx.imp b/libcxx/include/libcxx.imp index 8820fb8c0936..a4e2690fc55c 100644 --- a/libcxx/include/libcxx.imp +++ b/libcxx/include/libcxx.imp @@ -73,18 +73,12 @@ { include: [ "<__algorithm/pop_heap.h>", "private", "", "public" ] }, { include: [ "<__algorithm/prev_permutation.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_any_all_none_of.h>", "private", "", "public" ] }, - { include: [ "<__algorithm/pstl_backend.h>", "private", "", "public" ] }, - { include: [ "<__algorithm/pstl_backends/cpu_backend.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_backends/cpu_backends/any_of.h>", "private", "", "public" ] }, - { include: [ "<__algorithm/pstl_backends/cpu_backends/backend.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_backends/cpu_backends/fill.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_backends/cpu_backends/find_if.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_backends/cpu_backends/for_each.h>", "private", "", "public" ] }, - { include: [ "<__algorithm/pstl_backends/cpu_backends/libdispatch.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_backends/cpu_backends/merge.h>", "private", "", "public" ] }, - { include: [ "<__algorithm/pstl_backends/cpu_backends/serial.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_backends/cpu_backends/stable_sort.h>", "private", "", "public" ] }, - { include: [ "<__algorithm/pstl_backends/cpu_backends/thread.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_backends/cpu_backends/transform.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_backends/cpu_backends/transform_reduce.h>", "private", "", "public" ] }, { include: [ "<__algorithm/pstl_copy.h>", "private", "", "public" ] }, diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap index ce133e471deb..f996c2cc0545 100644 --- a/libcxx/include/module.modulemap +++ b/libcxx/include/module.modulemap @@ -714,32 +714,14 @@ module std_private_algorithm_partition_point [system module std_private_algorithm_pop_heap [system] { header "__algorithm/pop_heap.h" } module std_private_algorithm_prev_permutation [system] { header "__algorithm/prev_permutation.h" } module std_private_algorithm_pstl_any_all_none_of [system] { header "__algorithm/pstl_any_all_none_of.h" } -module std_private_algorithm_pstl_backend [system] { - header "__algorithm/pstl_backend.h" - export * -} -module std_private_algorithm_pstl_backends_cpu_backend [system] { - header "__algorithm/pstl_backends/cpu_backend.h" - export * -} -module std_private_algorithm_pstl_backends_cpu_backends_any_of [system] { header "__algorithm/pstl_backends/cpu_backends/any_of.h" } -module std_private_algorithm_pstl_backends_cpu_backends_backend [system] { - header "__algorithm/pstl_backends/cpu_backends/backend.h" - export * -} -module std_private_algorithm_pstl_backends_cpu_backends_fill [system] { header "__algorithm/pstl_backends/cpu_backends/fill.h" } -module std_private_algorithm_pstl_backends_cpu_backends_find_if [system] { header "__algorithm/pstl_backends/cpu_backends/find_if.h" } -module std_private_algorithm_pstl_backends_cpu_backends_for_each [system] { header "__algorithm/pstl_backends/cpu_backends/for_each.h" } -module std_private_algorithm_pstl_backends_cpu_backends_libdispatch [system] { header "__algorithm/pstl_backends/cpu_backends/libdispatch.h" } -module std_private_algorithm_pstl_backends_cpu_backends_merge [system] { header "__algorithm/pstl_backends/cpu_backends/merge.h" } -module std_private_algorithm_pstl_backends_cpu_backends_serial [system] { textual header "__algorithm/pstl_backends/cpu_backends/serial.h" } -module std_private_algorithm_pstl_backends_cpu_backends_stable_sort [system] { header "__algorithm/pstl_backends/cpu_backends/stable_sort.h" } -module std_private_algorithm_pstl_backends_cpu_backends_thread [system] { textual header "__algorithm/pstl_backends/cpu_backends/thread.h" } -module std_private_algorithm_pstl_backends_cpu_backends_transform [system] { - header "__algorithm/pstl_backends/cpu_backends/transform.h" - export std_private_algorithm_transform -} -module std_private_algorithm_pstl_backends_cpu_backends_transform_reduce [system] { header "__algorithm/pstl_backends/cpu_backends/transform_reduce.h" } +module std_private_algorithm_pstl_backends_cpu_backends_any_of [system] { textual header "__algorithm/pstl_backends/cpu_backends/any_of.h" } +module std_private_algorithm_pstl_backends_cpu_backends_fill [system] { textual header "__algorithm/pstl_backends/cpu_backends/fill.h" } +module std_private_algorithm_pstl_backends_cpu_backends_find_if [system] { textual header "__algorithm/pstl_backends/cpu_backends/find_if.h" } +module std_private_algorithm_pstl_backends_cpu_backends_for_each [system] { textual header "__algorithm/pstl_backends/cpu_backends/for_each.h" } +module std_private_algorithm_pstl_backends_cpu_backends_merge [system] { textual header "__algorithm/pstl_backends/cpu_backends/merge.h" } +module std_private_algorithm_pstl_backends_cpu_backends_stable_sort [system] { textual header "__algorithm/pstl_backends/cpu_backends/stable_sort.h" } +module std_private_algorithm_pstl_backends_cpu_backends_transform [system] { textual header "__algorithm/pstl_backends/cpu_backends/transform.h" } +module std_private_algorithm_pstl_backends_cpu_backends_transform_reduce [system] { textual header "__algorithm/pstl_backends/cpu_backends/transform_reduce.h" } module std_private_algorithm_pstl_copy [system] { header "__algorithm/pstl_copy.h" } module std_private_algorithm_pstl_count [system] { header "__algorithm/pstl_count.h" } module std_private_algorithm_pstl_equal [system] { header "__algorithm/pstl_equal.h" } @@ -1613,7 +1595,18 @@ module std_private_numeric_transform_exclusive_scan [system] { header "__numeric module std_private_numeric_transform_inclusive_scan [system] { header "__numeric/transform_inclusive_scan.h" } module std_private_numeric_transform_reduce [system] { header "__numeric/transform_reduce.h" } -module std_private_pstl_cpu_algos_cpu_traits [system] { header "__pstl/cpu_algos/cpu_traits.h" } +module std_private_pstl_backends_libdispatch [system] { header "__pstl/backends/libdispatch.h" } +module std_private_pstl_backends_serial [system] { header "__pstl/backends/serial.h" } +module std_private_pstl_backends_std_thread [system] { header "__pstl/backends/std_thread.h" } +module std_private_pstl_cpu_algos_cpu_traits [system] { header "__pstl/cpu_algos/cpu_traits.h" } +module std_private_pstl_configuration_fwd [system] { + header "__pstl/configuration_fwd.h" + export * +} +module std_private_pstl_configuration [system] { + header "__pstl/configuration.h" + export * +} module std_private_queue_fwd [system] { header "__fwd/queue.h" } diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt index a4a3fee86457..8b28d1b89189 100644 --- a/libcxx/src/CMakeLists.txt +++ b/libcxx/src/CMakeLists.txt @@ -327,7 +327,7 @@ set(LIBCXX_EXPERIMENTAL_SOURCES experimental/keep.cpp ) -if (LIBCXX_PSTL_CPU_BACKEND STREQUAL "libdispatch") +if (LIBCXX_PSTL_BACKEND STREQUAL "libdispatch") list(APPEND LIBCXX_EXPERIMENTAL_SOURCES pstl/libdispatch.cpp ) diff --git a/libcxx/src/pstl/libdispatch.cpp b/libcxx/src/pstl/libdispatch.cpp index d997a9c73463..3dca702341c8 100644 --- a/libcxx/src/pstl/libdispatch.cpp +++ b/libcxx/src/pstl/libdispatch.cpp @@ -7,8 +7,8 @@ //===----------------------------------------------------------------------===// #include <__algorithm/min.h> -#include <__algorithm/pstl_backends/cpu_backends/libdispatch.h> #include <__config> +#include <__pstl/backends/libdispatch.h> #include _LIBCPP_BEGIN_NAMESPACE_STD diff --git a/libcxx/test/libcxx/algorithms/pstl.libdispatch.chunk_partitions.pass.cpp b/libcxx/test/libcxx/algorithms/pstl.libdispatch.chunk_partitions.pass.cpp index 8c7016a80b81..b48ac02dd79c 100644 --- a/libcxx/test/libcxx/algorithms/pstl.libdispatch.chunk_partitions.pass.cpp +++ b/libcxx/test/libcxx/algorithms/pstl.libdispatch.chunk_partitions.pass.cpp @@ -8,11 +8,11 @@ // -// REQUIRES: libcpp-pstl-cpu-backend-libdispatch +// REQUIRES: libcpp-pstl-backend-libdispatch // __chunk_partitions __partition_chunks(ptrdiff_t); -#include <__algorithm/pstl_backends/cpu_backends/libdispatch.h> +#include <__pstl/backends/libdispatch.h> #include #include diff --git a/libcxx/test/libcxx/vendor/apple/system-install-properties.sh.cpp b/libcxx/test/libcxx/vendor/apple/system-install-properties.sh.cpp index 3e2e080368f4..4ea27401e35d 100644 --- a/libcxx/test/libcxx/vendor/apple/system-install-properties.sh.cpp +++ b/libcxx/test/libcxx/vendor/apple/system-install-properties.sh.cpp @@ -45,4 +45,4 @@ // Make sure we use the libdispatch backend for the PSTL. // -// RUN: grep "%{include-dir}/__config_site" -e '#define _LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH' +// RUN: grep "%{include-dir}/__config_site" -e '#define _LIBCPP_PSTL_BACKEND_LIBDISPATCH' diff --git a/libcxx/utils/libcxx/test/features.py b/libcxx/utils/libcxx/test/features.py index 6ff16309546b..c81b56b1af54 100644 --- a/libcxx/utils/libcxx/test/features.py +++ b/libcxx/utils/libcxx/test/features.py @@ -318,7 +318,7 @@ macros = { "_LIBCPP_HAS_NO_WIDE_CHARACTERS": "no-wide-characters", "_LIBCPP_HAS_NO_TIME_ZONE_DATABASE": "no-tzdb", "_LIBCPP_HAS_NO_UNICODE": "libcpp-has-no-unicode", - "_LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH": "libcpp-pstl-cpu-backend-libdispatch", + "_LIBCPP_PSTL_BACKEND_LIBDISPATCH": "libcpp-pstl-backend-libdispatch", } for macro, feature in macros.items(): DEFAULT_FEATURES.append( -- GitLab From 8d49ce176414cd4d0d5d276fd721d9226e17e810 Mon Sep 17 00:00:00 2001 From: David Green Date: Wed, 17 Apr 2024 18:38:24 +0100 Subject: [PATCH 046/714] [GlobalISel][AArch64] Add LLRINT support (#88702) This hooks up G_INTRINSIC_LLRINT instructions, very similar to the lrint nodes that already exist. On AArch64 they are treated the same as lrint with the default return types. --- llvm/include/llvm/Support/TargetOpcodes.def | 3 + llvm/include/llvm/Target/GenericOpcodes.td | 6 ++ .../Target/GlobalISel/SelectionDAGCompat.td | 1 + llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 2 + .../CodeGen/GlobalISel/LegalizerHelper.cpp | 6 +- .../AArch64/GISel/AArch64LegalizerInfo.cpp | 2 +- .../AArch64/GISel/AArch64RegisterBankInfo.cpp | 1 + .../AArch64/GlobalISel/arm64-irtranslator.ll | 10 ++ .../AArch64/GlobalISel/legalize-llrint.mir | 98 +++++++++++++++++++ .../GlobalISel/legalizer-info-validation.mir | 4 + llvm/test/CodeGen/AArch64/llrint-conv-fp16.ll | 2 + llvm/test/CodeGen/AArch64/llrint-conv.ll | 58 +++++++---- .../builtins/match-table-replacerreg.td | 22 ++--- .../match-table-imms.td | 30 +++--- .../match-table-intrinsics.td | 16 +-- .../match-table-patfrag-root.td | 30 +++--- .../GlobalISelCombinerEmitter/match-table.td | 62 ++++++------ llvm/test/TableGen/GlobalISelEmitter.td | 2 +- 18 files changed, 251 insertions(+), 104 deletions(-) create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/legalize-llrint.mir diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def index 5765926d6d93..cb98f96af522 100644 --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -351,6 +351,9 @@ HANDLE_TARGET_OPCODE(G_INTRINSIC_ROUND) /// INTRINSIC round to integer intrinsic. HANDLE_TARGET_OPCODE(G_INTRINSIC_LRINT) +/// INTRINSIC long round to integer intrinsic. +HANDLE_TARGET_OPCODE(G_INTRINSIC_LLRINT) + /// INTRINSIC roundeven intrinsic. HANDLE_TARGET_OPCODE(G_INTRINSIC_ROUNDEVEN) diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td index d0f471eb29b6..e8cf8fcb647f 100644 --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -1089,6 +1089,12 @@ def G_INTRINSIC_LRINT : GenericInstruction { let hasSideEffects = false; } +def G_INTRINSIC_LLRINT : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type1:$src); + let hasSideEffects = false; +} + def G_INTRINSIC_ROUNDEVEN : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type0:$src1); diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td index dd4e7d790bc6..8fa0e4b86d6d 100644 --- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -157,6 +157,7 @@ def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; +def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 9b4575f7f34d..0b6aae375975 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1955,6 +1955,8 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) { return TargetOpcode::G_PTRMASK; case Intrinsic::lrint: return TargetOpcode::G_INTRINSIC_LRINT; + case Intrinsic::llrint: + return TargetOpcode::G_INTRINSIC_LLRINT; // FADD/FMUL require checking the FMF, so are handled elsewhere. case Intrinsic::vector_reduce_fmin: return TargetOpcode::G_VECREDUCE_FMIN; diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 156353296cfc..d55091e2e717 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -474,6 +474,8 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) { RTLIBCASE(ROUNDEVEN_F); case TargetOpcode::G_INTRINSIC_LRINT: RTLIBCASE(LRINT_F); + case TargetOpcode::G_INTRINSIC_LLRINT: + RTLIBCASE(LLRINT_F); } llvm_unreachable("Unknown libcall function"); } @@ -1061,7 +1063,8 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) { return Status; break; } - case TargetOpcode::G_INTRINSIC_LRINT: { + case TargetOpcode::G_INTRINSIC_LRINT: + case TargetOpcode::G_INTRINSIC_LLRINT: { LLT LLTy = MRI.getType(MI.getOperand(1).getReg()); unsigned Size = LLTy.getSizeInBits(); Type *HLTy = getFloatTypeForLLT(Ctx, LLTy); @@ -2661,6 +2664,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { case TargetOpcode::G_FPTOSI: case TargetOpcode::G_FPTOUI: case TargetOpcode::G_INTRINSIC_LRINT: + case TargetOpcode::G_INTRINSIC_LLRINT: case TargetOpcode::G_IS_FPCLASS: Observer.changingInstr(MI); diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index 661ea151d1a0..85dd0f2eb192 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -262,7 +262,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) .minScalar(0, s32) .scalarize(0); - getActionDefinitionsBuilder(G_INTRINSIC_LRINT) + getActionDefinitionsBuilder({G_INTRINSIC_LRINT, G_INTRINSIC_LLRINT}) .legalFor({{s64, MinFPScalar}, {s64, s32}, {s64, s64}}) .libcallFor({{s64, s128}}) .minScalarOrElt(1, MinFPScalar); diff --git a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp index d5c4ce1888e7..44ba9f0429e6 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp @@ -793,6 +793,7 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case TargetOpcode::G_FPTOSI: case TargetOpcode::G_FPTOUI: case TargetOpcode::G_INTRINSIC_LRINT: + case TargetOpcode::G_INTRINSIC_LLRINT: if (MRI.getType(MI.getOperand(0).getReg()).isVector()) break; OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR}; diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll index a131f35e66d0..a61931b898ae 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll @@ -1385,6 +1385,16 @@ define i32 @test_intrinsic_lrint(float %a) { ret i32 %res } +declare i32 @llvm.llrint.i32.f32(float) +define i32 @test_intrinsic_llrint(float %a) { +; CHECK-LABEL: name: test_intrinsic_llrint +; CHECK: [[A:%[0-9]+]]:_(s32) = COPY $s0 +; CHECK: [[RES:%[0-9]+]]:_(s32) = G_INTRINSIC_LLRINT [[A]] +; CHECK: $w0 = COPY [[RES]] + %res = call i32 @llvm.llrint.i32.f32(float %a) + ret i32 %res +} + declare i32 @llvm.ctlz.i32(i32, i1) define i32 @test_ctlz_intrinsic_zero_not_undef(i32 %a) { ; CHECK-LABEL: name: test_ctlz_intrinsic_zero_not_undef diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-llrint.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-llrint.mir new file mode 100644 index 000000000000..f77649b79395 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-llrint.mir @@ -0,0 +1,98 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -verify-machineinstrs -mtriple aarch64-unknown-unknown -run-pass=legalizer %s -o - | FileCheck %s +--- +name: testmsws +alignment: 4 +tracksRegLiveness: true +liveins: + - { reg: '$s0' } +body: | + bb.1: + liveins: $s0 + + ; CHECK-LABEL: name: testmsws + ; CHECK: liveins: $s0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0 + ; CHECK-NEXT: [[INTRINSIC_LLRINT:%[0-9]+]]:_(s64) = G_INTRINSIC_LLRINT [[COPY]](s32) + ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[INTRINSIC_LLRINT]](s64) + ; CHECK-NEXT: $w0 = COPY [[TRUNC]](s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %0:_(s32) = COPY $s0 + %1:_(s64) = G_INTRINSIC_LLRINT %0(s32) + %2:_(s32) = G_TRUNC %1(s64) + $w0 = COPY %2(s32) + RET_ReallyLR implicit $w0 + +... +--- +name: testmsxs +alignment: 4 +tracksRegLiveness: true +liveins: + - { reg: '$s0' } +body: | + bb.1: + liveins: $s0 + + ; CHECK-LABEL: name: testmsxs + ; CHECK: liveins: $s0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0 + ; CHECK-NEXT: [[INTRINSIC_LLRINT:%[0-9]+]]:_(s64) = G_INTRINSIC_LLRINT [[COPY]](s32) + ; CHECK-NEXT: $x0 = COPY [[INTRINSIC_LLRINT]](s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %0:_(s32) = COPY $s0 + %1:_(s64) = G_INTRINSIC_LLRINT %0(s32) + $x0 = COPY %1(s64) + RET_ReallyLR implicit $x0 + +... +--- +name: testmswd +alignment: 4 +tracksRegLiveness: true +liveins: + - { reg: '$d0' } +body: | + bb.1: + liveins: $d0 + + ; CHECK-LABEL: name: testmswd + ; CHECK: liveins: $d0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $d0 + ; CHECK-NEXT: [[INTRINSIC_LLRINT:%[0-9]+]]:_(s64) = G_INTRINSIC_LLRINT [[COPY]](s64) + ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[INTRINSIC_LLRINT]](s64) + ; CHECK-NEXT: $w0 = COPY [[TRUNC]](s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %0:_(s64) = COPY $d0 + %1:_(s64) = G_INTRINSIC_LLRINT %0(s64) + %2:_(s32) = G_TRUNC %1(s64) + $w0 = COPY %2(s32) + RET_ReallyLR implicit $w0 + +... +--- +name: testmsxd +alignment: 4 +tracksRegLiveness: true +liveins: + - { reg: '$d0' } +body: | + bb.1: + liveins: $d0 + + ; CHECK-LABEL: name: testmsxd + ; CHECK: liveins: $d0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $d0 + ; CHECK-NEXT: [[INTRINSIC_LLRINT:%[0-9]+]]:_(s64) = G_INTRINSIC_LLRINT [[COPY]](s64) + ; CHECK-NEXT: $x0 = COPY [[INTRINSIC_LLRINT]](s64) + ; CHECK-NEXT: RET_ReallyLR implicit $x0 + %0:_(s64) = COPY $d0 + %1:_(s64) = G_INTRINSIC_LLRINT %0(s64) + $x0 = COPY %1(s64) + RET_ReallyLR implicit $x0 + +... diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir index 0793f3983c8e..098726b0a980 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir @@ -156,6 +156,10 @@ # DEBUG-NEXT: G_INTRINSIC_LRINT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices # DEBUG-NEXT: .. the first uncovered type index: 2, OK # DEBUG-NEXT: .. the first uncovered imm index: 0, OK +# DEBUG-NEXT: G_INTRINSIC_LLRINT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices +# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}} +# DEBUG-NEXT: .. the first uncovered type index: 2, OK +# DEBUG-NEXT: .. the first uncovered imm index: 0, OK # DEBUG-NEXT: G_INTRINSIC_ROUNDEVEN (opcode {{[0-9]+}}): 1 type index, 0 imm indices # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}} # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected diff --git a/llvm/test/CodeGen/AArch64/llrint-conv-fp16.ll b/llvm/test/CodeGen/AArch64/llrint-conv-fp16.ll index 1adbbab76abf..7e28c863b07a 100644 --- a/llvm/test/CodeGen/AArch64/llrint-conv-fp16.ll +++ b/llvm/test/CodeGen/AArch64/llrint-conv-fp16.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 ; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK-NOFP16 ; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK-FP16 +; RUN: llc < %s -mtriple=aarch64 -global-isel | FileCheck %s --check-prefixes=CHECK-NOFP16 +; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 -global-isel | FileCheck %s --check-prefixes=CHECK-FP16 define i16 @testmhhs(half %x) { ; CHECK-NOFP16-LABEL: testmhhs: diff --git a/llvm/test/CodeGen/AArch64/llrint-conv.ll b/llvm/test/CodeGen/AArch64/llrint-conv.ll index fa11b007eeb3..3a6396d120f7 100644 --- a/llvm/test/CodeGen/AArch64/llrint-conv.ll +++ b/llvm/test/CodeGen/AArch64/llrint-conv.ll @@ -1,59 +1,75 @@ -; RUN: llc < %s -mtriple=aarch64 -mattr=+neon | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 +; RUN: llc < %s -mtriple=aarch64 | FileCheck %s +; RUN: llc < %s -mtriple=aarch64 -global-isel | FileCheck %s -; CHECK-LABEL: testmsws: -; CHECK: frintx [[REG:s[0-9]]], s0 -; CHECK-NEXT: fcvtzs x0, [[REG]] -; CHECK: ret define i32 @testmsws(float %x) { +; CHECK-LABEL: testmsws: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: frintx s0, s0 +; CHECK-NEXT: fcvtzs x0, s0 +; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 +; CHECK-NEXT: ret entry: %0 = tail call i64 @llvm.llrint.f32(float %x) %conv = trunc i64 %0 to i32 ret i32 %conv } -; CHECK-LABEL: testmsxs: -; CHECK: frintx [[REG:s[0-9]]], s0 -; CHECK-NEXT: fcvtzs x0, [[REG]] -; CHECK-NEXT: ret define i64 @testmsxs(float %x) { +; CHECK-LABEL: testmsxs: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: frintx s0, s0 +; CHECK-NEXT: fcvtzs x0, s0 +; CHECK-NEXT: ret entry: %0 = tail call i64 @llvm.llrint.f32(float %x) ret i64 %0 } -; CHECK-LABEL: testmswd: -; CHECK: frintx [[REG:d[0-9]]], d0 -; CHECK-NEXT: fcvtzs x0, [[REG]] -; CHECK: ret define i32 @testmswd(double %x) { +; CHECK-LABEL: testmswd: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: frintx d0, d0 +; CHECK-NEXT: fcvtzs x0, d0 +; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 +; CHECK-NEXT: ret entry: %0 = tail call i64 @llvm.llrint.f64(double %x) %conv = trunc i64 %0 to i32 ret i32 %conv } -; CHECK-LABEL: testmsxd: -; CHECK: frintx [[REG:d[0-9]]], d0 -; CHECK-NEXT: fcvtzs x0, [[REG]] -; CHECK-nEXT: ret define i64 @testmsxd(double %x) { +; CHECK-LABEL: testmsxd: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: frintx d0, d0 +; CHECK-NEXT: fcvtzs x0, d0 +; CHECK-NEXT: ret entry: %0 = tail call i64 @llvm.llrint.f64(double %x) ret i64 %0 } -; CHECK-LABEL: testmswl: -; CHECK: bl llrintl define i32 @testmswl(fp128 %x) { +; CHECK-LABEL: testmswl: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: .cfi_offset w30, -16 +; CHECK-NEXT: bl llrintl +; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 +; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret entry: %0 = tail call i64 @llvm.llrint.f128(fp128 %x) %conv = trunc i64 %0 to i32 ret i32 %conv } -; CHECK-LABEL: testmsll: -; CHECK: b llrintl define i64 @testmsll(fp128 %x) { +; CHECK-LABEL: testmsll: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: b llrintl entry: %0 = tail call i64 @llvm.llrint.f128(fp128 %x) ret i64 %0 diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td index 40a831d7e9e8..ebb95ccb2104 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td @@ -28,11 +28,11 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(69), GIMT_Encode2(186), /*)*//*default:*//*Label 2*/ GIMT_Encode4(562), -// CHECK-NEXT: /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ GIMT_Encode4(478), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_FNEG*//*Label 1*/ GIMT_Encode4(530), -// CHECK-NEXT: // Label 0: @478 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(529), // Rule ID 1 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2({{[0-9]+}}), GIMT_Encode2({{[0-9]+}}), /*)*//*default:*//*Label 2*/ GIMT_Encode4([[L562:[0-9]+]]), +// CHECK-NEXT: /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ GIMT_Encode4([[L478:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_FNEG*//*Label 1*/ GIMT_Encode4([[L530:[0-9]+]]), +// CHECK-NEXT: // Label 0: @[[L478]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4([[L529:[0-9]+]]), // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: // MIs[0] a @@ -57,10 +57,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_ReplaceRegWithTempReg, /*OldInsnID*/0, /*OldOpIdx*/1, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 3: @529 +// CHECK-NEXT: // Label 3: @[[L529]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @530 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(561), // Rule ID 0 // +// CHECK-NEXT: // Label 1: @[[L530]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4([[L561:[0-9]+]]), // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: // MIs[0] dst // CHECK-NEXT: // No operand predicates @@ -75,10 +75,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_ReplaceReg, /*OldInsnID*/0, /*OldOpIdx*/0, /*NewInsnId*/1, /*NewOpIdx*/1, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 4: @561 +// CHECK-NEXT: // Label 4: @[[L561]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @562 +// CHECK-NEXT: // Label 2: @[[L562]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: }; // Size: 563 bytes +// CHECK-NEXT: }; // Size: {{[0-9]+}} bytes // CHECK-NEXT: return MatchTable0; // CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td index 751b1318ecc0..6004a17d351b 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td @@ -34,12 +34,12 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(132), /*)*//*default:*//*Label 3*/ GIMT_Encode4(579), -// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(462), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_CONSTANT*//*Label 1*/ GIMT_Encode4(493), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 2*/ GIMT_Encode4(539), -// CHECK-NEXT: // Label 0: @462 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(492), // Rule ID 0 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2({{[0-9]+}}), /*)*//*default:*//*Label 3*/ GIMT_Encode4([[L579:[0-9]+]]), +// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4([[L462:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_CONSTANT*//*Label 1*/ GIMT_Encode4([[L493:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 2*/ GIMT_Encode4({{[0-9]+}}), +// CHECK-NEXT: // Label 0: @[[L462]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4([[L492:[0-9]+]]), // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] a @@ -51,10 +51,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddImm8, /*InsnID*/0, /*Imm*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 4: @492 +// CHECK-NEXT: // Label 4: @[[L492]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @493 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(538), // Rule ID 2 // +// CHECK-NEXT: // Label 1: @[[L493]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4([[L538:[0-9]+]]), // Rule ID 2 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled), // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] a @@ -66,10 +66,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddCImm, /*InsnID*/0, /*Type*/GILLT_s32, /*Imm*/GIMT_Encode8(42), // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 5: @538 +// CHECK-NEXT: // Label 5: @[[L538]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @539 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(578), // Rule ID 1 // +// CHECK-NEXT: // Label 2: @{{[0-9]+}} +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4([[L578:[0-9]+]]), // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: // MIs[0] a // CHECK-NEXT: // No operand predicates @@ -83,10 +83,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 6: @578 +// CHECK-NEXT: // Label 6: @[[L578]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 3: @579 +// CHECK-NEXT: // Label 3: @[[L579]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: }; // Size: 580 bytes +// CHECK-NEXT: }; // Size: {{[0-9]+}} bytes // CHECK-NEXT: return MatchTable0; // CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td index e8e6d3e74f40..b2dd8b6684b1 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td @@ -29,11 +29,11 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(119), GIMT_Encode2(121), /*)*//*default:*//*Label 2*/ GIMT_Encode4(132), +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2({{[0-9]+}}), GIMT_Encode2({{[0-9]+}}), /*)*//*default:*//*Label 2*/ GIMT_Encode4([[L132:[0-9]+]]), // CHECK-NEXT: /*TargetOpcode::G_INTRINSIC*//*Label 0*/ GIMT_Encode4(18), -// CHECK-NEXT: /*TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS*//*Label 1*/ GIMT_Encode4(73), +// CHECK-NEXT: /*TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS*//*Label 1*/ GIMT_Encode4([[L73:[0-9]+]]), // CHECK-NEXT: // Label 0: @18 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(72), // Rule ID 0 // +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4([[L72:[0-9]+]]), // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::1in_1out), @@ -52,10 +52,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 3: @72 +// CHECK-NEXT: // Label 3: @[[L72]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @73 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(131), // Rule ID 1 // +// CHECK-NEXT: // Label 1: @[[L73]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4([[L131:[0-9]+]]), // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::sideeffects_1in_1out), @@ -76,9 +76,9 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_MergeMemOperands, /*InsnID*/1, /*NumInsns*/1, /*MergeInsnID's*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 4: @131 +// CHECK-NEXT: // Label 4: @[[L131]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @132 +// CHECK-NEXT: // Label 2: @[[L132]] // CHECK-NEXT: GIM_Reject, // CHECK-NEXT: }; // Size: 133 bytes // CHECK-NEXT: return MatchTable0; diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td index 26a0ec6235e3..016ab05ca01e 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td @@ -28,12 +28,12 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(124), GIMT_Encode2(187), /*)*//*default:*//*Label 3*/ GIMT_Encode4(380), -// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 0*/ GIMT_Encode4(262), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 1*/ GIMT_Encode4(298), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_FPEXT*//*Label 2*/ GIMT_Encode4(344), -// CHECK-NEXT: // Label 0: @262 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(297), // Rule ID 1 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2({{[0-9]+}}), GIMT_Encode2({{[0-9]+}}), /*)*//*default:*//*Label 3*/ GIMT_Encode4([[L380:[0-9]+]]), +// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 0*/ GIMT_Encode4([[L262:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 1*/ GIMT_Encode4([[L298:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_FPEXT*//*Label 2*/ GIMT_Encode4([[L344:[0-9]+]]), +// CHECK-NEXT: // Label 0: @[[L262]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4([[L297:[0-9]+]]), // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: // MIs[0] root // CHECK-NEXT: // No operand predicates @@ -47,10 +47,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 4: @297 +// CHECK-NEXT: // Label 4: @[[L297]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @298 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(343), // Rule ID 0 // +// CHECK-NEXT: // Label 1: @[[L298]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4([[L343:[0-9]+]]), // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: // MIs[0] root // CHECK-NEXT: // No operand predicates @@ -68,10 +68,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 5: @343 +// CHECK-NEXT: // Label 5: @[[L343]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @344 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(379), // Rule ID 2 // +// CHECK-NEXT: // Label 2: @[[L344]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4([[L379:[0-9]+]]), // Rule ID 2 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: // MIs[0] root // CHECK-NEXT: // No operand predicates @@ -85,10 +85,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 6: @379 +// CHECK-NEXT: // Label 6: @[[L379]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 3: @380 +// CHECK-NEXT: // Label 3: @[[L380]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: }; // Size: 381 bytes +// CHECK-NEXT: }; // Size: {{[0-9]+}} bytes // CHECK-NEXT: return MatchTable0; // CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td index 0189d3d056fc..02085c1fd266 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td @@ -135,15 +135,15 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // Verify match table. // CHECK: const uint8_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static uint8_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(132), /*)*//*default:*//*Label 6*/ GIMT_Encode4(677), -// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(462), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4(504), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4(557), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4(599), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4(624), GIMT_Encode4(0), -// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4(637), -// CHECK-NEXT: // Label 0: @462 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(491), // Rule ID 4 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2({{[0-9]+}}), /*)*//*default:*//*Label 6*/ GIMT_Encode4([[L677:[0-9]+]]), +// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4([[L462:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4([[L504:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4([[L557:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4([[L599:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4([[L624:[0-9]+]]), GIMT_Encode4(0), +// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4([[L637:[0-9]+]]), +// CHECK-NEXT: // Label 0: @[[L462]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4([[L491:[0-9]+]]), // Rule ID 4 // // CHECK-NEXT: GIM_CheckFeatures, GIMT_Encode2(GIFBS_HasAnswerToEverything), // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule3Enabled), // CHECK-NEXT: // MIs[0] a @@ -158,8 +158,8 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: // Combiner Rule #3: InstTest1 // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 7: @491 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(503), // Rule ID 3 // +// CHECK-NEXT: // Label 7: @[[L491]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4([[L503:[0-9]+]]), // Rule ID 3 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled), // CHECK-NEXT: // MIs[0] a // CHECK-NEXT: // No operand predicates @@ -168,10 +168,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: // Combiner Rule #2: InstTest0 // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner1), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 8: @503 +// CHECK-NEXT: // Label 8: @[[L503]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @504 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4(556), // Rule ID 6 // +// CHECK-NEXT: // Label 1: @[[L504]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4([[L556:[0-9]+]]), // Rule ID 6 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule5Enabled), // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] dst @@ -189,10 +189,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // z // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 9: @556 +// CHECK-NEXT: // Label 9: @[[L556]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @557 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4(598), // Rule ID 5 // +// CHECK-NEXT: // Label 2: @[[L557]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4([[L598:[0-9]+]]), // Rule ID 5 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule4Enabled), // CHECK-NEXT: // MIs[0] tmp // CHECK-NEXT: GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/0, // MIs[1] @@ -210,32 +210,32 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner2), // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 10: @598 +// CHECK-NEXT: // Label 10: @[[L598]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 3: @599 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4(611), // Rule ID 0 // +// CHECK-NEXT: // Label 3: @[[L599]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4([[L611:[0-9]+]]), // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled), // CHECK-NEXT: // Combiner Rule #0: WipOpcodeTest0; wip_match_opcode 'G_TRUNC' // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 11: @611 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4(623), // Rule ID 1 // +// CHECK-NEXT: // Label 11: @[[L611]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4([[L623:[0-9]+]]), // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_TRUNC' // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 12: @623 +// CHECK-NEXT: // Label 12: @[[L623]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 4: @624 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4(636), // Rule ID 2 // +// CHECK-NEXT: // Label 4: @[[L624]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4([[L636:[0-9]+]]), // Rule ID 2 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled), // CHECK-NEXT: // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_SEXT' // CHECK-NEXT: GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0), // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 13: @636 +// CHECK-NEXT: // Label 13: @[[L636]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 5: @637 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4(676), // Rule ID 7 // +// CHECK-NEXT: // Label 5: @[[L637]] +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4([[L676:[0-9]+]]), // Rule ID 7 // // CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule6Enabled), // CHECK-NEXT: // MIs[0] dst // CHECK-NEXT: // No operand predicates @@ -250,10 +250,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 14: @676 +// CHECK-NEXT: // Label 14: @[[L676]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 6: @677 +// CHECK-NEXT: // Label 6: @[[L677]] // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: }; // Size: 678 bytes +// CHECK-NEXT: }; // Size: {{[0-9]+}} bytes // CHECK-NEXT: return MatchTable0; // CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter.td index f79b792b37a3..82ecc4495e80 100644 --- a/llvm/test/TableGen/GlobalISelEmitter.td +++ b/llvm/test/TableGen/GlobalISelEmitter.td @@ -518,7 +518,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3), // R00O-NEXT: GIM_Reject, // R00O: // Label [[DEFAULT_NUM]]: @[[DEFAULT]] // R00O-NEXT: GIM_Reject, -// R00O-NEXT: }; // Size: 2023 bytes +// R00O-NEXT: }; // Size: 2027 bytes def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, GPR32:$src4), [(set GPR32:$dst, -- GitLab From 2c22a0c16d1cb844eac142156ba67098627a336c Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Fri, 12 Apr 2024 12:35:40 -0700 Subject: [PATCH 047/714] [InstCombine] Add test case for turning sub into xor using dominating condition. NFC I plan to disable using dominating conditions for turning sub into xor, but first we need that demonstrates it currently happens. --- llvm/test/Transforms/InstCombine/sub-xor.ll | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/llvm/test/Transforms/InstCombine/sub-xor.ll b/llvm/test/Transforms/InstCombine/sub-xor.ll index 71da73d51ae3..b4e87d0405fc 100644 --- a/llvm/test/Transforms/InstCombine/sub-xor.ll +++ b/llvm/test/Transforms/InstCombine/sub-xor.ll @@ -157,3 +157,26 @@ define <2 x i8> @xor_add_splat_undef(<2 x i8> %x) { %add = add <2 x i8> %xor, ret <2 x i8> %add } + +define i32 @xor_dominating_cond(i32 %x) { +; CHECK-LABEL: @xor_dominating_cond( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[X:%.*]], 256 +; CHECK-NEXT: br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[A:%.*]] = xor i32 [[X]], 255 +; CHECK-NEXT: ret i32 [[A]] +; CHECK: if.end: +; CHECK-NEXT: ret i32 [[X]] +; +entry: + %cond = icmp ult i32 %x, 256 + br i1 %cond, label %if.then, label %if.end + +if.then: + %a = sub i32 255, %x + ret i32 %a + +if.end: + ret i32 %x +} -- GitLab From 421a8c5892b7e59f27b2c21452f81fa789a758fd Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Tue, 16 Apr 2024 21:25:28 -0700 Subject: [PATCH 048/714] [InstCombine] Add phase ordering test for #88239. NFC --- .../Transforms/PhaseOrdering/X86/pr88239.ll | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll new file mode 100644 index 000000000000..3afa1904fb24 --- /dev/null +++ b/llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll @@ -0,0 +1,55 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -passes="default" -mcpu=skx -S | FileCheck %s + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +define void @foo(ptr noalias noundef %0, ptr noalias noundef %1) optsize { +; CHECK-LABEL: define void @foo( +; CHECK-SAME: ptr noalias nocapture noundef readonly [[TMP0:%.*]], ptr noalias nocapture noundef writeonly [[TMP1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: vector.ph: +; CHECK-NEXT: br label [[TMP4:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[TMP2:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[TMP4]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ , [[TMP2]] ], [ [[VEC_IND_NEXT:%.*]], [[TMP4]] ] +; CHECK-NEXT: [[TMP6:%.*]] = and <8 x i64> [[VEC_IND]], +; CHECK-NEXT: [[TMP3:%.*]] = xor <8 x i64> [[TMP6]], +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], <8 x i64> [[TMP3]] +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = tail call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> [[TMP7]], i32 4, <8 x i1> , <8 x i32> poison) +; CHECK-NEXT: [[TMP5:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_GATHER]], +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 [[INDVARS_IV]] +; CHECK-NEXT: store <8 x i32> [[TMP5]], ptr [[TMP10]], align 4 +; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw i64 [[INDVARS_IV]], 8 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 256 +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[MIDDLE_BLOCK:%.*]], label [[TMP4]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: middle.block: +; CHECK-NEXT: ret void +; + br label %3 + +3: ; preds = %7, %2 + %4 = phi i32 [ 0, %2 ], [ %15, %7 ] + %5 = icmp slt i32 %4, 256 + br i1 %5, label %7, label %6 + +6: ; preds = %3 + ret void + +7: ; preds = %3 + %8 = sub nsw i32 255, %4 + %9 = zext nneg i32 %8 to i64 + %10 = getelementptr inbounds i32, ptr %0, i64 %9 + %11 = load i32, ptr %10, align 4 + %12 = add nsw i32 %11, 5 + %13 = sext i32 %4 to i64 + %14 = getelementptr inbounds i32, ptr %1, i64 %13 + store i32 %12, ptr %14, align 4 + %15 = add nsw i32 %4, 1 + br label %3 +} +;. +; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} +; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} +; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} +;. -- GitLab From ed741ffe893698cd14c6785ac2ee7031d9d344a6 Mon Sep 17 00:00:00 2001 From: Nathan Lanza Date: Wed, 17 Apr 2024 14:06:01 -0400 Subject: [PATCH 049/714] [github] Add ClangIR to new-prs-labeler.yml (#86088) --- .github/new-prs-labeler.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/new-prs-labeler.yml b/.github/new-prs-labeler.yml index a0428336d300..1502d64a7d3e 100644 --- a/.github/new-prs-labeler.yml +++ b/.github/new-prs-labeler.yml @@ -1,3 +1,9 @@ +ClangIR: + - clang/include/clang/CIR/**/* + - clang/lib/CIR/**/* + - clang/tools/cir-*/**/* + - clang/test/CIR/**/* + clang:dataflow: - clang/include/clang/Analysis/FlowSensitive/**/* - clang/lib/Analysis/FlowSensitive/**/* -- GitLab From c02ed29ec151d1d555c3735efef2ab215126ddbf Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 17 Apr 2024 19:01:44 +0100 Subject: [PATCH 050/714] [CostModel][X86] Recognise vector rotation by uniform constant patterns Adds suitable costs for AVX512 targets (we still rely on default expansion for AVX2 and earlier) --- .../lib/Target/X86/X86TargetTransformInfo.cpp | 41 +++++++++++++--- .../Analysis/CostModel/X86/fshl-codesize.ll | 24 +++++----- .../Analysis/CostModel/X86/fshl-latency.ll | 24 +++++----- .../CostModel/X86/fshl-sizelatency.ll | 24 +++++----- llvm/test/Analysis/CostModel/X86/fshl.ll | 22 ++++----- .../Analysis/CostModel/X86/fshr-codesize.ll | 48 +++++++++---------- .../Analysis/CostModel/X86/fshr-latency.ll | 24 +++++----- .../CostModel/X86/fshr-sizelatency.ll | 48 +++++++++---------- llvm/test/Analysis/CostModel/X86/fshr.ll | 22 ++++----- .../SLPVectorizer/X86/arith-fshl-rot.ll | 40 ++++++++++------ .../SLPVectorizer/X86/arith-fshr-rot.ll | 40 ++++++++++------ 11 files changed, 204 insertions(+), 153 deletions(-) diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index 38064f979269..d111c4d4ecc1 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -3402,6 +3402,9 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, { ISD::ROTR, MVT::v32i16, { 1, 1, 1, 1 } }, { ISD::ROTR, MVT::v16i16, { 1, 1, 1, 1 } }, { ISD::ROTR, MVT::v8i16, { 1, 1, 1, 1 } }, + { X86ISD::VROTLI, MVT::v32i16, { 1, 1, 1, 1 } }, + { X86ISD::VROTLI, MVT::v16i16, { 1, 1, 1, 1 } }, + { X86ISD::VROTLI, MVT::v8i16, { 1, 1, 1, 1 } }, }; static const CostKindTblEntry AVX512BITALGCostTbl[] = { { ISD::CTPOP, MVT::v32i16, { 1, 1, 1, 1 } }, @@ -3498,6 +3501,12 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, { ISD::ROTR, MVT::v64i8, { 5, 6, 12, 14 } }, { ISD::ROTR, MVT::v32i8, { 5, 14, 6, 9 } }, { ISD::ROTR, MVT::v16i8, { 5, 14, 6, 9 } }, + { X86ISD::VROTLI, MVT::v32i16, { 2, 5, 3, 3 } }, + { X86ISD::VROTLI, MVT::v16i16, { 1, 5, 3, 3 } }, + { X86ISD::VROTLI, MVT::v8i16, { 1, 5, 3, 3 } }, + { X86ISD::VROTLI, MVT::v64i8, { 2, 9, 3, 4 } }, + { X86ISD::VROTLI, MVT::v32i8, { 1, 9, 3, 4 } }, + { X86ISD::VROTLI, MVT::v16i8, { 1, 8, 3, 4 } }, { ISD::SADDSAT, MVT::v32i16, { 1 } }, { ISD::SADDSAT, MVT::v64i8, { 1 } }, { ISD::SMAX, MVT::v32i16, { 1, 1, 1, 1 } }, @@ -3556,6 +3565,12 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, { ISD::ROTR, MVT::v16i32, { 1, 1, 1, 1 } }, { ISD::ROTR, MVT::v8i32, { 1, 1, 1, 1 } }, { ISD::ROTR, MVT::v4i32, { 1, 1, 1, 1 } }, + { X86ISD::VROTLI, MVT::v8i64, { 1, 1, 1, 1 } }, + { X86ISD::VROTLI, MVT::v4i64, { 1, 1, 1, 1 } }, + { X86ISD::VROTLI, MVT::v2i64, { 1, 1, 1, 1 } }, + { X86ISD::VROTLI, MVT::v16i32, { 1, 1, 1, 1 } }, + { X86ISD::VROTLI, MVT::v8i32, { 1, 1, 1, 1 } }, + { X86ISD::VROTLI, MVT::v4i32, { 1, 1, 1, 1 } }, { ISD::SMAX, MVT::v8i64, { 1, 3, 1, 1 } }, { ISD::SMAX, MVT::v16i32, { 1, 1, 1, 1 } }, { ISD::SMAX, MVT::v32i16, { 3, 7, 5, 5 } }, @@ -3642,7 +3657,15 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, { ISD::ROTR, MVT::v2i64, { 1, 3, 3, 3 } }, { ISD::ROTR, MVT::v4i32, { 1, 3, 3, 3 } }, { ISD::ROTR, MVT::v8i16, { 1, 3, 3, 3 } }, - { ISD::ROTR, MVT::v16i8, { 1, 3, 3, 3 } } + { ISD::ROTR, MVT::v16i8, { 1, 3, 3, 3 } }, + { X86ISD::VROTLI, MVT::v4i64, { 4, 7, 5, 6 } }, + { X86ISD::VROTLI, MVT::v8i32, { 4, 7, 5, 6 } }, + { X86ISD::VROTLI, MVT::v16i16, { 4, 7, 5, 6 } }, + { X86ISD::VROTLI, MVT::v32i8, { 4, 7, 5, 6 } }, + { X86ISD::VROTLI, MVT::v2i64, { 1, 3, 1, 1 } }, + { X86ISD::VROTLI, MVT::v4i32, { 1, 3, 1, 1 } }, + { X86ISD::VROTLI, MVT::v8i16, { 1, 3, 1, 1 } }, + { X86ISD::VROTLI, MVT::v16i8, { 1, 3, 1, 1 } }, }; static const CostKindTblEntry AVX2CostTbl[] = { { ISD::ABS, MVT::v2i64, { 2, 4, 3, 5 } }, // VBLENDVPD(X,VPSUBQ(0,X),X) @@ -4096,9 +4119,11 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, const SmallVectorImpl &Args = ICA.getArgs(); if (Args[0] == Args[1]) { ISD = ISD::ROTL; - // Handle scalar constant rotation amounts. - // TODO: Handle vector + funnel-shift cases. - if (isa_and_nonnull(Args[2])) + // Handle uniform constant rotation amounts. + // TODO: Handle funnel-shift cases. + const APInt *Amt; + if (Args[2] && + PatternMatch::match(Args[2], PatternMatch::m_APIntAllowUndef(Amt))) ISD = X86ISD::VROTLI; } } @@ -4109,10 +4134,12 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, if (!ICA.isTypeBasedOnly()) { const SmallVectorImpl &Args = ICA.getArgs(); if (Args[0] == Args[1]) { - // Handle scalar constant rotation amount. - // TODO: Handle vector + funnel-shift cases. ISD = ISD::ROTR; - if (isa_and_nonnull(Args[2])) + // Handle uniform constant rotation amount. + // TODO: Handle funnel-shift cases. + const APInt *Amt; + if (Args[2] && + PatternMatch::match(Args[2], PatternMatch::m_APIntAllowUndef(Amt))) ISD = X86ISD::VROTLI; } } diff --git a/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll b/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll index c46e32ffb4ad..a7585a4d9f39 100644 --- a/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll +++ b/llvm/test/Analysis/CostModel/X86/fshl-codesize.ll @@ -2744,9 +2744,9 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; AVX512BW-LABEL: 'splatconstant_rotate_i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i16' @@ -2829,9 +2829,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512BW-LABEL: 'splatconstant_rotate_i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i8' @@ -2843,9 +2843,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512VBMI2-LABEL: 'splatconstant_rotate_i8' ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; SLM-LABEL: 'splatconstant_rotate_i8' @@ -2871,9 +2871,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) diff --git a/llvm/test/Analysis/CostModel/X86/fshl-latency.ll b/llvm/test/Analysis/CostModel/X86/fshl-latency.ll index fa32497c63ec..7105f713fdc3 100644 --- a/llvm/test/Analysis/CostModel/X86/fshl-latency.ll +++ b/llvm/test/Analysis/CostModel/X86/fshl-latency.ll @@ -2696,9 +2696,9 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; AVX512BW-LABEL: 'splatconstant_rotate_i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i16' @@ -2781,9 +2781,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512BW-LABEL: 'splatconstant_rotate_i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i8' @@ -2795,9 +2795,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512VBMI2-LABEL: 'splatconstant_rotate_i8' ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; SLM-LABEL: 'splatconstant_rotate_i8' @@ -2823,9 +2823,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) diff --git a/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll index 832a574a9b33..5d7361e29317 100644 --- a/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll +++ b/llvm/test/Analysis/CostModel/X86/fshl-sizelatency.ll @@ -2984,9 +2984,9 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; AVX512BW-LABEL: 'splatconstant_rotate_i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i16' @@ -3069,9 +3069,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512BW-LABEL: 'splatconstant_rotate_i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i8' @@ -3083,9 +3083,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512VBMI2-LABEL: 'splatconstant_rotate_i8' ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; SLM-LABEL: 'splatconstant_rotate_i8' @@ -3111,9 +3111,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) diff --git a/llvm/test/Analysis/CostModel/X86/fshl.ll b/llvm/test/Analysis/CostModel/X86/fshl.ll index 311d8d5ed7d2..1cbdab09acd9 100644 --- a/llvm/test/Analysis/CostModel/X86/fshl.ll +++ b/llvm/test/Analysis/CostModel/X86/fshl.ll @@ -2682,8 +2682,8 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; AVX512BW-LABEL: 'splatconstant_rotate_i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %a16, i16 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; @@ -2767,9 +2767,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512BW-LABEL: 'splatconstant_rotate_i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i8' @@ -2781,9 +2781,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512VBMI2-LABEL: 'splatconstant_rotate_i8' ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; SLM-LABEL: 'splatconstant_rotate_i8' @@ -2809,9 +2809,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %a8, i8 3) diff --git a/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll b/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll index f9d30e4ced3e..ecc861dd7f8e 100644 --- a/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll +++ b/llvm/test/Analysis/CostModel/X86/fshr-codesize.ll @@ -2644,9 +2644,9 @@ define void @splatconstant_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; ; XOP-LABEL: 'splatconstant_rotate_i64' ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %a64, i64 7) -; XOP-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %a64, i64 7) @@ -2701,9 +2701,9 @@ define void @splatconstant_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; ; XOP-LABEL: 'splatconstant_rotate_i32' ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %a32, i32 5) -; XOP-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %a32, i32 5) @@ -2744,9 +2744,9 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; AVX512BW-LABEL: 'splatconstant_rotate_i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i16' @@ -2779,9 +2779,9 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; XOP-LABEL: 'splatconstant_rotate_i16' ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) -; XOP-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' @@ -2829,9 +2829,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512BW-LABEL: 'splatconstant_rotate_i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i8' @@ -2843,9 +2843,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512VBMI2-LABEL: 'splatconstant_rotate_i8' ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; SLM-LABEL: 'splatconstant_rotate_i8' @@ -2864,16 +2864,16 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; XOP-LABEL: 'splatconstant_rotate_i8' ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; XOP-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) diff --git a/llvm/test/Analysis/CostModel/X86/fshr-latency.ll b/llvm/test/Analysis/CostModel/X86/fshr-latency.ll index ed2227591847..0142ad77849c 100644 --- a/llvm/test/Analysis/CostModel/X86/fshr-latency.ll +++ b/llvm/test/Analysis/CostModel/X86/fshr-latency.ll @@ -2696,9 +2696,9 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; AVX512BW-LABEL: 'splatconstant_rotate_i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i16' @@ -2781,9 +2781,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512BW-LABEL: 'splatconstant_rotate_i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i8' @@ -2795,9 +2795,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512VBMI2-LABEL: 'splatconstant_rotate_i8' ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; SLM-LABEL: 'splatconstant_rotate_i8' @@ -2823,9 +2823,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) diff --git a/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll index 8931781f70bd..6dafb20a0aee 100644 --- a/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll +++ b/llvm/test/Analysis/CostModel/X86/fshr-sizelatency.ll @@ -2849,9 +2849,9 @@ define void @splatconstant_rotate_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256 ; ; XOP-LABEL: 'splatconstant_rotate_i64' ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %a64, i64 7) -; XOP-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %a128, <2 x i64> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %a256, <4 x i64> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %a512, <8 x i64> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i64' @@ -2934,9 +2934,9 @@ define void @splatconstant_rotate_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256 ; ; XOP-LABEL: 'splatconstant_rotate_i32' ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %a32, i32 5) -; XOP-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %a128, <4 x i32> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %a256, <8 x i32> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %a512, <16 x i32> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i32' @@ -2984,9 +2984,9 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; AVX512BW-LABEL: 'splatconstant_rotate_i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i16' @@ -3019,9 +3019,9 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; XOP-LABEL: 'splatconstant_rotate_i16' ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) -; XOP-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i16' @@ -3069,9 +3069,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512BW-LABEL: 'splatconstant_rotate_i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i8' @@ -3083,9 +3083,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512VBMI2-LABEL: 'splatconstant_rotate_i8' ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; SLM-LABEL: 'splatconstant_rotate_i8' @@ -3104,16 +3104,16 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; XOP-LABEL: 'splatconstant_rotate_i8' ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; XOP-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; XOP-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; XOP-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; XOP-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) diff --git a/llvm/test/Analysis/CostModel/X86/fshr.ll b/llvm/test/Analysis/CostModel/X86/fshr.ll index ca9ddcc52938..ada1b9c5bdc4 100644 --- a/llvm/test/Analysis/CostModel/X86/fshr.ll +++ b/llvm/test/Analysis/CostModel/X86/fshr.ll @@ -2682,8 +2682,8 @@ define void @splatconstant_rotate_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25 ; ; AVX512BW-LABEL: 'splatconstant_rotate_i16' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %a16, i16 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %a128, <8 x i16> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %a256, <16 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %a512, <32 x i16> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; @@ -2767,9 +2767,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512BW-LABEL: 'splatconstant_rotate_i8' ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; AVX512DQ-LABEL: 'splatconstant_rotate_i8' @@ -2781,9 +2781,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512VBMI2-LABEL: 'splatconstant_rotate_i8' ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512VBMI2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; SLM-LABEL: 'splatconstant_rotate_i8' @@ -2809,9 +2809,9 @@ define void @splatconstant_rotate_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, < ; ; AVX512GFNI-LABEL: 'splatconstant_rotate_i8' ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) -; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %a128, <16 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %a256, <32 x i8> ) +; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %a512, <64 x i8> ) ; AVX512GFNI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %a8, i8 3) diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fshl-rot.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fshl-rot.ll index 639aa0a1c6a2..9b8480cd0088 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/arith-fshl-rot.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fshl-rot.ll @@ -834,22 +834,34 @@ define void @fshl_v2i32_uniformconst() { ; SSE-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 ; SSE-NEXT: ret void ; -; AVX-LABEL: @fshl_v2i32_uniformconst( -; AVX-NEXT: [[A0:%.*]] = load i32, ptr @a32, align 4 -; AVX-NEXT: [[A1:%.*]] = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 1), align 4 -; AVX-NEXT: [[R0:%.*]] = call i32 @llvm.fshl.i32(i32 [[A0]], i32 [[A0]], i32 1) -; AVX-NEXT: [[R1:%.*]] = call i32 @llvm.fshl.i32(i32 [[A1]], i32 [[A1]], i32 1) -; AVX-NEXT: store i32 [[R0]], ptr @d32, align 4 -; AVX-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 -; AVX-NEXT: ret void +; AVX1-LABEL: @fshl_v2i32_uniformconst( +; AVX1-NEXT: [[A0:%.*]] = load i32, ptr @a32, align 4 +; AVX1-NEXT: [[A1:%.*]] = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 1), align 4 +; AVX1-NEXT: [[R0:%.*]] = call i32 @llvm.fshl.i32(i32 [[A0]], i32 [[A0]], i32 1) +; AVX1-NEXT: [[R1:%.*]] = call i32 @llvm.fshl.i32(i32 [[A1]], i32 [[A1]], i32 1) +; AVX1-NEXT: store i32 [[R0]], ptr @d32, align 4 +; AVX1-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 +; AVX1-NEXT: ret void +; +; AVX2-LABEL: @fshl_v2i32_uniformconst( +; AVX2-NEXT: [[A0:%.*]] = load i32, ptr @a32, align 4 +; AVX2-NEXT: [[A1:%.*]] = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 1), align 4 +; AVX2-NEXT: [[R0:%.*]] = call i32 @llvm.fshl.i32(i32 [[A0]], i32 [[A0]], i32 1) +; AVX2-NEXT: [[R1:%.*]] = call i32 @llvm.fshl.i32(i32 [[A1]], i32 [[A1]], i32 1) +; AVX2-NEXT: store i32 [[R0]], ptr @d32, align 4 +; AVX2-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 +; AVX2-NEXT: ret void +; +; AVX256-LABEL: @fshl_v2i32_uniformconst( +; AVX256-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr @a32, align 4 +; AVX256-NEXT: [[TMP2:%.*]] = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> [[TMP1]], <2 x i32> [[TMP1]], <2 x i32> ) +; AVX256-NEXT: store <2 x i32> [[TMP2]], ptr @d32, align 4 +; AVX256-NEXT: ret void ; ; AVX512-LABEL: @fshl_v2i32_uniformconst( -; AVX512-NEXT: [[A0:%.*]] = load i32, ptr @a32, align 4 -; AVX512-NEXT: [[A1:%.*]] = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 1), align 4 -; AVX512-NEXT: [[R0:%.*]] = call i32 @llvm.fshl.i32(i32 [[A0]], i32 [[A0]], i32 1) -; AVX512-NEXT: [[R1:%.*]] = call i32 @llvm.fshl.i32(i32 [[A1]], i32 [[A1]], i32 1) -; AVX512-NEXT: store i32 [[R0]], ptr @d32, align 4 -; AVX512-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 +; AVX512-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr @a32, align 4 +; AVX512-NEXT: [[TMP2:%.*]] = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> [[TMP1]], <2 x i32> [[TMP1]], <2 x i32> ) +; AVX512-NEXT: store <2 x i32> [[TMP2]], ptr @d32, align 4 ; AVX512-NEXT: ret void ; %a0 = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 0 ), align 4 diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fshr-rot.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fshr-rot.ll index c557c9647551..f3e73d0e6840 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/arith-fshr-rot.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fshr-rot.ll @@ -834,22 +834,34 @@ define void @fshr_v2i32_uniformconst() { ; SSE-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 ; SSE-NEXT: ret void ; -; AVX-LABEL: @fshr_v2i32_uniformconst( -; AVX-NEXT: [[A0:%.*]] = load i32, ptr @a32, align 4 -; AVX-NEXT: [[A1:%.*]] = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 1), align 4 -; AVX-NEXT: [[R0:%.*]] = call i32 @llvm.fshr.i32(i32 [[A0]], i32 [[A0]], i32 1) -; AVX-NEXT: [[R1:%.*]] = call i32 @llvm.fshr.i32(i32 [[A1]], i32 [[A1]], i32 1) -; AVX-NEXT: store i32 [[R0]], ptr @d32, align 4 -; AVX-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 -; AVX-NEXT: ret void +; AVX1-LABEL: @fshr_v2i32_uniformconst( +; AVX1-NEXT: [[A0:%.*]] = load i32, ptr @a32, align 4 +; AVX1-NEXT: [[A1:%.*]] = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 1), align 4 +; AVX1-NEXT: [[R0:%.*]] = call i32 @llvm.fshr.i32(i32 [[A0]], i32 [[A0]], i32 1) +; AVX1-NEXT: [[R1:%.*]] = call i32 @llvm.fshr.i32(i32 [[A1]], i32 [[A1]], i32 1) +; AVX1-NEXT: store i32 [[R0]], ptr @d32, align 4 +; AVX1-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 +; AVX1-NEXT: ret void +; +; AVX2-LABEL: @fshr_v2i32_uniformconst( +; AVX2-NEXT: [[A0:%.*]] = load i32, ptr @a32, align 4 +; AVX2-NEXT: [[A1:%.*]] = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 1), align 4 +; AVX2-NEXT: [[R0:%.*]] = call i32 @llvm.fshr.i32(i32 [[A0]], i32 [[A0]], i32 1) +; AVX2-NEXT: [[R1:%.*]] = call i32 @llvm.fshr.i32(i32 [[A1]], i32 [[A1]], i32 1) +; AVX2-NEXT: store i32 [[R0]], ptr @d32, align 4 +; AVX2-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 +; AVX2-NEXT: ret void +; +; AVX256-LABEL: @fshr_v2i32_uniformconst( +; AVX256-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr @a32, align 4 +; AVX256-NEXT: [[TMP2:%.*]] = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> [[TMP1]], <2 x i32> [[TMP1]], <2 x i32> ) +; AVX256-NEXT: store <2 x i32> [[TMP2]], ptr @d32, align 4 +; AVX256-NEXT: ret void ; ; AVX512-LABEL: @fshr_v2i32_uniformconst( -; AVX512-NEXT: [[A0:%.*]] = load i32, ptr @a32, align 4 -; AVX512-NEXT: [[A1:%.*]] = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 1), align 4 -; AVX512-NEXT: [[R0:%.*]] = call i32 @llvm.fshr.i32(i32 [[A0]], i32 [[A0]], i32 1) -; AVX512-NEXT: [[R1:%.*]] = call i32 @llvm.fshr.i32(i32 [[A1]], i32 [[A1]], i32 1) -; AVX512-NEXT: store i32 [[R0]], ptr @d32, align 4 -; AVX512-NEXT: store i32 [[R1]], ptr getelementptr inbounds ([16 x i32], ptr @d32, i32 0, i64 1), align 4 +; AVX512-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr @a32, align 4 +; AVX512-NEXT: [[TMP2:%.*]] = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> [[TMP1]], <2 x i32> [[TMP1]], <2 x i32> ) +; AVX512-NEXT: store <2 x i32> [[TMP2]], ptr @d32, align 4 ; AVX512-NEXT: ret void ; %a0 = load i32, ptr getelementptr inbounds ([16 x i32], ptr @a32, i32 0, i64 0 ), align 4 -- GitLab From 58a08e154c804051aaca9151a8053aea3ec15646 Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Wed, 17 Apr 2024 11:15:25 -0700 Subject: [PATCH 051/714] [RISCV] Add coverage for strength reduction of mul by small negative immediates --- llvm/test/CodeGen/RISCV/rv32zba.ll | 93 ++++++++++++++++++++++++++++++ llvm/test/CodeGen/RISCV/rv64zba.ll | 93 ++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) diff --git a/llvm/test/CodeGen/RISCV/rv32zba.ll b/llvm/test/CodeGen/RISCV/rv32zba.ll index cc632a09c805..a78f823d3184 100644 --- a/llvm/test/CodeGen/RISCV/rv32zba.ll +++ b/llvm/test/CodeGen/RISCV/rv32zba.ll @@ -645,3 +645,96 @@ define i32 @addshl_5_8(i32 %a, i32 %b) { %e = add i32 %c, %d ret i32 %e } + +define i32 @mul_neg1(i32 %a) { +; CHECK-LABEL: mul_neg1: +; CHECK: # %bb.0: +; CHECK-NEXT: neg a0, a0 +; CHECK-NEXT: ret + %c = mul i32 %a, -1 + ret i32 %c +} + +define i32 @mul_neg2(i32 %a) { +; CHECK-LABEL: mul_neg2: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a0, a0, 1 +; CHECK-NEXT: neg a0, a0 +; CHECK-NEXT: ret + %c = mul i32 %a, -2 + ret i32 %c +} + +define i32 @mul_neg3(i32 %a) { +; RV32I-LABEL: mul_neg3: +; RV32I: # %bb.0: +; RV32I-NEXT: slli a1, a0, 1 +; RV32I-NEXT: neg a0, a0 +; RV32I-NEXT: sub a0, a0, a1 +; RV32I-NEXT: ret +; +; RV32ZBA-LABEL: mul_neg3: +; RV32ZBA: # %bb.0: +; RV32ZBA-NEXT: sh1add a0, a0, a0 +; RV32ZBA-NEXT: neg a0, a0 +; RV32ZBA-NEXT: ret + %c = mul i32 %a, -3 + ret i32 %c +} + +define i32 @mul_neg4(i32 %a) { +; CHECK-LABEL: mul_neg4: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a0, a0, 2 +; CHECK-NEXT: neg a0, a0 +; CHECK-NEXT: ret + %c = mul i32 %a, -4 + ret i32 %c +} + +define i32 @mul_neg5(i32 %a) { +; RV32I-LABEL: mul_neg5: +; RV32I: # %bb.0: +; RV32I-NEXT: slli a1, a0, 2 +; RV32I-NEXT: neg a0, a0 +; RV32I-NEXT: sub a0, a0, a1 +; RV32I-NEXT: ret +; +; RV32ZBA-LABEL: mul_neg5: +; RV32ZBA: # %bb.0: +; RV32ZBA-NEXT: sh2add a0, a0, a0 +; RV32ZBA-NEXT: neg a0, a0 +; RV32ZBA-NEXT: ret + %c = mul i32 %a, -5 + ret i32 %c +} + +define i32 @mul_neg6(i32 %a) { +; CHECK-LABEL: mul_neg6: +; CHECK: # %bb.0: +; CHECK-NEXT: li a1, -6 +; CHECK-NEXT: mul a0, a0, a1 +; CHECK-NEXT: ret + %c = mul i32 %a, -6 + ret i32 %c +} + +define i32 @mul_neg7(i32 %a) { +; CHECK-LABEL: mul_neg7: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a1, a0, 3 +; CHECK-NEXT: sub a0, a0, a1 +; CHECK-NEXT: ret + %c = mul i32 %a, -7 + ret i32 %c +} + +define i32 @mul_neg8(i32 %a) { +; CHECK-LABEL: mul_neg8: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a0, a0, 3 +; CHECK-NEXT: neg a0, a0 +; CHECK-NEXT: ret + %c = mul i32 %a, -8 + ret i32 %c +} diff --git a/llvm/test/CodeGen/RISCV/rv64zba.ll b/llvm/test/CodeGen/RISCV/rv64zba.ll index b4c80b60e0ba..6939185947f4 100644 --- a/llvm/test/CodeGen/RISCV/rv64zba.ll +++ b/llvm/test/CodeGen/RISCV/rv64zba.ll @@ -2533,3 +2533,96 @@ define i64 @regression(i32 signext %x, i32 signext %y) { %res = mul nuw nsw i64 %ext, 24 ret i64 %res } + +define i64 @mul_neg1(i64 %a) { +; CHECK-LABEL: mul_neg1: +; CHECK: # %bb.0: +; CHECK-NEXT: neg a0, a0 +; CHECK-NEXT: ret + %c = mul i64 %a, -1 + ret i64 %c +} + +define i64 @mul_neg2(i64 %a) { +; CHECK-LABEL: mul_neg2: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a0, a0, 1 +; CHECK-NEXT: neg a0, a0 +; CHECK-NEXT: ret + %c = mul i64 %a, -2 + ret i64 %c +} + +define i64 @mul_neg3(i64 %a) { +; RV64I-LABEL: mul_neg3: +; RV64I: # %bb.0: +; RV64I-NEXT: slli a1, a0, 1 +; RV64I-NEXT: neg a0, a0 +; RV64I-NEXT: sub a0, a0, a1 +; RV64I-NEXT: ret +; +; RV64ZBA-LABEL: mul_neg3: +; RV64ZBA: # %bb.0: +; RV64ZBA-NEXT: sh1add a0, a0, a0 +; RV64ZBA-NEXT: neg a0, a0 +; RV64ZBA-NEXT: ret + %c = mul i64 %a, -3 + ret i64 %c +} + +define i64 @mul_neg4(i64 %a) { +; CHECK-LABEL: mul_neg4: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a0, a0, 2 +; CHECK-NEXT: neg a0, a0 +; CHECK-NEXT: ret + %c = mul i64 %a, -4 + ret i64 %c +} + +define i64 @mul_neg5(i64 %a) { +; RV64I-LABEL: mul_neg5: +; RV64I: # %bb.0: +; RV64I-NEXT: slli a1, a0, 2 +; RV64I-NEXT: neg a0, a0 +; RV64I-NEXT: sub a0, a0, a1 +; RV64I-NEXT: ret +; +; RV64ZBA-LABEL: mul_neg5: +; RV64ZBA: # %bb.0: +; RV64ZBA-NEXT: sh2add a0, a0, a0 +; RV64ZBA-NEXT: neg a0, a0 +; RV64ZBA-NEXT: ret + %c = mul i64 %a, -5 + ret i64 %c +} + +define i64 @mul_neg6(i64 %a) { +; CHECK-LABEL: mul_neg6: +; CHECK: # %bb.0: +; CHECK-NEXT: li a1, -6 +; CHECK-NEXT: mul a0, a0, a1 +; CHECK-NEXT: ret + %c = mul i64 %a, -6 + ret i64 %c +} + +define i64 @mul_neg7(i64 %a) { +; CHECK-LABEL: mul_neg7: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a1, a0, 3 +; CHECK-NEXT: sub a0, a0, a1 +; CHECK-NEXT: ret + %c = mul i64 %a, -7 + ret i64 %c +} + +define i64 @mul_neg8(i64 %a) { +; CHECK-LABEL: mul_neg8: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a0, a0, 3 +; CHECK-NEXT: neg a0, a0 +; CHECK-NEXT: ret + %c = mul i64 %a, -8 + ret i64 %c +} -- GitLab From cc82f1290a1e2157a6c0530d78d8cc84d2b8553d Mon Sep 17 00:00:00 2001 From: Usman Nadeem Date: Wed, 17 Apr 2024 11:42:52 -0700 Subject: [PATCH 052/714] [AArch64] Update latencies for Cortex-A510 scheduling model (#87293) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updated according to the Software Optimization Guide for Arm® Cortex®‑A510 Core Revision: r1p3 Issue 6.0. --- llvm/lib/Target/AArch64/AArch64SchedA510.td | 145 +- .../AArch64/GlobalISel/combine-udiv.ll | 8 +- llvm/test/CodeGen/AArch64/aarch64-addv.ll | 10 +- .../AArch64/aarch64-dup-ext-scalable.ll | 40 +- llvm/test/CodeGen/AArch64/aarch64-smull.ll | 2 +- llvm/test/CodeGen/AArch64/active_lane_mask.ll | 169 +- .../CodeGen/AArch64/arm64-convert-v4f64.ll | 14 +- llvm/test/CodeGen/AArch64/arm64-vabs.ll | 106 +- llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll | 28 +- llvm/test/CodeGen/AArch64/arm64-vhadd.ll | 8 +- ...interleaving-add-mull-scalable-contract.ll | 64 +- ...x-deinterleaving-add-mull-scalable-fast.ll | 63 +- ...complex-deinterleaving-f16-mul-scalable.ll | 23 +- ...complex-deinterleaving-f32-mul-scalable.ll | 18 +- ...complex-deinterleaving-f64-mul-scalable.ll | 18 +- ...complex-deinterleaving-i16-mul-scalable.ll | 3 +- ...rleaving-reductions-predicated-scalable.ll | 64 +- ...plex-deinterleaving-reductions-scalable.ll | 26 +- .../complex-deinterleaving-splat-scalable.ll | 16 +- .../AArch64/concat_vector-truncate-combine.ll | 2 +- .../AArch64/dag-combine-concat-vectors.ll | 12 +- .../div-rem-pair-recomposition-signed.ll | 6 +- .../div-rem-pair-recomposition-unsigned.ll | 6 +- llvm/test/CodeGen/AArch64/extbinopload.ll | 121 +- llvm/test/CodeGen/AArch64/fcmp.ll | 108 +- llvm/test/CodeGen/AArch64/fdiv-combine.ll | 2 +- .../fold-int-pow2-with-fmul-or-fdiv.ll | 2 +- .../CodeGen/AArch64/fp-veclib-expansion.ll | 8 +- llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll | 8 +- .../test/CodeGen/AArch64/fptosi-sat-vector.ll | 20 +- llvm/test/CodeGen/AArch64/funnel-shift-rot.ll | 4 +- ...st-and-by-const-from-lshr-in-eqcmp-zero.ll | 2 +- llvm/test/CodeGen/AArch64/icmp.ll | 4 +- llvm/test/CodeGen/AArch64/insert-extend.ll | 54 +- .../insert-subvector-res-legalization.ll | 70 +- .../AArch64/intrinsic-cttz-elts-sve.ll | 91 +- llvm/test/CodeGen/AArch64/itofp.ll | 1132 ++++++------ llvm/test/CodeGen/AArch64/ldexp.ll | 6 +- .../CodeGen/AArch64/llvm-ir-to-intrinsic.ll | 43 +- llvm/test/CodeGen/AArch64/load-insert-zero.ll | 28 +- llvm/test/CodeGen/AArch64/logic-shift.ll | 48 +- .../AArch64/named-vector-shuffles-sve.ll | 76 +- llvm/test/CodeGen/AArch64/neon-dotreduce.ll | 758 ++++---- llvm/test/CodeGen/AArch64/neon-extadd.ll | 12 +- llvm/test/CodeGen/AArch64/neon-shift-neg.ll | 24 +- .../CodeGen/AArch64/predicated-add-sub.ll | 22 +- .../AArch64/ragreedy-local-interval-cost.ll | 262 ++- llvm/test/CodeGen/AArch64/rcpc3-sve.ll | 4 +- llvm/test/CodeGen/AArch64/reassocmls.ll | 6 +- llvm/test/CodeGen/AArch64/reduce-shuffle.ll | 356 ++-- llvm/test/CodeGen/AArch64/sat-add.ll | 8 +- llvm/test/CodeGen/AArch64/sext.ll | 36 +- .../CodeGen/AArch64/sink-addsub-of-const.ll | 24 +- ...ate-sm-changing-call-disable-coalescing.ll | 130 +- .../sme-streaming-compatible-interface.ll | 2 +- .../AArch64/sme2-intrinsics-fp-dots.ll | 12 +- .../AArch64/sme2-intrinsics-int-dots.ll | 42 +- .../CodeGen/AArch64/sme2-intrinsics-max.ll | 305 ++- .../CodeGen/AArch64/sme2-intrinsics-min.ll | 163 +- .../CodeGen/AArch64/sme2-intrinsics-mlall.ll | 54 +- .../CodeGen/AArch64/sme2-intrinsics-rshl.ll | 161 +- .../AArch64/sme2-intrinsics-sqdmulh.ll | 89 +- .../CodeGen/AArch64/split-vector-insert.ll | 265 ++- .../CodeGen/AArch64/srem-seteq-vec-splat.ll | 16 +- llvm/test/CodeGen/AArch64/srem-vector-lkk.ll | 4 +- llvm/test/CodeGen/AArch64/sve-abd.ll | 18 +- llvm/test/CodeGen/AArch64/sve-bitcast.ll | 20 +- .../AArch64/sve-calling-convention-mixed.ll | 116 +- llvm/test/CodeGen/AArch64/sve-cmp-folds.ll | 4 +- llvm/test/CodeGen/AArch64/sve-doublereduct.ll | 8 +- llvm/test/CodeGen/AArch64/sve-expand-div.ll | 16 +- .../CodeGen/AArch64/sve-extract-element.ll | 10 +- .../sve-extract-fixed-from-scalable-vector.ll | 8 +- .../AArch64/sve-extract-fixed-vector.ll | 16 +- .../AArch64/sve-extract-scalable-vector.ll | 18 +- llvm/test/CodeGen/AArch64/sve-fcmp.ll | 2 +- llvm/test/CodeGen/AArch64/sve-fcopysign.ll | 18 +- llvm/test/CodeGen/AArch64/sve-fcvt.ll | 64 +- .../sve-fixed-length-addressing-modes.ll | 4 +- .../AArch64/sve-fixed-length-build-vector.ll | 13 +- .../AArch64/sve-fixed-length-concat.ll | 112 +- .../sve-fixed-length-extract-vector-elt.ll | 24 +- .../AArch64/sve-fixed-length-fcopysign.ll | 16 +- .../sve-fixed-length-fp-extend-trunc.ll | 6 +- .../AArch64/sve-fixed-length-fp-select.ll | 234 +-- .../AArch64/sve-fixed-length-fp-to-int.ll | 24 +- .../CodeGen/AArch64/sve-fixed-length-fp128.ll | 8 +- .../sve-fixed-length-frame-offests-crash.ll | 32 +- .../sve-fixed-length-insert-vector-elt.ll | 222 +-- .../AArch64/sve-fixed-length-int-arith.ll | 4 +- .../AArch64/sve-fixed-length-int-div.ll | 56 +- .../AArch64/sve-fixed-length-int-extends.ll | 12 +- .../AArch64/sve-fixed-length-int-rem.ll | 88 +- .../AArch64/sve-fixed-length-int-select.ll | 312 ++-- .../AArch64/sve-fixed-length-int-to-fp.ll | 24 +- .../AArch64/sve-fixed-length-mask-opt.ll | 6 +- .../sve-fixed-length-masked-128bit-loads.ll | 2 +- .../sve-fixed-length-masked-128bit-stores.ll | 18 +- .../AArch64/sve-fixed-length-masked-gather.ll | 148 +- .../AArch64/sve-fixed-length-masked-loads.ll | 32 +- .../sve-fixed-length-masked-scatter.ll | 156 +- .../AArch64/sve-fixed-length-masked-stores.ll | 42 +- .../AArch64/sve-fixed-length-shuffles.ll | 26 +- .../AArch64/sve-fixed-length-splat-vector.ll | 84 +- .../AArch64/sve-fixed-length-trunc-stores.ll | 16 +- .../sve-fixed-length-vector-shuffle-tbl.ll | 6 +- .../CodeGen/AArch64/sve-fp-int-min-max.ll | 2 +- .../test/CodeGen/AArch64/sve-fp-reciprocal.ll | 6 +- .../CodeGen/AArch64/sve-fp-reduce-fadda.ll | 4 +- llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll | 282 +-- llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll | 32 +- .../AArch64/sve-gather-scatter-addr-opts.ll | 28 +- llvm/test/CodeGen/AArch64/sve-hadd.ll | 20 +- .../AArch64/sve-implicit-zero-filling.ll | 8 +- .../CodeGen/AArch64/sve-insert-element.ll | 88 +- .../test/CodeGen/AArch64/sve-insert-vector.ll | 22 +- .../test/CodeGen/AArch64/sve-int-arith-imm.ll | 34 +- llvm/test/CodeGen/AArch64/sve-int-arith.ll | 20 +- llvm/test/CodeGen/AArch64/sve-int-reduce.ll | 62 +- .../CodeGen/AArch64/sve-intrinsics-index.ll | 2 +- .../AArch64/sve-intrinsics-int-arith-imm.ll | 38 +- .../AArch64/sve-intrinsics-logical-imm.ll | 2 +- llvm/test/CodeGen/AArch64/sve-ld-post-inc.ll | 30 +- llvm/test/CodeGen/AArch64/sve-ld1r.ll | 16 +- .../sve-lsr-scaled-index-addressing-mode.ll | 4 +- .../AArch64/sve-masked-gather-legalize.ll | 8 +- .../CodeGen/AArch64/sve-masked-ldst-sext.ll | 2 +- .../CodeGen/AArch64/sve-masked-ldst-zext.ll | 6 +- .../AArch64/sve-masked-scatter-legalize.ll | 20 +- .../CodeGen/AArch64/sve-masked-scatter.ll | 2 +- llvm/test/CodeGen/AArch64/sve-pr62151.ll | 2 +- llvm/test/CodeGen/AArch64/sve-pred-arith.ll | 40 +- .../test/CodeGen/AArch64/sve-pred-selectop.ll | 108 +- .../CodeGen/AArch64/sve-pred-selectop2.ll | 76 +- .../CodeGen/AArch64/sve-pred-selectop3.ll | 28 +- .../AArch64/sve-ptest-removal-cmple.ll | 8 +- .../CodeGen/AArch64/sve-redundant-store.ll | 2 +- .../CodeGen/AArch64/sve-split-extract-elt.ll | 42 +- llvm/test/CodeGen/AArch64/sve-split-fcvt.ll | 26 +- .../CodeGen/AArch64/sve-split-fp-reduce.ll | 2 +- .../CodeGen/AArch64/sve-split-insert-elt.ll | 26 +- .../CodeGen/AArch64/sve-split-int-reduce.ll | 8 +- llvm/test/CodeGen/AArch64/sve-split-load.ll | 4 +- llvm/test/CodeGen/AArch64/sve-split-store.ll | 8 +- .../CodeGen/AArch64/sve-srem-combine-loop.ll | 2 +- .../sve-st1-addressing-mode-reg-imm.ll | 8 +- llvm/test/CodeGen/AArch64/sve-stepvector.ll | 2 +- ...treaming-mode-fixed-length-bit-counting.ll | 36 +- ...e-streaming-mode-fixed-length-ext-loads.ll | 10 +- ...e-streaming-mode-fixed-length-fcopysign.ll | 26 +- ...ve-streaming-mode-fixed-length-fp-arith.ll | 48 +- ...streaming-mode-fixed-length-fp-compares.ll | 44 +- ...-streaming-mode-fixed-length-fp-convert.ll | 4 +- ...aming-mode-fixed-length-fp-extend-trunc.ll | 10 +- .../sve-streaming-mode-fixed-length-fp-fma.ll | 6 +- ...e-streaming-mode-fixed-length-fp-minmax.ll | 24 +- ...e-streaming-mode-fixed-length-fp-reduce.ll | 30 +- ...streaming-mode-fixed-length-fp-rounding.ll | 42 +- ...e-streaming-mode-fixed-length-fp-select.ll | 18 +- ...e-streaming-mode-fixed-length-fp-to-int.ll | 118 +- ...-streaming-mode-fixed-length-fp-vselect.ll | 6 +- ...ing-mode-fixed-length-insert-vector-elt.ll | 42 +- ...e-streaming-mode-fixed-length-int-arith.ll | 16 +- ...treaming-mode-fixed-length-int-compares.ll | 14 +- ...sve-streaming-mode-fixed-length-int-div.ll | 34 +- ...eaming-mode-fixed-length-int-immediates.ll | 8 +- ...-streaming-mode-fixed-length-int-minmax.ll | 32 +- ...ve-streaming-mode-fixed-length-int-mulh.ll | 20 +- ...-streaming-mode-fixed-length-int-reduce.ll | 40 +- ...sve-streaming-mode-fixed-length-int-rem.ll | 64 +- ...-streaming-mode-fixed-length-int-select.ll | 28 +- ...-streaming-mode-fixed-length-int-shifts.ll | 32 +- ...e-streaming-mode-fixed-length-int-to-fp.ll | 56 +- ...streaming-mode-fixed-length-int-vselect.ll | 10 +- ...-streaming-mode-fixed-length-log-reduce.ll | 24 +- ...treaming-mode-fixed-length-masked-store.ll | 16 +- ...eaming-mode-fixed-length-optimize-ptrue.ll | 10 +- ...streaming-mode-fixed-length-permute-rev.ll | 24 +- ...g-mode-fixed-length-permute-zip-uzp-trn.ll | 30 +- .../sve-streaming-mode-fixed-length-ptest.ll | 78 +- .../sve-streaming-mode-fixed-length-rev.ll | 14 +- ...e-streaming-mode-fixed-length-sdiv-pow2.ll | 8 +- ...sve-streaming-mode-fixed-length-shuffle.ll | 2 +- .../sve-streaming-mode-fixed-length-stores.ll | 4 +- .../sve-streaming-mode-fixed-length-trunc.ll | 354 ++-- llvm/test/CodeGen/AArch64/sve-trunc.ll | 36 +- llvm/test/CodeGen/AArch64/sve-umulo-sdnode.ll | 20 +- .../sve-uunpklo-load-uzp1-store-combine.ll | 8 +- .../test/CodeGen/AArch64/sve-vecreduce-dot.ll | 2 +- .../CodeGen/AArch64/sve-vecreduce-fold.ll | 2 +- llvm/test/CodeGen/AArch64/sve2-fcopysign.ll | 16 +- .../AArch64/sve2-fixed-length-fcopysign.ll | 16 +- .../AArch64/sve2-intrinsics-combine-rshrnb.ll | 22 +- .../AArch64/sve2-intrinsics-int-arith-imm.ll | 2 +- llvm/test/CodeGen/AArch64/sve2-rsh.ll | 2 +- llvm/test/CodeGen/AArch64/sve2-xar.ll | 2 +- .../AArch64/sve2p1-intrinsics-selx2.ll | 32 +- .../AArch64/sve2p1-intrinsics-selx4.ll | 64 +- .../AArch64/sve2p1-intrinsics-stores.ll | 64 +- llvm/test/CodeGen/AArch64/uadd_sat_vec.ll | 10 +- .../CodeGen/AArch64/urem-seteq-vec-nonzero.ll | 10 +- llvm/test/CodeGen/AArch64/vec_uaddo.ll | 8 +- llvm/test/CodeGen/AArch64/vecreduce-add.ll | 546 +++--- llvm/test/CodeGen/AArch64/vector-fcopysign.ll | 46 +- llvm/test/CodeGen/AArch64/vector-gep.ll | 4 +- .../test/CodeGen/AArch64/vselect-constants.ll | 5 +- llvm/test/CodeGen/AArch64/zext-to-tbl.ll | 50 +- .../AArch64/Cortex/A510-neon-instructions.s | 376 ++-- .../AArch64/Cortex/A510-sve-instructions.s | 1636 ++++++++--------- 209 files changed, 6488 insertions(+), 6481 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64SchedA510.td b/llvm/lib/Target/AArch64/AArch64SchedA510.td index 68343674bc81..945687894615 100644 --- a/llvm/lib/Target/AArch64/AArch64SchedA510.td +++ b/llvm/lib/Target/AArch64/AArch64SchedA510.td @@ -254,7 +254,7 @@ def : InstRW<[WriteIS], (instrs RBITWr, RBITXr)>; // Compute pointer authentication code for data address // Compute pointer authentication code, using generic key // Compute pointer authentication code for instruction address -def : InstRW<[CortexA510Write<3, CortexA510UnitPAC>], (instregex "^AUT", "^PAC")>; +def : InstRW<[CortexA510Write<5, CortexA510UnitPAC>], (instregex "^AUT", "^PAC")>; // Branch and link, register, with pointer authentication // Branch, register, with pointer authentication @@ -401,30 +401,30 @@ def : InstRW<[CortexA510WriteFPALU_F3], (instrs FCSELHrrr, FCSELSrrr, FCSELDrrr) def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU]ABDv(2i32|4i16|8i8)")>; def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU]ABDv(16i8|4i32|8i16)")>; // ASIMD absolute diff accum -def : InstRW<[CortexA510Write<8, CortexA510UnitVALU>], (instregex "[SU]ABAL?v")>; +def : InstRW<[CortexA510Write<6, CortexA510UnitVALU>], (instregex "[SU]ABAL?v")>; // ASIMD absolute diff long def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU]ABDLv")>; // ASIMD arith #1 -def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "(ADD|SUB|NEG)v(1i64|2i32|4i16|8i8)", - "[SU]R?HADDv(2i32|4i16|8i8)", "[SU]HSUBv(2i32|4i16|8i8)")>; -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "(ADD|SUB|NEG)v(2i64|4i32|8i16|16i8)", - "[SU]R?HADDv(8i16|4i32|16i8)", "[SU]HSUBv(8i16|4i32|16i8)")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "(ADD|SUB|NEG)v", + "[SU]R?HADDv", "[SU]HSUBv")>; // ASIMD arith #2 -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "ABSv(1i64|2i32|4i16|8i8)$", +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "ABSv(1i64|2i32|4i16|8i8)$", "[SU]ADDLPv(2i32_v1i64|4i16_v2i32|8i8_v4i16)$", - "([SU]QADD|[SU]QSUB|SQNEG|SUQADD|USQADD)v(1i16|1i32|1i64|1i8|2i32|4i16|8i8)$", "ADDPv(2i32|4i16|8i8)$")>; -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "ABSv(2i64|4i32|8i16|16i8)$", +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "([SU]QADD|[SU]QSUB|SQNEG|SUQADD|USQADD)v(1i16|1i32|1i64|1i8|2i32|4i16|8i8)$")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "ABSv(2i64|4i32|8i16|16i8)$", "[SU]ADDLPv(16i8_v8i16|4i32_v2i64|8i16_v4i32)$", - "([SU]QADD|[SU]QSUB|SQNEG|SUQADD|USQADD)v(16i8|2i64|4i32|8i16)$", "ADDPv(16i8|2i64|4i32|8i16)$")>; +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "([SU]QADD|[SU]QSUB|SQNEG|SUQADD|USQADD)v(16i8|2i64|4i32|8i16)$")>; // ASIMD arith #3 -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "SADDLv", "UADDLv", "SADDWv", - "UADDWv", "SSUBLv", "USUBLv", "SSUBWv", "USUBWv", "ADDHNv", "SUBHNv")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "SADDLv", "UADDLv", "SADDWv", + "UADDWv", "SSUBLv", "USUBLv", "SSUBWv", "USUBWv")>; +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "ADDHNv", "SUBHNv")>; // ASIMD arith #5 -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "RADDHNv", "RSUBHNv")>; +def : InstRW<[CortexA510Write<8, CortexA510UnitVALU>], (instregex "RADDHNv", "RSUBHNv")>; // ASIMD arith, reduce -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "ADDVv", "SADDLVv", "UADDLVv")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "ADDVv")>; +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "SADDLVv", "UADDLVv")>; // ASIMD compare #1 def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "CM(EQ|GE|GT|HI|HS|LE|LT)v(1i64|2i32|4i16|8i8)")>; def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "CM(EQ|GE|GT|HI|HS|LE|LT)v(2i64|4i32|8i16|16i8)")>; @@ -437,8 +437,8 @@ def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "(AND|EOR|NOT| def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "(AND|EOR|NOT|ORN)v16i8", "(ORR|BIC)v(16i8|4i32|8i16)$", "MVNIv(4i32|4s|8i16)")>; // ASIMD max/min, basic -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU](MIN|MAX)P?v(2i32|4i16|8i8)")>; -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU](MIN|MAX)P?v(16i8|4i132|8i16)")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU](MIN|MAX)P?v(2i32|4i16|8i8)")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU](MIN|MAX)P?v(16i8|4i132|8i16)")>; // SIMD max/min, reduce def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU](MAX|MIN)Vv")>; // ASIMD multiply, by element @@ -467,12 +467,12 @@ def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU]MULLv", " // ASIMD polynomial (8x8) multiply long def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instrs PMULLv8i8, PMULLv16i8)>; // ASIMD pairwise add and accumulate -def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "[SU]ADALPv")>; +def : InstRW<[CortexA510MCWrite<7, 2, CortexA510UnitVALU>], (instregex "[SU]ADALPv")>; // ASIMD shift accumulate -def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "[SU]SRA(d|v2i32|v4i16|v8i8)")>; -def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "[SU]SRAv(16i8|2i64|4i32|8i16)")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU]SRA(d|v2i32|v4i16|v8i8)")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU]SRAv(16i8|2i64|4i32|8i16)")>; // ASIMD shift accumulate #2 -def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "[SU]RSRA[vd]")>; +def : InstRW<[CortexA510MCWrite<7, 2, CortexA510UnitVALU>], (instregex "[SU]RSRA[vd]")>; // ASIMD shift by immed def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "SHLd$", "SHLv", "SLId$", "SRId$", "[SU]SHR[vd]", "SHRNv")>; @@ -504,7 +504,7 @@ def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU]QRSHLv(2i def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^AES[DE]rr$", "^AESI?MCrr")>; // Crypto polynomial (64x64) multiply long -def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instrs PMULLv1i64, PMULLv2i64)>; +def : InstRW<[CortexA510MCWrite<4, 0, CortexA510UnitVMC>], (instrs PMULLv1i64, PMULLv2i64)>; // Crypto SHA1 hash acceleration op // Crypto SHA1 schedule acceleration ops @@ -512,25 +512,26 @@ def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^SHA1(H|SU0|S // Crypto SHA1 hash acceleration ops // Crypto SHA256 hash acceleration ops -def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instregex "^SHA1[CMP]", "^SHA256H2?")>; +def : InstRW<[CortexA510MCWrite<4, 0, CortexA510UnitVMC>], (instregex "^SHA1[CMP]", "^SHA256H2?")>; // Crypto SHA256 schedule acceleration ops -def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instregex "^SHA256SU[01]")>; +def : InstRW<[CortexA510MCWrite<4, 0, CortexA510UnitVMC>], (instregex "^SHA256SU[01]")>; // Crypto SHA512 hash acceleration ops -def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instregex "^SHA512(H|H2|SU0|SU1)")>; +def : InstRW<[CortexA510MCWrite<9, 0, CortexA510UnitVMC>], (instregex "^SHA512(H|H2|SU0|SU1)")>; // Crypto SHA3 ops -def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instrs BCAX, EOR3, XAR)>; -def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instrs RAX1)>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instrs BCAX, EOR3)>; +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instrs XAR)>; +def : InstRW<[CortexA510MCWrite<9, 0, CortexA510UnitVMC>], (instrs RAX1)>; // Crypto SM3 ops -def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instregex "^SM3PARTW[12]$", "^SM3SS1$", +def : InstRW<[CortexA510MCWrite<9, 0, CortexA510UnitVMC>], (instregex "^SM3PARTW[12]$", "^SM3SS1$", "^SM3TT[12][AB]$")>; // Crypto SM4 ops -def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instrs SM4E, SM4ENCKEY)>; +def : InstRW<[CortexA510MCWrite<9, 0, CortexA510UnitVMC>], (instrs SM4E, SM4ENCKEY)>; // CRC // ----------------------------------------------------------------------------- @@ -540,25 +541,25 @@ def : InstRW<[CortexA510MCWrite<2, 0, CortexA510UnitMAC>], (instregex "^CRC32")> // SVE Predicate instructions // Loop control, based on predicate -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs BRKA_PPmP, BRKA_PPzP, +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instrs BRKA_PPmP, BRKA_PPzP, BRKB_PPmP, BRKB_PPzP)>; // Loop control, based on predicate and flag setting -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs BRKAS_PPzP, BRKBS_PPzP)>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instrs BRKAS_PPzP, BRKBS_PPzP)>; // Loop control, propagating -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs BRKN_PPzP, BRKPA_PPzPP, BRKPB_PPzPP)>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instrs BRKN_PPzP, BRKPA_PPzPP, BRKPB_PPzPP)>; // Loop control, propagating and flag setting -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs BRKNS_PPzP)>; -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs BRKPAS_PPzPP, BRKPBS_PPzPP)>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instrs BRKNS_PPzP)>; +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU0>], (instrs BRKPAS_PPzPP, BRKPBS_PPzPP)>; // Loop control, based on GPR -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^WHILE(GE|GT|HI|HS|LE|LO|LS|LT)_P(WW|XX)_[BHSD]")>; -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^WHILE(RW|WR)_PXX_[BHSD]")>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^WHILE(RW|WR)_PXX_[BHSD]")>; // Loop terminate def : InstRW<[CortexA510Write<1, CortexA510UnitALU>], (instregex "^CTERM(EQ|NE)_(WW|XX)")>; @@ -569,20 +570,20 @@ def : InstRW<[CortexA510Write<1, CortexA510UnitALU>], (instrs ADDPL_XXI, ADDVL_X def : InstRW<[CortexA510Write<1, CortexA510UnitALU>], (instregex "^CNT[BHWD]_XPiI")>; -def : InstRW<[CortexA510Write<1, CortexA510UnitALU>], +def : InstRW<[CortexA510Write<3, CortexA510UnitALU>], (instregex "^(INC|DEC)[BHWD]_XPiI")>; -def : InstRW<[CortexA510Write<1, CortexA510UnitALU>], +def : InstRW<[CortexA510Write<4, CortexA510UnitALU>], (instregex "^(SQINC|SQDEC|UQINC|UQDEC)[BHWD]_[XW]Pi(Wd)?I")>; // Predicate counting scalar, active predicate -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU0>], (instregex "^CNTP_XPP_[BHSD]")>; -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU0>], (instregex "^(DEC|INC)P_XP_[BHSD]")>; -def : InstRW<[CortexA510Write<8, CortexA510UnitVALU0>], +def : InstRW<[CortexA510Write<9, CortexA510UnitVALU0>], (instregex "^(SQDEC|SQINC|UQDEC|UQINC)P_XP_[BHSD]", "^(UQDEC|UQINC)P_WP_[BHSD]", "^(SQDEC|SQINC|UQDEC|UQINC)P_XPWd_[BHSD]")>; @@ -593,39 +594,39 @@ def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^(DEC|INC|SQDEC|SQINC|UQDEC|UQINC)P_ZP_[HSD]")>; // Predicate logical -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^(AND|BIC|EOR|NAND|NOR|ORN|ORR)_PPzPP")>; // Predicate logical, flag setting -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^(ANDS|BICS|EORS|NANDS|NORS|ORNS|ORRS)_PPzPP")>; // Predicate reverse -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^REV_PP_[BHSD]")>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^REV_PP_[BHSD]")>; // Predicate select -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs SEL_PPPP)>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instrs SEL_PPPP)>; // Predicate set -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^PFALSE", "^PTRUE_[BHSD]")>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^PFALSE", "^PTRUE_[BHSD]")>; // Predicate set/initialize, set flags -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^PTRUES_[BHSD]")>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^PTRUES_[BHSD]")>; // Predicate find first/next -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^PFIRST_B", "^PNEXT_[BHSD]")>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^PFIRST_B", "^PNEXT_[BHSD]")>; // Predicate test -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs PTEST_PP)>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instrs PTEST_PP)>; // Predicate transpose -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^TRN[12]_PPP_[BHSDQ]")>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^TRN[12]_PPP_[BHSDQ]")>; // Predicate unpack and widen -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs PUNPKHI_PP, PUNPKLO_PP)>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instrs PUNPKHI_PP, PUNPKLO_PP)>; // Predicate zip/unzip -def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^(ZIP|UZP)[12]_PPP_[BHSDQ]")>; +def : InstRW<[CortexA510Write<2, CortexA510UnitVALU0>], (instregex "^(ZIP|UZP)[12]_PPP_[BHSDQ]")>; // SVE integer instructions @@ -634,10 +635,10 @@ def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^(ZIP|UZP)[1 def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^[SU]ABD_(ZPmZ|ZPZZ)_[BHSD]")>; // Arithmetic, absolute diff accum -def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "^[SU]ABA_ZZZ_[BHSD]")>; +def : InstRW<[CortexA510MCWrite<6, 2, CortexA510UnitVALU>], (instregex "^[SU]ABA_ZZZ_[BHSD]")>; // Arithmetic, absolute diff accum long -def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "^[SU]ABAL[TB]_ZZZ_[HSD]")>; +def : InstRW<[CortexA510MCWrite<6, 2, CortexA510UnitVALU>], (instregex "^[SU]ABAL[TB]_ZZZ_[HSD]")>; // Arithmetic, absolute diff long def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^[SU]ABDL[TB]_ZZZ_[HSD]")>; @@ -651,20 +652,22 @@ def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], "^(ADD|SUB|SUBR)_ZI_[BHSD]", "^ADR_[SU]XTW_ZZZ_D_[0123]", "^ADR_LSL_ZZZ_[SD]_[0123]", - "^[SU](ADD|SUB)[LW][BT]_ZZZ_[HSD]", + "^[SU]H(ADD|SUB|SUBR)_ZPmZ_[BHSD]")>; +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], + (instregex "^[SU](ADD|SUB)[LW][BT]_ZZZ_[HSD]", "^SADDLBT_ZZZ_[HSD]", - "^[SU]H(ADD|SUB|SUBR)_ZPmZ_[BHSD]", "^SSUBL(BT|TB)_ZZZ_[HSD]")>; // Arithmetic, complex def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], - (instregex "^R?(ADD|SUB)HN[BT]_ZZZ_[BHS]", - "^SQ(ABS|NEG)_ZPmZ_[BHSD]", + (instregex "^SQ(ABS|NEG)_ZPmZ_[BHSD]", "^SQ(ADD|SUB|SUBR)_ZPmZ_?[BHSD]", "^[SU]Q(ADD|SUB)_ZZZ_[BHSD]", "^[SU]Q(ADD|SUB)_ZI_[BHSD]", "^(SRH|SUQ|UQ|USQ|URH)ADD_ZPmZ_[BHSD]", "^(UQSUB|UQSUBR)_ZPmZ_[BHSD]")>; +def : InstRW<[CortexA510Write<8, CortexA510UnitVALU>], + (instregex "^R?(ADD|SUB)HN[BT]_ZZZ_[BHS]")>; // Arithmetic, large integer def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^(AD|SB)CL[BT]_ZZZ_[SD]")>; @@ -735,14 +738,14 @@ def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^(BSL|BSL1N|B // Count/reverse bits def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^(CLS|CLZ|RBIT)_ZPmZ_[BHSD]")>; -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^CNT_ZPmZ_[BH]")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^CNT_ZPmZ_[BH]")>; def : InstRW<[CortexA510Write<8, CortexA510UnitVALU>], (instregex "^CNT_ZPmZ_S")>; def : InstRW<[CortexA510Write<12, CortexA510UnitVALU>], (instregex "^CNT_ZPmZ_D")>; // Broadcast logical bitmask immediate to vector def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instrs DUPM_ZI)>; // Compare and set flags -def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^CMP(EQ|GE|GT|HI|HS|LE|LO|LS|LT|NE)_PPzZ[IZ]_[BHSD]", "^CMP(EQ|GE|GT|HI|HS|LE|LO|LS|LT|NE)_WIDE_PPzZZ_[BHS]")>; @@ -939,12 +942,14 @@ def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^SQRDMULH_ZZZ // Multiply/multiply long, (8x8) polynomial def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^PMUL_ZZZ_B")>; -def : InstRW<[CortexA510Write<6, CortexA510UnitVMC>], (instregex "^PMULL[BT]_ZZZ_[HDQ]")>; +def : InstRW<[CortexA510Write<9, CortexA510UnitVMC>], (instregex "^PMULL[BT]_ZZZ_[HDQ]")>; // Predicate counting vector +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], + (instregex "^(DEC|INC)[HWD]_ZPiI")>; def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], - (instregex "^(DEC|INC|SQDEC|SQINC|UQDEC|UQINC)[HWD]_ZPiI")>; + (instregex "^(SQDEC|SQINC|UQDEC|UQINC)[HWD]_ZPiI")>; // Reciprocal estimate def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^URECPE_ZPmZ_S", "^URSQRTE_ZPmZ_S")>; @@ -965,7 +970,7 @@ def : InstRW<[CortexA510Write<4, CortexA510UnitVALU0>], (instregex "^[SU](ADD|MA def : InstRW<[CortexA510Write<4, CortexA510UnitVALU0>], (instregex "^(ANDV|EORV|ORV)_VPZ_[BHSD]")>; // Reverse, vector -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^REV_ZZ_[BHSD]", +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^REV_ZZ_[BHSD]", "^REVB_ZPmZ_[HSD]", "^REVH_ZPmZ_[SD]", "^REVW_ZPmZ_D")>; @@ -980,13 +985,13 @@ def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^TBL_ZZZZ?_[B def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^TBX_ZZZ_[BHSD]")>; // Transpose, vector form -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^TRN[12]_ZZZ_[BHSDQ]")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^TRN[12]_ZZZ_[BHSDQ]")>; // Unpack and extend def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^[SU]UNPK(HI|LO)_ZZ_[HSD]")>; // Zip/unzip -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^(UZP|ZIP)[12]_ZZZ_[BHSDQ]")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^(UZP|ZIP)[12]_ZZZ_[BHSDQ]")>; // SVE floating-point instructions // ----------------------------------------------------------------------------- @@ -1142,7 +1147,7 @@ def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^FTMAD_ZZI_[H // Floating point trigonometric, miscellaneous def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^FTSMUL_ZZZ_[HSD]")>; -def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^FTSSEL_ZZZ_[HSD]")>; +def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^FTSSEL_ZZZ_[HSD]")>; // SVE BFloat16 (BF16) instructions @@ -1251,12 +1256,12 @@ def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>], "^GLD(FF)?1D(_SCALED)?$")>; // Gather load, 32-bit scaled offset -def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLd>], +def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLd>], (instregex "^GLD(FF)?1S?[HW]_S_[SU]XTW_SCALED$", "^GLD(FF)?1W_[SU]XTW_SCALED")>; // Gather load, 32-bit unpacked unscaled offset -def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLd>], (instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW$", +def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLd>], (instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW$", "^GLD(FF)?1W_[SU]XTW$")>; def : InstRW<[CortexA510Write<0, CortexA510UnitVALU>], (instregex "^PRF(B|H|W|D).*")>; @@ -1377,12 +1382,12 @@ def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^AES[DE]_ZZZ_ "^AESI?MC_ZZ_B$")>; // Crypto SHA3 ops -def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^(BCAX|EOR3)_ZZZZ$", +def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^(BCAX|EOR3)_ZZZZ$", "^XAR_ZZZI_[BHSD]$")>; -def : InstRW<[CortexA510MC_RC0Write<8, CortexA510UnitVMC>], (instregex "^RAX1_ZZZ_D$")>; +def : InstRW<[CortexA510MC_RC0Write<9, CortexA510UnitVMC>], (instregex "^RAX1_ZZZ_D$")>; // Crypto SM4 ops -def : InstRW<[CortexA510MC_RC0Write<8, CortexA510UnitVMC>], (instregex "^SM4E(KEY)?_ZZZ_S$")>; +def : InstRW<[CortexA510MC_RC0Write<9, CortexA510UnitVMC>], (instregex "^SM4E(KEY)?_ZZZ_S$")>; } diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll b/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll index ceef0c49a45e..9a525151ca32 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll @@ -140,10 +140,10 @@ define <8 x i16> @combine_vec_udiv_nonuniform3(<8 x i16> %x) { ; GISEL-NEXT: umull2 v2.4s, v0.8h, v1.8h ; GISEL-NEXT: umull v1.4s, v0.4h, v1.4h ; GISEL-NEXT: uzp2 v1.8h, v1.8h, v2.8h +; GISEL-NEXT: ldr q2, [x8, :lo12:.LCPI3_0] ; GISEL-NEXT: sub v0.8h, v0.8h, v1.8h ; GISEL-NEXT: usra v1.8h, v0.8h, #1 -; GISEL-NEXT: ldr q0, [x8, :lo12:.LCPI3_0] -; GISEL-NEXT: neg v0.8h, v0.8h +; GISEL-NEXT: neg v0.8h, v2.8h ; GISEL-NEXT: ushl v0.8h, v1.8h, v0.8h ; GISEL-NEXT: ret %1 = udiv <8 x i16> %x, @@ -170,13 +170,13 @@ define <16 x i8> @combine_vec_udiv_nonuniform4(<16 x i8> %x) { ; GISEL-LABEL: combine_vec_udiv_nonuniform4: ; GISEL: // %bb.0: ; GISEL-NEXT: adrp x8, .LCPI4_2 +; GISEL-NEXT: adrp x9, .LCPI4_0 ; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI4_2] ; GISEL-NEXT: adrp x8, .LCPI4_1 +; GISEL-NEXT: ldr q4, [x9, :lo12:.LCPI4_0] ; GISEL-NEXT: ldr q3, [x8, :lo12:.LCPI4_1] -; GISEL-NEXT: adrp x8, .LCPI4_0 ; GISEL-NEXT: umull2 v2.8h, v0.16b, v1.16b ; GISEL-NEXT: umull v1.8h, v0.8b, v1.8b -; GISEL-NEXT: ldr q4, [x8, :lo12:.LCPI4_0] ; GISEL-NEXT: uzp2 v1.16b, v1.16b, v2.16b ; GISEL-NEXT: neg v2.16b, v3.16b ; GISEL-NEXT: shl v3.16b, v4.16b, #7 diff --git a/llvm/test/CodeGen/AArch64/aarch64-addv.ll b/llvm/test/CodeGen/AArch64/aarch64-addv.ll index b77d59134765..ee035ec1941d 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-addv.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-addv.ll @@ -101,12 +101,12 @@ define i32 @oversized_ADDV_256(ptr noalias nocapture readonly %arg1, ptr noalias ; GISEL-NEXT: ushll v2.8h, v2.8b, #0 ; GISEL-NEXT: usubl v3.4s, v1.4h, v2.4h ; GISEL-NEXT: usubl2 v1.4s, v1.8h, v2.8h -; GISEL-NEXT: neg v2.4s, v3.4s -; GISEL-NEXT: neg v4.4s, v1.4s -; GISEL-NEXT: cmgt v5.4s, v0.4s, v3.4s +; GISEL-NEXT: cmgt v2.4s, v0.4s, v3.4s ; GISEL-NEXT: cmgt v0.4s, v0.4s, v1.4s -; GISEL-NEXT: bif v2.16b, v3.16b, v5.16b -; GISEL-NEXT: bsl v0.16b, v4.16b, v1.16b +; GISEL-NEXT: neg v4.4s, v3.4s +; GISEL-NEXT: neg v5.4s, v1.4s +; GISEL-NEXT: bsl v2.16b, v4.16b, v3.16b +; GISEL-NEXT: bsl v0.16b, v5.16b, v1.16b ; GISEL-NEXT: add v0.4s, v2.4s, v0.4s ; GISEL-NEXT: addv s0, v0.4s ; GISEL-NEXT: fmov w0, s0 diff --git a/llvm/test/CodeGen/AArch64/aarch64-dup-ext-scalable.ll b/llvm/test/CodeGen/AArch64/aarch64-dup-ext-scalable.ll index fdeae9f326ad..36b81d8e495c 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-dup-ext-scalable.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-dup-ext-scalable.ll @@ -4,8 +4,8 @@ define @dupsext_v2i8_v2i16(i8 %src, %b) { ; CHECK-LABEL: dupsext_v2i8_v2i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: sxtb w8, w0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -20,8 +20,8 @@ entry: define @dupsext_v4i8_v4i16(i8 %src, %b) { ; CHECK-LABEL: dupsext_v4i8_v4i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sxtb w8, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -36,8 +36,8 @@ entry: define @dupsext_v8i8_v8i16(i8 %src, %b) { ; CHECK-LABEL: dupsext_v8i8_v8i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: sxtb w8, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: mul z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret @@ -52,8 +52,8 @@ entry: define @dupsext_v2i8_v2i32(i8 %src, %b) { ; CHECK-LABEL: dupsext_v2i8_v2i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: sxtb w8, w0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -68,8 +68,8 @@ entry: define @dupsext_v4i8_v4i32(i8 %src, %b) { ; CHECK-LABEL: dupsext_v4i8_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sxtb w8, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -84,9 +84,9 @@ entry: define @dupsext_v2i8_v2i64(i8 %src, %b) { ; CHECK-LABEL: dupsext_v2i8_v2i64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: sxtb x8, w0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -101,8 +101,8 @@ entry: define @dupsext_v2i16_v2i32(i16 %src, %b) { ; CHECK-LABEL: dupsext_v2i16_v2i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: sxth w8, w0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -117,8 +117,8 @@ entry: define @dupsext_v4i16_v4i32(i16 %src, %b) { ; CHECK-LABEL: dupsext_v4i16_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sxth w8, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -133,9 +133,9 @@ entry: define @dupsext_v2i16_v2i64(i16 %src, %b) { ; CHECK-LABEL: dupsext_v2i16_v2i64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: sxth x8, w0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -150,9 +150,9 @@ entry: define @dupsext_v2i32_v2i64(i32 %src, %b) { ; CHECK-LABEL: dupsext_v2i32_v2i64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: sxtw x8, w0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -167,8 +167,8 @@ entry: define @dupzext_v2i8_v2i16(i8 %src, %b) { ; CHECK-LABEL: dupzext_v2i8_v2i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and w8, w0, #0xff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -183,8 +183,8 @@ entry: define @dupzext_v4i8_v4i16(i8 %src, %b) { ; CHECK-LABEL: dupzext_v4i8_v4i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w0, #0xff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -199,8 +199,8 @@ entry: define @dupzext_v8i8_v8i16(i8 %src, %b) { ; CHECK-LABEL: dupzext_v8i8_v8i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and w8, w0, #0xff +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: mul z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret @@ -215,8 +215,8 @@ entry: define @dupzext_v2i8_v2i32(i8 %src, %b) { ; CHECK-LABEL: dupzext_v2i8_v2i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and w8, w0, #0xff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -231,8 +231,8 @@ entry: define @dupzext_v4i8_v4i32(i8 %src, %b) { ; CHECK-LABEL: dupzext_v4i8_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w0, #0xff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -247,9 +247,9 @@ entry: define @dupzext_v2i8_v2i64(i8 %src, %b) { ; CHECK-LABEL: dupzext_v2i8_v2i64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: and x8, x0, #0xff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -264,8 +264,8 @@ entry: define @dupzext_v2i16_v2i32(i16 %src, %b) { ; CHECK-LABEL: dupzext_v2i16_v2i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and w8, w0, #0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -280,8 +280,8 @@ entry: define @dupzext_v4i16_v4i32(i16 %src, %b) { ; CHECK-LABEL: dupzext_v4i16_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w0, #0xffff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -296,9 +296,9 @@ entry: define @dupzext_v2i16_v2i64(i16 %src, %b) { ; CHECK-LABEL: dupzext_v2i16_v2i64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: and x8, x0, #0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -313,8 +313,8 @@ entry: define @dupzext_v2i32_v2i64(i32 %src, %b) { ; CHECK-LABEL: dupzext_v2i32_v2i64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/aarch64-smull.ll b/llvm/test/CodeGen/AArch64/aarch64-smull.ll index 61a4f64ac2bf..540471a05901 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-smull.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-smull.ll @@ -257,8 +257,8 @@ define <2 x i64> @smull_zext_v2i32_v2i64(ptr %A, ptr %B) nounwind { ; CHECK-SVE-LABEL: smull_zext_v2i32_v2i64: ; CHECK-SVE: // %bb.0: ; CHECK-SVE-NEXT: ldrh w8, [x0] -; CHECK-SVE-NEXT: ptrue p0.d, vl2 ; CHECK-SVE-NEXT: ldrh w9, [x0, #2] +; CHECK-SVE-NEXT: ptrue p0.d, vl2 ; CHECK-SVE-NEXT: ldr d0, [x1] ; CHECK-SVE-NEXT: fmov d1, x8 ; CHECK-SVE-NEXT: sshll v0.2d, v0.2s, #0 diff --git a/llvm/test/CodeGen/AArch64/active_lane_mask.ll b/llvm/test/CodeGen/AArch64/active_lane_mask.ll index a65c5d666779..43122c8c953f 100644 --- a/llvm/test/CodeGen/AArch64/active_lane_mask.ll +++ b/llvm/test/CodeGen/AArch64/active_lane_mask.ll @@ -131,9 +131,9 @@ define @lane_mask_nxv2i1_i8(i8 %index, i8 %TC) { ; CHECK-NEXT: index z0.d, #0, #1 ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: and x8, x0, #0xff +; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1 ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 -; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1 ; CHECK-NEXT: and x8, x1, #0xff ; CHECK-NEXT: and z0.d, z0.d, #0xff ; CHECK-NEXT: add z0.d, z0.d, z1.d @@ -153,6 +153,7 @@ define @lane_mask_nxv32i1_i32(i32 %index, i32 %TC) { ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: str p7, [sp, #4, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p6, [sp, #5, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p5, [sp, #6, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill @@ -160,16 +161,16 @@ define @lane_mask_nxv32i1_i32(i32 %index, i32 %TC) { ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: index z0.s, #0, #1 ; CHECK-NEXT: mov z1.s, w0 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z25.s, w1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.d, z0.d ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: uqadd z6.s, z0.s, z1.s ; CHECK-NEXT: incw z0.s, all, mul #4 ; CHECK-NEXT: incw z2.s ; CHECK-NEXT: incw z3.s, all, mul #2 -; CHECK-NEXT: cmphi p2.s, p0/z, z25.s, z6.s ; CHECK-NEXT: uqadd z0.s, z0.s, z1.s +; CHECK-NEXT: cmphi p2.s, p0/z, z25.s, z6.s ; CHECK-NEXT: mov z4.d, z2.d ; CHECK-NEXT: uqadd z5.s, z2.s, z1.s ; CHECK-NEXT: uqadd z7.s, z3.s, z1.s @@ -177,25 +178,26 @@ define @lane_mask_nxv32i1_i32(i32 %index, i32 %TC) { ; CHECK-NEXT: incw z3.s, all, mul #4 ; CHECK-NEXT: cmphi p5.s, p0/z, z25.s, z0.s ; CHECK-NEXT: incw z4.s, all, mul #2 -; CHECK-NEXT: cmphi p1.s, p0/z, z25.s, z5.s -; CHECK-NEXT: cmphi p3.s, p0/z, z25.s, z7.s ; CHECK-NEXT: uqadd z2.s, z2.s, z1.s ; CHECK-NEXT: uqadd z3.s, z3.s, z1.s +; CHECK-NEXT: cmphi p1.s, p0/z, z25.s, z5.s +; CHECK-NEXT: cmphi p3.s, p0/z, z25.s, z7.s ; CHECK-NEXT: uqadd z24.s, z4.s, z1.s ; CHECK-NEXT: incw z4.s, all, mul #4 -; CHECK-NEXT: uzp1 p1.h, p2.h, p1.h ; CHECK-NEXT: cmphi p6.s, p0/z, z25.s, z2.s -; CHECK-NEXT: cmphi p2.s, p0/z, z25.s, z3.s +; CHECK-NEXT: cmphi p7.s, p0/z, z25.s, z3.s +; CHECK-NEXT: uzp1 p1.h, p2.h, p1.h ; CHECK-NEXT: uqadd z1.s, z4.s, z1.s ; CHECK-NEXT: cmphi p4.s, p0/z, z25.s, z24.s -; CHECK-NEXT: uzp1 p3.h, p3.h, p4.h ; CHECK-NEXT: cmphi p0.s, p0/z, z25.s, z1.s -; CHECK-NEXT: uzp1 p4.h, p5.h, p6.h +; CHECK-NEXT: uzp1 p2.h, p3.h, p4.h +; CHECK-NEXT: uzp1 p3.h, p5.h, p6.h ; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p2.h, p2.h, p0.h -; CHECK-NEXT: uzp1 p0.b, p1.b, p3.b -; CHECK-NEXT: uzp1 p1.b, p4.b, p2.b +; CHECK-NEXT: uzp1 p4.h, p7.h, p0.h +; CHECK-NEXT: ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: uzp1 p0.b, p1.b, p2.b +; CHECK-NEXT: uzp1 p1.b, p3.b, p4.b ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -208,96 +210,97 @@ define @lane_mask_nxv32i1_i64(i64 %index, i64 %TC) { ; CHECK-LABEL: lane_mask_nxv32i1_i64: ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill -; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p10, [sp, #1, mul vl] // 2-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: str p9, [sp, #2, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p8, [sp, #3, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p7, [sp, #4, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p6, [sp, #5, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p5, [sp, #6, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG +; CHECK-NEXT: str z8, [sp, #1, mul vl] // 16-byte Folded Spill +; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: index z1.d, #0, #1 +; CHECK-NEXT: .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG +; CHECK-NEXT: index z5.d, #0, #1 ; CHECK-NEXT: mov z0.d, x0 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z3.d, x1 -; CHECK-NEXT: mov z2.d, z1.d -; CHECK-NEXT: mov z4.d, z1.d -; CHECK-NEXT: mov z6.d, z1.d -; CHECK-NEXT: uqadd z25.d, z1.d, z0.d -; CHECK-NEXT: incd z1.d, all, mul #8 +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: mov z2.d, z5.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: mov z4.d, z5.d +; CHECK-NEXT: uqadd z25.d, z5.d, z0.d +; CHECK-NEXT: incd z5.d, all, mul #8 ; CHECK-NEXT: incd z2.d -; CHECK-NEXT: incd z4.d, all, mul #2 -; CHECK-NEXT: incd z6.d, all, mul #4 -; CHECK-NEXT: cmphi p1.d, p0/z, z3.d, z25.d -; CHECK-NEXT: uqadd z1.d, z1.d, z0.d -; CHECK-NEXT: mov z5.d, z2.d -; CHECK-NEXT: uqadd z26.d, z2.d, z0.d +; CHECK-NEXT: incd z1.d, all, mul #2 +; CHECK-NEXT: incd z4.d, all, mul #4 +; CHECK-NEXT: uqadd z5.d, z5.d, z0.d +; CHECK-NEXT: cmphi p3.d, p0/z, z3.d, z25.d +; CHECK-NEXT: mov z6.d, z2.d ; CHECK-NEXT: mov z7.d, z2.d -; CHECK-NEXT: mov z24.d, z4.d -; CHECK-NEXT: uqadd z27.d, z4.d, z0.d -; CHECK-NEXT: uqadd z28.d, z6.d, z0.d +; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: uqadd z26.d, z2.d, z0.d +; CHECK-NEXT: uqadd z27.d, z1.d, z0.d +; CHECK-NEXT: uqadd z28.d, z4.d, z0.d ; CHECK-NEXT: incd z2.d, all, mul #8 +; CHECK-NEXT: incd z1.d, all, mul #8 ; CHECK-NEXT: incd z4.d, all, mul #8 -; CHECK-NEXT: incd z6.d, all, mul #8 -; CHECK-NEXT: incd z5.d, all, mul #2 +; CHECK-NEXT: incd z6.d, all, mul #2 ; CHECK-NEXT: incd z7.d, all, mul #4 -; CHECK-NEXT: cmphi p2.d, p0/z, z3.d, z26.d ; CHECK-NEXT: incd z24.d, all, mul #4 -; CHECK-NEXT: cmphi p3.d, p0/z, z3.d, z27.d -; CHECK-NEXT: cmphi p5.d, p0/z, z3.d, z28.d +; CHECK-NEXT: cmphi p4.d, p0/z, z3.d, z26.d +; CHECK-NEXT: cmphi p2.d, p0/z, z3.d, z27.d +; CHECK-NEXT: cmphi p1.d, p0/z, z3.d, z28.d +; CHECK-NEXT: mov z31.d, z6.d +; CHECK-NEXT: uqadd z29.d, z6.d, z0.d +; CHECK-NEXT: uqadd z30.d, z7.d, z0.d +; CHECK-NEXT: uqadd z8.d, z24.d, z0.d +; CHECK-NEXT: incd z6.d, all, mul #8 +; CHECK-NEXT: incd z7.d, all, mul #8 +; CHECK-NEXT: incd z24.d, all, mul #8 ; CHECK-NEXT: uqadd z2.d, z2.d, z0.d +; CHECK-NEXT: uqadd z1.d, z1.d, z0.d +; CHECK-NEXT: incd z31.d, all, mul #4 ; CHECK-NEXT: uqadd z4.d, z4.d, z0.d +; CHECK-NEXT: uzp1 p3.s, p3.s, p4.s +; CHECK-NEXT: cmphi p5.d, p0/z, z3.d, z29.d +; CHECK-NEXT: cmphi p7.d, p0/z, z3.d, z30.d ; CHECK-NEXT: uqadd z6.d, z6.d, z0.d -; CHECK-NEXT: mov z26.d, z5.d -; CHECK-NEXT: uqadd z25.d, z5.d, z0.d -; CHECK-NEXT: uqadd z27.d, z7.d, z0.d -; CHECK-NEXT: incd z5.d, all, mul #8 -; CHECK-NEXT: incd z7.d, all, mul #8 -; CHECK-NEXT: uzp1 p1.s, p1.s, p2.s -; CHECK-NEXT: incd z26.d, all, mul #4 -; CHECK-NEXT: cmphi p8.d, p0/z, z3.d, z2.d -; CHECK-NEXT: cmphi p4.d, p0/z, z3.d, z25.d -; CHECK-NEXT: uqadd z25.d, z24.d, z0.d -; CHECK-NEXT: incd z24.d, all, mul #8 -; CHECK-NEXT: uqadd z5.d, z5.d, z0.d +; CHECK-NEXT: cmphi p6.d, p0/z, z3.d, z8.d +; CHECK-NEXT: ldr z8, [sp, #1, mul vl] // 16-byte Folded Reload ; CHECK-NEXT: uqadd z7.d, z7.d, z0.d -; CHECK-NEXT: cmphi p6.d, p0/z, z3.d, z27.d -; CHECK-NEXT: uqadd z28.d, z26.d, z0.d -; CHECK-NEXT: incd z26.d, all, mul #8 -; CHECK-NEXT: uzp1 p3.s, p3.s, p4.s +; CHECK-NEXT: uqadd z25.d, z31.d, z0.d +; CHECK-NEXT: incd z31.d, all, mul #8 ; CHECK-NEXT: uqadd z24.d, z24.d, z0.d -; CHECK-NEXT: cmphi p7.d, p0/z, z3.d, z25.d -; CHECK-NEXT: cmphi p4.d, p0/z, z3.d, z1.d -; CHECK-NEXT: uzp1 p5.s, p5.s, p6.s -; CHECK-NEXT: cmphi p6.d, p0/z, z3.d, z4.d -; CHECK-NEXT: cmphi p9.d, p0/z, z3.d, z5.d -; CHECK-NEXT: cmphi p10.d, p0/z, z3.d, z7.d -; CHECK-NEXT: uqadd z0.d, z26.d, z0.d -; CHECK-NEXT: cmphi p2.d, p0/z, z3.d, z28.d -; CHECK-NEXT: uzp1 p4.s, p4.s, p8.s -; CHECK-NEXT: cmphi p8.d, p0/z, z3.d, z24.d -; CHECK-NEXT: uzp1 p6.s, p6.s, p9.s +; CHECK-NEXT: cmphi p4.d, p0/z, z3.d, z5.d +; CHECK-NEXT: uzp1 p2.s, p2.s, p5.s +; CHECK-NEXT: cmphi p5.d, p0/z, z3.d, z2.d +; CHECK-NEXT: cmphi p9.d, p0/z, z3.d, z6.d +; CHECK-NEXT: uqadd z0.d, z31.d, z0.d +; CHECK-NEXT: uzp1 p1.s, p1.s, p7.s +; CHECK-NEXT: cmphi p7.d, p0/z, z3.d, z1.d +; CHECK-NEXT: cmphi p8.d, p0/z, z3.d, z25.d +; CHECK-NEXT: uzp1 p2.h, p3.h, p2.h +; CHECK-NEXT: cmphi p3.d, p0/z, z3.d, z7.d +; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s +; CHECK-NEXT: uzp1 p5.s, p7.s, p9.s ; CHECK-NEXT: ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p1.h, p1.h, p3.h -; CHECK-NEXT: uzp1 p2.s, p7.s, p2.s -; CHECK-NEXT: cmphi p7.d, p0/z, z3.d, z6.d +; CHECK-NEXT: uzp1 p6.s, p6.s, p8.s +; CHECK-NEXT: cmphi p8.d, p0/z, z3.d, z4.d +; CHECK-NEXT: ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h +; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: uzp1 p1.h, p1.h, p6.h +; CHECK-NEXT: cmphi p6.d, p0/z, z3.d, z24.d ; CHECK-NEXT: cmphi p0.d, p0/z, z3.d, z0.d -; CHECK-NEXT: uzp1 p7.s, p7.s, p10.s -; CHECK-NEXT: ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p0.s, p8.s, p0.s +; CHECK-NEXT: uzp1 p3.s, p8.s, p3.s ; CHECK-NEXT: ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p3.h, p4.h, p6.h +; CHECK-NEXT: uzp1 p0.s, p6.s, p0.s ; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p2.h, p5.h, p2.h -; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p4.h, p7.h, p0.h -; CHECK-NEXT: ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p0.b, p1.b, p2.b -; CHECK-NEXT: uzp1 p1.b, p3.b, p4.b +; CHECK-NEXT: uzp1 p3.h, p3.h, p0.h +; CHECK-NEXT: uzp1 p0.b, p2.b, p1.b +; CHECK-NEXT: uzp1 p1.b, p4.b, p3.b ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: addvl sp, sp, #1 +; CHECK-NEXT: addvl sp, sp, #2 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-NEXT: ret %active.lane.mask = call @llvm.get.active.lane.mask.nxv32i1.i64(i64 %index, i64 %TC) @@ -459,12 +462,12 @@ define <4 x i1> @lane_mask_v4i1_i8(i8 %index, i8 %TC) { ; CHECK-NEXT: adrp x8, .LCPI26_0 ; CHECK-NEXT: movi d2, #0xff00ff00ff00ff ; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI26_0] +; CHECK-NEXT: dup v3.4h, w1 ; CHECK-NEXT: bic v0.4h, #255, lsl #8 +; CHECK-NEXT: bic v3.4h, #255, lsl #8 ; CHECK-NEXT: add v0.4h, v0.4h, v1.4h -; CHECK-NEXT: dup v1.4h, w1 ; CHECK-NEXT: umin v0.4h, v0.4h, v2.4h -; CHECK-NEXT: bic v1.4h, #255, lsl #8 -; CHECK-NEXT: cmhi v0.4h, v1.4h, v0.4h +; CHECK-NEXT: cmhi v0.4h, v3.4h, v0.4h ; CHECK-NEXT: ret %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i8(i8 %index, i8 %TC) ret <4 x i1> %active.lane.mask @@ -480,9 +483,9 @@ define <2 x i1> @lane_mask_v2i1_i8(i8 %index, i8 %TC) { ; CHECK-NEXT: dup v3.2s, w1 ; CHECK-NEXT: and v1.8b, v1.8b, v0.8b ; CHECK-NEXT: add v1.2s, v1.2s, v2.2s -; CHECK-NEXT: umin v1.2s, v1.2s, v0.2s -; CHECK-NEXT: and v0.8b, v3.8b, v0.8b -; CHECK-NEXT: cmhi v0.2s, v0.2s, v1.2s +; CHECK-NEXT: and v2.8b, v3.8b, v0.8b +; CHECK-NEXT: umin v0.2s, v1.2s, v0.2s +; CHECK-NEXT: cmhi v0.2s, v2.2s, v0.2s ; CHECK-NEXT: ret %active.lane.mask = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i8(i8 %index, i8 %TC) ret <2 x i1> %active.lane.mask diff --git a/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll b/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll index 3007e7ce771e..508f68d6f14d 100644 --- a/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll +++ b/llvm/test/CodeGen/AArch64/arm64-convert-v4f64.ll @@ -54,19 +54,19 @@ define <4 x bfloat> @uitofp_v4i64_to_v4bf16(ptr %ptr) { ; CHECK-LABEL: uitofp_v4i64_to_v4bf16: ; CHECK: // %bb.0: ; CHECK-NEXT: ldp q0, q1, [x0] -; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: ucvtf v0.2d, v0.2d ; CHECK-NEXT: ucvtf v1.2d, v1.2d ; CHECK-NEXT: fcvtn v0.2s, v0.2d ; CHECK-NEXT: fcvtn2 v0.4s, v1.2d -; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ushr v3.4s, v0.4s, #16 -; CHECK-NEXT: add v1.4s, v0.4s, v1.4s -; CHECK-NEXT: and v2.16b, v3.16b, v2.16b -; CHECK-NEXT: add v1.4s, v2.4s, v1.4s -; CHECK-NEXT: fcmeq v2.4s, v0.4s, v0.4s +; CHECK-NEXT: add v2.4s, v0.4s, v2.4s +; CHECK-NEXT: and v1.16b, v3.16b, v1.16b +; CHECK-NEXT: fcmeq v3.4s, v0.4s, v0.4s ; CHECK-NEXT: orr v0.4s, #64, lsl #16 -; CHECK-NEXT: bit v0.16b, v1.16b, v2.16b +; CHECK-NEXT: add v1.4s, v1.4s, v2.4s +; CHECK-NEXT: bit v0.16b, v1.16b, v3.16b ; CHECK-NEXT: shrn v0.4h, v0.4s, #16 ; CHECK-NEXT: ret %tmp1 = load <4 x i64>, ptr %ptr diff --git a/llvm/test/CodeGen/AArch64/arm64-vabs.ll b/llvm/test/CodeGen/AArch64/arm64-vabs.ll index fe4da2e7cf36..5b45ba2552ce 100644 --- a/llvm/test/CodeGen/AArch64/arm64-vabs.ll +++ b/llvm/test/CodeGen/AArch64/arm64-vabs.ll @@ -257,12 +257,12 @@ define i16 @uabd16b_rdx(ptr %a, ptr %b) { ; CHECK-GI-NEXT: movi.2d v0, #0000000000000000 ; CHECK-GI-NEXT: usubl.8h v3, v1, v2 ; CHECK-GI-NEXT: usubl2.8h v1, v1, v2 -; CHECK-GI-NEXT: neg.8h v2, v3 -; CHECK-GI-NEXT: neg.8h v4, v1 -; CHECK-GI-NEXT: cmgt.8h v5, v0, v3 +; CHECK-GI-NEXT: cmgt.8h v2, v0, v3 ; CHECK-GI-NEXT: cmgt.8h v0, v0, v1 -; CHECK-GI-NEXT: bif.16b v2, v3, v5 -; CHECK-GI-NEXT: bsl.16b v0, v4, v1 +; CHECK-GI-NEXT: neg.8h v4, v3 +; CHECK-GI-NEXT: neg.8h v5, v1 +; CHECK-GI-NEXT: bsl.16b v2, v4, v3 +; CHECK-GI-NEXT: bsl.16b v0, v5, v1 ; CHECK-GI-NEXT: add.8h v0, v2, v0 ; CHECK-GI-NEXT: addv.8h h0, v0 ; CHECK-GI-NEXT: fmov w0, s0 @@ -299,18 +299,18 @@ define i32 @uabd16b_rdx_i32(<16 x i8> %a, <16 x i8> %b) { ; CHECK-GI-NEXT: usubl2.4s v3, v3, v4 ; CHECK-GI-NEXT: usubl.4s v4, v0, v1 ; CHECK-GI-NEXT: usubl2.4s v0, v0, v1 -; CHECK-GI-NEXT: neg.4s v6, v5 -; CHECK-GI-NEXT: neg.4s v7, v3 ; CHECK-GI-NEXT: cmgt.4s v1, v2, v5 -; CHECK-GI-NEXT: neg.4s v16, v4 -; CHECK-GI-NEXT: neg.4s v17, v0 -; CHECK-GI-NEXT: cmgt.4s v18, v2, v3 -; CHECK-GI-NEXT: cmgt.4s v19, v2, v4 +; CHECK-GI-NEXT: cmgt.4s v6, v2, v3 +; CHECK-GI-NEXT: neg.4s v16, v5 +; CHECK-GI-NEXT: cmgt.4s v7, v2, v4 ; CHECK-GI-NEXT: cmgt.4s v2, v2, v0 -; CHECK-GI-NEXT: bsl.16b v1, v6, v5 -; CHECK-GI-NEXT: bit.16b v3, v7, v18 -; CHECK-GI-NEXT: bit.16b v4, v16, v19 -; CHECK-GI-NEXT: bit.16b v0, v17, v2 +; CHECK-GI-NEXT: neg.4s v17, v3 +; CHECK-GI-NEXT: neg.4s v18, v4 +; CHECK-GI-NEXT: neg.4s v19, v0 +; CHECK-GI-NEXT: bsl.16b v1, v16, v5 +; CHECK-GI-NEXT: bit.16b v3, v17, v6 +; CHECK-GI-NEXT: bit.16b v4, v18, v7 +; CHECK-GI-NEXT: bit.16b v0, v19, v2 ; CHECK-GI-NEXT: add.4s v1, v1, v3 ; CHECK-GI-NEXT: add.4s v0, v4, v0 ; CHECK-GI-NEXT: add.4s v0, v1, v0 @@ -347,18 +347,18 @@ define i32 @sabd16b_rdx_i32(<16 x i8> %a, <16 x i8> %b) { ; CHECK-GI-NEXT: ssubl2.4s v3, v3, v4 ; CHECK-GI-NEXT: ssubl.4s v4, v0, v1 ; CHECK-GI-NEXT: ssubl2.4s v0, v0, v1 -; CHECK-GI-NEXT: neg.4s v6, v5 -; CHECK-GI-NEXT: neg.4s v7, v3 ; CHECK-GI-NEXT: cmgt.4s v1, v2, v5 -; CHECK-GI-NEXT: neg.4s v16, v4 -; CHECK-GI-NEXT: neg.4s v17, v0 -; CHECK-GI-NEXT: cmgt.4s v18, v2, v3 -; CHECK-GI-NEXT: cmgt.4s v19, v2, v4 +; CHECK-GI-NEXT: cmgt.4s v6, v2, v3 +; CHECK-GI-NEXT: neg.4s v16, v5 +; CHECK-GI-NEXT: cmgt.4s v7, v2, v4 ; CHECK-GI-NEXT: cmgt.4s v2, v2, v0 -; CHECK-GI-NEXT: bsl.16b v1, v6, v5 -; CHECK-GI-NEXT: bit.16b v3, v7, v18 -; CHECK-GI-NEXT: bit.16b v4, v16, v19 -; CHECK-GI-NEXT: bit.16b v0, v17, v2 +; CHECK-GI-NEXT: neg.4s v17, v3 +; CHECK-GI-NEXT: neg.4s v18, v4 +; CHECK-GI-NEXT: neg.4s v19, v0 +; CHECK-GI-NEXT: bsl.16b v1, v16, v5 +; CHECK-GI-NEXT: bit.16b v3, v17, v6 +; CHECK-GI-NEXT: bit.16b v4, v18, v7 +; CHECK-GI-NEXT: bit.16b v0, v19, v2 ; CHECK-GI-NEXT: add.4s v1, v1, v3 ; CHECK-GI-NEXT: add.4s v0, v4, v0 ; CHECK-GI-NEXT: add.4s v0, v1, v0 @@ -396,12 +396,12 @@ define i32 @uabd8h_rdx(ptr %a, ptr %b) { ; CHECK-GI-NEXT: movi.2d v0, #0000000000000000 ; CHECK-GI-NEXT: usubl.4s v3, v1, v2 ; CHECK-GI-NEXT: usubl2.4s v1, v1, v2 -; CHECK-GI-NEXT: neg.4s v2, v3 -; CHECK-GI-NEXT: neg.4s v4, v1 -; CHECK-GI-NEXT: cmgt.4s v5, v0, v3 +; CHECK-GI-NEXT: cmgt.4s v2, v0, v3 ; CHECK-GI-NEXT: cmgt.4s v0, v0, v1 -; CHECK-GI-NEXT: bif.16b v2, v3, v5 -; CHECK-GI-NEXT: bsl.16b v0, v4, v1 +; CHECK-GI-NEXT: neg.4s v4, v3 +; CHECK-GI-NEXT: neg.4s v5, v1 +; CHECK-GI-NEXT: bsl.16b v2, v4, v3 +; CHECK-GI-NEXT: bsl.16b v0, v5, v1 ; CHECK-GI-NEXT: add.4s v0, v2, v0 ; CHECK-GI-NEXT: addv.4s s0, v0 ; CHECK-GI-NEXT: fmov w0, s0 @@ -428,15 +428,15 @@ define i32 @sabd8h_rdx(<8 x i16> %a, <8 x i16> %b) { ; ; CHECK-GI-LABEL: sabd8h_rdx: ; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.2d v2, #0000000000000000 ; CHECK-GI-NEXT: ssubl.4s v3, v0, v1 ; CHECK-GI-NEXT: ssubl2.4s v0, v0, v1 -; CHECK-GI-NEXT: movi.2d v2, #0000000000000000 -; CHECK-GI-NEXT: neg.4s v1, v3 -; CHECK-GI-NEXT: neg.4s v4, v0 -; CHECK-GI-NEXT: cmgt.4s v5, v2, v3 +; CHECK-GI-NEXT: neg.4s v4, v3 +; CHECK-GI-NEXT: neg.4s v5, v0 +; CHECK-GI-NEXT: cmgt.4s v1, v2, v3 ; CHECK-GI-NEXT: cmgt.4s v2, v2, v0 -; CHECK-GI-NEXT: bif.16b v1, v3, v5 -; CHECK-GI-NEXT: bit.16b v0, v4, v2 +; CHECK-GI-NEXT: bsl.16b v1, v4, v3 +; CHECK-GI-NEXT: bit.16b v0, v5, v2 ; CHECK-GI-NEXT: add.4s v0, v1, v0 ; CHECK-GI-NEXT: addv.4s s0, v0 ; CHECK-GI-NEXT: fmov w0, s0 @@ -461,10 +461,10 @@ define i32 @uabdl4s_rdx_i32(<4 x i16> %a, <4 x i16> %b) { ; ; CHECK-GI-LABEL: uabdl4s_rdx_i32: ; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.2d v2, #0000000000000000 ; CHECK-GI-NEXT: usubl.4s v0, v0, v1 -; CHECK-GI-NEXT: movi.2d v1, #0000000000000000 +; CHECK-GI-NEXT: cmgt.4s v1, v2, v0 ; CHECK-GI-NEXT: neg.4s v2, v0 -; CHECK-GI-NEXT: cmgt.4s v1, v1, v0 ; CHECK-GI-NEXT: bit.16b v0, v2, v1 ; CHECK-GI-NEXT: addv.4s s0, v0 ; CHECK-GI-NEXT: fmov w0, s0 @@ -499,12 +499,12 @@ define i64 @uabd4s_rdx(ptr %a, ptr %b, i32 %h) { ; CHECK-GI-NEXT: movi.2d v0, #0000000000000000 ; CHECK-GI-NEXT: usubl.2d v3, v1, v2 ; CHECK-GI-NEXT: usubl2.2d v1, v1, v2 -; CHECK-GI-NEXT: neg.2d v2, v3 -; CHECK-GI-NEXT: neg.2d v4, v1 -; CHECK-GI-NEXT: cmgt.2d v5, v0, v3 +; CHECK-GI-NEXT: cmgt.2d v2, v0, v3 ; CHECK-GI-NEXT: cmgt.2d v0, v0, v1 -; CHECK-GI-NEXT: bif.16b v2, v3, v5 -; CHECK-GI-NEXT: bsl.16b v0, v4, v1 +; CHECK-GI-NEXT: neg.2d v4, v3 +; CHECK-GI-NEXT: neg.2d v5, v1 +; CHECK-GI-NEXT: bsl.16b v2, v4, v3 +; CHECK-GI-NEXT: bsl.16b v0, v5, v1 ; CHECK-GI-NEXT: add.2d v0, v2, v0 ; CHECK-GI-NEXT: addp.2d d0, v0 ; CHECK-GI-NEXT: fmov x0, d0 @@ -531,15 +531,15 @@ define i64 @sabd4s_rdx(<4 x i32> %a, <4 x i32> %b) { ; ; CHECK-GI-LABEL: sabd4s_rdx: ; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.2d v2, #0000000000000000 ; CHECK-GI-NEXT: ssubl.2d v3, v0, v1 ; CHECK-GI-NEXT: ssubl2.2d v0, v0, v1 -; CHECK-GI-NEXT: movi.2d v2, #0000000000000000 -; CHECK-GI-NEXT: neg.2d v1, v3 -; CHECK-GI-NEXT: neg.2d v4, v0 -; CHECK-GI-NEXT: cmgt.2d v5, v2, v3 +; CHECK-GI-NEXT: neg.2d v4, v3 +; CHECK-GI-NEXT: neg.2d v5, v0 +; CHECK-GI-NEXT: cmgt.2d v1, v2, v3 ; CHECK-GI-NEXT: cmgt.2d v2, v2, v0 -; CHECK-GI-NEXT: bif.16b v1, v3, v5 -; CHECK-GI-NEXT: bit.16b v0, v4, v2 +; CHECK-GI-NEXT: bsl.16b v1, v4, v3 +; CHECK-GI-NEXT: bit.16b v0, v5, v2 ; CHECK-GI-NEXT: add.2d v0, v1, v0 ; CHECK-GI-NEXT: addp.2d d0, v0 ; CHECK-GI-NEXT: fmov x0, d0 @@ -564,10 +564,10 @@ define i64 @uabdl2d_rdx_i64(<2 x i32> %a, <2 x i32> %b) { ; ; CHECK-GI-LABEL: uabdl2d_rdx_i64: ; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.2d v2, #0000000000000000 ; CHECK-GI-NEXT: usubl.2d v0, v0, v1 -; CHECK-GI-NEXT: movi.2d v1, #0000000000000000 +; CHECK-GI-NEXT: cmgt.2d v1, v2, v0 ; CHECK-GI-NEXT: neg.2d v2, v0 -; CHECK-GI-NEXT: cmgt.2d v1, v1, v0 ; CHECK-GI-NEXT: bit.16b v0, v2, v1 ; CHECK-GI-NEXT: addp.2d d0, v0 ; CHECK-GI-NEXT: fmov x0, d0 @@ -1796,10 +1796,10 @@ define <2 x i64> @uabd_i32(<2 x i32> %a, <2 x i32> %b) { ; ; CHECK-GI-LABEL: uabd_i32: ; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi.2d v2, #0000000000000000 ; CHECK-GI-NEXT: ssubl.2d v0, v0, v1 -; CHECK-GI-NEXT: movi.2d v1, #0000000000000000 +; CHECK-GI-NEXT: cmgt.2d v1, v2, v0 ; CHECK-GI-NEXT: neg.2d v2, v0 -; CHECK-GI-NEXT: cmgt.2d v1, v1, v0 ; CHECK-GI-NEXT: bit.16b v0, v2, v1 ; CHECK-GI-NEXT: ret %aext = sext <2 x i32> %a to <2 x i64> diff --git a/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll b/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll index cafee32ada68..d4cc154ac6af 100644 --- a/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll +++ b/llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll @@ -205,15 +205,15 @@ define <2 x bfloat> @test_vcvt_bf16_f64(<2 x double> %v) nounwind readnone ssp { ; GENERIC-LABEL: test_vcvt_bf16_f64: ; GENERIC: // %bb.0: ; GENERIC-NEXT: fcvtxn v0.2s, v0.2d -; GENERIC-NEXT: movi.4s v1, #127, msl #8 -; GENERIC-NEXT: movi.4s v2, #1 +; GENERIC-NEXT: movi.4s v1, #1 +; GENERIC-NEXT: movi.4s v2, #127, msl #8 ; GENERIC-NEXT: ushr.4s v3, v0, #16 -; GENERIC-NEXT: add.4s v1, v0, v1 -; GENERIC-NEXT: and.16b v2, v3, v2 -; GENERIC-NEXT: add.4s v1, v2, v1 -; GENERIC-NEXT: fcmeq.4s v2, v0, v0 +; GENERIC-NEXT: add.4s v2, v0, v2 +; GENERIC-NEXT: and.16b v1, v3, v1 +; GENERIC-NEXT: fcmeq.4s v3, v0, v0 ; GENERIC-NEXT: orr.4s v0, #64, lsl #16 -; GENERIC-NEXT: bit.16b v0, v1, v2 +; GENERIC-NEXT: add.4s v1, v1, v2 +; GENERIC-NEXT: bit.16b v0, v1, v3 ; GENERIC-NEXT: shrn.4h v0, v0, #16 ; GENERIC-NEXT: ret ; @@ -238,15 +238,15 @@ define <2 x bfloat> @test_vcvt_bf16_f64(<2 x double> %v) nounwind readnone ssp { ; GISEL-LABEL: test_vcvt_bf16_f64: ; GISEL: // %bb.0: ; GISEL-NEXT: fcvtxn v0.2s, v0.2d -; GISEL-NEXT: movi.4s v1, #127, msl #8 -; GISEL-NEXT: movi.4s v2, #1 +; GISEL-NEXT: movi.4s v1, #1 +; GISEL-NEXT: movi.4s v2, #127, msl #8 ; GISEL-NEXT: ushr.4s v3, v0, #16 -; GISEL-NEXT: add.4s v1, v0, v1 -; GISEL-NEXT: and.16b v2, v3, v2 -; GISEL-NEXT: add.4s v1, v2, v1 -; GISEL-NEXT: fcmeq.4s v2, v0, v0 +; GISEL-NEXT: add.4s v2, v0, v2 +; GISEL-NEXT: and.16b v1, v3, v1 +; GISEL-NEXT: fcmeq.4s v3, v0, v0 ; GISEL-NEXT: orr.4s v0, #64, lsl #16 -; GISEL-NEXT: bit.16b v0, v1, v2 +; GISEL-NEXT: add.4s v1, v1, v2 +; GISEL-NEXT: bit.16b v0, v1, v3 ; GISEL-NEXT: shrn.4h v0, v0, #16 ; GISEL-NEXT: ret %vcvt1.i = fptrunc <2 x double> %v to <2 x bfloat> diff --git a/llvm/test/CodeGen/AArch64/arm64-vhadd.ll b/llvm/test/CodeGen/AArch64/arm64-vhadd.ll index dda610e5dd3c..e754f01daa2a 100644 --- a/llvm/test/CodeGen/AArch64/arm64-vhadd.ll +++ b/llvm/test/CodeGen/AArch64/arm64-vhadd.ll @@ -903,10 +903,10 @@ define <2 x i16> @hadd8x2_sext_lsr(<2 x i8> %src1, <2 x i8> %src2) { ; CHECK: // %bb.0: ; CHECK-NEXT: shl.2s v0, v0, #24 ; CHECK-NEXT: shl.2s v1, v1, #24 +; CHECK-NEXT: movi d2, #0x00ffff0000ffff ; CHECK-NEXT: sshr.2s v0, v0, #24 ; CHECK-NEXT: ssra.2s v0, v1, #24 -; CHECK-NEXT: movi d1, #0x00ffff0000ffff -; CHECK-NEXT: and.8b v0, v0, v1 +; CHECK-NEXT: and.8b v0, v0, v2 ; CHECK-NEXT: ushr.2s v0, v0, #1 ; CHECK-NEXT: ret %zextsrc1 = sext <2 x i8> %src1 to <2 x i16> @@ -968,10 +968,10 @@ define <4 x i16> @rhadd8_sext_lsr(<4 x i8> %src1, <4 x i8> %src2) { ; CHECK: // %bb.0: ; CHECK-NEXT: shl.4h v0, v0, #8 ; CHECK-NEXT: shl.4h v1, v1, #8 +; CHECK-NEXT: movi.4h v2, #1 ; CHECK-NEXT: sshr.4h v0, v0, #8 ; CHECK-NEXT: ssra.4h v0, v1, #8 -; CHECK-NEXT: movi.4h v1, #1 -; CHECK-NEXT: add.4h v0, v0, v1 +; CHECK-NEXT: add.4h v0, v0, v2 ; CHECK-NEXT: ushr.4h v0, v0, #1 ; CHECK-NEXT: ret %zextsrc1 = sext <4 x i8> %src1 to <4 x i16> diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-add-mull-scalable-contract.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-add-mull-scalable-contract.ll index ebf5ce20d4ec..86b1d5d195ff 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-add-mull-scalable-contract.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-add-mull-scalable-contract.ll @@ -7,21 +7,22 @@ target triple = "aarch64-unknown-linux-gnu" define @mull_add( %a, %b, %c) { ; CHECK-LABEL: mull_add: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: uzp1 z6.d, z0.d, z1.d -; CHECK-NEXT: uzp2 z7.d, z2.d, z3.d -; CHECK-NEXT: uzp2 z0.d, z0.d, z1.d +; CHECK-NEXT: uzp2 z6.d, z0.d, z1.d +; CHECK-NEXT: uzp1 z0.d, z0.d, z1.d +; CHECK-NEXT: uzp2 z1.d, z2.d, z3.d +; CHECK-NEXT: uzp1 z2.d, z2.d, z3.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: uzp1 z1.d, z2.d, z3.d -; CHECK-NEXT: fmul z2.d, z6.d, z7.d -; CHECK-NEXT: fmul z3.d, z0.d, z7.d -; CHECK-NEXT: fmad z0.d, p0/m, z1.d, z2.d -; CHECK-NEXT: fnmsb z1.d, p0/m, z6.d, z3.d -; CHECK-NEXT: uzp2 z2.d, z4.d, z5.d -; CHECK-NEXT: uzp1 z3.d, z4.d, z5.d -; CHECK-NEXT: fadd z2.d, z0.d, z2.d +; CHECK-NEXT: fmul z7.d, z0.d, z1.d +; CHECK-NEXT: fmul z1.d, z6.d, z1.d +; CHECK-NEXT: movprfx z3, z7 +; CHECK-NEXT: fmla z3.d, p0/m, z6.d, z2.d +; CHECK-NEXT: fnmsb z0.d, p0/m, z2.d, z1.d +; CHECK-NEXT: uzp2 z1.d, z4.d, z5.d +; CHECK-NEXT: uzp1 z2.d, z4.d, z5.d +; CHECK-NEXT: fadd z2.d, z2.d, z0.d ; CHECK-NEXT: fadd z1.d, z3.d, z1.d -; CHECK-NEXT: zip1 z0.d, z1.d, z2.d -; CHECK-NEXT: zip2 z1.d, z1.d, z2.d +; CHECK-NEXT: zip1 z0.d, z2.d, z1.d +; CHECK-NEXT: zip2 z1.d, z2.d, z1.d ; CHECK-NEXT: ret entry: %strided.vec = tail call { , } @llvm.experimental.vector.deinterleave2.nxv4f64( %a) @@ -49,21 +50,21 @@ entry: define @mul_add_mull( %a, %b, %c, %d) { ; CHECK-LABEL: mul_add_mull: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z24.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z25.d, z24.d ; CHECK-NEXT: mov z26.d, z24.d ; CHECK-NEXT: mov z27.d, z24.d +; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #0 ; CHECK-NEXT: fcmla z25.d, p0/m, z2.d, z0.d, #0 ; CHECK-NEXT: fcmla z26.d, p0/m, z3.d, z1.d, #0 ; CHECK-NEXT: fcmla z27.d, p0/m, z6.d, z4.d, #0 -; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #0 +; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #90 ; CHECK-NEXT: fcmla z25.d, p0/m, z2.d, z0.d, #90 ; CHECK-NEXT: fcmla z26.d, p0/m, z3.d, z1.d, #90 ; CHECK-NEXT: fcmla z27.d, p0/m, z6.d, z4.d, #90 -; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #90 -; CHECK-NEXT: fadd z0.d, z25.d, z27.d ; CHECK-NEXT: fadd z1.d, z26.d, z24.d +; CHECK-NEXT: fadd z0.d, z25.d, z27.d ; CHECK-NEXT: ret entry: %strided.vec = tail call { , } @llvm.experimental.vector.deinterleave2.nxv4f64( %a) @@ -100,21 +101,21 @@ entry: define @mul_sub_mull( %a, %b, %c, %d) { ; CHECK-LABEL: mul_sub_mull: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z24.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z25.d, z24.d ; CHECK-NEXT: mov z26.d, z24.d ; CHECK-NEXT: mov z27.d, z24.d +; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #0 ; CHECK-NEXT: fcmla z25.d, p0/m, z2.d, z0.d, #0 ; CHECK-NEXT: fcmla z26.d, p0/m, z3.d, z1.d, #0 ; CHECK-NEXT: fcmla z27.d, p0/m, z6.d, z4.d, #0 -; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #0 +; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #90 ; CHECK-NEXT: fcmla z25.d, p0/m, z2.d, z0.d, #90 ; CHECK-NEXT: fcmla z26.d, p0/m, z3.d, z1.d, #90 ; CHECK-NEXT: fcmla z27.d, p0/m, z6.d, z4.d, #90 -; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #90 -; CHECK-NEXT: fsub z0.d, z25.d, z27.d ; CHECK-NEXT: fsub z1.d, z26.d, z24.d +; CHECK-NEXT: fsub z0.d, z25.d, z27.d ; CHECK-NEXT: ret entry: %strided.vec = tail call { , } @llvm.experimental.vector.deinterleave2.nxv4f64( %a) @@ -151,21 +152,21 @@ entry: define @mul_conj_mull( %a, %b, %c, %d) { ; CHECK-LABEL: mul_conj_mull: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z24.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z25.d, z24.d ; CHECK-NEXT: mov z26.d, z24.d ; CHECK-NEXT: mov z27.d, z24.d +; CHECK-NEXT: fcmla z24.d, p0/m, z5.d, z7.d, #0 ; CHECK-NEXT: fcmla z25.d, p0/m, z2.d, z0.d, #0 ; CHECK-NEXT: fcmla z26.d, p0/m, z3.d, z1.d, #0 ; CHECK-NEXT: fcmla z27.d, p0/m, z4.d, z6.d, #0 -; CHECK-NEXT: fcmla z24.d, p0/m, z5.d, z7.d, #0 +; CHECK-NEXT: fcmla z24.d, p0/m, z5.d, z7.d, #270 ; CHECK-NEXT: fcmla z25.d, p0/m, z2.d, z0.d, #90 ; CHECK-NEXT: fcmla z26.d, p0/m, z3.d, z1.d, #90 ; CHECK-NEXT: fcmla z27.d, p0/m, z4.d, z6.d, #270 -; CHECK-NEXT: fcmla z24.d, p0/m, z5.d, z7.d, #270 -; CHECK-NEXT: fadd z0.d, z25.d, z27.d ; CHECK-NEXT: fadd z1.d, z26.d, z24.d +; CHECK-NEXT: fadd z0.d, z25.d, z27.d ; CHECK-NEXT: ret entry: %strided.vec = tail call { , } @llvm.experimental.vector.deinterleave2.nxv4f64( %a) @@ -206,8 +207,8 @@ define @mul_add_rot_mull( %a, @mul_add_rot_mull( %a, @mul_add_mull( %a, %b, %c, %d) { ; CHECK-LABEL: mul_add_mull: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z24.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z25.d, z24.d -; CHECK-NEXT: fcmla z25.d, p0/m, z6.d, z4.d, #0 ; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #0 -; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #0 +; CHECK-NEXT: fcmla z25.d, p0/m, z6.d, z4.d, #0 ; CHECK-NEXT: fcmla z24.d, p0/m, z1.d, z3.d, #0 -; CHECK-NEXT: fcmla z25.d, p0/m, z6.d, z4.d, #90 +; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #0 ; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #90 -; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #90 +; CHECK-NEXT: fcmla z25.d, p0/m, z6.d, z4.d, #90 ; CHECK-NEXT: fcmla z24.d, p0/m, z1.d, z3.d, #90 -; CHECK-NEXT: mov z0.d, z25.d +; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #90 ; CHECK-NEXT: mov z1.d, z24.d +; CHECK-NEXT: mov z0.d, z25.d ; CHECK-NEXT: ret entry: %strided.vec = tail call { , } @llvm.experimental.vector.deinterleave2.nxv4f64( %a) @@ -90,19 +90,19 @@ entry: define @mul_sub_mull( %a, %b, %c, %d) { ; CHECK-LABEL: mul_sub_mull: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z24.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z25.d, z24.d -; CHECK-NEXT: fcmla z25.d, p0/m, z6.d, z4.d, #270 ; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #270 -; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #0 +; CHECK-NEXT: fcmla z25.d, p0/m, z6.d, z4.d, #270 ; CHECK-NEXT: fcmla z24.d, p0/m, z1.d, z3.d, #0 -; CHECK-NEXT: fcmla z25.d, p0/m, z6.d, z4.d, #180 +; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #0 ; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z5.d, #180 -; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #90 +; CHECK-NEXT: fcmla z25.d, p0/m, z6.d, z4.d, #180 ; CHECK-NEXT: fcmla z24.d, p0/m, z1.d, z3.d, #90 -; CHECK-NEXT: mov z0.d, z25.d +; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #90 ; CHECK-NEXT: mov z1.d, z24.d +; CHECK-NEXT: mov z0.d, z25.d ; CHECK-NEXT: ret entry: %strided.vec = tail call { , } @llvm.experimental.vector.deinterleave2.nxv4f64( %a) @@ -139,19 +139,19 @@ entry: define @mul_conj_mull( %a, %b, %c, %d) { ; CHECK-LABEL: mul_conj_mull: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z24.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z25.d, z24.d -; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #0 ; CHECK-NEXT: fcmla z24.d, p0/m, z1.d, z3.d, #0 -; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #90 +; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #0 ; CHECK-NEXT: fcmla z24.d, p0/m, z1.d, z3.d, #90 -; CHECK-NEXT: fcmla z25.d, p0/m, z4.d, z6.d, #0 +; CHECK-NEXT: fcmla z25.d, p0/m, z0.d, z2.d, #90 ; CHECK-NEXT: fcmla z24.d, p0/m, z5.d, z7.d, #0 -; CHECK-NEXT: fcmla z25.d, p0/m, z4.d, z6.d, #270 +; CHECK-NEXT: fcmla z25.d, p0/m, z4.d, z6.d, #0 ; CHECK-NEXT: fcmla z24.d, p0/m, z5.d, z7.d, #270 -; CHECK-NEXT: mov z0.d, z25.d +; CHECK-NEXT: fcmla z25.d, p0/m, z4.d, z6.d, #270 ; CHECK-NEXT: mov z1.d, z24.d +; CHECK-NEXT: mov z0.d, z25.d ; CHECK-NEXT: ret entry: %strided.vec = tail call { , } @llvm.experimental.vector.deinterleave2.nxv4f64( %a) @@ -188,24 +188,25 @@ entry: define @mul_add_rot_mull( %a, %b, %c, %d) { ; CHECK-LABEL: mul_add_rot_mull: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: uzp1 z24.d, z2.d, z3.d +; CHECK-NEXT: uzp2 z24.d, z2.d, z3.d ; CHECK-NEXT: uzp2 z25.d, z0.d, z1.d -; CHECK-NEXT: uzp2 z2.d, z2.d, z3.d -; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: uzp1 z2.d, z2.d, z3.d ; CHECK-NEXT: uzp1 z0.d, z0.d, z1.d +; CHECK-NEXT: uzp2 z1.d, z4.d, z5.d ; CHECK-NEXT: uzp1 z26.d, z6.d, z7.d -; CHECK-NEXT: fmul z1.d, z24.d, z25.d -; CHECK-NEXT: fmul z3.d, z2.d, z25.d -; CHECK-NEXT: uzp2 z25.d, z4.d, z5.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uzp1 z4.d, z4.d, z5.d ; CHECK-NEXT: uzp2 z5.d, z6.d, z7.d -; CHECK-NEXT: fmla z3.d, p0/m, z26.d, z25.d -; CHECK-NEXT: fmla z1.d, p0/m, z2.d, z0.d -; CHECK-NEXT: movprfx z2, z3 -; CHECK-NEXT: fmla z2.d, p0/m, z5.d, z4.d -; CHECK-NEXT: fmla z1.d, p0/m, z26.d, z4.d -; CHECK-NEXT: fnmls z2.d, p0/m, z24.d, z0.d -; CHECK-NEXT: fmls z1.d, p0/m, z5.d, z25.d +; CHECK-NEXT: fmul z3.d, z2.d, z25.d +; CHECK-NEXT: fmul z25.d, z24.d, z25.d +; CHECK-NEXT: fmla z3.d, p0/m, z24.d, z0.d +; CHECK-NEXT: movprfx z24, z25 +; CHECK-NEXT: fmla z24.d, p0/m, z26.d, z1.d +; CHECK-NEXT: movprfx z6, z24 +; CHECK-NEXT: fmla z6.d, p0/m, z5.d, z4.d +; CHECK-NEXT: fmla z3.d, p0/m, z26.d, z4.d +; CHECK-NEXT: fnmsb z2.d, p0/m, z0.d, z6.d +; CHECK-NEXT: fmsb z1.d, p0/m, z5.d, z3.d ; CHECK-NEXT: zip1 z0.d, z2.d, z1.d ; CHECK-NEXT: zip2 z1.d, z2.d, z1.d ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-f16-mul-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-f16-mul-scalable.ll index 611cf44ea7ee..cb285c05b2e8 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-f16-mul-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-f16-mul-scalable.ll @@ -16,9 +16,10 @@ define @complex_mul_v4f16( %a, @complex_mul_v8f16( %a, %b) { ; CHECK-LABEL: complex_mul_v8f16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, #0 // =0x0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: fcmla z2.h, p0/m, z1.h, z0.h, #0 ; CHECK-NEXT: fcmla z2.h, p0/m, z1.h, z0.h, #90 ; CHECK-NEXT: mov z0.d, z2.d @@ -72,15 +73,15 @@ entry: define @complex_mul_v16f16( %a, %b) { ; CHECK-LABEL: complex_mul_v16f16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z4.h, #0 // =0x0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z5.d, z4.d -; CHECK-NEXT: fcmla z5.h, p0/m, z2.h, z0.h, #0 ; CHECK-NEXT: fcmla z4.h, p0/m, z3.h, z1.h, #0 -; CHECK-NEXT: fcmla z5.h, p0/m, z2.h, z0.h, #90 +; CHECK-NEXT: fcmla z5.h, p0/m, z2.h, z0.h, #0 ; CHECK-NEXT: fcmla z4.h, p0/m, z3.h, z1.h, #90 -; CHECK-NEXT: mov z0.d, z5.d +; CHECK-NEXT: fcmla z5.h, p0/m, z2.h, z0.h, #90 ; CHECK-NEXT: mov z1.d, z4.d +; CHECK-NEXT: mov z0.d, z5.d ; CHECK-NEXT: ret entry: %a.deinterleaved = tail call { , } @llvm.experimental.vector.deinterleave2.nxv16f16( %a) @@ -103,23 +104,23 @@ entry: define @complex_mul_v32f16( %a, %b) { ; CHECK-LABEL: complex_mul_v32f16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z24.h, #0 // =0x0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z25.d, z24.d ; CHECK-NEXT: mov z26.d, z24.d ; CHECK-NEXT: mov z27.d, z24.d +; CHECK-NEXT: fcmla z24.h, p0/m, z7.h, z3.h, #0 ; CHECK-NEXT: fcmla z25.h, p0/m, z4.h, z0.h, #0 ; CHECK-NEXT: fcmla z26.h, p0/m, z5.h, z1.h, #0 ; CHECK-NEXT: fcmla z27.h, p0/m, z6.h, z2.h, #0 -; CHECK-NEXT: fcmla z24.h, p0/m, z7.h, z3.h, #0 +; CHECK-NEXT: fcmla z24.h, p0/m, z7.h, z3.h, #90 ; CHECK-NEXT: fcmla z25.h, p0/m, z4.h, z0.h, #90 ; CHECK-NEXT: fcmla z26.h, p0/m, z5.h, z1.h, #90 ; CHECK-NEXT: fcmla z27.h, p0/m, z6.h, z2.h, #90 -; CHECK-NEXT: fcmla z24.h, p0/m, z7.h, z3.h, #90 +; CHECK-NEXT: mov z3.d, z24.d ; CHECK-NEXT: mov z0.d, z25.d ; CHECK-NEXT: mov z1.d, z26.d ; CHECK-NEXT: mov z2.d, z27.d -; CHECK-NEXT: mov z3.d, z24.d ; CHECK-NEXT: ret entry: %a.deinterleaved = tail call { , } @llvm.experimental.vector.deinterleave2.nxv32f16( %a) diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-f32-mul-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-f32-mul-scalable.ll index 0f5e9a2202dd..1e2afb78de1b 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-f32-mul-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-f32-mul-scalable.ll @@ -7,8 +7,8 @@ target triple = "aarch64" define @complex_mul_v4f32( %a, %b) { ; CHECK-LABEL: complex_mul_v4f32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, #0 // =0x0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: fcmla z2.s, p0/m, z1.s, z0.s, #0 ; CHECK-NEXT: fcmla z2.s, p0/m, z1.s, z0.s, #90 ; CHECK-NEXT: mov z0.d, z2.d @@ -34,15 +34,15 @@ entry: define @complex_mul_v8f32( %a, %b) { ; CHECK-LABEL: complex_mul_v8f32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z4.s, #0 // =0x0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z5.d, z4.d -; CHECK-NEXT: fcmla z5.s, p0/m, z2.s, z0.s, #0 ; CHECK-NEXT: fcmla z4.s, p0/m, z3.s, z1.s, #0 -; CHECK-NEXT: fcmla z5.s, p0/m, z2.s, z0.s, #90 +; CHECK-NEXT: fcmla z5.s, p0/m, z2.s, z0.s, #0 ; CHECK-NEXT: fcmla z4.s, p0/m, z3.s, z1.s, #90 -; CHECK-NEXT: mov z0.d, z5.d +; CHECK-NEXT: fcmla z5.s, p0/m, z2.s, z0.s, #90 ; CHECK-NEXT: mov z1.d, z4.d +; CHECK-NEXT: mov z0.d, z5.d ; CHECK-NEXT: ret entry: %a.deinterleaved = tail call { , } @llvm.experimental.vector.deinterleave2.nxv8f32( %a) @@ -65,23 +65,23 @@ entry: define @complex_mul_v16f32( %a, %b) { ; CHECK-LABEL: complex_mul_v16f32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z24.s, #0 // =0x0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z25.d, z24.d ; CHECK-NEXT: mov z26.d, z24.d ; CHECK-NEXT: mov z27.d, z24.d +; CHECK-NEXT: fcmla z24.s, p0/m, z7.s, z3.s, #0 ; CHECK-NEXT: fcmla z25.s, p0/m, z4.s, z0.s, #0 ; CHECK-NEXT: fcmla z26.s, p0/m, z5.s, z1.s, #0 ; CHECK-NEXT: fcmla z27.s, p0/m, z6.s, z2.s, #0 -; CHECK-NEXT: fcmla z24.s, p0/m, z7.s, z3.s, #0 +; CHECK-NEXT: fcmla z24.s, p0/m, z7.s, z3.s, #90 ; CHECK-NEXT: fcmla z25.s, p0/m, z4.s, z0.s, #90 ; CHECK-NEXT: fcmla z26.s, p0/m, z5.s, z1.s, #90 ; CHECK-NEXT: fcmla z27.s, p0/m, z6.s, z2.s, #90 -; CHECK-NEXT: fcmla z24.s, p0/m, z7.s, z3.s, #90 +; CHECK-NEXT: mov z3.d, z24.d ; CHECK-NEXT: mov z0.d, z25.d ; CHECK-NEXT: mov z1.d, z26.d ; CHECK-NEXT: mov z2.d, z27.d -; CHECK-NEXT: mov z3.d, z24.d ; CHECK-NEXT: ret entry: %a.deinterleaved = tail call { , } @llvm.experimental.vector.deinterleave2.nxv16f32( %a) diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-f64-mul-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-f64-mul-scalable.ll index 1fe554bdc616..17a239a09a03 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-f64-mul-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-f64-mul-scalable.ll @@ -7,8 +7,8 @@ target triple = "aarch64" define @complex_mul_v2f64( %a, %b) { ; CHECK-LABEL: complex_mul_v2f64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: fcmla z2.d, p0/m, z1.d, z0.d, #0 ; CHECK-NEXT: fcmla z2.d, p0/m, z1.d, z0.d, #90 ; CHECK-NEXT: mov z0.d, z2.d @@ -34,15 +34,15 @@ entry: define @complex_mul_v4f64( %a, %b) { ; CHECK-LABEL: complex_mul_v4f64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z4.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z5.d, z4.d -; CHECK-NEXT: fcmla z5.d, p0/m, z2.d, z0.d, #0 ; CHECK-NEXT: fcmla z4.d, p0/m, z3.d, z1.d, #0 -; CHECK-NEXT: fcmla z5.d, p0/m, z2.d, z0.d, #90 +; CHECK-NEXT: fcmla z5.d, p0/m, z2.d, z0.d, #0 ; CHECK-NEXT: fcmla z4.d, p0/m, z3.d, z1.d, #90 -; CHECK-NEXT: mov z0.d, z5.d +; CHECK-NEXT: fcmla z5.d, p0/m, z2.d, z0.d, #90 ; CHECK-NEXT: mov z1.d, z4.d +; CHECK-NEXT: mov z0.d, z5.d ; CHECK-NEXT: ret entry: %a.deinterleaved = tail call { , } @llvm.experimental.vector.deinterleave2.nxv4f64( %a) @@ -65,23 +65,23 @@ entry: define @complex_mul_v8f64( %a, %b) { ; CHECK-LABEL: complex_mul_v8f64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z24.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z25.d, z24.d ; CHECK-NEXT: mov z26.d, z24.d ; CHECK-NEXT: mov z27.d, z24.d +; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z3.d, #0 ; CHECK-NEXT: fcmla z25.d, p0/m, z4.d, z0.d, #0 ; CHECK-NEXT: fcmla z26.d, p0/m, z5.d, z1.d, #0 ; CHECK-NEXT: fcmla z27.d, p0/m, z6.d, z2.d, #0 -; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z3.d, #0 +; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z3.d, #90 ; CHECK-NEXT: fcmla z25.d, p0/m, z4.d, z0.d, #90 ; CHECK-NEXT: fcmla z26.d, p0/m, z5.d, z1.d, #90 ; CHECK-NEXT: fcmla z27.d, p0/m, z6.d, z2.d, #90 -; CHECK-NEXT: fcmla z24.d, p0/m, z7.d, z3.d, #90 +; CHECK-NEXT: mov z3.d, z24.d ; CHECK-NEXT: mov z0.d, z25.d ; CHECK-NEXT: mov z1.d, z26.d ; CHECK-NEXT: mov z2.d, z27.d -; CHECK-NEXT: mov z3.d, z24.d ; CHECK-NEXT: ret entry: %a.deinterleaved = tail call { , } @llvm.experimental.vector.deinterleave2.nxv8f64( %a) diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-i16-mul-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-i16-mul-scalable.ll index 1b8a21b66ade..07488b623b98 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-i16-mul-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-i16-mul-scalable.ll @@ -16,8 +16,9 @@ define @complex_mul_v4i16( %a, This Inner Loop Header: Depth=1 -; CHECK-NEXT: zip2 p3.d, p1.d, p1.d +; CHECK-NEXT: zip2 p2.d, p1.d, p1.d ; CHECK-NEXT: add x13, x0, x8 ; CHECK-NEXT: add x14, x1, x8 -; CHECK-NEXT: zip1 p2.d, p1.d, p1.d +; CHECK-NEXT: zip1 p1.d, p1.d, p1.d ; CHECK-NEXT: mov z6.d, z1.d ; CHECK-NEXT: mov z7.d, z0.d -; CHECK-NEXT: whilelo p1.d, x12, x9 +; CHECK-NEXT: ld1d { z2.d }, p2/z, [x13, #1, mul vl] +; CHECK-NEXT: ld1d { z4.d }, p2/z, [x14, #1, mul vl] ; CHECK-NEXT: add x8, x8, x11 -; CHECK-NEXT: add x12, x12, x10 -; CHECK-NEXT: ld1d { z2.d }, p3/z, [x13, #1, mul vl] -; CHECK-NEXT: ld1d { z4.d }, p3/z, [x14, #1, mul vl] -; CHECK-NEXT: ld1d { z3.d }, p2/z, [x13] -; CHECK-NEXT: ld1d { z5.d }, p2/z, [x14] +; CHECK-NEXT: ld1d { z3.d }, p1/z, [x13] +; CHECK-NEXT: ld1d { z5.d }, p1/z, [x14] ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #0 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #0 ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #90 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #90 -; CHECK-NEXT: mov z0.d, p3/m, z7.d -; CHECK-NEXT: mov z1.d, p2/m, z6.d +; CHECK-NEXT: mov z0.d, p2/m, z7.d +; CHECK-NEXT: mov z1.d, p1/m, z6.d +; CHECK-NEXT: whilelo p1.d, x12, x9 +; CHECK-NEXT: add x12, x12, x10 ; CHECK-NEXT: b.mi .LBB0_1 ; CHECK-NEXT: // %bb.2: // %exit.block ; CHECK-NEXT: uzp1 z2.d, z1.d, z0.d @@ -114,10 +114,10 @@ define %"class.std::complex" @complex_mul_predicated_v2f64(ptr %a, ptr %b, ptr % ; CHECK-LABEL: complex_mul_predicated_v2f64: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: mov z1.d, #0 // =0x0 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cntd x10 -; CHECK-NEXT: neg x11, x10 ; CHECK-NEXT: mov w12, #100 // =0x64 +; CHECK-NEXT: neg x11, x10 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: mov x9, xzr ; CHECK-NEXT: and x11, x11, x12 @@ -133,20 +133,20 @@ define %"class.std::complex" @complex_mul_predicated_v2f64(ptr %a, ptr %b, ptr % ; CHECK-NEXT: mov z7.d, z0.d ; CHECK-NEXT: add x9, x9, x10 ; CHECK-NEXT: add x8, x8, x12 -; CHECK-NEXT: cmpne p1.d, p0/z, z2.d, #0 +; CHECK-NEXT: cmpne p2.d, p0/z, z2.d, #0 ; CHECK-NEXT: cmp x11, x9 -; CHECK-NEXT: zip2 p2.d, p1.d, p1.d -; CHECK-NEXT: zip1 p1.d, p1.d, p1.d -; CHECK-NEXT: ld1d { z2.d }, p2/z, [x13, #1, mul vl] -; CHECK-NEXT: ld1d { z4.d }, p2/z, [x14, #1, mul vl] -; CHECK-NEXT: ld1d { z3.d }, p1/z, [x13] -; CHECK-NEXT: ld1d { z5.d }, p1/z, [x14] +; CHECK-NEXT: zip2 p1.d, p2.d, p2.d +; CHECK-NEXT: zip1 p2.d, p2.d, p2.d +; CHECK-NEXT: ld1d { z2.d }, p1/z, [x13, #1, mul vl] +; CHECK-NEXT: ld1d { z4.d }, p1/z, [x14, #1, mul vl] +; CHECK-NEXT: ld1d { z3.d }, p2/z, [x13] +; CHECK-NEXT: ld1d { z5.d }, p2/z, [x14] ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #0 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #0 ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #90 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #90 -; CHECK-NEXT: mov z0.d, p2/m, z7.d -; CHECK-NEXT: mov z1.d, p1/m, z6.d +; CHECK-NEXT: mov z0.d, p1/m, z7.d +; CHECK-NEXT: mov z1.d, p2/m, z6.d ; CHECK-NEXT: b.ne .LBB1_1 ; CHECK-NEXT: // %bb.2: // %exit.block ; CHECK-NEXT: uzp1 z2.d, z1.d, z0.d @@ -217,8 +217,8 @@ exit.block: ; preds = %vector.body define %"class.std::complex" @complex_mul_predicated_x2_v2f64(ptr %a, ptr %b, ptr %cond) { ; CHECK-LABEL: complex_mul_predicated_x2_v2f64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: mov w10, #100 // =0x64 ; CHECK-NEXT: mov z1.d, #0 // =0x0 +; CHECK-NEXT: mov w10, #100 // =0x64 ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: whilelo p1.d, xzr, x10 ; CHECK-NEXT: mov x8, xzr @@ -236,19 +236,19 @@ define %"class.std::complex" @complex_mul_predicated_x2_v2f64(ptr %a, ptr %b, pt ; CHECK-NEXT: mov z7.d, z0.d ; CHECK-NEXT: add x9, x9, x11 ; CHECK-NEXT: add x8, x8, x12 -; CHECK-NEXT: cmpne p1.d, p1/z, z2.d, #0 -; CHECK-NEXT: zip2 p3.d, p1.d, p1.d -; CHECK-NEXT: zip1 p2.d, p1.d, p1.d -; CHECK-NEXT: whilelo p1.d, x9, x10 -; CHECK-NEXT: ld1d { z2.d }, p3/z, [x13, #1, mul vl] -; CHECK-NEXT: ld1d { z4.d }, p3/z, [x14, #1, mul vl] +; CHECK-NEXT: cmpne p2.d, p1/z, z2.d, #0 +; CHECK-NEXT: zip2 p1.d, p2.d, p2.d +; CHECK-NEXT: zip1 p2.d, p2.d, p2.d +; CHECK-NEXT: ld1d { z2.d }, p1/z, [x13, #1, mul vl] +; CHECK-NEXT: ld1d { z4.d }, p1/z, [x14, #1, mul vl] ; CHECK-NEXT: ld1d { z3.d }, p2/z, [x13] ; CHECK-NEXT: ld1d { z5.d }, p2/z, [x14] ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #0 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #0 ; CHECK-NEXT: fcmla z7.d, p0/m, z4.d, z2.d, #90 ; CHECK-NEXT: fcmla z6.d, p0/m, z5.d, z3.d, #90 -; CHECK-NEXT: mov z0.d, p3/m, z7.d +; CHECK-NEXT: mov z0.d, p1/m, z7.d +; CHECK-NEXT: whilelo p1.d, x9, x10 ; CHECK-NEXT: mov z1.d, p2/m, z6.d ; CHECK-NEXT: b.mi .LBB2_1 ; CHECK-NEXT: // %bb.2: // %exit.block diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll index 1696ac8709d4..664d99a3627b 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll @@ -15,11 +15,11 @@ define %"class.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) { ; CHECK-LABEL: complex_mul_v2f64: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: mov z1.d, #0 // =0x0 -; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: cntd x9 -; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: neg x9, x9 ; CHECK-NEXT: mov w10, #100 // =0x64 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: and x10, x9, x10 ; CHECK-NEXT: rdvl x11, #2 @@ -101,18 +101,18 @@ exit.block: ; preds = %vector.body define %"class.std::complex" @complex_mul_nonzero_init_v2f64(ptr %a, ptr %b) { ; CHECK-LABEL: complex_mul_nonzero_init_v2f64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: fmov d0, #1.00000000 ; CHECK-NEXT: mov z1.d, #0 // =0x0 -; CHECK-NEXT: fmov d2, #2.00000000 ; CHECK-NEXT: cntd x9 -; CHECK-NEXT: mov w10, #100 // =0x64 -; CHECK-NEXT: ptrue p1.b +; CHECK-NEXT: fmov d2, #2.00000000 +; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: neg x9, x9 +; CHECK-NEXT: ptrue p1.b +; CHECK-NEXT: mov w10, #100 // =0x64 ; CHECK-NEXT: mov x8, xzr +; CHECK-NEXT: sel z3.d, p0, z0.d, z1.d ; CHECK-NEXT: and x10, x9, x10 ; CHECK-NEXT: rdvl x11, #2 -; CHECK-NEXT: sel z3.d, p0, z0.d, z1.d ; CHECK-NEXT: mov z1.d, p0/m, z2.d ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: zip2 z0.d, z1.d, z3.d @@ -190,12 +190,12 @@ define %"class.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) { ; CHECK-LABEL: complex_mul_v2f64_unrolled: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: mov z1.d, #0 // =0x0 -; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: cntw x9 -; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: neg x9, x9 ; CHECK-NEXT: mov w10, #1000 // =0x3e8 +; CHECK-NEXT: neg x9, x9 ; CHECK-NEXT: rdvl x12, #2 +; CHECK-NEXT: ptrue p1.b +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: and x10, x9, x10 ; CHECK-NEXT: zip2 z0.d, z1.d, z1.d @@ -324,10 +324,10 @@ define dso_local %"class.std::complex" @reduction_mix(ptr %a, ptr %b, ptr noalia ; CHECK-LABEL: reduction_mix: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: mov z2.d, #0 // =0x0 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cntd x9 -; CHECK-NEXT: neg x10, x9 ; CHECK-NEXT: mov w11, #100 // =0x64 +; CHECK-NEXT: neg x10, x9 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: and x10, x10, x11 ; CHECK-NEXT: rdvl x11, #2 @@ -349,8 +349,8 @@ define dso_local %"class.std::complex" @reduction_mix(ptr %a, ptr %b, ptr noalia ; CHECK-NEXT: uaddv d2, p0, z2.d ; CHECK-NEXT: uzp2 z3.d, z1.d, z0.d ; CHECK-NEXT: uzp1 z1.d, z1.d, z0.d -; CHECK-NEXT: fmov x8, d2 ; CHECK-NEXT: faddv d0, p0, z3.d +; CHECK-NEXT: fmov x8, d2 ; CHECK-NEXT: faddv d1, p0, z1.d ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 killed $z1 diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll index 742a7099559f..17bf5ba6eb48 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll @@ -8,8 +8,8 @@ target triple = "aarch64" define @complex_mul_const( %a, %b) { ; CHECK-LABEL: complex_mul_const: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z4.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: fmov z7.d, #3.00000000 ; CHECK-NEXT: fmov z24.d, #11.00000000 ; CHECK-NEXT: mov z6.d, z4.d @@ -55,25 +55,25 @@ entry: define @complex_mul_non_const( %a, %b, [2 x double] %c) { ; CHECK-LABEL: complex_mul_non_const: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z6.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $d5 killed $d5 def $z5 ; CHECK-NEXT: // kill: def $d4 killed $d4 def $z4 ; CHECK-NEXT: mov z5.d, d5 ; CHECK-NEXT: mov z4.d, d4 ; CHECK-NEXT: mov z24.d, z6.d ; CHECK-NEXT: mov z7.d, z6.d -; CHECK-NEXT: zip2 z25.d, z4.d, z5.d -; CHECK-NEXT: zip1 z4.d, z4.d, z5.d ; CHECK-NEXT: fcmla z24.d, p0/m, z1.d, z3.d, #0 ; CHECK-NEXT: fcmla z7.d, p0/m, z0.d, z2.d, #0 ; CHECK-NEXT: fcmla z24.d, p0/m, z1.d, z3.d, #90 +; CHECK-NEXT: zip2 z1.d, z4.d, z5.d ; CHECK-NEXT: fcmla z7.d, p0/m, z0.d, z2.d, #90 +; CHECK-NEXT: zip1 z2.d, z4.d, z5.d ; CHECK-NEXT: mov z0.d, z6.d -; CHECK-NEXT: fcmla z6.d, p0/m, z24.d, z25.d, #0 -; CHECK-NEXT: fcmla z0.d, p0/m, z7.d, z4.d, #0 -; CHECK-NEXT: fcmla z6.d, p0/m, z24.d, z25.d, #90 -; CHECK-NEXT: fcmla z0.d, p0/m, z7.d, z4.d, #90 +; CHECK-NEXT: fcmla z6.d, p0/m, z24.d, z1.d, #0 +; CHECK-NEXT: fcmla z0.d, p0/m, z7.d, z2.d, #0 +; CHECK-NEXT: fcmla z6.d, p0/m, z24.d, z1.d, #90 +; CHECK-NEXT: fcmla z0.d, p0/m, z7.d, z2.d, #90 ; CHECK-NEXT: mov z1.d, z6.d ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/concat_vector-truncate-combine.ll b/llvm/test/CodeGen/AArch64/concat_vector-truncate-combine.ll index 1c254f9ed935..e6d5a2ac0fd7 100644 --- a/llvm/test/CodeGen/AArch64/concat_vector-truncate-combine.ll +++ b/llvm/test/CodeGen/AArch64/concat_vector-truncate-combine.ll @@ -96,8 +96,8 @@ entry: define void @test_concat_fptrunc_v4f64_to_v4f32(ptr %ptr) #1 { ; CHECK-LABEL: test_concat_fptrunc_v4f64_to_v4f32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: fmov z0.s, #1.00000000 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/dag-combine-concat-vectors.ll b/llvm/test/CodeGen/AArch64/dag-combine-concat-vectors.ll index 4a2e85c715f7..83c7f73800af 100644 --- a/llvm/test/CodeGen/AArch64/dag-combine-concat-vectors.ll +++ b/llvm/test/CodeGen/AArch64/dag-combine-concat-vectors.ll @@ -9,8 +9,8 @@ define fastcc i8 @allocno_reload_assign() { ; CHECK-LABEL: allocno_reload_assign: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z0.b, #0 // =0x0 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z16.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: uunpklo z1.h, z0.b ; CHECK-NEXT: uunpkhi z0.h, z0.b @@ -48,12 +48,12 @@ define fastcc i8 @allocno_reload_assign() { ; CHECK-NEXT: punpklo p4.h, p3.b ; CHECK-NEXT: punpkhi p3.h, p3.b ; CHECK-NEXT: st1b { z2.d }, p4, [z16.d] -; CHECK-NEXT: punpklo p4.h, p2.b -; CHECK-NEXT: punpkhi p2.h, p2.b ; CHECK-NEXT: st1b { z3.d }, p3, [z16.d] -; CHECK-NEXT: punpklo p3.h, p4.b -; CHECK-NEXT: st1b { z4.d }, p3, [z16.d] -; CHECK-NEXT: punpkhi p3.h, p4.b +; CHECK-NEXT: punpklo p3.h, p2.b +; CHECK-NEXT: punpkhi p2.h, p2.b +; CHECK-NEXT: punpklo p4.h, p3.b +; CHECK-NEXT: punpkhi p3.h, p3.b +; CHECK-NEXT: st1b { z4.d }, p4, [z16.d] ; CHECK-NEXT: st1b { z5.d }, p3, [z16.d] ; CHECK-NEXT: punpklo p3.h, p2.b ; CHECK-NEXT: punpkhi p2.h, p2.b diff --git a/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-signed.ll index 49ad3ae7d629..6e13ae6feb66 100644 --- a/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-signed.ll +++ b/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-signed.ll @@ -228,13 +228,13 @@ define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y, ptr %divdst) nounw ; ALL-NEXT: sdiv x9, x9, x8 ; ALL-NEXT: sdiv x11, x11, x10 ; ALL-NEXT: mul x8, x9, x8 +; ALL-NEXT: fmov d2, x9 ; ALL-NEXT: fmov d1, x8 ; ALL-NEXT: mul x10, x11, x10 +; ALL-NEXT: mov v2.d[1], x11 +; ALL-NEXT: str q2, [x0] ; ALL-NEXT: mov v1.d[1], x10 ; ALL-NEXT: sub v0.2d, v0.2d, v1.2d -; ALL-NEXT: fmov d1, x9 -; ALL-NEXT: mov v1.d[1], x11 -; ALL-NEXT: str q1, [x0] ; ALL-NEXT: ret %div = sdiv <2 x i64> %x, %y store <2 x i64> %div, ptr %divdst, align 16 diff --git a/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-unsigned.ll b/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-unsigned.ll index 3bc50b2f03d8..650219e03b8a 100644 --- a/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-unsigned.ll +++ b/llvm/test/CodeGen/AArch64/div-rem-pair-recomposition-unsigned.ll @@ -228,13 +228,13 @@ define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y, ptr %divdst) nounw ; ALL-NEXT: udiv x9, x9, x8 ; ALL-NEXT: udiv x11, x11, x10 ; ALL-NEXT: mul x8, x9, x8 +; ALL-NEXT: fmov d2, x9 ; ALL-NEXT: fmov d1, x8 ; ALL-NEXT: mul x10, x11, x10 +; ALL-NEXT: mov v2.d[1], x11 +; ALL-NEXT: str q2, [x0] ; ALL-NEXT: mov v1.d[1], x10 ; ALL-NEXT: sub v0.2d, v0.2d, v1.2d -; ALL-NEXT: fmov d1, x9 -; ALL-NEXT: mov v1.d[1], x11 -; ALL-NEXT: str q1, [x0] ; ALL-NEXT: ret %div = udiv <2 x i64> %x, %y store <2 x i64> %div, ptr %divdst, align 16 diff --git a/llvm/test/CodeGen/AArch64/extbinopload.ll b/llvm/test/CodeGen/AArch64/extbinopload.ll index dff4831330de..bd9d9b99622e 100644 --- a/llvm/test/CodeGen/AArch64/extbinopload.ll +++ b/llvm/test/CodeGen/AArch64/extbinopload.ll @@ -502,9 +502,9 @@ define <16 x i32> @double_bv_4xv4i8_i32(ptr %p, ptr %q, ptr %r, ptr %s, ptr %t, ; CHECK-NEXT: usubl v4.8h, v4.8b, v5.8b ; CHECK-NEXT: ld1 { v6.s }[1], [x7], #4 ; CHECK-NEXT: ld1 { v7.s }[1], [x7] -; CHECK-NEXT: usubl v5.8h, v6.8b, v7.8b ; CHECK-NEXT: shll v0.4s, v4.4h, #16 ; CHECK-NEXT: shll2 v4.4s, v4.8h, #16 +; CHECK-NEXT: usubl v5.8h, v6.8b, v7.8b ; CHECK-NEXT: saddw v0.4s, v0.4s, v1.4h ; CHECK-NEXT: saddw2 v1.4s, v4.4s, v1.8h ; CHECK-NEXT: shll v6.4s, v5.4h, #16 @@ -647,10 +647,10 @@ define <16 x i32> @extrause_load(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK: // %bb.0: ; CHECK-NEXT: ldr s1, [x0] ; CHECK-NEXT: add x8, x3, #8 -; CHECK-NEXT: add x11, x3, #12 +; CHECK-NEXT: add x11, x1, #12 ; CHECK-NEXT: str s1, [x4] ; CHECK-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-NEXT: ldp s0, s4, [x2] +; CHECK-NEXT: ldr s0, [x2] ; CHECK-NEXT: ushll v2.8h, v0.8b, #0 ; CHECK-NEXT: umov w9, v2.h[0] ; CHECK-NEXT: umov w10, v2.h[1] @@ -664,31 +664,32 @@ define <16 x i32> @extrause_load(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK-NEXT: add x9, x1, #4 ; CHECK-NEXT: mov v1.d[1], v2.d[0] ; CHECK-NEXT: mov v0.b[11], w10 -; CHECK-NEXT: add x10, x1, #12 +; CHECK-NEXT: add x10, x3, #12 ; CHECK-NEXT: bic v1.8h, #255, lsl #8 ; CHECK-NEXT: ld1 { v0.s }[3], [x3], #4 -; CHECK-NEXT: ldr s3, [x0, #12] -; CHECK-NEXT: ldp s2, s7, [x0, #4] -; CHECK-NEXT: ld1 { v4.s }[1], [x3] -; CHECK-NEXT: ldp s5, s6, [x2, #8] -; CHECK-NEXT: ld1 { v3.s }[1], [x10] -; CHECK-NEXT: ld1 { v2.s }[1], [x9] -; CHECK-NEXT: ld1 { v5.s }[1], [x8] -; CHECK-NEXT: ld1 { v6.s }[1], [x11] +; CHECK-NEXT: ldr s5, [x0, #4] +; CHECK-NEXT: ldp s2, s3, [x2, #4] +; CHECK-NEXT: ldr s7, [x2, #12] +; CHECK-NEXT: ldp s6, s4, [x0, #8] +; CHECK-NEXT: ld1 { v5.s }[1], [x9] +; CHECK-NEXT: ld1 { v7.s }[1], [x10] +; CHECK-NEXT: ld1 { v3.s }[1], [x8] +; CHECK-NEXT: ld1 { v2.s }[1], [x3] ; CHECK-NEXT: add x8, x1, #8 -; CHECK-NEXT: ld1 { v7.s }[1], [x8] -; CHECK-NEXT: uaddl v2.8h, v2.8b, v3.8b -; CHECK-NEXT: ushll v3.8h, v5.8b, #0 -; CHECK-NEXT: uaddl v4.8h, v4.8b, v6.8b -; CHECK-NEXT: uaddw v1.8h, v1.8h, v7.8b +; CHECK-NEXT: ld1 { v4.s }[1], [x11] +; CHECK-NEXT: ld1 { v6.s }[1], [x8] +; CHECK-NEXT: ushll v3.8h, v3.8b, #0 +; CHECK-NEXT: uaddl v2.8h, v2.8b, v7.8b +; CHECK-NEXT: uaddl v4.8h, v5.8b, v4.8b +; CHECK-NEXT: uaddw v1.8h, v1.8h, v6.8b ; CHECK-NEXT: uaddw2 v5.8h, v3.8h, v0.16b -; CHECK-NEXT: ushll v0.4s, v2.4h, #3 +; CHECK-NEXT: ushll v6.4s, v2.4h, #3 ; CHECK-NEXT: ushll2 v2.4s, v2.8h, #3 -; CHECK-NEXT: ushll v6.4s, v4.4h, #3 +; CHECK-NEXT: ushll v0.4s, v4.4h, #3 ; CHECK-NEXT: ushll2 v3.4s, v4.8h, #3 ; CHECK-NEXT: uaddw v0.4s, v0.4s, v1.4h -; CHECK-NEXT: uaddw2 v1.4s, v2.4s, v1.8h -; CHECK-NEXT: uaddw2 v3.4s, v3.4s, v5.8h +; CHECK-NEXT: uaddw2 v1.4s, v3.4s, v1.8h +; CHECK-NEXT: uaddw2 v3.4s, v2.4s, v5.8h ; CHECK-NEXT: uaddw v2.4s, v6.4s, v5.4h ; CHECK-NEXT: ret %lp1 = load <4 x i8>, ptr %p @@ -762,35 +763,35 @@ define <16 x i32> @extrause_shuffle(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK-NEXT: add x8, x1, #8 ; CHECK-NEXT: ldr s6, [x1, #12] ; CHECK-NEXT: ldp s17, s18, [x2, #8] -; CHECK-NEXT: ldp s2, s3, [x2] +; CHECK-NEXT: ldp s3, s5, [x2] ; CHECK-NEXT: add x9, x3, #8 ; CHECK-NEXT: mov v4.16b, v1.16b ; CHECK-NEXT: ldp s7, s16, [x0] -; CHECK-NEXT: ldr s5, [x3, #12] +; CHECK-NEXT: ldr s2, [x3, #12] ; CHECK-NEXT: mov v1.s[1], v6.s[0] -; CHECK-NEXT: ld1 { v2.s }[1], [x3], #4 +; CHECK-NEXT: ld1 { v3.s }[1], [x3], #4 ; CHECK-NEXT: mov v4.s[1], v6.s[0] ; CHECK-NEXT: ld1 { v7.s }[1], [x1], #4 ; CHECK-NEXT: ld1 { v16.s }[1], [x1] -; CHECK-NEXT: ld1 { v3.s }[1], [x3] -; CHECK-NEXT: ld1 { v0.s }[1], [x8] +; CHECK-NEXT: ld1 { v5.s }[1], [x3] ; CHECK-NEXT: ld1 { v17.s }[1], [x9] +; CHECK-NEXT: ld1 { v0.s }[1], [x8] ; CHECK-NEXT: mov v4.s[2], v18.s[0] -; CHECK-NEXT: mov v18.s[1], v5.s[0] +; CHECK-NEXT: mov v18.s[1], v2.s[0] ; CHECK-NEXT: uaddl v1.8h, v16.8b, v1.8b ; CHECK-NEXT: uaddl v6.8h, v7.8b, v0.8b -; CHECK-NEXT: uaddl v2.8h, v2.8b, v17.8b -; CHECK-NEXT: uaddl v3.8h, v3.8b, v18.8b +; CHECK-NEXT: uaddl v7.8h, v3.8b, v17.8b ; CHECK-NEXT: ushll v0.4s, v1.4h, #3 ; CHECK-NEXT: ushll2 v1.4s, v1.8h, #3 -; CHECK-NEXT: mov v4.s[3], v5.s[0] +; CHECK-NEXT: uaddl v5.8h, v5.8b, v18.8b +; CHECK-NEXT: mov v4.s[3], v2.s[0] ; CHECK-NEXT: uaddw v0.4s, v0.4s, v6.4h ; CHECK-NEXT: uaddw2 v1.4s, v1.4s, v6.8h -; CHECK-NEXT: ushll v7.4s, v3.4h, #3 -; CHECK-NEXT: ushll2 v3.4s, v3.8h, #3 +; CHECK-NEXT: ushll v16.4s, v5.4h, #3 +; CHECK-NEXT: ushll2 v3.4s, v5.8h, #3 ; CHECK-NEXT: str q4, [x4] -; CHECK-NEXT: uaddw2 v3.4s, v3.4s, v2.8h -; CHECK-NEXT: uaddw v2.4s, v7.4s, v2.4h +; CHECK-NEXT: uaddw2 v3.4s, v3.4s, v7.8h +; CHECK-NEXT: uaddw v2.4s, v16.4s, v7.4h ; CHECK-NEXT: ret %lp1 = load <4 x i8>, ptr %p %p2 = getelementptr i8, ptr %p, i32 4 @@ -873,8 +874,8 @@ define <16 x i32> @extrause_ext(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK-NEXT: ld1 { v4.s }[1], [x11] ; CHECK-NEXT: ld1 { v2.s }[1], [x3] ; CHECK-NEXT: ld1 { v0.s }[1], [x10] -; CHECK-NEXT: ld1 { v7.s }[1], [x9] ; CHECK-NEXT: ld1 { v6.s }[1], [x8] +; CHECK-NEXT: ld1 { v7.s }[1], [x9] ; CHECK-NEXT: uaddl v5.8h, v5.8b, v4.8b ; CHECK-NEXT: uaddl v2.8h, v2.8b, v0.8b ; CHECK-NEXT: ushll v16.8h, v0.8b, #0 @@ -972,8 +973,8 @@ define <16 x i32> @extrause_add(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK-NEXT: ld1 { v5.s }[1], [x11] ; CHECK-NEXT: ld1 { v3.s }[1], [x3] ; CHECK-NEXT: ld1 { v7.s }[1], [x10] -; CHECK-NEXT: ld1 { v4.s }[1], [x9] ; CHECK-NEXT: ld1 { v6.s }[1], [x8] +; CHECK-NEXT: ld1 { v4.s }[1], [x9] ; CHECK-NEXT: uaddl v5.8h, v1.8b, v5.8b ; CHECK-NEXT: uaddl v7.8h, v3.8b, v7.8b ; CHECK-NEXT: uaddl v1.8h, v0.8b, v4.8b @@ -1072,23 +1073,23 @@ define <16 x i32> @extrause_ext2(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK-NEXT: ld1 { v6.s }[1], [x9] ; CHECK-NEXT: ld1 { v4.s }[1], [x8] ; CHECK-NEXT: uaddl v7.8h, v3.8b, v7.8b -; CHECK-NEXT: uaddl v3.8h, v1.8b, v5.8b +; CHECK-NEXT: uaddl v1.8h, v1.8b, v5.8b ; CHECK-NEXT: uaddl v2.8h, v2.8b, v6.8b ; CHECK-NEXT: uaddl v4.8h, v0.8b, v4.8b ; CHECK-NEXT: ushll v0.4s, v7.4h, #3 -; CHECK-NEXT: ushll2 v1.4s, v7.8h, #3 -; CHECK-NEXT: ushll v5.4s, v3.4h, #3 -; CHECK-NEXT: ushll2 v6.4s, v3.8h, #3 -; CHECK-NEXT: ushll2 v16.4s, v3.8h, #0 -; CHECK-NEXT: ushll v17.4s, v3.4h, #0 -; CHECK-NEXT: uaddw2 v1.4s, v1.4s, v2.8h +; CHECK-NEXT: ushll2 v3.4s, v7.8h, #3 +; CHECK-NEXT: ushll v6.4s, v1.4h, #3 +; CHECK-NEXT: ushll2 v16.4s, v1.8h, #3 +; CHECK-NEXT: ushll2 v5.4s, v1.8h, #0 +; CHECK-NEXT: ushll v17.4s, v1.4h, #0 +; CHECK-NEXT: ushll2 v18.4s, v7.8h, #0 +; CHECK-NEXT: uaddw2 v1.4s, v3.4s, v2.8h ; CHECK-NEXT: uaddw v0.4s, v0.4s, v2.4h -; CHECK-NEXT: uaddw v2.4s, v5.4s, v4.4h -; CHECK-NEXT: uaddw2 v3.4s, v6.4s, v4.8h -; CHECK-NEXT: ushll2 v4.4s, v7.8h, #0 -; CHECK-NEXT: ushll v5.4s, v7.4h, #0 -; CHECK-NEXT: stp q17, q16, [x4, #32] -; CHECK-NEXT: stp q5, q4, [x4] +; CHECK-NEXT: uaddw v2.4s, v6.4s, v4.4h +; CHECK-NEXT: uaddw2 v3.4s, v16.4s, v4.8h +; CHECK-NEXT: ushll v4.4s, v7.4h, #0 +; CHECK-NEXT: stp q17, q5, [x4, #32] +; CHECK-NEXT: stp q4, q18, [x4] ; CHECK-NEXT: ret %lp1 = load <4 x i8>, ptr %p %p2 = getelementptr i8, ptr %p, i32 4 @@ -1157,32 +1158,32 @@ define <16 x i32> @extrause_ext2(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { define <16 x i32> @extrause_shl(ptr %p, ptr %q, ptr %r, ptr %s, ptr %z) { ; CHECK-LABEL: extrause_shl: ; CHECK: // %bb.0: -; CHECK-NEXT: ldp s0, s1, [x0] +; CHECK-NEXT: ldp s1, s2, [x0] ; CHECK-NEXT: add x10, x3, #12 -; CHECK-NEXT: ldp s2, s3, [x2] +; CHECK-NEXT: ldp s0, s3, [x2] ; CHECK-NEXT: add x11, x1, #12 ; CHECK-NEXT: ldp s4, s5, [x0, #8] ; CHECK-NEXT: add x8, x3, #8 ; CHECK-NEXT: ldp s6, s7, [x2, #8] ; CHECK-NEXT: add x9, x1, #8 -; CHECK-NEXT: ld1 { v2.s }[1], [x3], #4 -; CHECK-NEXT: ld1 { v0.s }[1], [x1], #4 -; CHECK-NEXT: ld1 { v1.s }[1], [x1] +; CHECK-NEXT: ld1 { v0.s }[1], [x3], #4 +; CHECK-NEXT: ld1 { v1.s }[1], [x1], #4 +; CHECK-NEXT: ld1 { v2.s }[1], [x1] ; CHECK-NEXT: ld1 { v5.s }[1], [x11] ; CHECK-NEXT: ld1 { v3.s }[1], [x3] ; CHECK-NEXT: ld1 { v7.s }[1], [x10] ; CHECK-NEXT: ld1 { v4.s }[1], [x9] ; CHECK-NEXT: ld1 { v6.s }[1], [x8] -; CHECK-NEXT: uaddl v1.8h, v1.8b, v5.8b +; CHECK-NEXT: uaddl v2.8h, v2.8b, v5.8b ; CHECK-NEXT: uaddl v3.8h, v3.8b, v7.8b -; CHECK-NEXT: uaddl v4.8h, v0.8b, v4.8b -; CHECK-NEXT: uaddl v2.8h, v2.8b, v6.8b -; CHECK-NEXT: ushll v5.4s, v1.4h, #3 +; CHECK-NEXT: uaddl v4.8h, v1.8b, v4.8b +; CHECK-NEXT: ushll v5.4s, v2.4h, #3 +; CHECK-NEXT: ushll2 v7.4s, v2.8h, #3 +; CHECK-NEXT: uaddl v2.8h, v0.8b, v6.8b ; CHECK-NEXT: ushll v6.4s, v3.4h, #3 -; CHECK-NEXT: ushll2 v7.4s, v1.8h, #3 ; CHECK-NEXT: ushll2 v16.4s, v3.8h, #3 -; CHECK-NEXT: uaddw v0.4s, v5.4s, v4.4h ; CHECK-NEXT: uaddw2 v1.4s, v7.4s, v4.8h +; CHECK-NEXT: uaddw v0.4s, v5.4s, v4.4h ; CHECK-NEXT: stp q5, q7, [x4] ; CHECK-NEXT: uaddw2 v3.4s, v16.4s, v2.8h ; CHECK-NEXT: uaddw v2.4s, v6.4s, v2.4h diff --git a/llvm/test/CodeGen/AArch64/fcmp.ll b/llvm/test/CodeGen/AArch64/fcmp.ll index 9916aeeab1ca..b1ca88975a62 100644 --- a/llvm/test/CodeGen/AArch64/fcmp.ll +++ b/llvm/test/CodeGen/AArch64/fcmp.ll @@ -280,10 +280,10 @@ define <3 x i32> @v3f64_i32(<3 x double> %a, <3 x double> %b, <3 x i32> %d, <3 x ; CHECK-GI-NEXT: uzp1 v0.4s, v0.4s, v2.4s ; CHECK-GI-NEXT: fmov s2, w8 ; CHECK-GI-NEXT: mov v2.s[1], w8 -; CHECK-GI-NEXT: neg v3.4s, v1.4s ; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: neg v1.4s, v1.4s ; CHECK-GI-NEXT: mov v2.s[2], w8 -; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v3.4s +; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v1.4s ; CHECK-GI-NEXT: eor v1.16b, v0.16b, v2.16b ; CHECK-GI-NEXT: and v0.16b, v6.16b, v0.16b ; CHECK-GI-NEXT: and v1.16b, v7.16b, v1.16b @@ -348,10 +348,10 @@ define <3 x float> @v3f32_float(<3 x float> %a, <3 x float> %b, <3 x float> %d, ; CHECK-GI-NEXT: mov w8, #-1 // =0xffffffff ; CHECK-GI-NEXT: fmov s1, w8 ; CHECK-GI-NEXT: mov v1.s[1], w8 -; CHECK-GI-NEXT: neg v5.4s, v4.4s ; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v4.4s +; CHECK-GI-NEXT: neg v4.4s, v4.4s +; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v4.4s ; CHECK-GI-NEXT: mov v1.s[2], w8 -; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v5.4s ; CHECK-GI-NEXT: eor v1.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: and v0.16b, v2.16b, v0.16b ; CHECK-GI-NEXT: and v1.16b, v3.16b, v1.16b @@ -426,10 +426,10 @@ define <3 x i32> @v3f32_i32(<3 x float> %a, <3 x float> %b, <3 x i32> %d, <3 x i ; CHECK-GI-NEXT: mov w8, #-1 // =0xffffffff ; CHECK-GI-NEXT: fmov s1, w8 ; CHECK-GI-NEXT: mov v1.s[1], w8 -; CHECK-GI-NEXT: neg v5.4s, v4.4s ; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v4.4s +; CHECK-GI-NEXT: neg v4.4s, v4.4s +; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v4.4s ; CHECK-GI-NEXT: mov v1.s[2], w8 -; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v5.4s ; CHECK-GI-NEXT: eor v1.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: and v0.16b, v2.16b, v0.16b ; CHECK-GI-NEXT: and v1.16b, v3.16b, v1.16b @@ -545,8 +545,8 @@ define <7 x half> @v7f16_half(<7 x half> %a, <7 x half> %b, <7 x half> %d, <7 x ; CHECK-GI-NOFP16-LABEL: v7f16_half: ; CHECK-GI-NOFP16: // %bb.0: // %entry ; CHECK-GI-NOFP16-NEXT: mov w8, #15 // =0xf -; CHECK-GI-NOFP16-NEXT: mov h6, v0.h[4] -; CHECK-GI-NOFP16-NEXT: mov h7, v0.h[5] +; CHECK-GI-NOFP16-NEXT: mov h5, v0.h[4] +; CHECK-GI-NOFP16-NEXT: mov h6, v0.h[5] ; CHECK-GI-NOFP16-NEXT: fmov s4, w8 ; CHECK-GI-NOFP16-NEXT: mov h16, v1.h[4] ; CHECK-GI-NOFP16-NEXT: mov w8, #65535 // =0xffff @@ -555,32 +555,32 @@ define <7 x half> @v7f16_half(<7 x half> %a, <7 x half> %b, <7 x half> %d, <7 x ; CHECK-GI-NOFP16-NEXT: mov h19, v1.h[6] ; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v0.4h ; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h -; CHECK-GI-NOFP16-NEXT: mov v5.16b, v4.16b -; CHECK-GI-NOFP16-NEXT: mov v6.h[1], v7.h[0] -; CHECK-GI-NOFP16-NEXT: fmov s7, w8 +; CHECK-GI-NOFP16-NEXT: mov v7.16b, v4.16b +; CHECK-GI-NOFP16-NEXT: mov v5.h[1], v6.h[0] +; CHECK-GI-NOFP16-NEXT: fmov s6, w8 ; CHECK-GI-NOFP16-NEXT: mov v16.h[1], v17.h[0] -; CHECK-GI-NOFP16-NEXT: mov v5.h[1], v4.h[0] -; CHECK-GI-NOFP16-NEXT: mov v17.16b, v7.16b +; CHECK-GI-NOFP16-NEXT: mov v7.h[1], v4.h[0] +; CHECK-GI-NOFP16-NEXT: mov v17.16b, v6.16b ; CHECK-GI-NOFP16-NEXT: fcmgt v0.4s, v1.4s, v0.4s -; CHECK-GI-NOFP16-NEXT: mov v6.h[2], v18.h[0] -; CHECK-GI-NOFP16-NEXT: mov v17.h[1], v7.h[0] +; CHECK-GI-NOFP16-NEXT: mov v5.h[2], v18.h[0] +; CHECK-GI-NOFP16-NEXT: mov v17.h[1], v6.h[0] ; CHECK-GI-NOFP16-NEXT: mov v16.h[2], v19.h[0] -; CHECK-GI-NOFP16-NEXT: mov v5.h[2], v4.h[0] -; CHECK-GI-NOFP16-NEXT: fcvtl v6.4s, v6.4h -; CHECK-GI-NOFP16-NEXT: mov v17.h[2], v7.h[0] +; CHECK-GI-NOFP16-NEXT: mov v7.h[2], v4.h[0] +; CHECK-GI-NOFP16-NEXT: fcvtl v5.4s, v5.4h +; CHECK-GI-NOFP16-NEXT: mov v17.h[2], v6.h[0] ; CHECK-GI-NOFP16-NEXT: fcvtl v16.4s, v16.4h -; CHECK-GI-NOFP16-NEXT: mov v5.h[3], v4.h[0] -; CHECK-GI-NOFP16-NEXT: mov v17.h[3], v7.h[0] -; CHECK-GI-NOFP16-NEXT: fcmgt v1.4s, v16.4s, v6.4s -; CHECK-GI-NOFP16-NEXT: mov v5.h[4], v4.h[0] -; CHECK-GI-NOFP16-NEXT: mov v17.h[4], v7.h[0] +; CHECK-GI-NOFP16-NEXT: mov v7.h[3], v4.h[0] +; CHECK-GI-NOFP16-NEXT: mov v17.h[3], v6.h[0] +; CHECK-GI-NOFP16-NEXT: fcmgt v1.4s, v16.4s, v5.4s +; CHECK-GI-NOFP16-NEXT: mov v7.h[4], v4.h[0] +; CHECK-GI-NOFP16-NEXT: mov v17.h[4], v6.h[0] ; CHECK-GI-NOFP16-NEXT: uzp1 v0.8h, v0.8h, v1.8h -; CHECK-GI-NOFP16-NEXT: mov v5.h[5], v4.h[0] -; CHECK-GI-NOFP16-NEXT: mov v17.h[5], v7.h[0] -; CHECK-GI-NOFP16-NEXT: mov v5.h[6], v4.h[0] -; CHECK-GI-NOFP16-NEXT: mov v17.h[6], v7.h[0] -; CHECK-GI-NOFP16-NEXT: neg v1.8h, v5.8h -; CHECK-GI-NOFP16-NEXT: ushl v0.8h, v0.8h, v5.8h +; CHECK-GI-NOFP16-NEXT: mov v7.h[5], v4.h[0] +; CHECK-GI-NOFP16-NEXT: mov v17.h[5], v6.h[0] +; CHECK-GI-NOFP16-NEXT: mov v7.h[6], v4.h[0] +; CHECK-GI-NOFP16-NEXT: mov v17.h[6], v6.h[0] +; CHECK-GI-NOFP16-NEXT: ushl v0.8h, v0.8h, v7.8h +; CHECK-GI-NOFP16-NEXT: neg v1.8h, v7.8h ; CHECK-GI-NOFP16-NEXT: sshl v0.8h, v0.8h, v1.8h ; CHECK-GI-NOFP16-NEXT: eor v1.16b, v0.16b, v17.16b ; CHECK-GI-NOFP16-NEXT: and v0.16b, v2.16b, v0.16b @@ -609,8 +609,8 @@ define <7 x half> @v7f16_half(<7 x half> %a, <7 x half> %b, <7 x half> %d, <7 x ; CHECK-GI-FP16-NEXT: mov v7.h[5], v6.h[0] ; CHECK-GI-FP16-NEXT: mov v5.h[6], v4.h[0] ; CHECK-GI-FP16-NEXT: mov v7.h[6], v6.h[0] -; CHECK-GI-FP16-NEXT: neg v1.8h, v5.8h ; CHECK-GI-FP16-NEXT: ushl v0.8h, v0.8h, v5.8h +; CHECK-GI-FP16-NEXT: neg v1.8h, v5.8h ; CHECK-GI-FP16-NEXT: sshl v0.8h, v0.8h, v1.8h ; CHECK-GI-FP16-NEXT: eor v1.16b, v0.16b, v7.16b ; CHECK-GI-FP16-NEXT: and v0.16b, v2.16b, v0.16b @@ -1047,6 +1047,7 @@ define <7 x i32> @v7f16_i32(<7 x half> %a, <7 x half> %b, <7 x i32> %d, <7 x i32 ; CHECK-GI-NOFP16-NEXT: mov h6, v0.h[6] ; CHECK-GI-NOFP16-NEXT: mov h7, v1.h[6] ; CHECK-GI-NOFP16-NEXT: fmov s16, w0 +; CHECK-GI-NOFP16-NEXT: fmov s18, w4 ; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v0.4h ; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h ; CHECK-GI-NOFP16-NEXT: mov v2.h[1], v3.h[0] @@ -1054,6 +1055,7 @@ define <7 x i32> @v7f16_i32(<7 x half> %a, <7 x half> %b, <7 x i32> %d, <7 x i32 ; CHECK-GI-NOFP16-NEXT: mov v4.h[1], v5.h[0] ; CHECK-GI-NOFP16-NEXT: ldr s5, [sp] ; CHECK-GI-NOFP16-NEXT: mov v16.s[1], w1 +; CHECK-GI-NOFP16-NEXT: mov v18.s[1], w5 ; CHECK-GI-NOFP16-NEXT: mov v3.s[1], w8 ; CHECK-GI-NOFP16-NEXT: fmov w9, s5 ; CHECK-GI-NOFP16-NEXT: fmov s5, w7 @@ -1069,27 +1071,25 @@ define <7 x i32> @v7f16_i32(<7 x half> %a, <7 x half> %b, <7 x i32> %d, <7 x i32 ; CHECK-GI-NOFP16-NEXT: mov v3.s[2], w8 ; CHECK-GI-NOFP16-NEXT: mov w8, #-1 // =0xffffffff ; CHECK-GI-NOFP16-NEXT: mov v7.s[1], v17.s[0] -; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v2.4h ; CHECK-GI-NOFP16-NEXT: ldr s17, [sp, #40] +; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v2.4h +; CHECK-GI-NOFP16-NEXT: mov v18.s[2], w6 ; CHECK-GI-NOFP16-NEXT: fcvtl v4.4s, v4.4h ; CHECK-GI-NOFP16-NEXT: mov v16.s[3], w3 ; CHECK-GI-NOFP16-NEXT: mov v5.s[2], w9 -; CHECK-GI-NOFP16-NEXT: neg v18.4s, v3.4s ; CHECK-GI-NOFP16-NEXT: mov v7.s[2], v17.s[0] ; CHECK-GI-NOFP16-NEXT: fcmgt v2.4s, v4.4s, v2.4s ; CHECK-GI-NOFP16-NEXT: fmov s4, w8 ; CHECK-GI-NOFP16-NEXT: mov v4.s[1], w8 ; CHECK-GI-NOFP16-NEXT: ushl v2.4s, v2.4s, v3.4s -; CHECK-GI-NOFP16-NEXT: fmov s3, w4 -; CHECK-GI-NOFP16-NEXT: mov v3.s[1], w5 +; CHECK-GI-NOFP16-NEXT: neg v3.4s, v3.4s ; CHECK-GI-NOFP16-NEXT: mov v4.s[2], w8 -; CHECK-GI-NOFP16-NEXT: sshl v2.4s, v2.4s, v18.4s +; CHECK-GI-NOFP16-NEXT: sshl v2.4s, v2.4s, v3.4s ; CHECK-GI-NOFP16-NEXT: fmov w8, s6 -; CHECK-GI-NOFP16-NEXT: mov v3.s[2], w6 -; CHECK-GI-NOFP16-NEXT: eor v1.16b, v2.16b, v4.16b ; CHECK-GI-NOFP16-NEXT: mov v5.s[3], w8 +; CHECK-GI-NOFP16-NEXT: eor v1.16b, v2.16b, v4.16b +; CHECK-GI-NOFP16-NEXT: and v2.16b, v18.16b, v2.16b ; CHECK-GI-NOFP16-NEXT: and v1.16b, v7.16b, v1.16b -; CHECK-GI-NOFP16-NEXT: and v2.16b, v3.16b, v2.16b ; CHECK-GI-NOFP16-NEXT: bsl v0.16b, v16.16b, v5.16b ; CHECK-GI-NOFP16-NEXT: orr v1.16b, v2.16b, v1.16b ; CHECK-GI-NOFP16-NEXT: mov s2, v0.s[1] @@ -1111,30 +1111,32 @@ define <7 x i32> @v7f16_i32(<7 x half> %a, <7 x half> %b, <7 x i32> %d, <7 x i32 ; CHECK-GI-FP16-NEXT: fcmgt v0.8h, v1.8h, v0.8h ; CHECK-GI-FP16-NEXT: mov w10, #31 // =0x1f ; CHECK-GI-FP16-NEXT: ldr s3, [sp] -; CHECK-GI-FP16-NEXT: fmov s1, w10 +; CHECK-GI-FP16-NEXT: fmov s2, w10 ; CHECK-GI-FP16-NEXT: fmov s6, w0 ; CHECK-GI-FP16-NEXT: ldr s4, [sp, #8] +; CHECK-GI-FP16-NEXT: fmov s17, w4 ; CHECK-GI-FP16-NEXT: ldr s7, [sp, #24] ; CHECK-GI-FP16-NEXT: ldr s16, [sp, #32] ; CHECK-GI-FP16-NEXT: umov w8, v0.h[4] ; CHECK-GI-FP16-NEXT: umov w9, v0.h[5] -; CHECK-GI-FP16-NEXT: mov v1.s[1], w10 +; CHECK-GI-FP16-NEXT: mov v2.s[1], w10 ; CHECK-GI-FP16-NEXT: mov v6.s[1], w1 +; CHECK-GI-FP16-NEXT: mov v17.s[1], w5 ; CHECK-GI-FP16-NEXT: mov v7.s[1], v16.s[0] ; CHECK-GI-FP16-NEXT: ldr s16, [sp, #40] -; CHECK-GI-FP16-NEXT: fmov s2, w8 +; CHECK-GI-FP16-NEXT: fmov s1, w8 ; CHECK-GI-FP16-NEXT: umov w8, v0.h[6] -; CHECK-GI-FP16-NEXT: mov v1.s[2], w10 +; CHECK-GI-FP16-NEXT: mov v2.s[2], w10 ; CHECK-GI-FP16-NEXT: ushll v0.4s, v0.4h, #0 ; CHECK-GI-FP16-NEXT: mov v6.s[2], w2 +; CHECK-GI-FP16-NEXT: mov v17.s[2], w6 ; CHECK-GI-FP16-NEXT: mov v7.s[2], v16.s[0] -; CHECK-GI-FP16-NEXT: mov v2.s[1], w9 +; CHECK-GI-FP16-NEXT: mov v1.s[1], w9 ; CHECK-GI-FP16-NEXT: mov w9, #-1 // =0xffffffff ; CHECK-GI-FP16-NEXT: fmov s5, w9 -; CHECK-GI-FP16-NEXT: neg v17.4s, v1.4s ; CHECK-GI-FP16-NEXT: shl v0.4s, v0.4s, #31 ; CHECK-GI-FP16-NEXT: mov v6.s[3], w3 -; CHECK-GI-FP16-NEXT: mov v2.s[2], w8 +; CHECK-GI-FP16-NEXT: mov v1.s[2], w8 ; CHECK-GI-FP16-NEXT: fmov w8, s3 ; CHECK-GI-FP16-NEXT: fmov s3, w7 ; CHECK-GI-FP16-NEXT: mov v5.s[1], w9 @@ -1142,26 +1144,24 @@ define <7 x i32> @v7f16_i32(<7 x half> %a, <7 x half> %b, <7 x i32> %d, <7 x i32 ; CHECK-GI-FP16-NEXT: mov v3.s[1], w8 ; CHECK-GI-FP16-NEXT: fmov w8, s4 ; CHECK-GI-FP16-NEXT: ldr s4, [sp, #16] -; CHECK-GI-FP16-NEXT: ushl v1.4s, v2.4s, v1.4s -; CHECK-GI-FP16-NEXT: fmov s2, w4 +; CHECK-GI-FP16-NEXT: ushl v1.4s, v1.4s, v2.4s +; CHECK-GI-FP16-NEXT: neg v2.4s, v2.4s ; CHECK-GI-FP16-NEXT: mov v5.s[2], w9 -; CHECK-GI-FP16-NEXT: mov v2.s[1], w5 ; CHECK-GI-FP16-NEXT: mov v3.s[2], w8 -; CHECK-GI-FP16-NEXT: sshl v1.4s, v1.4s, v17.4s +; CHECK-GI-FP16-NEXT: sshl v1.4s, v1.4s, v2.4s ; CHECK-GI-FP16-NEXT: fmov w8, s4 -; CHECK-GI-FP16-NEXT: eor v4.16b, v1.16b, v5.16b -; CHECK-GI-FP16-NEXT: mov v2.s[2], w6 +; CHECK-GI-FP16-NEXT: eor v2.16b, v1.16b, v5.16b +; CHECK-GI-FP16-NEXT: and v1.16b, v17.16b, v1.16b ; CHECK-GI-FP16-NEXT: mov v3.s[3], w8 -; CHECK-GI-FP16-NEXT: and v1.16b, v2.16b, v1.16b -; CHECK-GI-FP16-NEXT: and v2.16b, v7.16b, v4.16b +; CHECK-GI-FP16-NEXT: and v2.16b, v7.16b, v2.16b ; CHECK-GI-FP16-NEXT: bsl v0.16b, v6.16b, v3.16b ; CHECK-GI-FP16-NEXT: orr v1.16b, v1.16b, v2.16b ; CHECK-GI-FP16-NEXT: mov s2, v0.s[1] ; CHECK-GI-FP16-NEXT: mov s3, v0.s[2] ; CHECK-GI-FP16-NEXT: mov s4, v0.s[3] -; CHECK-GI-FP16-NEXT: fmov w0, s0 ; CHECK-GI-FP16-NEXT: mov s5, v1.s[1] ; CHECK-GI-FP16-NEXT: mov s6, v1.s[2] +; CHECK-GI-FP16-NEXT: fmov w0, s0 ; CHECK-GI-FP16-NEXT: fmov w4, s1 ; CHECK-GI-FP16-NEXT: fmov w1, s2 ; CHECK-GI-FP16-NEXT: fmov w2, s3 diff --git a/llvm/test/CodeGen/AArch64/fdiv-combine.ll b/llvm/test/CodeGen/AArch64/fdiv-combine.ll index 1ed63f3ef250..0627250d0779 100644 --- a/llvm/test/CodeGen/AArch64/fdiv-combine.ll +++ b/llvm/test/CodeGen/AArch64/fdiv-combine.ll @@ -171,8 +171,8 @@ entry: define @splat_fdiv_nxv2f64(double %D, %a) #1 { ; CHECK-LABEL: splat_fdiv_nxv2f64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, d0 ; CHECK-NEXT: fdivr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/fold-int-pow2-with-fmul-or-fdiv.ll b/llvm/test/CodeGen/AArch64/fold-int-pow2-with-fmul-or-fdiv.ll index 03e64f8b785b..a78addc49008 100644 --- a/llvm/test/CodeGen/AArch64/fold-int-pow2-with-fmul-or-fdiv.ll +++ b/llvm/test/CodeGen/AArch64/fold-int-pow2-with-fmul-or-fdiv.ll @@ -604,8 +604,8 @@ define fastcc i1 @quantum_hadamard(i32 %0) { define @fdiv_pow2_nx4xfloat( %i) "target-features"="+sve" { ; CHECK-LABEL: fdiv_pow2_nx4xfloat: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: lslr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: fmov z1.s, #9.00000000 ; CHECK-NEXT: ucvtf z0.s, p0/m, z0.s diff --git a/llvm/test/CodeGen/AArch64/fp-veclib-expansion.ll b/llvm/test/CodeGen/AArch64/fp-veclib-expansion.ll index 67c056c780cc..2c8e2190f820 100644 --- a/llvm/test/CodeGen/AArch64/fp-veclib-expansion.ll +++ b/llvm/test/CodeGen/AArch64/fp-veclib-expansion.ll @@ -62,9 +62,9 @@ define @frem_nxv4f32( %unused, @frem_nxv4f32( %unused, @frem_strict_nxv2f64( %unused, ; ARMPL-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill ; ARMPL-NEXT: .cfi_def_cfa_offset 16 ; ARMPL-NEXT: .cfi_offset w30, -16 -; ARMPL-NEXT: ptrue p0.d ; ARMPL-NEXT: mov z0.d, z1.d ; ARMPL-NEXT: mov z1.d, z2.d +; ARMPL-NEXT: ptrue p0.d ; ARMPL-NEXT: bl armpl_svfmod_f64_x ; ARMPL-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload ; ARMPL-NEXT: ret @@ -102,9 +102,9 @@ define @frem_strict_nxv2f64( %unused, ; SLEEF-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill ; SLEEF-NEXT: .cfi_def_cfa_offset 16 ; SLEEF-NEXT: .cfi_offset w30, -16 -; SLEEF-NEXT: ptrue p0.d ; SLEEF-NEXT: mov z0.d, z1.d ; SLEEF-NEXT: mov z1.d, z2.d +; SLEEF-NEXT: ptrue p0.d ; SLEEF-NEXT: bl _ZGVsMxvv_fmod ; SLEEF-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload ; SLEEF-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll index 301d28fd7be5..2ea581359af6 100644 --- a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll +++ b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll @@ -194,10 +194,10 @@ define <2 x i16> @ustest_f64i16(<2 x double> %x) { ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: fcvtzs v0.2d, v0.2d ; CHECK-NEXT: movi d1, #0x00ffff0000ffff +; CHECK-NEXT: movi v2.2d, #0000000000000000 ; CHECK-NEXT: xtn v0.2s, v0.2d ; CHECK-NEXT: smin v0.2s, v0.2s, v1.2s -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: smax v0.2s, v0.2s, v1.2s +; CHECK-NEXT: smax v0.2s, v0.2s, v2.2s ; CHECK-NEXT: ret entry: %conv = fptosi <2 x double> %x to <2 x i32> @@ -833,10 +833,10 @@ define <2 x i16> @ustest_f64i16_mm(<2 x double> %x) { ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: fcvtzs v0.2d, v0.2d ; CHECK-NEXT: movi d1, #0x00ffff0000ffff +; CHECK-NEXT: movi v2.2d, #0000000000000000 ; CHECK-NEXT: xtn v0.2s, v0.2d ; CHECK-NEXT: smin v0.2s, v0.2s, v1.2s -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: smax v0.2s, v0.2s, v1.2s +; CHECK-NEXT: smax v0.2s, v0.2s, v2.2s ; CHECK-NEXT: ret entry: %conv = fptosi <2 x double> %x to <2 x i32> diff --git a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll index 92fd3183393e..c45885a38f15 100644 --- a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll +++ b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll @@ -697,9 +697,9 @@ define <2 x i1> @test_signed_v2f32_v2i1(<2 x float> %f) { ; CHECK: // %bb.0: ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: fcvtzs v0.2s, v0.2s +; CHECK-NEXT: movi v2.2d, #0xffffffffffffffff ; CHECK-NEXT: smin v0.2s, v0.2s, v1.2s -; CHECK-NEXT: movi v1.2d, #0xffffffffffffffff -; CHECK-NEXT: smax v0.2s, v0.2s, v1.2s +; CHECK-NEXT: smax v0.2s, v0.2s, v2.2s ; CHECK-NEXT: ret %x = call <2 x i1> @llvm.fptosi.sat.v2f32.v2i1(<2 x float> %f) ret <2 x i1> %x @@ -1620,9 +1620,9 @@ define <4 x i1> @test_signed_v4f16_v4i1(<4 x half> %f) { ; CHECK-FP16: // %bb.0: ; CHECK-FP16-NEXT: movi v1.2d, #0000000000000000 ; CHECK-FP16-NEXT: fcvtzs v0.4h, v0.4h +; CHECK-FP16-NEXT: movi v2.2d, #0xffffffffffffffff ; CHECK-FP16-NEXT: smin v0.4h, v0.4h, v1.4h -; CHECK-FP16-NEXT: movi v1.2d, #0xffffffffffffffff -; CHECK-FP16-NEXT: smax v0.4h, v0.4h, v1.4h +; CHECK-FP16-NEXT: smax v0.4h, v0.4h, v2.4h ; CHECK-FP16-NEXT: ret %x = call <4 x i1> @llvm.fptosi.sat.v4f16.v4i1(<4 x half> %f) ret <4 x i1> %x @@ -1668,9 +1668,9 @@ define <4 x i13> @test_signed_v4f16_v4i13(<4 x half> %f) { ; CHECK-FP16: // %bb.0: ; CHECK-FP16-NEXT: fcvtzs v0.4h, v0.4h ; CHECK-FP16-NEXT: mvni v1.4h, #240, lsl #8 +; CHECK-FP16-NEXT: movi v2.4h, #240, lsl #8 ; CHECK-FP16-NEXT: smin v0.4h, v0.4h, v1.4h -; CHECK-FP16-NEXT: movi v1.4h, #240, lsl #8 -; CHECK-FP16-NEXT: smax v0.4h, v0.4h, v1.4h +; CHECK-FP16-NEXT: smax v0.4h, v0.4h, v2.4h ; CHECK-FP16-NEXT: ret %x = call <4 x i13> @llvm.fptosi.sat.v4f16.v4i13(<4 x half> %f) ret <4 x i13> %x @@ -2103,9 +2103,9 @@ define <8 x i1> @test_signed_v8f16_v8i1(<8 x half> %f) { ; CHECK-FP16: // %bb.0: ; CHECK-FP16-NEXT: movi v1.2d, #0000000000000000 ; CHECK-FP16-NEXT: fcvtzs v0.8h, v0.8h +; CHECK-FP16-NEXT: movi v2.2d, #0xffffffffffffffff ; CHECK-FP16-NEXT: smin v0.8h, v0.8h, v1.8h -; CHECK-FP16-NEXT: movi v1.2d, #0xffffffffffffffff -; CHECK-FP16-NEXT: smax v0.8h, v0.8h, v1.8h +; CHECK-FP16-NEXT: smax v0.8h, v0.8h, v2.8h ; CHECK-FP16-NEXT: xtn v0.8b, v0.8h ; CHECK-FP16-NEXT: ret %x = call <8 x i1> @llvm.fptosi.sat.v8f16.v8i1(<8 x half> %f) @@ -2254,9 +2254,9 @@ define <8 x i13> @test_signed_v8f16_v8i13(<8 x half> %f) { ; CHECK-FP16: // %bb.0: ; CHECK-FP16-NEXT: fcvtzs v0.8h, v0.8h ; CHECK-FP16-NEXT: mvni v1.8h, #240, lsl #8 +; CHECK-FP16-NEXT: movi v2.8h, #240, lsl #8 ; CHECK-FP16-NEXT: smin v0.8h, v0.8h, v1.8h -; CHECK-FP16-NEXT: movi v1.8h, #240, lsl #8 -; CHECK-FP16-NEXT: smax v0.8h, v0.8h, v1.8h +; CHECK-FP16-NEXT: smax v0.8h, v0.8h, v2.8h ; CHECK-FP16-NEXT: ret %x = call <8 x i13> @llvm.fptosi.sat.v8f16.v8i13(<8 x half> %f) ret <8 x i13> %x diff --git a/llvm/test/CodeGen/AArch64/funnel-shift-rot.ll b/llvm/test/CodeGen/AArch64/funnel-shift-rot.ll index 181f2185893e..d39c09524e1a 100644 --- a/llvm/test/CodeGen/AArch64/funnel-shift-rot.ll +++ b/llvm/test/CodeGen/AArch64/funnel-shift-rot.ll @@ -78,9 +78,9 @@ define <4 x i32> @rotl_v4i32(<4 x i32> %x, <4 x i32> %z) { ; CHECK: // %bb.0: ; CHECK-NEXT: movi v2.4s, #31 ; CHECK-NEXT: neg v3.4s, v1.4s +; CHECK-NEXT: and v3.16b, v3.16b, v2.16b ; CHECK-NEXT: and v1.16b, v1.16b, v2.16b -; CHECK-NEXT: and v2.16b, v3.16b, v2.16b -; CHECK-NEXT: neg v2.4s, v2.4s +; CHECK-NEXT: neg v2.4s, v3.4s ; CHECK-NEXT: ushl v1.4s, v0.4s, v1.4s ; CHECK-NEXT: ushl v0.4s, v0.4s, v2.4s ; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b diff --git a/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll b/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll index 97511639ec8c..cb9f04a7fac4 100644 --- a/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll +++ b/llvm/test/CodeGen/AArch64/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll @@ -202,8 +202,8 @@ define <4 x i1> @vec_4xi32_splat_eq(<4 x i32> %x, <4 x i32> %y) nounwind { define <4 x i1> @vec_4xi32_nonsplat_eq(<4 x i32> %x, <4 x i32> %y) nounwind { ; CHECK-LABEL: vec_4xi32_nonsplat_eq: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v1.4s, v1.4s ; CHECK-NEXT: adrp x8, .LCPI13_0 +; CHECK-NEXT: neg v1.4s, v1.4s ; CHECK-NEXT: ldr q2, [x8, :lo12:.LCPI13_0] ; CHECK-NEXT: ushl v1.4s, v2.4s, v1.4s ; CHECK-NEXT: and v0.16b, v1.16b, v0.16b diff --git a/llvm/test/CodeGen/AArch64/icmp.ll b/llvm/test/CodeGen/AArch64/icmp.ll index e7352fe03d01..8e10847e7aae 100644 --- a/llvm/test/CodeGen/AArch64/icmp.ll +++ b/llvm/test/CodeGen/AArch64/icmp.ll @@ -179,10 +179,10 @@ define <3 x i32> @v3i32_i32(<3 x i32> %a, <3 x i32> %b, <3 x i32> %d, <3 x i32> ; CHECK-GI-NEXT: mov w8, #-1 // =0xffffffff ; CHECK-GI-NEXT: fmov s1, w8 ; CHECK-GI-NEXT: mov v1.s[1], w8 -; CHECK-GI-NEXT: neg v5.4s, v4.4s ; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v4.4s +; CHECK-GI-NEXT: neg v4.4s, v4.4s +; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v4.4s ; CHECK-GI-NEXT: mov v1.s[2], w8 -; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v5.4s ; CHECK-GI-NEXT: eor v1.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: and v0.16b, v2.16b, v0.16b ; CHECK-GI-NEXT: and v1.16b, v3.16b, v1.16b diff --git a/llvm/test/CodeGen/AArch64/insert-extend.ll b/llvm/test/CodeGen/AArch64/insert-extend.ll index e4d2b516b8fb..0b730f6e7715 100644 --- a/llvm/test/CodeGen/AArch64/insert-extend.ll +++ b/llvm/test/CodeGen/AArch64/insert-extend.ll @@ -64,104 +64,104 @@ define i32 @large(ptr nocapture noundef readonly %p1, i32 noundef %st1, ptr noca ; CHECK-NEXT: ldr d3, [x11] ; CHECK-NEXT: ldr d4, [x10, x8] ; CHECK-NEXT: ldr d5, [x11, x9] +; CHECK-NEXT: shll2 v6.4s, v0.8h, #16 ; CHECK-NEXT: usubl v2.8h, v2.8b, v3.8b ; CHECK-NEXT: usubl v3.8h, v4.8b, v5.8b -; CHECK-NEXT: shll2 v4.4s, v0.8h, #16 -; CHECK-NEXT: shll2 v5.4s, v1.8h, #16 -; CHECK-NEXT: saddw v0.4s, v4.4s, v0.4h +; CHECK-NEXT: shll2 v4.4s, v1.8h, #16 +; CHECK-NEXT: saddw v0.4s, v6.4s, v0.4h ; CHECK-NEXT: shll2 v6.4s, v2.8h, #16 -; CHECK-NEXT: saddw v1.4s, v5.4s, v1.4h -; CHECK-NEXT: shll2 v4.4s, v3.8h, #16 -; CHECK-NEXT: saddw v2.4s, v6.4s, v2.4h -; CHECK-NEXT: saddw v3.4s, v4.4s, v3.4h +; CHECK-NEXT: shll2 v5.4s, v3.8h, #16 +; CHECK-NEXT: saddw v1.4s, v4.4s, v1.4h ; CHECK-NEXT: rev64 v4.4s, v0.4s +; CHECK-NEXT: saddw v2.4s, v6.4s, v2.4h +; CHECK-NEXT: saddw v3.4s, v5.4s, v3.4h ; CHECK-NEXT: rev64 v5.4s, v1.4s ; CHECK-NEXT: rev64 v6.4s, v2.4s -; CHECK-NEXT: rev64 v7.4s, v3.4s ; CHECK-NEXT: sub v4.4s, v0.4s, v4.4s ; CHECK-NEXT: addp v0.4s, v1.4s, v0.4s +; CHECK-NEXT: rev64 v7.4s, v3.4s ; CHECK-NEXT: sub v5.4s, v1.4s, v5.4s ; CHECK-NEXT: sub v6.4s, v2.4s, v6.4s ; CHECK-NEXT: addp v2.4s, v3.4s, v2.4s -; CHECK-NEXT: sub v7.4s, v3.4s, v7.4s ; CHECK-NEXT: zip1 v16.4s, v5.4s, v4.4s -; CHECK-NEXT: ext v1.16b, v2.16b, v2.16b, #8 +; CHECK-NEXT: sub v7.4s, v3.4s, v7.4s ; CHECK-NEXT: zip2 v3.4s, v6.4s, v7.4s ; CHECK-NEXT: mov v6.s[1], v7.s[0] ; CHECK-NEXT: ext v7.16b, v5.16b, v16.16b, #8 ; CHECK-NEXT: mov v5.s[3], v4.s[2] -; CHECK-NEXT: uzp1 v4.4s, v1.4s, v0.4s -; CHECK-NEXT: uzp2 v1.4s, v1.4s, v0.4s +; CHECK-NEXT: ext v4.16b, v2.16b, v2.16b, #8 ; CHECK-NEXT: mov v6.d[1], v7.d[1] ; CHECK-NEXT: mov v3.d[1], v5.d[1] +; CHECK-NEXT: uzp1 v1.4s, v4.4s, v0.4s +; CHECK-NEXT: uzp2 v4.4s, v4.4s, v0.4s ; CHECK-NEXT: uzp2 v5.4s, v2.4s, v0.4s ; CHECK-NEXT: uzp1 v0.4s, v2.4s, v0.4s -; CHECK-NEXT: sub v1.4s, v4.4s, v1.4s ; CHECK-NEXT: add v2.4s, v3.4s, v6.4s ; CHECK-NEXT: sub v3.4s, v6.4s, v3.4s +; CHECK-NEXT: sub v1.4s, v1.4s, v4.4s ; CHECK-NEXT: add v0.4s, v5.4s, v0.4s -; CHECK-NEXT: rev64 v6.4s, v1.4s ; CHECK-NEXT: rev64 v4.4s, v2.4s ; CHECK-NEXT: rev64 v5.4s, v3.4s -; CHECK-NEXT: addp v16.4s, v1.4s, v3.4s +; CHECK-NEXT: rev64 v6.4s, v1.4s ; CHECK-NEXT: rev64 v7.4s, v0.4s +; CHECK-NEXT: addp v16.4s, v1.4s, v3.4s ; CHECK-NEXT: addp v17.4s, v0.4s, v2.4s -; CHECK-NEXT: sub v1.4s, v1.4s, v6.4s ; CHECK-NEXT: sub v3.4s, v3.4s, v5.4s ; CHECK-NEXT: sub v2.4s, v2.4s, v4.4s +; CHECK-NEXT: sub v1.4s, v1.4s, v6.4s ; CHECK-NEXT: sub v0.4s, v0.4s, v7.4s ; CHECK-NEXT: zip1 v18.4s, v17.4s, v17.4s -; CHECK-NEXT: ext v6.16b, v1.16b, v16.16b, #8 ; CHECK-NEXT: ext v4.16b, v17.16b, v2.16b, #4 ; CHECK-NEXT: ext v5.16b, v16.16b, v3.16b, #4 ; CHECK-NEXT: mov v20.16b, v3.16b +; CHECK-NEXT: ext v6.16b, v1.16b, v16.16b, #8 ; CHECK-NEXT: ext v7.16b, v0.16b, v17.16b, #4 ; CHECK-NEXT: mov v21.16b, v2.16b ; CHECK-NEXT: trn2 v0.4s, v18.4s, v0.4s -; CHECK-NEXT: ext v19.16b, v6.16b, v1.16b, #4 -; CHECK-NEXT: mov v1.s[2], v16.s[1] ; CHECK-NEXT: mov v20.s[2], v16.s[3] ; CHECK-NEXT: zip2 v4.4s, v4.4s, v17.4s ; CHECK-NEXT: zip2 v5.4s, v5.4s, v16.4s ; CHECK-NEXT: mov v21.s[2], v17.s[3] +; CHECK-NEXT: ext v19.16b, v6.16b, v1.16b, #4 ; CHECK-NEXT: ext v7.16b, v7.16b, v7.16b, #4 -; CHECK-NEXT: mov v18.16b, v1.16b +; CHECK-NEXT: mov v1.s[2], v16.s[1] ; CHECK-NEXT: ext v2.16b, v2.16b, v4.16b, #12 ; CHECK-NEXT: ext v3.16b, v3.16b, v5.16b, #12 ; CHECK-NEXT: uzp2 v4.4s, v6.4s, v19.4s ; CHECK-NEXT: mov v5.16b, v7.16b ; CHECK-NEXT: mov v6.16b, v20.16b +; CHECK-NEXT: mov v18.16b, v1.16b ; CHECK-NEXT: mov v19.16b, v21.16b -; CHECK-NEXT: mov v18.s[1], v16.s[0] ; CHECK-NEXT: sub v7.4s, v0.4s, v7.4s ; CHECK-NEXT: mov v6.s[1], v16.s[2] ; CHECK-NEXT: mov v5.s[0], v17.s[1] +; CHECK-NEXT: mov v18.s[1], v16.s[0] ; CHECK-NEXT: mov v19.s[1], v17.s[2] ; CHECK-NEXT: sub v1.4s, v1.4s, v4.4s ; CHECK-NEXT: sub v16.4s, v20.4s, v3.4s ; CHECK-NEXT: sub v17.4s, v21.4s, v2.4s -; CHECK-NEXT: add v4.4s, v18.4s, v4.4s ; CHECK-NEXT: add v3.4s, v6.4s, v3.4s ; CHECK-NEXT: add v0.4s, v0.4s, v5.4s +; CHECK-NEXT: add v4.4s, v18.4s, v4.4s ; CHECK-NEXT: add v2.4s, v19.4s, v2.4s -; CHECK-NEXT: mov v4.d[1], v1.d[1] ; CHECK-NEXT: mov v3.d[1], v16.d[1] ; CHECK-NEXT: mov v0.d[1], v7.d[1] +; CHECK-NEXT: mov v4.d[1], v1.d[1] ; CHECK-NEXT: mov v2.d[1], v17.d[1] -; CHECK-NEXT: cmlt v6.8h, v4.8h, #0 ; CHECK-NEXT: cmlt v1.8h, v3.8h, #0 ; CHECK-NEXT: cmlt v5.8h, v0.8h, #0 +; CHECK-NEXT: cmlt v6.8h, v4.8h, #0 ; CHECK-NEXT: cmlt v7.8h, v2.8h, #0 -; CHECK-NEXT: add v4.4s, v6.4s, v4.4s ; CHECK-NEXT: add v3.4s, v1.4s, v3.4s ; CHECK-NEXT: add v0.4s, v5.4s, v0.4s +; CHECK-NEXT: add v4.4s, v6.4s, v4.4s ; CHECK-NEXT: add v2.4s, v7.4s, v2.4s ; CHECK-NEXT: eor v1.16b, v3.16b, v1.16b +; CHECK-NEXT: eor v0.16b, v0.16b, v5.16b ; CHECK-NEXT: eor v2.16b, v2.16b, v7.16b ; CHECK-NEXT: eor v3.16b, v4.16b, v6.16b -; CHECK-NEXT: eor v0.16b, v0.16b, v5.16b -; CHECK-NEXT: add v1.4s, v2.4s, v1.4s ; CHECK-NEXT: add v0.4s, v0.4s, v3.4s +; CHECK-NEXT: add v1.4s, v2.4s, v1.4s ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 diff --git a/llvm/test/CodeGen/AArch64/insert-subvector-res-legalization.ll b/llvm/test/CodeGen/AArch64/insert-subvector-res-legalization.ll index 29f9c0336bbc..542b2e90ffc1 100644 --- a/llvm/test/CodeGen/AArch64/insert-subvector-res-legalization.ll +++ b/llvm/test/CodeGen/AArch64/insert-subvector-res-legalization.ll @@ -9,9 +9,9 @@ define @vec_scalable_subvec_scalable_idx_zero_i8(ptr %a, ptr % ; CHECK-LABEL: vec_scalable_subvec_scalable_idx_zero_i8: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: ld1b { z0.h }, p0/z, [x0] -; CHECK-NEXT: ld1b { z1.s }, p1/z, [x1] +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ld1b { z1.s }, p0/z, [x1] ; CHECK-NEXT: uunpkhi z0.s, z0.h ; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h ; CHECK-NEXT: ret @@ -25,9 +25,9 @@ define @vec_scalable_subvec_scalable_idx_nonzero_i8(ptr %a, pt ; CHECK-LABEL: vec_scalable_subvec_scalable_idx_nonzero_i8: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: ld1b { z0.h }, p0/z, [x0] -; CHECK-NEXT: ld1b { z1.s }, p1/z, [x1] +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ld1b { z1.s }, p0/z, [x1] ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h ; CHECK-NEXT: ret @@ -41,9 +41,9 @@ define @vec_scalable_subvec_scalable_idx_zero_i16(ptr %a, ptr ; CHECK-LABEL: vec_scalable_subvec_scalable_idx_zero_i16: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0] -; CHECK-NEXT: ld1h { z1.d }, p1/z, [x1] +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: ld1h { z1.d }, p0/z, [x1] ; CHECK-NEXT: uunpkhi z0.d, z0.s ; CHECK-NEXT: uzp1 z0.s, z1.s, z0.s ; CHECK-NEXT: ret @@ -57,9 +57,9 @@ define @vec_scalable_subvec_scalable_idx_nonzero_i16(ptr %a, ; CHECK-LABEL: vec_scalable_subvec_scalable_idx_nonzero_i16: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0] -; CHECK-NEXT: ld1h { z1.d }, p1/z, [x1] +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: ld1h { z1.d }, p0/z, [x1] ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z1.s ; CHECK-NEXT: ret @@ -76,10 +76,10 @@ define @vec_scalable_subvec_fixed_idx_zero_i8(ptr %a, ptr %b) ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldr d0, [x1] -; CHECK-NEXT: ptrue p1.h, vl8 -; CHECK-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-NEXT: ld1b { z1.h }, p0/z, [x0] -; CHECK-NEXT: sel z0.h, p1, z0.h, z1.h +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h ; CHECK-NEXT: ret %vec = load , ptr %a %subvec = load <8 x i8>, ptr %b @@ -92,19 +92,19 @@ define @vec_scalable_subvec_fixed_idx_nonzero_i8(ptr %a, ptr % ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cnth x8 -; CHECK-NEXT: ldr d0, [x1] +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: ldr d1, [x1] ; CHECK-NEXT: sub x8, x8, #8 ; CHECK-NEXT: mov w9, #8 // =0x8 ; CHECK-NEXT: cmp x8, #8 -; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ld1b { z0.h }, p0/z, [x0] +; CHECK-NEXT: ushll v1.8h, v1.8b, #0 ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #1 -; CHECK-NEXT: ld1b { z1.h }, p0/z, [x0] -; CHECK-NEXT: st1h { z1.h }, p0, [sp] -; CHECK-NEXT: str q0, [x9, x8] +; CHECK-NEXT: st1h { z0.h }, p0, [sp] +; CHECK-NEXT: str q1, [x9, x8] ; CHECK-NEXT: ld1h { z0.h }, p0/z, [sp] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -120,10 +120,10 @@ define @vec_scalable_subvec_fixed_idx_zero_i16(ptr %a, ptr %b ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr d0, [x1] -; CHECK-NEXT: ptrue p1.s, vl4 -; CHECK-NEXT: ushll v0.4s, v0.4h, #0 ; CHECK-NEXT: ld1h { z1.s }, p0/z, [x0] -; CHECK-NEXT: sel z0.s, p1, z0.s, z1.s +; CHECK-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-NEXT: ptrue p0.s, vl4 +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s ; CHECK-NEXT: ret %vec = load , ptr %a %subvec = load <4 x i16>, ptr %b @@ -136,19 +136,19 @@ define @vec_scalable_subvec_fixed_idx_nonzero_i16(ptr %a, ptr ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cntw x8 -; CHECK-NEXT: ldr d0, [x1] +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ldr d1, [x1] ; CHECK-NEXT: sub x8, x8, #4 ; CHECK-NEXT: mov w9, #4 // =0x4 ; CHECK-NEXT: cmp x8, #4 -; CHECK-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0] +; CHECK-NEXT: ushll v1.4s, v1.4h, #0 ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #2 -; CHECK-NEXT: ld1h { z1.s }, p0/z, [x0] -; CHECK-NEXT: st1w { z1.s }, p0, [sp] -; CHECK-NEXT: str q0, [x9, x8] +; CHECK-NEXT: st1w { z0.s }, p0, [sp] +; CHECK-NEXT: str q1, [x9, x8] ; CHECK-NEXT: ld1w { z0.s }, p0/z, [sp] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -164,10 +164,10 @@ define @vec_scalable_subvec_fixed_idx_zero_i32(ptr %a, ptr %b ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr d0, [x1] -; CHECK-NEXT: ptrue p1.d, vl2 -; CHECK-NEXT: ushll v0.2d, v0.2s, #0 ; CHECK-NEXT: ld1w { z1.d }, p0/z, [x0] -; CHECK-NEXT: sel z0.d, p1, z0.d, z1.d +; CHECK-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-NEXT: ptrue p0.d, vl2 +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d ; CHECK-NEXT: ret %vec = load , ptr %a %subvec = load <2 x i32>, ptr %b @@ -180,19 +180,19 @@ define @vec_scalable_subvec_fixed_idx_nonzero_i32(ptr %a, ptr ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cntd x8 -; CHECK-NEXT: ldr d0, [x1] +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: ldr d1, [x1] ; CHECK-NEXT: sub x8, x8, #2 ; CHECK-NEXT: mov w9, #2 // =0x2 ; CHECK-NEXT: cmp x8, #2 -; CHECK-NEXT: ushll v0.2d, v0.2s, #0 +; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0] +; CHECK-NEXT: ushll v1.2d, v1.2s, #0 ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #3 -; CHECK-NEXT: ld1w { z1.d }, p0/z, [x0] -; CHECK-NEXT: st1d { z1.d }, p0, [sp] -; CHECK-NEXT: str q0, [x9, x8] +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: str q1, [x9, x8] ; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload diff --git a/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll b/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll index 1a4ab6ab334a..9bd2ed240810 100644 --- a/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll +++ b/llvm/test/CodeGen/AArch64/intrinsic-cttz-elts-sve.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme < %s | FileCheck %s @@ -31,28 +32,28 @@ define i32 @ctz_nxv32i1( %a) #0 { ; CHECK-NEXT: neg x8, x8 ; CHECK-NEXT: punpklo p3.h, p1.b ; CHECK-NEXT: rdvl x9, #2 -; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: rdvl x8, #-1 -; CHECK-NEXT: punpkhi p1.h, p1.b +; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: mov z2.h, w8 -; CHECK-NEXT: inch z0.h, all, mul #4 +; CHECK-NEXT: punpkhi p1.h, p1.b ; CHECK-NEXT: mov z3.h, p2/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: ptrue p2.h +; CHECK-NEXT: inch z0.h, all, mul #4 ; CHECK-NEXT: mov z5.h, p3/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: add z1.h, z0.h, z1.h -; CHECK-NEXT: add z4.h, z0.h, z2.h ; CHECK-NEXT: mov z6.h, p0/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z7.h, p1/z, #-1 // =0xffffffffffffffff +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: add z1.h, z0.h, z1.h +; CHECK-NEXT: add z4.h, z0.h, z2.h ; CHECK-NEXT: and z0.d, z0.d, z3.d ; CHECK-NEXT: add z2.h, z1.h, z2.h ; CHECK-NEXT: and z3.d, z4.d, z5.d ; CHECK-NEXT: and z1.d, z1.d, z6.d ; CHECK-NEXT: and z2.d, z2.d, z7.d -; CHECK-NEXT: umax z0.h, p2/m, z0.h, z3.h -; CHECK-NEXT: umax z1.h, p2/m, z1.h, z2.h -; CHECK-NEXT: umax z0.h, p2/m, z0.h, z1.h -; CHECK-NEXT: umaxv h0, p2, z0.h +; CHECK-NEXT: umax z0.h, p0/m, z0.h, z3.h +; CHECK-NEXT: umax z1.h, p0/m, z1.h, z2.h +; CHECK-NEXT: umax z0.h, p0/m, z0.h, z1.h +; CHECK-NEXT: umaxv h0, p0, z0.h ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: sub w8, w9, w8 ; CHECK-NEXT: and w0, w8, #0xffff @@ -65,12 +66,12 @@ define i32 @ctz_nxv4i32( %a) #0 { ; CHECK-LABEL: ctz_nxv4i32: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: index z1.s, #0, #-1 ; CHECK-NEXT: cntw x9 -; CHECK-NEXT: incw z1.s ; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0 -; CHECK-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: and z0.d, z1.d, z0.d +; CHECK-NEXT: index z0.s, #0, #-1 +; CHECK-NEXT: mov z1.s, p1/z, #-1 // =0xffffffffffffffff +; CHECK-NEXT: incw z0.s +; CHECK-NEXT: and z0.d, z0.d, z1.d ; CHECK-NEXT: and z0.s, z0.s, #0xff ; CHECK-NEXT: umaxv s0, p0, z0.s ; CHECK-NEXT: fmov w8, s0 @@ -87,38 +88,38 @@ define i64 @vscale_4096( %a) #1 { ; CHECK-LABEL: vscale_4096: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: index z1.s, #0, #-1 ; CHECK-NEXT: cntw x8 -; CHECK-NEXT: cnth x9 ; CHECK-NEXT: neg x8, x8 -; CHECK-NEXT: mov z1.s, w8 -; CHECK-NEXT: neg x8, x9 ; CHECK-NEXT: rdvl x9, #1 -; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 -; CHECK-NEXT: index z0.s, #0, #-1 +; CHECK-NEXT: mov z0.s, w8 +; CHECK-NEXT: cnth x8 +; CHECK-NEXT: neg x8, x8 +; CHECK-NEXT: incw z1.s, all, mul #4 +; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: punpkhi p0.h, p0.b -; CHECK-NEXT: incw z0.s, all, mul #4 -; CHECK-NEXT: add z1.s, z0.s, z1.s -; CHECK-NEXT: add z5.s, z0.s, z2.s +; CHECK-NEXT: add z0.s, z1.s, z0.s +; CHECK-NEXT: add z4.s, z1.s, z2.s ; CHECK-NEXT: punpkhi p2.h, p1.b ; CHECK-NEXT: punpkhi p3.h, p0.b ; CHECK-NEXT: punpklo p0.h, p0.b -; CHECK-NEXT: add z2.s, z1.s, z2.s -; CHECK-NEXT: punpklo p1.h, p1.b ; CHECK-NEXT: mov z3.s, p2/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: ptrue p2.s -; CHECK-NEXT: mov z4.s, p3/z, #-1 // =0xffffffffffffffff +; CHECK-NEXT: add z2.s, z0.s, z2.s +; CHECK-NEXT: punpklo p1.h, p1.b +; CHECK-NEXT: mov z5.s, p3/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z6.s, p0/z, #-1 // =0xffffffffffffffff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z7.s, p1/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: and z1.d, z1.d, z3.d -; CHECK-NEXT: and z2.d, z2.d, z4.d -; CHECK-NEXT: and z3.d, z5.d, z6.d -; CHECK-NEXT: and z0.d, z0.d, z7.d -; CHECK-NEXT: umax z1.s, p2/m, z1.s, z2.s -; CHECK-NEXT: umax z0.s, p2/m, z0.s, z3.s -; CHECK-NEXT: umax z0.s, p2/m, z0.s, z1.s -; CHECK-NEXT: umaxv s0, p2, z0.s +; CHECK-NEXT: and z0.d, z0.d, z3.d +; CHECK-NEXT: and z2.d, z2.d, z5.d +; CHECK-NEXT: and z3.d, z4.d, z6.d +; CHECK-NEXT: and z1.d, z1.d, z7.d +; CHECK-NEXT: umax z0.s, p0/m, z0.s, z2.s +; CHECK-NEXT: umax z1.s, p0/m, z1.s, z3.s +; CHECK-NEXT: umax z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: umaxv s0, p0, z0.s ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: sub w0, w9, w8 ; CHECK-NEXT: ret @@ -130,21 +131,21 @@ define i64 @vscale_4096_poison( %a) #1 { ; CHECK-LABEL: vscale_4096_poison: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: index z1.h, #0, #-1 ; CHECK-NEXT: cnth x8 -; CHECK-NEXT: rdvl x9, #1 ; CHECK-NEXT: neg x8, x8 -; CHECK-NEXT: mov z1.h, w8 +; CHECK-NEXT: rdvl x9, #1 ; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 -; CHECK-NEXT: index z0.h, #0, #-1 +; CHECK-NEXT: mov z0.h, w8 +; CHECK-NEXT: inch z1.h, all, mul #2 ; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: punpklo p0.h, p0.b -; CHECK-NEXT: inch z0.h, all, mul #2 -; CHECK-NEXT: add z1.h, z0.h, z1.h +; CHECK-NEXT: add z0.h, z1.h, z0.h ; CHECK-NEXT: mov z2.h, p1/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z3.h, p0/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: and z1.d, z1.d, z2.d -; CHECK-NEXT: and z0.d, z0.d, z3.d +; CHECK-NEXT: and z0.d, z0.d, z2.d +; CHECK-NEXT: and z1.d, z1.d, z3.d ; CHECK-NEXT: umax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: umaxv h0, p0, z0.h ; CHECK-NEXT: fmov w8, s0 @@ -161,16 +162,16 @@ define i32 @ctz_nxv8i1_no_range( %a) { ; CHECK-LABEL: ctz_nxv8i1_no_range: ; CHECK: // %bb.0: ; CHECK-NEXT: index z0.s, #0, #-1 -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: cntw x8 -; CHECK-NEXT: punpkhi p0.h, p0.b +; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: neg x8, x8 +; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: cnth x9 ; CHECK-NEXT: mov z1.s, w8 -; CHECK-NEXT: incw z0.s, all, mul #2 ; CHECK-NEXT: mov z2.s, p1/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z3.s, p0/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: incw z0.s, all, mul #2 ; CHECK-NEXT: add z1.s, z0.s, z1.s ; CHECK-NEXT: and z0.d, z0.d, z2.d ; CHECK-NEXT: and z1.d, z1.d, z3.d @@ -212,8 +213,8 @@ define i32 @ctz_nxv16i1_poison( %pg, %a) { define i32 @ctz_and_nxv16i1( %pg, %a, %b) { ; CHECK-LABEL: ctz_and_nxv16i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, z1.b +; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: brkb p0.b, p1/z, p0.b ; CHECK-NEXT: cntp x0, p0, p0.b ; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 diff --git a/llvm/test/CodeGen/AArch64/itofp.ll b/llvm/test/CodeGen/AArch64/itofp.ll index f5a7b5dc9f49..ae4ced258bb8 100644 --- a/llvm/test/CodeGen/AArch64/itofp.ll +++ b/llvm/test/CodeGen/AArch64/itofp.ll @@ -6555,18 +6555,18 @@ define <3 x bfloat> @stofp_v3i64_v3bf16(<3 x i64> %a) { ; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2 ; CHECK-NEXT: mov v0.d[1], v1.d[0] ; CHECK-NEXT: scvtf v1.2d, v2.2d -; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: scvtf v0.2d, v0.2d ; CHECK-NEXT: fcvtn v0.2s, v0.2d ; CHECK-NEXT: fcvtn2 v0.4s, v1.2d -; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ushr v3.4s, v0.4s, #16 -; CHECK-NEXT: add v1.4s, v0.4s, v1.4s -; CHECK-NEXT: and v2.16b, v3.16b, v2.16b -; CHECK-NEXT: add v1.4s, v2.4s, v1.4s -; CHECK-NEXT: fcmeq v2.4s, v0.4s, v0.4s +; CHECK-NEXT: add v2.4s, v0.4s, v2.4s +; CHECK-NEXT: and v1.16b, v3.16b, v1.16b +; CHECK-NEXT: fcmeq v3.4s, v0.4s, v0.4s ; CHECK-NEXT: orr v0.4s, #64, lsl #16 -; CHECK-NEXT: bit v0.16b, v1.16b, v2.16b +; CHECK-NEXT: add v1.4s, v1.4s, v2.4s +; CHECK-NEXT: bit v0.16b, v1.16b, v3.16b ; CHECK-NEXT: shrn v0.4h, v0.4s, #16 ; CHECK-NEXT: ret entry: @@ -6582,18 +6582,18 @@ define <3 x bfloat> @utofp_v3i64_v3bf16(<3 x i64> %a) { ; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2 ; CHECK-NEXT: mov v0.d[1], v1.d[0] ; CHECK-NEXT: ucvtf v1.2d, v2.2d -; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: ucvtf v0.2d, v0.2d ; CHECK-NEXT: fcvtn v0.2s, v0.2d ; CHECK-NEXT: fcvtn2 v0.4s, v1.2d -; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ushr v3.4s, v0.4s, #16 -; CHECK-NEXT: add v1.4s, v0.4s, v1.4s -; CHECK-NEXT: and v2.16b, v3.16b, v2.16b -; CHECK-NEXT: add v1.4s, v2.4s, v1.4s -; CHECK-NEXT: fcmeq v2.4s, v0.4s, v0.4s +; CHECK-NEXT: add v2.4s, v0.4s, v2.4s +; CHECK-NEXT: and v1.16b, v3.16b, v1.16b +; CHECK-NEXT: fcmeq v3.4s, v0.4s, v0.4s ; CHECK-NEXT: orr v0.4s, #64, lsl #16 -; CHECK-NEXT: bit v0.16b, v1.16b, v2.16b +; CHECK-NEXT: add v1.4s, v1.4s, v2.4s +; CHECK-NEXT: bit v0.16b, v1.16b, v3.16b ; CHECK-NEXT: shrn v0.4h, v0.4s, #16 ; CHECK-NEXT: ret entry: @@ -6606,17 +6606,17 @@ define <4 x bfloat> @stofp_v4i64_v4bf16(<4 x i64> %a) { ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: scvtf v0.2d, v0.2d ; CHECK-NEXT: scvtf v1.2d, v1.2d -; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: fcvtn v0.2s, v0.2d ; CHECK-NEXT: fcvtn2 v0.4s, v1.2d -; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ushr v3.4s, v0.4s, #16 -; CHECK-NEXT: add v1.4s, v0.4s, v1.4s -; CHECK-NEXT: and v2.16b, v3.16b, v2.16b -; CHECK-NEXT: add v1.4s, v2.4s, v1.4s -; CHECK-NEXT: fcmeq v2.4s, v0.4s, v0.4s +; CHECK-NEXT: add v2.4s, v0.4s, v2.4s +; CHECK-NEXT: and v1.16b, v3.16b, v1.16b +; CHECK-NEXT: fcmeq v3.4s, v0.4s, v0.4s ; CHECK-NEXT: orr v0.4s, #64, lsl #16 -; CHECK-NEXT: bit v0.16b, v1.16b, v2.16b +; CHECK-NEXT: add v1.4s, v1.4s, v2.4s +; CHECK-NEXT: bit v0.16b, v1.16b, v3.16b ; CHECK-NEXT: shrn v0.4h, v0.4s, #16 ; CHECK-NEXT: ret entry: @@ -6629,17 +6629,17 @@ define <4 x bfloat> @utofp_v4i64_v4bf16(<4 x i64> %a) { ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ucvtf v0.2d, v0.2d ; CHECK-NEXT: ucvtf v1.2d, v1.2d -; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: fcvtn v0.2s, v0.2d ; CHECK-NEXT: fcvtn2 v0.4s, v1.2d -; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ushr v3.4s, v0.4s, #16 -; CHECK-NEXT: add v1.4s, v0.4s, v1.4s -; CHECK-NEXT: and v2.16b, v3.16b, v2.16b -; CHECK-NEXT: add v1.4s, v2.4s, v1.4s -; CHECK-NEXT: fcmeq v2.4s, v0.4s, v0.4s +; CHECK-NEXT: add v2.4s, v0.4s, v2.4s +; CHECK-NEXT: and v1.16b, v3.16b, v1.16b +; CHECK-NEXT: fcmeq v3.4s, v0.4s, v0.4s ; CHECK-NEXT: orr v0.4s, #64, lsl #16 -; CHECK-NEXT: bit v0.16b, v1.16b, v2.16b +; CHECK-NEXT: add v1.4s, v1.4s, v2.4s +; CHECK-NEXT: bit v0.16b, v1.16b, v3.16b ; CHECK-NEXT: shrn v0.4h, v0.4s, #16 ; CHECK-NEXT: ret entry: @@ -6658,22 +6658,22 @@ define <8 x bfloat> @stofp_v8i64_v8bf16(<8 x i64> %a) { ; CHECK-NEXT: fcvtn v0.2s, v0.2d ; CHECK-NEXT: fcvtn2 v2.4s, v3.2d ; CHECK-NEXT: fcvtn2 v0.4s, v1.2d -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: movi v3.4s, #1 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: movi v3.4s, #127, msl #8 ; CHECK-NEXT: ushr v4.4s, v2.4s, #16 ; CHECK-NEXT: ushr v5.4s, v0.4s, #16 -; CHECK-NEXT: add v6.4s, v2.4s, v1.4s -; CHECK-NEXT: add v1.4s, v0.4s, v1.4s -; CHECK-NEXT: and v4.16b, v4.16b, v3.16b -; CHECK-NEXT: and v3.16b, v5.16b, v3.16b +; CHECK-NEXT: add v6.4s, v2.4s, v3.4s +; CHECK-NEXT: add v3.4s, v0.4s, v3.4s +; CHECK-NEXT: and v4.16b, v4.16b, v1.16b +; CHECK-NEXT: and v1.16b, v5.16b, v1.16b ; CHECK-NEXT: fcmeq v5.4s, v2.4s, v2.4s ; CHECK-NEXT: orr v2.4s, #64, lsl #16 ; CHECK-NEXT: add v4.4s, v4.4s, v6.4s -; CHECK-NEXT: add v1.4s, v3.4s, v1.4s -; CHECK-NEXT: fcmeq v3.4s, v0.4s, v0.4s +; CHECK-NEXT: fcmeq v6.4s, v0.4s, v0.4s +; CHECK-NEXT: add v1.4s, v1.4s, v3.4s ; CHECK-NEXT: orr v0.4s, #64, lsl #16 ; CHECK-NEXT: bit v2.16b, v4.16b, v5.16b -; CHECK-NEXT: bit v0.16b, v1.16b, v3.16b +; CHECK-NEXT: bit v0.16b, v1.16b, v6.16b ; CHECK-NEXT: uzp2 v0.8h, v0.8h, v2.8h ; CHECK-NEXT: ret entry: @@ -6692,22 +6692,22 @@ define <8 x bfloat> @utofp_v8i64_v8bf16(<8 x i64> %a) { ; CHECK-NEXT: fcvtn v0.2s, v0.2d ; CHECK-NEXT: fcvtn2 v2.4s, v3.2d ; CHECK-NEXT: fcvtn2 v0.4s, v1.2d -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: movi v3.4s, #1 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: movi v3.4s, #127, msl #8 ; CHECK-NEXT: ushr v4.4s, v2.4s, #16 ; CHECK-NEXT: ushr v5.4s, v0.4s, #16 -; CHECK-NEXT: add v6.4s, v2.4s, v1.4s -; CHECK-NEXT: add v1.4s, v0.4s, v1.4s -; CHECK-NEXT: and v4.16b, v4.16b, v3.16b -; CHECK-NEXT: and v3.16b, v5.16b, v3.16b +; CHECK-NEXT: add v6.4s, v2.4s, v3.4s +; CHECK-NEXT: add v3.4s, v0.4s, v3.4s +; CHECK-NEXT: and v4.16b, v4.16b, v1.16b +; CHECK-NEXT: and v1.16b, v5.16b, v1.16b ; CHECK-NEXT: fcmeq v5.4s, v2.4s, v2.4s ; CHECK-NEXT: orr v2.4s, #64, lsl #16 ; CHECK-NEXT: add v4.4s, v4.4s, v6.4s -; CHECK-NEXT: add v1.4s, v3.4s, v1.4s -; CHECK-NEXT: fcmeq v3.4s, v0.4s, v0.4s +; CHECK-NEXT: fcmeq v6.4s, v0.4s, v0.4s +; CHECK-NEXT: add v1.4s, v1.4s, v3.4s ; CHECK-NEXT: orr v0.4s, #64, lsl #16 ; CHECK-NEXT: bit v2.16b, v4.16b, v5.16b -; CHECK-NEXT: bit v0.16b, v1.16b, v3.16b +; CHECK-NEXT: bit v0.16b, v1.16b, v6.16b ; CHECK-NEXT: uzp2 v0.8h, v0.8h, v2.8h ; CHECK-NEXT: ret entry: @@ -6718,51 +6718,51 @@ entry: define <16 x bfloat> @stofp_v16i64_v16bf16(<16 x i64> %a) { ; CHECK-LABEL: stofp_v16i64_v16bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: scvtf v2.2d, v2.2d ; CHECK-NEXT: scvtf v0.2d, v0.2d +; CHECK-NEXT: scvtf v2.2d, v2.2d ; CHECK-NEXT: scvtf v6.2d, v6.2d ; CHECK-NEXT: scvtf v4.2d, v4.2d -; CHECK-NEXT: scvtf v3.2d, v3.2d ; CHECK-NEXT: scvtf v1.2d, v1.2d +; CHECK-NEXT: scvtf v3.2d, v3.2d ; CHECK-NEXT: scvtf v7.2d, v7.2d ; CHECK-NEXT: scvtf v5.2d, v5.2d -; CHECK-NEXT: fcvtn v2.2s, v2.2d ; CHECK-NEXT: fcvtn v0.2s, v0.2d +; CHECK-NEXT: fcvtn v2.2s, v2.2d ; CHECK-NEXT: fcvtn v6.2s, v6.2d ; CHECK-NEXT: fcvtn v4.2s, v4.2d -; CHECK-NEXT: fcvtn2 v2.4s, v3.2d ; CHECK-NEXT: fcvtn2 v0.4s, v1.2d -; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: fcvtn2 v2.4s, v3.2d ; CHECK-NEXT: fcvtn2 v6.4s, v7.2d ; CHECK-NEXT: fcvtn2 v4.4s, v5.2d -; CHECK-NEXT: movi v3.4s, #1 -; CHECK-NEXT: ushr v5.4s, v2.4s, #16 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: movi v3.4s, #127, msl #8 ; CHECK-NEXT: ushr v7.4s, v0.4s, #16 -; CHECK-NEXT: add v17.4s, v2.4s, v1.4s -; CHECK-NEXT: add v19.4s, v0.4s, v1.4s +; CHECK-NEXT: ushr v5.4s, v2.4s, #16 ; CHECK-NEXT: ushr v16.4s, v6.4s, #16 -; CHECK-NEXT: ushr v18.4s, v4.4s, #16 -; CHECK-NEXT: add v20.4s, v6.4s, v1.4s -; CHECK-NEXT: add v1.4s, v4.4s, v1.4s -; CHECK-NEXT: and v5.16b, v5.16b, v3.16b -; CHECK-NEXT: and v7.16b, v7.16b, v3.16b -; CHECK-NEXT: and v16.16b, v16.16b, v3.16b -; CHECK-NEXT: and v3.16b, v18.16b, v3.16b -; CHECK-NEXT: fcmeq v18.4s, v0.4s, v0.4s -; CHECK-NEXT: orr v0.4s, #64, lsl #16 -; CHECK-NEXT: add v5.4s, v5.4s, v17.4s +; CHECK-NEXT: ushr v17.4s, v4.4s, #16 +; CHECK-NEXT: add v19.4s, v0.4s, v3.4s +; CHECK-NEXT: add v18.4s, v2.4s, v3.4s +; CHECK-NEXT: add v20.4s, v6.4s, v3.4s +; CHECK-NEXT: add v3.4s, v4.4s, v3.4s +; CHECK-NEXT: and v7.16b, v7.16b, v1.16b +; CHECK-NEXT: and v5.16b, v5.16b, v1.16b +; CHECK-NEXT: and v16.16b, v16.16b, v1.16b +; CHECK-NEXT: and v1.16b, v17.16b, v1.16b ; CHECK-NEXT: fcmeq v17.4s, v2.4s, v2.4s +; CHECK-NEXT: orr v2.4s, #64, lsl #16 ; CHECK-NEXT: add v7.4s, v7.4s, v19.4s ; CHECK-NEXT: fcmeq v19.4s, v6.4s, v6.4s -; CHECK-NEXT: orr v2.4s, #64, lsl #16 -; CHECK-NEXT: add v16.4s, v16.4s, v20.4s -; CHECK-NEXT: add v1.4s, v3.4s, v1.4s +; CHECK-NEXT: add v5.4s, v5.4s, v18.4s +; CHECK-NEXT: fcmeq v18.4s, v0.4s, v0.4s +; CHECK-NEXT: add v1.4s, v1.4s, v3.4s ; CHECK-NEXT: fcmeq v3.4s, v4.4s, v4.4s +; CHECK-NEXT: add v16.4s, v16.4s, v20.4s +; CHECK-NEXT: orr v0.4s, #64, lsl #16 ; CHECK-NEXT: orr v6.4s, #64, lsl #16 ; CHECK-NEXT: orr v4.4s, #64, lsl #16 ; CHECK-NEXT: bit v2.16b, v5.16b, v17.16b -; CHECK-NEXT: bit v0.16b, v7.16b, v18.16b ; CHECK-NEXT: mov v5.16b, v19.16b +; CHECK-NEXT: bit v0.16b, v7.16b, v18.16b ; CHECK-NEXT: bif v1.16b, v4.16b, v3.16b ; CHECK-NEXT: bsl v5.16b, v16.16b, v6.16b ; CHECK-NEXT: uzp2 v0.8h, v0.8h, v2.8h @@ -6776,51 +6776,51 @@ entry: define <16 x bfloat> @utofp_v16i64_v16bf16(<16 x i64> %a) { ; CHECK-LABEL: utofp_v16i64_v16bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ucvtf v2.2d, v2.2d ; CHECK-NEXT: ucvtf v0.2d, v0.2d +; CHECK-NEXT: ucvtf v2.2d, v2.2d ; CHECK-NEXT: ucvtf v6.2d, v6.2d ; CHECK-NEXT: ucvtf v4.2d, v4.2d -; CHECK-NEXT: ucvtf v3.2d, v3.2d ; CHECK-NEXT: ucvtf v1.2d, v1.2d +; CHECK-NEXT: ucvtf v3.2d, v3.2d ; CHECK-NEXT: ucvtf v7.2d, v7.2d ; CHECK-NEXT: ucvtf v5.2d, v5.2d -; CHECK-NEXT: fcvtn v2.2s, v2.2d ; CHECK-NEXT: fcvtn v0.2s, v0.2d +; CHECK-NEXT: fcvtn v2.2s, v2.2d ; CHECK-NEXT: fcvtn v6.2s, v6.2d ; CHECK-NEXT: fcvtn v4.2s, v4.2d -; CHECK-NEXT: fcvtn2 v2.4s, v3.2d ; CHECK-NEXT: fcvtn2 v0.4s, v1.2d -; CHECK-NEXT: movi v1.4s, #127, msl #8 +; CHECK-NEXT: fcvtn2 v2.4s, v3.2d ; CHECK-NEXT: fcvtn2 v6.4s, v7.2d ; CHECK-NEXT: fcvtn2 v4.4s, v5.2d -; CHECK-NEXT: movi v3.4s, #1 -; CHECK-NEXT: ushr v5.4s, v2.4s, #16 +; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: movi v3.4s, #127, msl #8 ; CHECK-NEXT: ushr v7.4s, v0.4s, #16 -; CHECK-NEXT: add v17.4s, v2.4s, v1.4s -; CHECK-NEXT: add v19.4s, v0.4s, v1.4s +; CHECK-NEXT: ushr v5.4s, v2.4s, #16 ; CHECK-NEXT: ushr v16.4s, v6.4s, #16 -; CHECK-NEXT: ushr v18.4s, v4.4s, #16 -; CHECK-NEXT: add v20.4s, v6.4s, v1.4s -; CHECK-NEXT: add v1.4s, v4.4s, v1.4s -; CHECK-NEXT: and v5.16b, v5.16b, v3.16b -; CHECK-NEXT: and v7.16b, v7.16b, v3.16b -; CHECK-NEXT: and v16.16b, v16.16b, v3.16b -; CHECK-NEXT: and v3.16b, v18.16b, v3.16b -; CHECK-NEXT: fcmeq v18.4s, v0.4s, v0.4s -; CHECK-NEXT: orr v0.4s, #64, lsl #16 -; CHECK-NEXT: add v5.4s, v5.4s, v17.4s +; CHECK-NEXT: ushr v17.4s, v4.4s, #16 +; CHECK-NEXT: add v19.4s, v0.4s, v3.4s +; CHECK-NEXT: add v18.4s, v2.4s, v3.4s +; CHECK-NEXT: add v20.4s, v6.4s, v3.4s +; CHECK-NEXT: add v3.4s, v4.4s, v3.4s +; CHECK-NEXT: and v7.16b, v7.16b, v1.16b +; CHECK-NEXT: and v5.16b, v5.16b, v1.16b +; CHECK-NEXT: and v16.16b, v16.16b, v1.16b +; CHECK-NEXT: and v1.16b, v17.16b, v1.16b ; CHECK-NEXT: fcmeq v17.4s, v2.4s, v2.4s +; CHECK-NEXT: orr v2.4s, #64, lsl #16 ; CHECK-NEXT: add v7.4s, v7.4s, v19.4s ; CHECK-NEXT: fcmeq v19.4s, v6.4s, v6.4s -; CHECK-NEXT: orr v2.4s, #64, lsl #16 -; CHECK-NEXT: add v16.4s, v16.4s, v20.4s -; CHECK-NEXT: add v1.4s, v3.4s, v1.4s +; CHECK-NEXT: add v5.4s, v5.4s, v18.4s +; CHECK-NEXT: fcmeq v18.4s, v0.4s, v0.4s +; CHECK-NEXT: add v1.4s, v1.4s, v3.4s ; CHECK-NEXT: fcmeq v3.4s, v4.4s, v4.4s +; CHECK-NEXT: add v16.4s, v16.4s, v20.4s +; CHECK-NEXT: orr v0.4s, #64, lsl #16 ; CHECK-NEXT: orr v6.4s, #64, lsl #16 ; CHECK-NEXT: orr v4.4s, #64, lsl #16 ; CHECK-NEXT: bit v2.16b, v5.16b, v17.16b -; CHECK-NEXT: bit v0.16b, v7.16b, v18.16b ; CHECK-NEXT: mov v5.16b, v19.16b +; CHECK-NEXT: bit v0.16b, v7.16b, v18.16b ; CHECK-NEXT: bif v1.16b, v4.16b, v3.16b ; CHECK-NEXT: bsl v5.16b, v16.16b, v6.16b ; CHECK-NEXT: uzp2 v0.8h, v0.8h, v2.8h @@ -6834,107 +6834,107 @@ entry: define <32 x bfloat> @stofp_v32i64_v32bf16(<32 x i64> %a) { ; CHECK-LABEL: stofp_v32i64_v32bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: scvtf v16.2d, v2.2d -; CHECK-NEXT: scvtf v17.2d, v0.2d -; CHECK-NEXT: scvtf v18.2d, v3.2d -; CHECK-NEXT: scvtf v19.2d, v6.2d -; CHECK-NEXT: ldp q24, q23, [sp, #96] -; CHECK-NEXT: scvtf v21.2d, v1.2d -; CHECK-NEXT: scvtf v22.2d, v4.2d +; CHECK-NEXT: scvtf v17.2d, v2.2d +; CHECK-NEXT: scvtf v18.2d, v0.2d +; CHECK-NEXT: scvtf v19.2d, v3.2d +; CHECK-NEXT: scvtf v3.2d, v6.2d +; CHECK-NEXT: ldp q21, q20, [sp, #32] +; CHECK-NEXT: scvtf v4.2d, v4.2d ; CHECK-NEXT: scvtf v6.2d, v7.2d -; CHECK-NEXT: scvtf v7.2d, v5.2d -; CHECK-NEXT: movi v3.4s, #127, msl #8 -; CHECK-NEXT: movi v2.4s, #1 -; CHECK-NEXT: fcvtn v0.2s, v16.2d -; CHECK-NEXT: ldp q20, q16, [sp, #32] -; CHECK-NEXT: fcvtn v1.2s, v17.2d -; CHECK-NEXT: ldp q5, q17, [sp] -; CHECK-NEXT: fcvtn v4.2s, v19.2d -; CHECK-NEXT: scvtf v23.2d, v23.2d +; CHECK-NEXT: scvtf v5.2d, v5.2d +; CHECK-NEXT: ldp q24, q23, [sp, #64] +; CHECK-NEXT: movi v16.4s, #1 +; CHECK-NEXT: fcvtn v0.2s, v17.2d +; CHECK-NEXT: scvtf v17.2d, v1.2d +; CHECK-NEXT: fcvtn v1.2s, v18.2d +; CHECK-NEXT: fcvtn v3.2s, v3.2d +; CHECK-NEXT: ldp q18, q7, [sp] +; CHECK-NEXT: scvtf v21.2d, v21.2d +; CHECK-NEXT: fcvtn v4.2s, v4.2d +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: scvtf v20.2d, v20.2d -; CHECK-NEXT: scvtf v16.2d, v16.2d -; CHECK-NEXT: fcvtn2 v0.4s, v18.2d -; CHECK-NEXT: ldp q19, q18, [sp, #64] -; CHECK-NEXT: scvtf v25.2d, v5.2d -; CHECK-NEXT: fcvtn v5.2s, v22.2d -; CHECK-NEXT: fcvtn2 v1.4s, v21.2d -; CHECK-NEXT: scvtf v21.2d, v24.2d -; CHECK-NEXT: scvtf v17.2d, v17.2d -; CHECK-NEXT: fcvtn2 v4.4s, v6.2d +; CHECK-NEXT: fcvtn2 v0.4s, v19.2d +; CHECK-NEXT: ldp q22, q19, [sp, #96] +; CHECK-NEXT: fcvtn2 v1.4s, v17.2d +; CHECK-NEXT: fcvtn2 v3.4s, v6.2d +; CHECK-NEXT: scvtf v18.2d, v18.2d +; CHECK-NEXT: scvtf v17.2d, v24.2d +; CHECK-NEXT: fcvtn v6.2s, v21.2d +; CHECK-NEXT: fcvtn2 v4.4s, v5.2d +; CHECK-NEXT: scvtf v22.2d, v22.2d +; CHECK-NEXT: scvtf v21.2d, v23.2d +; CHECK-NEXT: scvtf v7.2d, v7.2d +; CHECK-NEXT: ushr v24.4s, v0.4s, #16 +; CHECK-NEXT: add v5.4s, v0.4s, v2.4s ; CHECK-NEXT: scvtf v19.2d, v19.2d -; CHECK-NEXT: scvtf v6.2d, v18.2d -; CHECK-NEXT: fcvtn v18.2s, v20.2d -; CHECK-NEXT: ushr v22.4s, v0.4s, #16 -; CHECK-NEXT: add v20.4s, v0.4s, v3.4s -; CHECK-NEXT: fcvtn2 v5.4s, v7.2d -; CHECK-NEXT: fcvtn v24.2s, v25.2d -; CHECK-NEXT: ushr v7.4s, v1.4s, #16 -; CHECK-NEXT: fcvtn v21.2s, v21.2d -; CHECK-NEXT: add v26.4s, v1.4s, v3.4s -; CHECK-NEXT: ushr v27.4s, v4.4s, #16 -; CHECK-NEXT: fcvtn v19.2s, v19.2d -; CHECK-NEXT: fcvtn2 v18.4s, v16.2d -; CHECK-NEXT: and v22.16b, v22.16b, v2.16b -; CHECK-NEXT: and v7.16b, v7.16b, v2.16b -; CHECK-NEXT: fcmeq v25.4s, v0.4s, v0.4s +; CHECK-NEXT: ushr v23.4s, v1.4s, #16 +; CHECK-NEXT: ushr v25.4s, v3.4s, #16 +; CHECK-NEXT: fcvtn v18.2s, v18.2d +; CHECK-NEXT: fcvtn2 v6.4s, v20.2d +; CHECK-NEXT: add v26.4s, v1.4s, v2.4s +; CHECK-NEXT: fcvtn v17.2s, v17.2d +; CHECK-NEXT: and v24.16b, v24.16b, v16.16b +; CHECK-NEXT: fcvtn v22.2s, v22.2d +; CHECK-NEXT: fcmeq v20.4s, v0.4s, v0.4s +; CHECK-NEXT: and v23.16b, v23.16b, v16.16b ; CHECK-NEXT: orr v0.4s, #64, lsl #16 -; CHECK-NEXT: ushr v28.4s, v5.4s, #16 -; CHECK-NEXT: fcvtn2 v24.4s, v17.2d -; CHECK-NEXT: add v17.4s, v5.4s, v3.4s -; CHECK-NEXT: fcvtn2 v21.4s, v23.2d -; CHECK-NEXT: and v16.16b, v27.16b, v2.16b -; CHECK-NEXT: add v20.4s, v22.4s, v20.4s -; CHECK-NEXT: fcvtn2 v19.4s, v6.2d -; CHECK-NEXT: add v7.4s, v7.4s, v26.4s -; CHECK-NEXT: ushr v26.4s, v18.4s, #16 -; CHECK-NEXT: and v23.16b, v28.16b, v2.16b -; CHECK-NEXT: add v22.4s, v4.4s, v3.4s -; CHECK-NEXT: fcmeq v6.4s, v1.4s, v1.4s -; CHECK-NEXT: ushr v27.4s, v24.4s, #16 -; CHECK-NEXT: add v30.4s, v24.4s, v3.4s +; CHECK-NEXT: fcmeq v27.4s, v3.4s, v3.4s +; CHECK-NEXT: fcvtn2 v18.4s, v7.2d +; CHECK-NEXT: add v7.4s, v3.4s, v2.4s +; CHECK-NEXT: orr v3.4s, #64, lsl #16 +; CHECK-NEXT: add v5.4s, v24.4s, v5.4s +; CHECK-NEXT: and v24.16b, v25.16b, v16.16b +; CHECK-NEXT: ushr v25.4s, v4.4s, #16 +; CHECK-NEXT: fcvtn2 v22.4s, v19.2d +; CHECK-NEXT: add v19.4s, v23.4s, v26.4s +; CHECK-NEXT: ushr v26.4s, v6.4s, #16 +; CHECK-NEXT: fcvtn2 v17.4s, v21.2d +; CHECK-NEXT: fcmeq v21.4s, v1.4s, v1.4s ; CHECK-NEXT: orr v1.4s, #64, lsl #16 -; CHECK-NEXT: ushr v28.4s, v21.4s, #16 -; CHECK-NEXT: add v31.4s, v21.4s, v3.4s -; CHECK-NEXT: and v26.16b, v26.16b, v2.16b -; CHECK-NEXT: add v17.4s, v23.4s, v17.4s -; CHECK-NEXT: add v23.4s, v18.4s, v3.4s -; CHECK-NEXT: ushr v29.4s, v19.4s, #16 -; CHECK-NEXT: and v27.16b, v27.16b, v2.16b -; CHECK-NEXT: add v3.4s, v19.4s, v3.4s -; CHECK-NEXT: add v16.4s, v16.4s, v22.4s -; CHECK-NEXT: and v28.16b, v28.16b, v2.16b -; CHECK-NEXT: fcmeq v22.4s, v4.4s, v4.4s -; CHECK-NEXT: orr v4.4s, #64, lsl #16 -; CHECK-NEXT: and v2.16b, v29.16b, v2.16b -; CHECK-NEXT: fcmeq v29.4s, v5.4s, v5.4s -; CHECK-NEXT: orr v5.4s, #64, lsl #16 -; CHECK-NEXT: add v23.4s, v26.4s, v23.4s -; CHECK-NEXT: fcmeq v26.4s, v18.4s, v18.4s -; CHECK-NEXT: add v27.4s, v27.4s, v30.4s -; CHECK-NEXT: fcmeq v30.4s, v24.4s, v24.4s +; CHECK-NEXT: and v23.16b, v25.16b, v16.16b +; CHECK-NEXT: add v25.4s, v4.4s, v2.4s +; CHECK-NEXT: add v7.4s, v24.4s, v7.4s +; CHECK-NEXT: ushr v24.4s, v18.4s, #16 +; CHECK-NEXT: add v30.4s, v18.4s, v2.4s +; CHECK-NEXT: bit v0.16b, v5.16b, v20.16b +; CHECK-NEXT: ushr v28.4s, v22.4s, #16 +; CHECK-NEXT: add v31.4s, v22.4s, v2.4s +; CHECK-NEXT: add v23.4s, v23.4s, v25.4s +; CHECK-NEXT: and v25.16b, v26.16b, v16.16b +; CHECK-NEXT: add v26.4s, v6.4s, v2.4s +; CHECK-NEXT: ushr v29.4s, v17.4s, #16 +; CHECK-NEXT: and v24.16b, v24.16b, v16.16b +; CHECK-NEXT: add v2.4s, v17.4s, v2.4s +; CHECK-NEXT: and v28.16b, v28.16b, v16.16b +; CHECK-NEXT: bit v3.16b, v7.16b, v27.16b +; CHECK-NEXT: bit v1.16b, v19.16b, v21.16b +; CHECK-NEXT: add v25.4s, v25.4s, v26.4s +; CHECK-NEXT: fcmeq v26.4s, v6.4s, v6.4s +; CHECK-NEXT: orr v6.4s, #64, lsl #16 +; CHECK-NEXT: and v16.16b, v29.16b, v16.16b +; CHECK-NEXT: add v24.4s, v24.4s, v30.4s +; CHECK-NEXT: fcmeq v30.4s, v18.4s, v18.4s ; CHECK-NEXT: add v28.4s, v28.4s, v31.4s -; CHECK-NEXT: fcmeq v31.4s, v21.4s, v21.4s -; CHECK-NEXT: add v2.4s, v2.4s, v3.4s -; CHECK-NEXT: fcmeq v3.4s, v19.4s, v19.4s +; CHECK-NEXT: fcmeq v31.4s, v22.4s, v22.4s +; CHECK-NEXT: fcmeq v29.4s, v4.4s, v4.4s +; CHECK-NEXT: orr v4.4s, #64, lsl #16 ; CHECK-NEXT: orr v18.4s, #64, lsl #16 -; CHECK-NEXT: orr v24.4s, #64, lsl #16 -; CHECK-NEXT: orr v21.4s, #64, lsl #16 -; CHECK-NEXT: orr v19.4s, #64, lsl #16 -; CHECK-NEXT: bit v1.16b, v7.16b, v6.16b -; CHECK-NEXT: bit v4.16b, v16.16b, v22.16b -; CHECK-NEXT: mov v6.16b, v26.16b -; CHECK-NEXT: mov v7.16b, v30.16b -; CHECK-NEXT: mov v16.16b, v31.16b -; CHECK-NEXT: bit v0.16b, v20.16b, v25.16b -; CHECK-NEXT: bit v5.16b, v17.16b, v29.16b -; CHECK-NEXT: bsl v3.16b, v2.16b, v19.16b -; CHECK-NEXT: bsl v6.16b, v23.16b, v18.16b -; CHECK-NEXT: bsl v7.16b, v27.16b, v24.16b -; CHECK-NEXT: bsl v16.16b, v28.16b, v21.16b +; CHECK-NEXT: orr v22.4s, #64, lsl #16 +; CHECK-NEXT: mov v5.16b, v26.16b +; CHECK-NEXT: add v2.4s, v16.4s, v2.4s +; CHECK-NEXT: fcmeq v16.4s, v17.4s, v17.4s +; CHECK-NEXT: orr v17.4s, #64, lsl #16 ; CHECK-NEXT: uzp2 v0.8h, v1.8h, v0.8h -; CHECK-NEXT: uzp2 v1.8h, v5.8h, v4.8h -; CHECK-NEXT: uzp2 v2.8h, v7.8h, v6.8h -; CHECK-NEXT: uzp2 v3.8h, v3.8h, v16.8h +; CHECK-NEXT: mov v7.16b, v31.16b +; CHECK-NEXT: bit v4.16b, v23.16b, v29.16b +; CHECK-NEXT: bsl v5.16b, v25.16b, v6.16b +; CHECK-NEXT: mov v6.16b, v30.16b +; CHECK-NEXT: bsl v16.16b, v2.16b, v17.16b +; CHECK-NEXT: bsl v7.16b, v28.16b, v22.16b +; CHECK-NEXT: bsl v6.16b, v24.16b, v18.16b +; CHECK-NEXT: uzp2 v1.8h, v4.8h, v3.8h +; CHECK-NEXT: uzp2 v3.8h, v16.8h, v7.8h +; CHECK-NEXT: uzp2 v2.8h, v6.8h, v5.8h ; CHECK-NEXT: ret entry: %c = sitofp <32 x i64> %a to <32 x bfloat> @@ -6944,107 +6944,107 @@ entry: define <32 x bfloat> @utofp_v32i64_v32bf16(<32 x i64> %a) { ; CHECK-LABEL: utofp_v32i64_v32bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ucvtf v16.2d, v2.2d -; CHECK-NEXT: ucvtf v17.2d, v0.2d -; CHECK-NEXT: ucvtf v18.2d, v3.2d -; CHECK-NEXT: ucvtf v19.2d, v6.2d -; CHECK-NEXT: ldp q24, q23, [sp, #96] -; CHECK-NEXT: ucvtf v21.2d, v1.2d -; CHECK-NEXT: ucvtf v22.2d, v4.2d +; CHECK-NEXT: ucvtf v17.2d, v2.2d +; CHECK-NEXT: ucvtf v18.2d, v0.2d +; CHECK-NEXT: ucvtf v19.2d, v3.2d +; CHECK-NEXT: ucvtf v3.2d, v6.2d +; CHECK-NEXT: ldp q21, q20, [sp, #32] +; CHECK-NEXT: ucvtf v4.2d, v4.2d ; CHECK-NEXT: ucvtf v6.2d, v7.2d -; CHECK-NEXT: ucvtf v7.2d, v5.2d -; CHECK-NEXT: movi v3.4s, #127, msl #8 -; CHECK-NEXT: movi v2.4s, #1 -; CHECK-NEXT: fcvtn v0.2s, v16.2d -; CHECK-NEXT: ldp q20, q16, [sp, #32] -; CHECK-NEXT: fcvtn v1.2s, v17.2d -; CHECK-NEXT: ldp q5, q17, [sp] -; CHECK-NEXT: fcvtn v4.2s, v19.2d -; CHECK-NEXT: ucvtf v23.2d, v23.2d +; CHECK-NEXT: ucvtf v5.2d, v5.2d +; CHECK-NEXT: ldp q24, q23, [sp, #64] +; CHECK-NEXT: movi v16.4s, #1 +; CHECK-NEXT: fcvtn v0.2s, v17.2d +; CHECK-NEXT: ucvtf v17.2d, v1.2d +; CHECK-NEXT: fcvtn v1.2s, v18.2d +; CHECK-NEXT: fcvtn v3.2s, v3.2d +; CHECK-NEXT: ldp q18, q7, [sp] +; CHECK-NEXT: ucvtf v21.2d, v21.2d +; CHECK-NEXT: fcvtn v4.2s, v4.2d +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: ucvtf v20.2d, v20.2d -; CHECK-NEXT: ucvtf v16.2d, v16.2d -; CHECK-NEXT: fcvtn2 v0.4s, v18.2d -; CHECK-NEXT: ldp q19, q18, [sp, #64] -; CHECK-NEXT: ucvtf v25.2d, v5.2d -; CHECK-NEXT: fcvtn v5.2s, v22.2d -; CHECK-NEXT: fcvtn2 v1.4s, v21.2d -; CHECK-NEXT: ucvtf v21.2d, v24.2d -; CHECK-NEXT: ucvtf v17.2d, v17.2d -; CHECK-NEXT: fcvtn2 v4.4s, v6.2d +; CHECK-NEXT: fcvtn2 v0.4s, v19.2d +; CHECK-NEXT: ldp q22, q19, [sp, #96] +; CHECK-NEXT: fcvtn2 v1.4s, v17.2d +; CHECK-NEXT: fcvtn2 v3.4s, v6.2d +; CHECK-NEXT: ucvtf v18.2d, v18.2d +; CHECK-NEXT: ucvtf v17.2d, v24.2d +; CHECK-NEXT: fcvtn v6.2s, v21.2d +; CHECK-NEXT: fcvtn2 v4.4s, v5.2d +; CHECK-NEXT: ucvtf v22.2d, v22.2d +; CHECK-NEXT: ucvtf v21.2d, v23.2d +; CHECK-NEXT: ucvtf v7.2d, v7.2d +; CHECK-NEXT: ushr v24.4s, v0.4s, #16 +; CHECK-NEXT: add v5.4s, v0.4s, v2.4s ; CHECK-NEXT: ucvtf v19.2d, v19.2d -; CHECK-NEXT: ucvtf v6.2d, v18.2d -; CHECK-NEXT: fcvtn v18.2s, v20.2d -; CHECK-NEXT: ushr v22.4s, v0.4s, #16 -; CHECK-NEXT: add v20.4s, v0.4s, v3.4s -; CHECK-NEXT: fcvtn2 v5.4s, v7.2d -; CHECK-NEXT: fcvtn v24.2s, v25.2d -; CHECK-NEXT: ushr v7.4s, v1.4s, #16 -; CHECK-NEXT: fcvtn v21.2s, v21.2d -; CHECK-NEXT: add v26.4s, v1.4s, v3.4s -; CHECK-NEXT: ushr v27.4s, v4.4s, #16 -; CHECK-NEXT: fcvtn v19.2s, v19.2d -; CHECK-NEXT: fcvtn2 v18.4s, v16.2d -; CHECK-NEXT: and v22.16b, v22.16b, v2.16b -; CHECK-NEXT: and v7.16b, v7.16b, v2.16b -; CHECK-NEXT: fcmeq v25.4s, v0.4s, v0.4s +; CHECK-NEXT: ushr v23.4s, v1.4s, #16 +; CHECK-NEXT: ushr v25.4s, v3.4s, #16 +; CHECK-NEXT: fcvtn v18.2s, v18.2d +; CHECK-NEXT: fcvtn2 v6.4s, v20.2d +; CHECK-NEXT: add v26.4s, v1.4s, v2.4s +; CHECK-NEXT: fcvtn v17.2s, v17.2d +; CHECK-NEXT: and v24.16b, v24.16b, v16.16b +; CHECK-NEXT: fcvtn v22.2s, v22.2d +; CHECK-NEXT: fcmeq v20.4s, v0.4s, v0.4s +; CHECK-NEXT: and v23.16b, v23.16b, v16.16b ; CHECK-NEXT: orr v0.4s, #64, lsl #16 -; CHECK-NEXT: ushr v28.4s, v5.4s, #16 -; CHECK-NEXT: fcvtn2 v24.4s, v17.2d -; CHECK-NEXT: add v17.4s, v5.4s, v3.4s -; CHECK-NEXT: fcvtn2 v21.4s, v23.2d -; CHECK-NEXT: and v16.16b, v27.16b, v2.16b -; CHECK-NEXT: add v20.4s, v22.4s, v20.4s -; CHECK-NEXT: fcvtn2 v19.4s, v6.2d -; CHECK-NEXT: add v7.4s, v7.4s, v26.4s -; CHECK-NEXT: ushr v26.4s, v18.4s, #16 -; CHECK-NEXT: and v23.16b, v28.16b, v2.16b -; CHECK-NEXT: add v22.4s, v4.4s, v3.4s -; CHECK-NEXT: fcmeq v6.4s, v1.4s, v1.4s -; CHECK-NEXT: ushr v27.4s, v24.4s, #16 -; CHECK-NEXT: add v30.4s, v24.4s, v3.4s +; CHECK-NEXT: fcmeq v27.4s, v3.4s, v3.4s +; CHECK-NEXT: fcvtn2 v18.4s, v7.2d +; CHECK-NEXT: add v7.4s, v3.4s, v2.4s +; CHECK-NEXT: orr v3.4s, #64, lsl #16 +; CHECK-NEXT: add v5.4s, v24.4s, v5.4s +; CHECK-NEXT: and v24.16b, v25.16b, v16.16b +; CHECK-NEXT: ushr v25.4s, v4.4s, #16 +; CHECK-NEXT: fcvtn2 v22.4s, v19.2d +; CHECK-NEXT: add v19.4s, v23.4s, v26.4s +; CHECK-NEXT: ushr v26.4s, v6.4s, #16 +; CHECK-NEXT: fcvtn2 v17.4s, v21.2d +; CHECK-NEXT: fcmeq v21.4s, v1.4s, v1.4s ; CHECK-NEXT: orr v1.4s, #64, lsl #16 -; CHECK-NEXT: ushr v28.4s, v21.4s, #16 -; CHECK-NEXT: add v31.4s, v21.4s, v3.4s -; CHECK-NEXT: and v26.16b, v26.16b, v2.16b -; CHECK-NEXT: add v17.4s, v23.4s, v17.4s -; CHECK-NEXT: add v23.4s, v18.4s, v3.4s -; CHECK-NEXT: ushr v29.4s, v19.4s, #16 -; CHECK-NEXT: and v27.16b, v27.16b, v2.16b -; CHECK-NEXT: add v3.4s, v19.4s, v3.4s -; CHECK-NEXT: add v16.4s, v16.4s, v22.4s -; CHECK-NEXT: and v28.16b, v28.16b, v2.16b -; CHECK-NEXT: fcmeq v22.4s, v4.4s, v4.4s -; CHECK-NEXT: orr v4.4s, #64, lsl #16 -; CHECK-NEXT: and v2.16b, v29.16b, v2.16b -; CHECK-NEXT: fcmeq v29.4s, v5.4s, v5.4s -; CHECK-NEXT: orr v5.4s, #64, lsl #16 -; CHECK-NEXT: add v23.4s, v26.4s, v23.4s -; CHECK-NEXT: fcmeq v26.4s, v18.4s, v18.4s -; CHECK-NEXT: add v27.4s, v27.4s, v30.4s -; CHECK-NEXT: fcmeq v30.4s, v24.4s, v24.4s +; CHECK-NEXT: and v23.16b, v25.16b, v16.16b +; CHECK-NEXT: add v25.4s, v4.4s, v2.4s +; CHECK-NEXT: add v7.4s, v24.4s, v7.4s +; CHECK-NEXT: ushr v24.4s, v18.4s, #16 +; CHECK-NEXT: add v30.4s, v18.4s, v2.4s +; CHECK-NEXT: bit v0.16b, v5.16b, v20.16b +; CHECK-NEXT: ushr v28.4s, v22.4s, #16 +; CHECK-NEXT: add v31.4s, v22.4s, v2.4s +; CHECK-NEXT: add v23.4s, v23.4s, v25.4s +; CHECK-NEXT: and v25.16b, v26.16b, v16.16b +; CHECK-NEXT: add v26.4s, v6.4s, v2.4s +; CHECK-NEXT: ushr v29.4s, v17.4s, #16 +; CHECK-NEXT: and v24.16b, v24.16b, v16.16b +; CHECK-NEXT: add v2.4s, v17.4s, v2.4s +; CHECK-NEXT: and v28.16b, v28.16b, v16.16b +; CHECK-NEXT: bit v3.16b, v7.16b, v27.16b +; CHECK-NEXT: bit v1.16b, v19.16b, v21.16b +; CHECK-NEXT: add v25.4s, v25.4s, v26.4s +; CHECK-NEXT: fcmeq v26.4s, v6.4s, v6.4s +; CHECK-NEXT: orr v6.4s, #64, lsl #16 +; CHECK-NEXT: and v16.16b, v29.16b, v16.16b +; CHECK-NEXT: add v24.4s, v24.4s, v30.4s +; CHECK-NEXT: fcmeq v30.4s, v18.4s, v18.4s ; CHECK-NEXT: add v28.4s, v28.4s, v31.4s -; CHECK-NEXT: fcmeq v31.4s, v21.4s, v21.4s -; CHECK-NEXT: add v2.4s, v2.4s, v3.4s -; CHECK-NEXT: fcmeq v3.4s, v19.4s, v19.4s +; CHECK-NEXT: fcmeq v31.4s, v22.4s, v22.4s +; CHECK-NEXT: fcmeq v29.4s, v4.4s, v4.4s +; CHECK-NEXT: orr v4.4s, #64, lsl #16 ; CHECK-NEXT: orr v18.4s, #64, lsl #16 -; CHECK-NEXT: orr v24.4s, #64, lsl #16 -; CHECK-NEXT: orr v21.4s, #64, lsl #16 -; CHECK-NEXT: orr v19.4s, #64, lsl #16 -; CHECK-NEXT: bit v1.16b, v7.16b, v6.16b -; CHECK-NEXT: bit v4.16b, v16.16b, v22.16b -; CHECK-NEXT: mov v6.16b, v26.16b -; CHECK-NEXT: mov v7.16b, v30.16b -; CHECK-NEXT: mov v16.16b, v31.16b -; CHECK-NEXT: bit v0.16b, v20.16b, v25.16b -; CHECK-NEXT: bit v5.16b, v17.16b, v29.16b -; CHECK-NEXT: bsl v3.16b, v2.16b, v19.16b -; CHECK-NEXT: bsl v6.16b, v23.16b, v18.16b -; CHECK-NEXT: bsl v7.16b, v27.16b, v24.16b -; CHECK-NEXT: bsl v16.16b, v28.16b, v21.16b +; CHECK-NEXT: orr v22.4s, #64, lsl #16 +; CHECK-NEXT: mov v5.16b, v26.16b +; CHECK-NEXT: add v2.4s, v16.4s, v2.4s +; CHECK-NEXT: fcmeq v16.4s, v17.4s, v17.4s +; CHECK-NEXT: orr v17.4s, #64, lsl #16 ; CHECK-NEXT: uzp2 v0.8h, v1.8h, v0.8h -; CHECK-NEXT: uzp2 v1.8h, v5.8h, v4.8h -; CHECK-NEXT: uzp2 v2.8h, v7.8h, v6.8h -; CHECK-NEXT: uzp2 v3.8h, v3.8h, v16.8h +; CHECK-NEXT: mov v7.16b, v31.16b +; CHECK-NEXT: bit v4.16b, v23.16b, v29.16b +; CHECK-NEXT: bsl v5.16b, v25.16b, v6.16b +; CHECK-NEXT: mov v6.16b, v30.16b +; CHECK-NEXT: bsl v16.16b, v2.16b, v17.16b +; CHECK-NEXT: bsl v7.16b, v28.16b, v22.16b +; CHECK-NEXT: bsl v6.16b, v24.16b, v18.16b +; CHECK-NEXT: uzp2 v1.8h, v4.8h, v3.8h +; CHECK-NEXT: uzp2 v3.8h, v16.8h, v7.8h +; CHECK-NEXT: uzp2 v2.8h, v6.8h, v5.8h ; CHECK-NEXT: ret entry: %c = uitofp <32 x i64> %a to <32 x bfloat> @@ -7059,9 +7059,9 @@ define <2 x bfloat> @stofp_v2i32_v2bf16(<2 x i32> %a) { ; CHECK-NEXT: scvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = sitofp <2 x i32> %a to <2 x bfloat> @@ -7076,9 +7076,9 @@ define <2 x bfloat> @utofp_v2i32_v2bf16(<2 x i32> %a) { ; CHECK-NEXT: ucvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = uitofp <2 x i32> %a to <2 x bfloat> @@ -7092,9 +7092,9 @@ define <3 x bfloat> @stofp_v3i32_v3bf16(<3 x i32> %a) { ; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = sitofp <3 x i32> %a to <3 x bfloat> @@ -7108,9 +7108,9 @@ define <3 x bfloat> @utofp_v3i32_v3bf16(<3 x i32> %a) { ; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = uitofp <3 x i32> %a to <3 x bfloat> @@ -7124,9 +7124,9 @@ define <4 x bfloat> @stofp_v4i32_v4bf16(<4 x i32> %a) { ; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = sitofp <4 x i32> %a to <4 x bfloat> @@ -7140,9 +7140,9 @@ define <4 x bfloat> @utofp_v4i32_v4bf16(<4 x i32> %a) { ; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = uitofp <4 x i32> %a to <4 x bfloat> @@ -7155,15 +7155,15 @@ define <8 x bfloat> @stofp_v8i32_v8bf16(<8 x i32> %a) { ; CHECK-NEXT: scvtf v0.4s, v0.4s ; CHECK-NEXT: movi v2.4s, #1 ; CHECK-NEXT: scvtf v1.4s, v1.4s +; CHECK-NEXT: movi v5.4s, #127, msl #8 ; CHECK-NEXT: ushr v3.4s, v0.4s, #16 ; CHECK-NEXT: ushr v4.4s, v1.4s, #16 ; CHECK-NEXT: and v3.16b, v3.16b, v2.16b ; CHECK-NEXT: and v2.16b, v4.16b, v2.16b ; CHECK-NEXT: add v0.4s, v3.4s, v0.4s -; CHECK-NEXT: movi v3.4s, #127, msl #8 ; CHECK-NEXT: add v1.4s, v2.4s, v1.4s -; CHECK-NEXT: addhn v0.4h, v0.4s, v3.4s -; CHECK-NEXT: addhn2 v0.8h, v1.4s, v3.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v5.4s +; CHECK-NEXT: addhn2 v0.8h, v1.4s, v5.4s ; CHECK-NEXT: ret entry: %c = sitofp <8 x i32> %a to <8 x bfloat> @@ -7176,15 +7176,15 @@ define <8 x bfloat> @utofp_v8i32_v8bf16(<8 x i32> %a) { ; CHECK-NEXT: ucvtf v0.4s, v0.4s ; CHECK-NEXT: movi v2.4s, #1 ; CHECK-NEXT: ucvtf v1.4s, v1.4s +; CHECK-NEXT: movi v5.4s, #127, msl #8 ; CHECK-NEXT: ushr v3.4s, v0.4s, #16 ; CHECK-NEXT: ushr v4.4s, v1.4s, #16 ; CHECK-NEXT: and v3.16b, v3.16b, v2.16b ; CHECK-NEXT: and v2.16b, v4.16b, v2.16b ; CHECK-NEXT: add v0.4s, v3.4s, v0.4s -; CHECK-NEXT: movi v3.4s, #127, msl #8 ; CHECK-NEXT: add v1.4s, v2.4s, v1.4s -; CHECK-NEXT: addhn v0.4h, v0.4s, v3.4s -; CHECK-NEXT: addhn2 v0.8h, v1.4s, v3.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v5.4s +; CHECK-NEXT: addhn2 v0.8h, v1.4s, v5.4s ; CHECK-NEXT: ret entry: %c = uitofp <8 x i32> %a to <8 x bfloat> @@ -7194,28 +7194,28 @@ entry: define <16 x bfloat> @stofp_v16i32_v16bf16(<16 x i32> %a) { ; CHECK-LABEL: stofp_v16i32_v16bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: scvtf v0.4s, v0.4s ; CHECK-NEXT: scvtf v2.4s, v2.4s -; CHECK-NEXT: movi v4.4s, #1 -; CHECK-NEXT: scvtf v1.4s, v1.4s +; CHECK-NEXT: scvtf v0.4s, v0.4s +; CHECK-NEXT: scvtf v4.4s, v1.4s +; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: scvtf v3.4s, v3.4s +; CHECK-NEXT: movi v17.4s, #127, msl #8 ; CHECK-NEXT: ushr v5.4s, v0.4s, #16 ; CHECK-NEXT: ushr v6.4s, v2.4s, #16 -; CHECK-NEXT: ushr v7.4s, v1.4s, #16 +; CHECK-NEXT: ushr v7.4s, v4.4s, #16 ; CHECK-NEXT: ushr v16.4s, v3.4s, #16 -; CHECK-NEXT: and v5.16b, v5.16b, v4.16b -; CHECK-NEXT: and v6.16b, v6.16b, v4.16b +; CHECK-NEXT: and v5.16b, v5.16b, v1.16b +; CHECK-NEXT: and v6.16b, v6.16b, v1.16b ; CHECK-NEXT: add v0.4s, v5.4s, v0.4s ; CHECK-NEXT: add v2.4s, v6.4s, v2.4s -; CHECK-NEXT: movi v6.4s, #127, msl #8 -; CHECK-NEXT: and v5.16b, v7.16b, v4.16b -; CHECK-NEXT: and v4.16b, v16.16b, v4.16b -; CHECK-NEXT: add v5.4s, v5.4s, v1.4s -; CHECK-NEXT: addhn v0.4h, v0.4s, v6.4s -; CHECK-NEXT: add v3.4s, v4.4s, v3.4s -; CHECK-NEXT: addhn v1.4h, v2.4s, v6.4s -; CHECK-NEXT: addhn2 v0.8h, v5.4s, v6.4s -; CHECK-NEXT: addhn2 v1.8h, v3.4s, v6.4s +; CHECK-NEXT: and v5.16b, v7.16b, v1.16b +; CHECK-NEXT: and v6.16b, v16.16b, v1.16b +; CHECK-NEXT: addhn v0.4h, v0.4s, v17.4s +; CHECK-NEXT: addhn v1.4h, v2.4s, v17.4s +; CHECK-NEXT: add v2.4s, v5.4s, v4.4s +; CHECK-NEXT: add v3.4s, v6.4s, v3.4s +; CHECK-NEXT: addhn2 v0.8h, v2.4s, v17.4s +; CHECK-NEXT: addhn2 v1.8h, v3.4s, v17.4s ; CHECK-NEXT: ret entry: %c = sitofp <16 x i32> %a to <16 x bfloat> @@ -7225,28 +7225,28 @@ entry: define <16 x bfloat> @utofp_v16i32_v16bf16(<16 x i32> %a) { ; CHECK-LABEL: utofp_v16i32_v16bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ucvtf v0.4s, v0.4s ; CHECK-NEXT: ucvtf v2.4s, v2.4s -; CHECK-NEXT: movi v4.4s, #1 -; CHECK-NEXT: ucvtf v1.4s, v1.4s +; CHECK-NEXT: ucvtf v0.4s, v0.4s +; CHECK-NEXT: ucvtf v4.4s, v1.4s +; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: ucvtf v3.4s, v3.4s +; CHECK-NEXT: movi v17.4s, #127, msl #8 ; CHECK-NEXT: ushr v5.4s, v0.4s, #16 ; CHECK-NEXT: ushr v6.4s, v2.4s, #16 -; CHECK-NEXT: ushr v7.4s, v1.4s, #16 +; CHECK-NEXT: ushr v7.4s, v4.4s, #16 ; CHECK-NEXT: ushr v16.4s, v3.4s, #16 -; CHECK-NEXT: and v5.16b, v5.16b, v4.16b -; CHECK-NEXT: and v6.16b, v6.16b, v4.16b +; CHECK-NEXT: and v5.16b, v5.16b, v1.16b +; CHECK-NEXT: and v6.16b, v6.16b, v1.16b ; CHECK-NEXT: add v0.4s, v5.4s, v0.4s ; CHECK-NEXT: add v2.4s, v6.4s, v2.4s -; CHECK-NEXT: movi v6.4s, #127, msl #8 -; CHECK-NEXT: and v5.16b, v7.16b, v4.16b -; CHECK-NEXT: and v4.16b, v16.16b, v4.16b -; CHECK-NEXT: add v5.4s, v5.4s, v1.4s -; CHECK-NEXT: addhn v0.4h, v0.4s, v6.4s -; CHECK-NEXT: add v3.4s, v4.4s, v3.4s -; CHECK-NEXT: addhn v1.4h, v2.4s, v6.4s -; CHECK-NEXT: addhn2 v0.8h, v5.4s, v6.4s -; CHECK-NEXT: addhn2 v1.8h, v3.4s, v6.4s +; CHECK-NEXT: and v5.16b, v7.16b, v1.16b +; CHECK-NEXT: and v6.16b, v16.16b, v1.16b +; CHECK-NEXT: addhn v0.4h, v0.4s, v17.4s +; CHECK-NEXT: addhn v1.4h, v2.4s, v17.4s +; CHECK-NEXT: add v2.4s, v5.4s, v4.4s +; CHECK-NEXT: add v3.4s, v6.4s, v3.4s +; CHECK-NEXT: addhn2 v0.8h, v2.4s, v17.4s +; CHECK-NEXT: addhn2 v1.8h, v3.4s, v17.4s ; CHECK-NEXT: ret entry: %c = uitofp <16 x i32> %a to <16 x bfloat> @@ -7262,42 +7262,42 @@ define <32 x bfloat> @stofp_v32i32_v32bf16(<32 x i32> %a) { ; CHECK-NEXT: scvtf v6.4s, v6.4s ; CHECK-NEXT: movi v16.4s, #1 ; CHECK-NEXT: scvtf v1.4s, v1.4s -; CHECK-NEXT: scvtf v3.4s, v3.4s +; CHECK-NEXT: scvtf v17.4s, v3.4s ; CHECK-NEXT: scvtf v5.4s, v5.4s ; CHECK-NEXT: scvtf v7.4s, v7.4s -; CHECK-NEXT: ushr v17.4s, v0.4s, #16 +; CHECK-NEXT: movi v21.4s, #127, msl #8 +; CHECK-NEXT: ushr v3.4s, v0.4s, #16 ; CHECK-NEXT: ushr v18.4s, v2.4s, #16 ; CHECK-NEXT: ushr v19.4s, v4.4s, #16 ; CHECK-NEXT: ushr v20.4s, v6.4s, #16 -; CHECK-NEXT: ushr v21.4s, v1.4s, #16 -; CHECK-NEXT: ushr v22.4s, v3.4s, #16 -; CHECK-NEXT: ushr v23.4s, v5.4s, #16 -; CHECK-NEXT: and v17.16b, v17.16b, v16.16b +; CHECK-NEXT: ushr v22.4s, v1.4s, #16 +; CHECK-NEXT: ushr v23.4s, v17.4s, #16 +; CHECK-NEXT: ushr v24.4s, v5.4s, #16 +; CHECK-NEXT: ushr v25.4s, v7.4s, #16 +; CHECK-NEXT: and v3.16b, v3.16b, v16.16b ; CHECK-NEXT: and v18.16b, v18.16b, v16.16b ; CHECK-NEXT: and v19.16b, v19.16b, v16.16b ; CHECK-NEXT: and v20.16b, v20.16b, v16.16b -; CHECK-NEXT: and v21.16b, v21.16b, v16.16b -; CHECK-NEXT: and v22.16b, v22.16b, v16.16b -; CHECK-NEXT: add v0.4s, v17.4s, v0.4s -; CHECK-NEXT: ushr v17.4s, v7.4s, #16 +; CHECK-NEXT: add v0.4s, v3.4s, v0.4s +; CHECK-NEXT: and v3.16b, v22.16b, v16.16b ; CHECK-NEXT: add v2.4s, v18.4s, v2.4s -; CHECK-NEXT: movi v18.4s, #127, msl #8 ; CHECK-NEXT: add v4.4s, v19.4s, v4.4s ; CHECK-NEXT: add v6.4s, v20.4s, v6.4s -; CHECK-NEXT: and v19.16b, v23.16b, v16.16b -; CHECK-NEXT: add v20.4s, v22.4s, v3.4s -; CHECK-NEXT: and v16.16b, v17.16b, v16.16b -; CHECK-NEXT: add v17.4s, v21.4s, v1.4s +; CHECK-NEXT: and v18.16b, v23.16b, v16.16b +; CHECK-NEXT: and v19.16b, v24.16b, v16.16b +; CHECK-NEXT: and v16.16b, v25.16b, v16.16b +; CHECK-NEXT: add v20.4s, v3.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v21.4s +; CHECK-NEXT: addhn v1.4h, v2.4s, v21.4s +; CHECK-NEXT: addhn v2.4h, v4.4s, v21.4s +; CHECK-NEXT: addhn v3.4h, v6.4s, v21.4s +; CHECK-NEXT: add v4.4s, v18.4s, v17.4s ; CHECK-NEXT: add v5.4s, v19.4s, v5.4s -; CHECK-NEXT: addhn v0.4h, v0.4s, v18.4s -; CHECK-NEXT: addhn v1.4h, v2.4s, v18.4s -; CHECK-NEXT: addhn v2.4h, v4.4s, v18.4s -; CHECK-NEXT: add v4.4s, v16.4s, v7.4s -; CHECK-NEXT: addhn v3.4h, v6.4s, v18.4s -; CHECK-NEXT: addhn2 v0.8h, v17.4s, v18.4s -; CHECK-NEXT: addhn2 v1.8h, v20.4s, v18.4s -; CHECK-NEXT: addhn2 v2.8h, v5.4s, v18.4s -; CHECK-NEXT: addhn2 v3.8h, v4.4s, v18.4s +; CHECK-NEXT: add v6.4s, v16.4s, v7.4s +; CHECK-NEXT: addhn2 v0.8h, v20.4s, v21.4s +; CHECK-NEXT: addhn2 v1.8h, v4.4s, v21.4s +; CHECK-NEXT: addhn2 v2.8h, v5.4s, v21.4s +; CHECK-NEXT: addhn2 v3.8h, v6.4s, v21.4s ; CHECK-NEXT: ret entry: %c = sitofp <32 x i32> %a to <32 x bfloat> @@ -7313,42 +7313,42 @@ define <32 x bfloat> @utofp_v32i32_v32bf16(<32 x i32> %a) { ; CHECK-NEXT: ucvtf v6.4s, v6.4s ; CHECK-NEXT: movi v16.4s, #1 ; CHECK-NEXT: ucvtf v1.4s, v1.4s -; CHECK-NEXT: ucvtf v3.4s, v3.4s +; CHECK-NEXT: ucvtf v17.4s, v3.4s ; CHECK-NEXT: ucvtf v5.4s, v5.4s ; CHECK-NEXT: ucvtf v7.4s, v7.4s -; CHECK-NEXT: ushr v17.4s, v0.4s, #16 +; CHECK-NEXT: movi v21.4s, #127, msl #8 +; CHECK-NEXT: ushr v3.4s, v0.4s, #16 ; CHECK-NEXT: ushr v18.4s, v2.4s, #16 ; CHECK-NEXT: ushr v19.4s, v4.4s, #16 ; CHECK-NEXT: ushr v20.4s, v6.4s, #16 -; CHECK-NEXT: ushr v21.4s, v1.4s, #16 -; CHECK-NEXT: ushr v22.4s, v3.4s, #16 -; CHECK-NEXT: ushr v23.4s, v5.4s, #16 -; CHECK-NEXT: and v17.16b, v17.16b, v16.16b +; CHECK-NEXT: ushr v22.4s, v1.4s, #16 +; CHECK-NEXT: ushr v23.4s, v17.4s, #16 +; CHECK-NEXT: ushr v24.4s, v5.4s, #16 +; CHECK-NEXT: ushr v25.4s, v7.4s, #16 +; CHECK-NEXT: and v3.16b, v3.16b, v16.16b ; CHECK-NEXT: and v18.16b, v18.16b, v16.16b ; CHECK-NEXT: and v19.16b, v19.16b, v16.16b ; CHECK-NEXT: and v20.16b, v20.16b, v16.16b -; CHECK-NEXT: and v21.16b, v21.16b, v16.16b -; CHECK-NEXT: and v22.16b, v22.16b, v16.16b -; CHECK-NEXT: add v0.4s, v17.4s, v0.4s -; CHECK-NEXT: ushr v17.4s, v7.4s, #16 +; CHECK-NEXT: add v0.4s, v3.4s, v0.4s +; CHECK-NEXT: and v3.16b, v22.16b, v16.16b ; CHECK-NEXT: add v2.4s, v18.4s, v2.4s -; CHECK-NEXT: movi v18.4s, #127, msl #8 ; CHECK-NEXT: add v4.4s, v19.4s, v4.4s ; CHECK-NEXT: add v6.4s, v20.4s, v6.4s -; CHECK-NEXT: and v19.16b, v23.16b, v16.16b -; CHECK-NEXT: add v20.4s, v22.4s, v3.4s -; CHECK-NEXT: and v16.16b, v17.16b, v16.16b -; CHECK-NEXT: add v17.4s, v21.4s, v1.4s +; CHECK-NEXT: and v18.16b, v23.16b, v16.16b +; CHECK-NEXT: and v19.16b, v24.16b, v16.16b +; CHECK-NEXT: and v16.16b, v25.16b, v16.16b +; CHECK-NEXT: add v20.4s, v3.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v21.4s +; CHECK-NEXT: addhn v1.4h, v2.4s, v21.4s +; CHECK-NEXT: addhn v2.4h, v4.4s, v21.4s +; CHECK-NEXT: addhn v3.4h, v6.4s, v21.4s +; CHECK-NEXT: add v4.4s, v18.4s, v17.4s ; CHECK-NEXT: add v5.4s, v19.4s, v5.4s -; CHECK-NEXT: addhn v0.4h, v0.4s, v18.4s -; CHECK-NEXT: addhn v1.4h, v2.4s, v18.4s -; CHECK-NEXT: addhn v2.4h, v4.4s, v18.4s -; CHECK-NEXT: add v4.4s, v16.4s, v7.4s -; CHECK-NEXT: addhn v3.4h, v6.4s, v18.4s -; CHECK-NEXT: addhn2 v0.8h, v17.4s, v18.4s -; CHECK-NEXT: addhn2 v1.8h, v20.4s, v18.4s -; CHECK-NEXT: addhn2 v2.8h, v5.4s, v18.4s -; CHECK-NEXT: addhn2 v3.8h, v4.4s, v18.4s +; CHECK-NEXT: add v6.4s, v16.4s, v7.4s +; CHECK-NEXT: addhn2 v0.8h, v20.4s, v21.4s +; CHECK-NEXT: addhn2 v1.8h, v4.4s, v21.4s +; CHECK-NEXT: addhn2 v2.8h, v5.4s, v21.4s +; CHECK-NEXT: addhn2 v3.8h, v6.4s, v21.4s ; CHECK-NEXT: ret entry: %c = uitofp <32 x i32> %a to <32 x bfloat> @@ -7364,9 +7364,9 @@ define <2 x bfloat> @stofp_v2i16_v2bf16(<2 x i16> %a) { ; CHECK-NEXT: scvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = sitofp <2 x i16> %a to <2 x bfloat> @@ -7382,9 +7382,9 @@ define <2 x bfloat> @utofp_v2i16_v2bf16(<2 x i16> %a) { ; CHECK-NEXT: ucvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = uitofp <2 x i16> %a to <2 x bfloat> @@ -7399,9 +7399,9 @@ define <3 x bfloat> @stofp_v3i16_v3bf16(<3 x i16> %a) { ; CHECK-NEXT: scvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = sitofp <3 x i16> %a to <3 x bfloat> @@ -7416,9 +7416,9 @@ define <3 x bfloat> @utofp_v3i16_v3bf16(<3 x i16> %a) { ; CHECK-NEXT: ucvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = uitofp <3 x i16> %a to <3 x bfloat> @@ -7433,9 +7433,9 @@ define <4 x bfloat> @stofp_v4i16_v4bf16(<4 x i16> %a) { ; CHECK-NEXT: scvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = sitofp <4 x i16> %a to <4 x bfloat> @@ -7450,9 +7450,9 @@ define <4 x bfloat> @utofp_v4i16_v4bf16(<4 x i16> %a) { ; CHECK-NEXT: ucvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = uitofp <4 x i16> %a to <4 x bfloat> @@ -7513,27 +7513,27 @@ define <16 x bfloat> @stofp_v16i16_v16bf16(<16 x i16> %a) { ; CHECK-NEXT: sshll2 v0.4s, v0.8h, #0 ; CHECK-NEXT: sshll2 v1.4s, v1.8h, #0 ; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: movi v7.4s, #127, msl #8 ; CHECK-NEXT: scvtf v3.4s, v3.4s ; CHECK-NEXT: scvtf v4.4s, v4.4s -; CHECK-NEXT: scvtf v6.4s, v0.4s -; CHECK-NEXT: scvtf v7.4s, v1.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: ushr v5.4s, v3.4s, #16 -; CHECK-NEXT: ushr v0.4s, v4.4s, #16 -; CHECK-NEXT: ushr v16.4s, v6.4s, #16 -; CHECK-NEXT: ushr v17.4s, v7.4s, #16 -; CHECK-NEXT: and v5.16b, v5.16b, v2.16b +; CHECK-NEXT: scvtf v5.4s, v0.4s +; CHECK-NEXT: scvtf v6.4s, v1.4s +; CHECK-NEXT: ushr v0.4s, v3.4s, #16 +; CHECK-NEXT: ushr v1.4s, v4.4s, #16 +; CHECK-NEXT: ushr v16.4s, v5.4s, #16 +; CHECK-NEXT: ushr v17.4s, v6.4s, #16 ; CHECK-NEXT: and v0.16b, v0.16b, v2.16b +; CHECK-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-NEXT: and v16.16b, v16.16b, v2.16b ; CHECK-NEXT: and v2.16b, v17.16b, v2.16b -; CHECK-NEXT: add v5.4s, v5.4s, v1.4s -; CHECK-NEXT: add v18.4s, v0.4s, v1.4s -; CHECK-NEXT: add v2.4s, v2.4s, v1.4s -; CHECK-NEXT: addhn v0.4h, v3.4s, v5.4s -; CHECK-NEXT: add v3.4s, v16.4s, v1.4s -; CHECK-NEXT: addhn v1.4h, v4.4s, v18.4s -; CHECK-NEXT: addhn2 v0.8h, v6.4s, v3.4s -; CHECK-NEXT: addhn2 v1.8h, v7.4s, v2.4s +; CHECK-NEXT: add v0.4s, v0.4s, v7.4s +; CHECK-NEXT: add v1.4s, v1.4s, v7.4s +; CHECK-NEXT: add v2.4s, v2.4s, v7.4s +; CHECK-NEXT: addhn v0.4h, v3.4s, v0.4s +; CHECK-NEXT: addhn v1.4h, v4.4s, v1.4s +; CHECK-NEXT: add v3.4s, v16.4s, v7.4s +; CHECK-NEXT: addhn2 v0.8h, v5.4s, v3.4s +; CHECK-NEXT: addhn2 v1.8h, v6.4s, v2.4s ; CHECK-NEXT: ret entry: %c = sitofp <16 x i16> %a to <16 x bfloat> @@ -7548,27 +7548,27 @@ define <16 x bfloat> @utofp_v16i16_v16bf16(<16 x i16> %a) { ; CHECK-NEXT: ushll2 v0.4s, v0.8h, #0 ; CHECK-NEXT: ushll2 v1.4s, v1.8h, #0 ; CHECK-NEXT: movi v2.4s, #1 +; CHECK-NEXT: movi v7.4s, #127, msl #8 ; CHECK-NEXT: ucvtf v3.4s, v3.4s ; CHECK-NEXT: ucvtf v4.4s, v4.4s -; CHECK-NEXT: ucvtf v6.4s, v0.4s -; CHECK-NEXT: ucvtf v7.4s, v1.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: ushr v5.4s, v3.4s, #16 -; CHECK-NEXT: ushr v0.4s, v4.4s, #16 -; CHECK-NEXT: ushr v16.4s, v6.4s, #16 -; CHECK-NEXT: ushr v17.4s, v7.4s, #16 -; CHECK-NEXT: and v5.16b, v5.16b, v2.16b +; CHECK-NEXT: ucvtf v5.4s, v0.4s +; CHECK-NEXT: ucvtf v6.4s, v1.4s +; CHECK-NEXT: ushr v0.4s, v3.4s, #16 +; CHECK-NEXT: ushr v1.4s, v4.4s, #16 +; CHECK-NEXT: ushr v16.4s, v5.4s, #16 +; CHECK-NEXT: ushr v17.4s, v6.4s, #16 ; CHECK-NEXT: and v0.16b, v0.16b, v2.16b +; CHECK-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-NEXT: and v16.16b, v16.16b, v2.16b ; CHECK-NEXT: and v2.16b, v17.16b, v2.16b -; CHECK-NEXT: add v5.4s, v5.4s, v1.4s -; CHECK-NEXT: add v18.4s, v0.4s, v1.4s -; CHECK-NEXT: add v2.4s, v2.4s, v1.4s -; CHECK-NEXT: addhn v0.4h, v3.4s, v5.4s -; CHECK-NEXT: add v3.4s, v16.4s, v1.4s -; CHECK-NEXT: addhn v1.4h, v4.4s, v18.4s -; CHECK-NEXT: addhn2 v0.8h, v6.4s, v3.4s -; CHECK-NEXT: addhn2 v1.8h, v7.4s, v2.4s +; CHECK-NEXT: add v0.4s, v0.4s, v7.4s +; CHECK-NEXT: add v1.4s, v1.4s, v7.4s +; CHECK-NEXT: add v2.4s, v2.4s, v7.4s +; CHECK-NEXT: addhn v0.4h, v3.4s, v0.4s +; CHECK-NEXT: addhn v1.4h, v4.4s, v1.4s +; CHECK-NEXT: add v3.4s, v16.4s, v7.4s +; CHECK-NEXT: addhn2 v0.8h, v5.4s, v3.4s +; CHECK-NEXT: addhn2 v1.8h, v6.4s, v2.4s ; CHECK-NEXT: ret entry: %c = uitofp <16 x i16> %a to <16 x bfloat> @@ -7578,56 +7578,56 @@ entry: define <32 x bfloat> @stofp_v32i16_v32bf16(<32 x i16> %a) { ; CHECK-LABEL: stofp_v32i16_v32bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: sshll v5.4s, v1.4h, #0 -; CHECK-NEXT: sshll v4.4s, v0.4h, #0 +; CHECK-NEXT: sshll v4.4s, v1.4h, #0 +; CHECK-NEXT: sshll v5.4s, v0.4h, #0 ; CHECK-NEXT: sshll v6.4s, v2.4h, #0 ; CHECK-NEXT: sshll v7.4s, v3.4h, #0 ; CHECK-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-NEXT: movi v16.4s, #1 ; CHECK-NEXT: sshll2 v1.4s, v1.8h, #0 ; CHECK-NEXT: sshll2 v2.4s, v2.8h, #0 ; CHECK-NEXT: sshll2 v3.4s, v3.8h, #0 +; CHECK-NEXT: movi v16.4s, #1 ; CHECK-NEXT: scvtf v5.4s, v5.4s ; CHECK-NEXT: scvtf v4.4s, v4.4s ; CHECK-NEXT: scvtf v6.4s, v6.4s ; CHECK-NEXT: scvtf v7.4s, v7.4s -; CHECK-NEXT: scvtf v19.4s, v0.4s -; CHECK-NEXT: movi v18.4s, #127, msl #8 -; CHECK-NEXT: scvtf v20.4s, v1.4s -; CHECK-NEXT: scvtf v21.4s, v2.4s -; CHECK-NEXT: scvtf v22.4s, v3.4s +; CHECK-NEXT: scvtf v17.4s, v0.4s +; CHECK-NEXT: scvtf v18.4s, v1.4s +; CHECK-NEXT: scvtf v19.4s, v2.4s +; CHECK-NEXT: scvtf v20.4s, v3.4s +; CHECK-NEXT: movi v21.4s, #127, msl #8 ; CHECK-NEXT: ushr v0.4s, v5.4s, #16 -; CHECK-NEXT: ushr v17.4s, v4.4s, #16 -; CHECK-NEXT: ushr v1.4s, v6.4s, #16 -; CHECK-NEXT: ushr v2.4s, v7.4s, #16 -; CHECK-NEXT: ushr v23.4s, v20.4s, #16 -; CHECK-NEXT: ushr v25.4s, v22.4s, #16 +; CHECK-NEXT: ushr v1.4s, v4.4s, #16 +; CHECK-NEXT: ushr v2.4s, v6.4s, #16 +; CHECK-NEXT: ushr v3.4s, v7.4s, #16 +; CHECK-NEXT: ushr v22.4s, v17.4s, #16 +; CHECK-NEXT: ushr v23.4s, v18.4s, #16 +; CHECK-NEXT: ushr v24.4s, v19.4s, #16 +; CHECK-NEXT: ushr v25.4s, v20.4s, #16 ; CHECK-NEXT: and v0.16b, v0.16b, v16.16b -; CHECK-NEXT: and v3.16b, v17.16b, v16.16b ; CHECK-NEXT: and v1.16b, v1.16b, v16.16b ; CHECK-NEXT: and v2.16b, v2.16b, v16.16b -; CHECK-NEXT: ushr v17.4s, v19.4s, #16 +; CHECK-NEXT: and v3.16b, v3.16b, v16.16b +; CHECK-NEXT: and v22.16b, v22.16b, v16.16b ; CHECK-NEXT: and v23.16b, v23.16b, v16.16b -; CHECK-NEXT: add v24.4s, v0.4s, v18.4s -; CHECK-NEXT: ushr v0.4s, v21.4s, #16 -; CHECK-NEXT: add v3.4s, v3.4s, v18.4s -; CHECK-NEXT: add v26.4s, v1.4s, v18.4s -; CHECK-NEXT: add v27.4s, v2.4s, v18.4s -; CHECK-NEXT: and v17.16b, v17.16b, v16.16b -; CHECK-NEXT: and v28.16b, v0.16b, v16.16b +; CHECK-NEXT: and v24.16b, v24.16b, v16.16b ; CHECK-NEXT: and v16.16b, v25.16b, v16.16b -; CHECK-NEXT: addhn v0.4h, v4.4s, v3.4s -; CHECK-NEXT: addhn v1.4h, v5.4s, v24.4s -; CHECK-NEXT: add v4.4s, v17.4s, v18.4s -; CHECK-NEXT: addhn v2.4h, v6.4s, v26.4s -; CHECK-NEXT: add v5.4s, v23.4s, v18.4s -; CHECK-NEXT: addhn v3.4h, v7.4s, v27.4s -; CHECK-NEXT: add v6.4s, v28.4s, v18.4s -; CHECK-NEXT: add v16.4s, v16.4s, v18.4s -; CHECK-NEXT: addhn2 v0.8h, v19.4s, v4.4s -; CHECK-NEXT: addhn2 v1.8h, v20.4s, v5.4s -; CHECK-NEXT: addhn2 v2.8h, v21.4s, v6.4s -; CHECK-NEXT: addhn2 v3.8h, v22.4s, v16.4s +; CHECK-NEXT: add v0.4s, v0.4s, v21.4s +; CHECK-NEXT: add v1.4s, v1.4s, v21.4s +; CHECK-NEXT: add v2.4s, v2.4s, v21.4s +; CHECK-NEXT: add v3.4s, v3.4s, v21.4s +; CHECK-NEXT: addhn v0.4h, v5.4s, v0.4s +; CHECK-NEXT: addhn v1.4h, v4.4s, v1.4s +; CHECK-NEXT: addhn v2.4h, v6.4s, v2.4s +; CHECK-NEXT: addhn v3.4h, v7.4s, v3.4s +; CHECK-NEXT: add v4.4s, v22.4s, v21.4s +; CHECK-NEXT: add v5.4s, v23.4s, v21.4s +; CHECK-NEXT: add v6.4s, v24.4s, v21.4s +; CHECK-NEXT: add v7.4s, v16.4s, v21.4s +; CHECK-NEXT: addhn2 v0.8h, v17.4s, v4.4s +; CHECK-NEXT: addhn2 v1.8h, v18.4s, v5.4s +; CHECK-NEXT: addhn2 v2.8h, v19.4s, v6.4s +; CHECK-NEXT: addhn2 v3.8h, v20.4s, v7.4s ; CHECK-NEXT: ret entry: %c = sitofp <32 x i16> %a to <32 x bfloat> @@ -7637,56 +7637,56 @@ entry: define <32 x bfloat> @utofp_v32i16_v32bf16(<32 x i16> %a) { ; CHECK-LABEL: utofp_v32i16_v32bf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ushll v5.4s, v1.4h, #0 -; CHECK-NEXT: ushll v4.4s, v0.4h, #0 +; CHECK-NEXT: ushll v4.4s, v1.4h, #0 +; CHECK-NEXT: ushll v5.4s, v0.4h, #0 ; CHECK-NEXT: ushll v6.4s, v2.4h, #0 ; CHECK-NEXT: ushll v7.4s, v3.4h, #0 ; CHECK-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-NEXT: movi v16.4s, #1 ; CHECK-NEXT: ushll2 v1.4s, v1.8h, #0 ; CHECK-NEXT: ushll2 v2.4s, v2.8h, #0 ; CHECK-NEXT: ushll2 v3.4s, v3.8h, #0 +; CHECK-NEXT: movi v16.4s, #1 ; CHECK-NEXT: ucvtf v5.4s, v5.4s ; CHECK-NEXT: ucvtf v4.4s, v4.4s ; CHECK-NEXT: ucvtf v6.4s, v6.4s ; CHECK-NEXT: ucvtf v7.4s, v7.4s -; CHECK-NEXT: ucvtf v19.4s, v0.4s -; CHECK-NEXT: movi v18.4s, #127, msl #8 -; CHECK-NEXT: ucvtf v20.4s, v1.4s -; CHECK-NEXT: ucvtf v21.4s, v2.4s -; CHECK-NEXT: ucvtf v22.4s, v3.4s +; CHECK-NEXT: ucvtf v17.4s, v0.4s +; CHECK-NEXT: ucvtf v18.4s, v1.4s +; CHECK-NEXT: ucvtf v19.4s, v2.4s +; CHECK-NEXT: ucvtf v20.4s, v3.4s +; CHECK-NEXT: movi v21.4s, #127, msl #8 ; CHECK-NEXT: ushr v0.4s, v5.4s, #16 -; CHECK-NEXT: ushr v17.4s, v4.4s, #16 -; CHECK-NEXT: ushr v1.4s, v6.4s, #16 -; CHECK-NEXT: ushr v2.4s, v7.4s, #16 -; CHECK-NEXT: ushr v23.4s, v20.4s, #16 -; CHECK-NEXT: ushr v25.4s, v22.4s, #16 +; CHECK-NEXT: ushr v1.4s, v4.4s, #16 +; CHECK-NEXT: ushr v2.4s, v6.4s, #16 +; CHECK-NEXT: ushr v3.4s, v7.4s, #16 +; CHECK-NEXT: ushr v22.4s, v17.4s, #16 +; CHECK-NEXT: ushr v23.4s, v18.4s, #16 +; CHECK-NEXT: ushr v24.4s, v19.4s, #16 +; CHECK-NEXT: ushr v25.4s, v20.4s, #16 ; CHECK-NEXT: and v0.16b, v0.16b, v16.16b -; CHECK-NEXT: and v3.16b, v17.16b, v16.16b ; CHECK-NEXT: and v1.16b, v1.16b, v16.16b ; CHECK-NEXT: and v2.16b, v2.16b, v16.16b -; CHECK-NEXT: ushr v17.4s, v19.4s, #16 +; CHECK-NEXT: and v3.16b, v3.16b, v16.16b +; CHECK-NEXT: and v22.16b, v22.16b, v16.16b ; CHECK-NEXT: and v23.16b, v23.16b, v16.16b -; CHECK-NEXT: add v24.4s, v0.4s, v18.4s -; CHECK-NEXT: ushr v0.4s, v21.4s, #16 -; CHECK-NEXT: add v3.4s, v3.4s, v18.4s -; CHECK-NEXT: add v26.4s, v1.4s, v18.4s -; CHECK-NEXT: add v27.4s, v2.4s, v18.4s -; CHECK-NEXT: and v17.16b, v17.16b, v16.16b -; CHECK-NEXT: and v28.16b, v0.16b, v16.16b +; CHECK-NEXT: and v24.16b, v24.16b, v16.16b ; CHECK-NEXT: and v16.16b, v25.16b, v16.16b -; CHECK-NEXT: addhn v0.4h, v4.4s, v3.4s -; CHECK-NEXT: addhn v1.4h, v5.4s, v24.4s -; CHECK-NEXT: add v4.4s, v17.4s, v18.4s -; CHECK-NEXT: addhn v2.4h, v6.4s, v26.4s -; CHECK-NEXT: add v5.4s, v23.4s, v18.4s -; CHECK-NEXT: addhn v3.4h, v7.4s, v27.4s -; CHECK-NEXT: add v6.4s, v28.4s, v18.4s -; CHECK-NEXT: add v16.4s, v16.4s, v18.4s -; CHECK-NEXT: addhn2 v0.8h, v19.4s, v4.4s -; CHECK-NEXT: addhn2 v1.8h, v20.4s, v5.4s -; CHECK-NEXT: addhn2 v2.8h, v21.4s, v6.4s -; CHECK-NEXT: addhn2 v3.8h, v22.4s, v16.4s +; CHECK-NEXT: add v0.4s, v0.4s, v21.4s +; CHECK-NEXT: add v1.4s, v1.4s, v21.4s +; CHECK-NEXT: add v2.4s, v2.4s, v21.4s +; CHECK-NEXT: add v3.4s, v3.4s, v21.4s +; CHECK-NEXT: addhn v0.4h, v5.4s, v0.4s +; CHECK-NEXT: addhn v1.4h, v4.4s, v1.4s +; CHECK-NEXT: addhn v2.4h, v6.4s, v2.4s +; CHECK-NEXT: addhn v3.4h, v7.4s, v3.4s +; CHECK-NEXT: add v4.4s, v22.4s, v21.4s +; CHECK-NEXT: add v5.4s, v23.4s, v21.4s +; CHECK-NEXT: add v6.4s, v24.4s, v21.4s +; CHECK-NEXT: add v7.4s, v16.4s, v21.4s +; CHECK-NEXT: addhn2 v0.8h, v17.4s, v4.4s +; CHECK-NEXT: addhn2 v1.8h, v18.4s, v5.4s +; CHECK-NEXT: addhn2 v2.8h, v19.4s, v6.4s +; CHECK-NEXT: addhn2 v3.8h, v20.4s, v7.4s ; CHECK-NEXT: ret entry: %c = uitofp <32 x i16> %a to <32 x bfloat> @@ -7768,9 +7768,9 @@ define <3 x bfloat> @stofp_v3i8_v3bf16(<3 x i8> %a) { ; CHECK-NEXT: scvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = sitofp <3 x i8> %a to <3 x bfloat> @@ -7789,9 +7789,9 @@ define <3 x bfloat> @utofp_v3i8_v3bf16(<3 x i8> %a) { ; CHECK-NEXT: ucvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = uitofp <3 x i8> %a to <3 x bfloat> @@ -7808,9 +7808,9 @@ define <4 x bfloat> @stofp_v4i8_v4bf16(<4 x i8> %a) { ; CHECK-NEXT: scvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = sitofp <4 x i8> %a to <4 x bfloat> @@ -7826,9 +7826,9 @@ define <4 x bfloat> @utofp_v4i8_v4bf16(<4 x i8> %a) { ; CHECK-NEXT: ucvtf v0.4s, v0.4s ; CHECK-NEXT: ushr v2.4s, v0.4s, #16 ; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: movi v2.4s, #127, msl #8 ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: movi v1.4s, #127, msl #8 -; CHECK-NEXT: addhn v0.4h, v0.4s, v1.4s +; CHECK-NEXT: addhn v0.4h, v0.4s, v2.4s ; CHECK-NEXT: ret entry: %c = uitofp <4 x i8> %a to <4 x bfloat> @@ -7909,11 +7909,11 @@ define <16 x bfloat> @stofp_v16i8_v16bf16(<16 x i8> %a) { ; CHECK-NEXT: add v5.4s, v5.4s, v7.4s ; CHECK-NEXT: add v0.4s, v0.4s, v7.4s ; CHECK-NEXT: addhn v1.4h, v3.4s, v5.4s -; CHECK-NEXT: add v3.4s, v16.4s, v7.4s -; CHECK-NEXT: add v5.4s, v17.4s, v7.4s ; CHECK-NEXT: addhn v0.4h, v4.4s, v0.4s +; CHECK-NEXT: add v3.4s, v16.4s, v7.4s +; CHECK-NEXT: add v4.4s, v17.4s, v7.4s ; CHECK-NEXT: addhn2 v1.8h, v2.4s, v3.4s -; CHECK-NEXT: addhn2 v0.8h, v6.4s, v5.4s +; CHECK-NEXT: addhn2 v0.8h, v6.4s, v4.4s ; CHECK-NEXT: ret entry: %c = sitofp <16 x i8> %a to <16 x bfloat> @@ -7946,11 +7946,11 @@ define <16 x bfloat> @utofp_v16i8_v16bf16(<16 x i8> %a) { ; CHECK-NEXT: add v5.4s, v5.4s, v7.4s ; CHECK-NEXT: add v0.4s, v0.4s, v7.4s ; CHECK-NEXT: addhn v1.4h, v3.4s, v5.4s -; CHECK-NEXT: add v3.4s, v16.4s, v7.4s -; CHECK-NEXT: add v5.4s, v17.4s, v7.4s ; CHECK-NEXT: addhn v0.4h, v4.4s, v0.4s +; CHECK-NEXT: add v3.4s, v16.4s, v7.4s +; CHECK-NEXT: add v4.4s, v17.4s, v7.4s ; CHECK-NEXT: addhn2 v1.8h, v2.4s, v3.4s -; CHECK-NEXT: addhn2 v0.8h, v6.4s, v5.4s +; CHECK-NEXT: addhn2 v0.8h, v6.4s, v4.4s ; CHECK-NEXT: ret entry: %c = uitofp <16 x i8> %a to <16 x bfloat> @@ -7961,14 +7961,14 @@ define <32 x bfloat> @stofp_v32i8_v32bf16(<32 x i8> %a) { ; CHECK-LABEL: stofp_v32i8_v32bf16: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: sshll2 v3.8h, v0.16b, #0 -; CHECK-NEXT: sshll2 v4.8h, v1.16b, #0 ; CHECK-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-NEXT: sshll2 v4.8h, v1.16b, #0 ; CHECK-NEXT: sshll v1.8h, v1.8b, #0 ; CHECK-NEXT: movi v2.4s, #1 -; CHECK-NEXT: movi v20.4s, #127, msl #8 +; CHECK-NEXT: movi v21.4s, #127, msl #8 ; CHECK-NEXT: sshll v5.4s, v3.4h, #0 -; CHECK-NEXT: sshll v6.4s, v4.4h, #0 -; CHECK-NEXT: sshll v7.4s, v0.4h, #0 +; CHECK-NEXT: sshll v6.4s, v0.4h, #0 +; CHECK-NEXT: sshll v7.4s, v4.4h, #0 ; CHECK-NEXT: sshll v16.4s, v1.4h, #0 ; CHECK-NEXT: sshll2 v3.4s, v3.8h, #0 ; CHECK-NEXT: sshll2 v4.4s, v4.8h, #0 @@ -7980,40 +7980,40 @@ define <32 x bfloat> @stofp_v32i8_v32bf16(<32 x i8> %a) { ; CHECK-NEXT: scvtf v16.4s, v16.4s ; CHECK-NEXT: scvtf v17.4s, v3.4s ; CHECK-NEXT: scvtf v4.4s, v4.4s -; CHECK-NEXT: scvtf v19.4s, v0.4s -; CHECK-NEXT: scvtf v21.4s, v1.4s -; CHECK-NEXT: ushr v3.4s, v5.4s, #16 -; CHECK-NEXT: ushr v18.4s, v6.4s, #16 -; CHECK-NEXT: ushr v0.4s, v7.4s, #16 -; CHECK-NEXT: ushr v1.4s, v16.4s, #16 -; CHECK-NEXT: ushr v22.4s, v17.4s, #16 -; CHECK-NEXT: ushr v23.4s, v4.4s, #16 -; CHECK-NEXT: ushr v24.4s, v19.4s, #16 -; CHECK-NEXT: ushr v25.4s, v21.4s, #16 -; CHECK-NEXT: and v3.16b, v3.16b, v2.16b -; CHECK-NEXT: and v18.16b, v18.16b, v2.16b +; CHECK-NEXT: scvtf v18.4s, v0.4s +; CHECK-NEXT: scvtf v19.4s, v1.4s +; CHECK-NEXT: ushr v0.4s, v5.4s, #16 +; CHECK-NEXT: ushr v3.4s, v6.4s, #16 +; CHECK-NEXT: ushr v1.4s, v7.4s, #16 +; CHECK-NEXT: ushr v20.4s, v16.4s, #16 +; CHECK-NEXT: ushr v23.4s, v17.4s, #16 +; CHECK-NEXT: ushr v24.4s, v4.4s, #16 +; CHECK-NEXT: ushr v22.4s, v18.4s, #16 +; CHECK-NEXT: ushr v25.4s, v19.4s, #16 ; CHECK-NEXT: and v0.16b, v0.16b, v2.16b +; CHECK-NEXT: and v3.16b, v3.16b, v2.16b ; CHECK-NEXT: and v1.16b, v1.16b, v2.16b -; CHECK-NEXT: and v22.16b, v22.16b, v2.16b +; CHECK-NEXT: and v20.16b, v20.16b, v2.16b ; CHECK-NEXT: and v23.16b, v23.16b, v2.16b ; CHECK-NEXT: and v24.16b, v24.16b, v2.16b -; CHECK-NEXT: and v2.16b, v25.16b, v2.16b -; CHECK-NEXT: add v3.4s, v3.4s, v20.4s -; CHECK-NEXT: add v18.4s, v18.4s, v20.4s -; CHECK-NEXT: add v0.4s, v0.4s, v20.4s -; CHECK-NEXT: add v26.4s, v1.4s, v20.4s -; CHECK-NEXT: addhn v1.4h, v5.4s, v3.4s -; CHECK-NEXT: addhn v3.4h, v6.4s, v18.4s -; CHECK-NEXT: addhn v0.4h, v7.4s, v0.4s -; CHECK-NEXT: add v5.4s, v22.4s, v20.4s -; CHECK-NEXT: add v6.4s, v24.4s, v20.4s -; CHECK-NEXT: add v7.4s, v23.4s, v20.4s -; CHECK-NEXT: add v18.4s, v2.4s, v20.4s -; CHECK-NEXT: addhn v2.4h, v16.4s, v26.4s -; CHECK-NEXT: addhn2 v0.8h, v19.4s, v6.4s -; CHECK-NEXT: addhn2 v1.8h, v17.4s, v5.4s +; CHECK-NEXT: and v22.16b, v22.16b, v2.16b +; CHECK-NEXT: and v25.16b, v25.16b, v2.16b +; CHECK-NEXT: add v0.4s, v0.4s, v21.4s +; CHECK-NEXT: add v3.4s, v3.4s, v21.4s +; CHECK-NEXT: add v26.4s, v1.4s, v21.4s +; CHECK-NEXT: add v20.4s, v20.4s, v21.4s +; CHECK-NEXT: addhn v1.4h, v5.4s, v0.4s +; CHECK-NEXT: addhn v0.4h, v6.4s, v3.4s +; CHECK-NEXT: addhn v3.4h, v7.4s, v26.4s +; CHECK-NEXT: addhn v2.4h, v16.4s, v20.4s +; CHECK-NEXT: add v5.4s, v22.4s, v21.4s +; CHECK-NEXT: add v6.4s, v23.4s, v21.4s +; CHECK-NEXT: add v7.4s, v24.4s, v21.4s +; CHECK-NEXT: add v16.4s, v25.4s, v21.4s +; CHECK-NEXT: addhn2 v0.8h, v18.4s, v5.4s +; CHECK-NEXT: addhn2 v1.8h, v17.4s, v6.4s ; CHECK-NEXT: addhn2 v3.8h, v4.4s, v7.4s -; CHECK-NEXT: addhn2 v2.8h, v21.4s, v18.4s +; CHECK-NEXT: addhn2 v2.8h, v19.4s, v16.4s ; CHECK-NEXT: ret entry: %c = sitofp <32 x i8> %a to <32 x bfloat> @@ -8024,14 +8024,14 @@ define <32 x bfloat> @utofp_v32i8_v32bf16(<32 x i8> %a) { ; CHECK-LABEL: utofp_v32i8_v32bf16: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ushll2 v3.8h, v0.16b, #0 -; CHECK-NEXT: ushll2 v4.8h, v1.16b, #0 ; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ushll2 v4.8h, v1.16b, #0 ; CHECK-NEXT: ushll v1.8h, v1.8b, #0 ; CHECK-NEXT: movi v2.4s, #1 -; CHECK-NEXT: movi v20.4s, #127, msl #8 +; CHECK-NEXT: movi v21.4s, #127, msl #8 ; CHECK-NEXT: ushll v5.4s, v3.4h, #0 -; CHECK-NEXT: ushll v6.4s, v4.4h, #0 -; CHECK-NEXT: ushll v7.4s, v0.4h, #0 +; CHECK-NEXT: ushll v6.4s, v0.4h, #0 +; CHECK-NEXT: ushll v7.4s, v4.4h, #0 ; CHECK-NEXT: ushll v16.4s, v1.4h, #0 ; CHECK-NEXT: ushll2 v3.4s, v3.8h, #0 ; CHECK-NEXT: ushll2 v4.4s, v4.8h, #0 @@ -8043,40 +8043,40 @@ define <32 x bfloat> @utofp_v32i8_v32bf16(<32 x i8> %a) { ; CHECK-NEXT: ucvtf v16.4s, v16.4s ; CHECK-NEXT: ucvtf v17.4s, v3.4s ; CHECK-NEXT: ucvtf v4.4s, v4.4s -; CHECK-NEXT: ucvtf v19.4s, v0.4s -; CHECK-NEXT: ucvtf v21.4s, v1.4s -; CHECK-NEXT: ushr v3.4s, v5.4s, #16 -; CHECK-NEXT: ushr v18.4s, v6.4s, #16 -; CHECK-NEXT: ushr v0.4s, v7.4s, #16 -; CHECK-NEXT: ushr v1.4s, v16.4s, #16 -; CHECK-NEXT: ushr v22.4s, v17.4s, #16 -; CHECK-NEXT: ushr v23.4s, v4.4s, #16 -; CHECK-NEXT: ushr v24.4s, v19.4s, #16 -; CHECK-NEXT: ushr v25.4s, v21.4s, #16 -; CHECK-NEXT: and v3.16b, v3.16b, v2.16b -; CHECK-NEXT: and v18.16b, v18.16b, v2.16b +; CHECK-NEXT: ucvtf v18.4s, v0.4s +; CHECK-NEXT: ucvtf v19.4s, v1.4s +; CHECK-NEXT: ushr v0.4s, v5.4s, #16 +; CHECK-NEXT: ushr v3.4s, v6.4s, #16 +; CHECK-NEXT: ushr v1.4s, v7.4s, #16 +; CHECK-NEXT: ushr v20.4s, v16.4s, #16 +; CHECK-NEXT: ushr v23.4s, v17.4s, #16 +; CHECK-NEXT: ushr v24.4s, v4.4s, #16 +; CHECK-NEXT: ushr v22.4s, v18.4s, #16 +; CHECK-NEXT: ushr v25.4s, v19.4s, #16 ; CHECK-NEXT: and v0.16b, v0.16b, v2.16b +; CHECK-NEXT: and v3.16b, v3.16b, v2.16b ; CHECK-NEXT: and v1.16b, v1.16b, v2.16b -; CHECK-NEXT: and v22.16b, v22.16b, v2.16b +; CHECK-NEXT: and v20.16b, v20.16b, v2.16b ; CHECK-NEXT: and v23.16b, v23.16b, v2.16b ; CHECK-NEXT: and v24.16b, v24.16b, v2.16b -; CHECK-NEXT: and v2.16b, v25.16b, v2.16b -; CHECK-NEXT: add v3.4s, v3.4s, v20.4s -; CHECK-NEXT: add v18.4s, v18.4s, v20.4s -; CHECK-NEXT: add v0.4s, v0.4s, v20.4s -; CHECK-NEXT: add v26.4s, v1.4s, v20.4s -; CHECK-NEXT: addhn v1.4h, v5.4s, v3.4s -; CHECK-NEXT: addhn v3.4h, v6.4s, v18.4s -; CHECK-NEXT: addhn v0.4h, v7.4s, v0.4s -; CHECK-NEXT: add v5.4s, v22.4s, v20.4s -; CHECK-NEXT: add v6.4s, v24.4s, v20.4s -; CHECK-NEXT: add v7.4s, v23.4s, v20.4s -; CHECK-NEXT: add v18.4s, v2.4s, v20.4s -; CHECK-NEXT: addhn v2.4h, v16.4s, v26.4s -; CHECK-NEXT: addhn2 v0.8h, v19.4s, v6.4s -; CHECK-NEXT: addhn2 v1.8h, v17.4s, v5.4s +; CHECK-NEXT: and v22.16b, v22.16b, v2.16b +; CHECK-NEXT: and v25.16b, v25.16b, v2.16b +; CHECK-NEXT: add v0.4s, v0.4s, v21.4s +; CHECK-NEXT: add v3.4s, v3.4s, v21.4s +; CHECK-NEXT: add v26.4s, v1.4s, v21.4s +; CHECK-NEXT: add v20.4s, v20.4s, v21.4s +; CHECK-NEXT: addhn v1.4h, v5.4s, v0.4s +; CHECK-NEXT: addhn v0.4h, v6.4s, v3.4s +; CHECK-NEXT: addhn v3.4h, v7.4s, v26.4s +; CHECK-NEXT: addhn v2.4h, v16.4s, v20.4s +; CHECK-NEXT: add v5.4s, v22.4s, v21.4s +; CHECK-NEXT: add v6.4s, v23.4s, v21.4s +; CHECK-NEXT: add v7.4s, v24.4s, v21.4s +; CHECK-NEXT: add v16.4s, v25.4s, v21.4s +; CHECK-NEXT: addhn2 v0.8h, v18.4s, v5.4s +; CHECK-NEXT: addhn2 v1.8h, v17.4s, v6.4s ; CHECK-NEXT: addhn2 v3.8h, v4.4s, v7.4s -; CHECK-NEXT: addhn2 v2.8h, v21.4s, v18.4s +; CHECK-NEXT: addhn2 v2.8h, v19.4s, v16.4s ; CHECK-NEXT: ret entry: %c = uitofp <32 x i8> %a to <32 x bfloat> diff --git a/llvm/test/CodeGen/AArch64/ldexp.ll b/llvm/test/CodeGen/AArch64/ldexp.ll index 4b491051a88a..ba04ba1d7bb6 100644 --- a/llvm/test/CodeGen/AArch64/ldexp.ll +++ b/llvm/test/CodeGen/AArch64/ldexp.ll @@ -4,9 +4,9 @@ define double @testExp(double %val, i32 %a) { ; CHECK-LABEL: testExp: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: sxtw x8, w0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: fmov d1, x8 ; CHECK-NEXT: fscale z0.d, p0/m, z0.d, z1.d @@ -22,8 +22,8 @@ declare double @ldexp(double, i32) memory(none) define float @testExpf(float %val, i32 %a) { ; CHECK-LABEL: testExpf: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: fmov s1, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: // kill: def $s0 killed $s0 def $z0 ; CHECK-NEXT: fscale z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: // kill: def $s0 killed $s0 killed $z0 @@ -49,9 +49,9 @@ declare fp128 @ldexpl(fp128, i32) memory(none) define half @testExpf16(half %val, i32 %a) { ; CHECK-LABEL: testExpf16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: fcvt s0, h0 ; CHECK-NEXT: fmov s1, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: fscale z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: fcvt h0, s0 ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll b/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll index ab15bf564ec4..59a460923e8b 100644 --- a/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll +++ b/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll @@ -10,9 +10,9 @@ define @sdiv_i8( %a, %b) ; CHECK: // %bb.0: ; CHECK-NEXT: sunpkhi z2.h, z1.b ; CHECK-NEXT: sunpkhi z3.h, z0.b -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpklo z1.h, z1.b ; CHECK-NEXT: sunpklo z0.h, z0.b +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpkhi z4.s, z2.h ; CHECK-NEXT: sunpkhi z5.s, z3.h ; CHECK-NEXT: sunpklo z2.s, z2.h @@ -36,11 +36,11 @@ define @sdiv_i8( %a, %b) define @sdiv_i16( %a, %b) { ; CHECK-LABEL: sdiv_i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpkhi z2.s, z1.h ; CHECK-NEXT: sunpkhi z3.s, z0.h ; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: sunpklo z0.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h @@ -140,9 +140,9 @@ define @srem_i8( %a, %b) define @srem_i16( %a, %b) { ; CHECK-LABEL: srem_i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpkhi z2.s, z1.h ; CHECK-NEXT: sunpkhi z3.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpklo z4.s, z0.h ; CHECK-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; CHECK-NEXT: sunpklo z3.s, z1.h @@ -188,9 +188,9 @@ define @udiv_i8( %a, %b) ; CHECK: // %bb.0: ; CHECK-NEXT: uunpkhi z2.h, z1.b ; CHECK-NEXT: uunpkhi z3.h, z0.b -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpklo z1.h, z1.b ; CHECK-NEXT: uunpklo z0.h, z0.b +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpkhi z4.s, z2.h ; CHECK-NEXT: uunpkhi z5.s, z3.h ; CHECK-NEXT: uunpklo z2.s, z2.h @@ -214,11 +214,11 @@ define @udiv_i8( %a, %b) define @udiv_i16( %a, %b) { ; CHECK-LABEL: udiv_i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpkhi z2.s, z1.h ; CHECK-NEXT: uunpkhi z3.s, z0.h ; CHECK-NEXT: uunpklo z1.s, z1.h ; CHECK-NEXT: uunpklo z0.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h @@ -261,9 +261,9 @@ define @udiv_split_i32( %a, @udiv_widen_i32( %a, %b) { ; CHECK-LABEL: udiv_widen_i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z1.d, z1.d, #0xffffffff ; CHECK-NEXT: and z0.d, z0.d, #0xffffffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: udiv z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %div = udiv %a, %b @@ -319,9 +319,9 @@ define @urem_i8( %a, %b) define @urem_i16( %a, %b) { ; CHECK-LABEL: urem_i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpkhi z2.s, z1.h ; CHECK-NEXT: uunpkhi z3.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpklo z4.s, z0.h ; CHECK-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; CHECK-NEXT: uunpklo z3.s, z1.h @@ -558,9 +558,9 @@ define @umin_split_i64( %a, @umin_promote_i8( %a, %b) { ; CHECK-LABEL: umin_promote_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and z1.h, z1.h, #0xff ; CHECK-NEXT: and z0.h, z0.h, #0xff +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: umin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %cmp = icmp ult %a, %b @@ -704,9 +704,9 @@ define @umax_split_i16( %a, @umax_promote_i32( %a, %b) { ; CHECK-LABEL: umax_promote_i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z1.d, z1.d, #0xffffffff ; CHECK-NEXT: and z0.d, z0.d, #0xffffffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: umax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %cmp = icmp ugt %a, %b @@ -883,8 +883,8 @@ define @lsl_split_i64( %a, @lsl_promote_i16( %a, %b){ ; CHECK-LABEL: lsl_promote_i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z1.s, z1.s, #0xffff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: lsl z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %shl = shl %a, %b @@ -982,9 +982,9 @@ define @lsr_i64( %a, %b) define @lsr_promote_i8( %a, %b){ ; CHECK-LABEL: lsr_promote_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and z1.h, z1.h, #0xff ; CHECK-NEXT: and z0.h, z0.h, #0xff +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: lsr z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %shr = lshr %a, %b @@ -1081,10 +1081,10 @@ declare @llvm.fshr.nxv2i64(, @fshl_i64( %a, %b, %c){ ; CHECK-LABEL: fshl_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z3.d, #63 // =0x3f ; CHECK-NEXT: mov z4.d, z2.d ; CHECK-NEXT: lsr z1.d, z1.d, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: bic z2.d, z3.d, z2.d ; CHECK-NEXT: and z4.d, z4.d, #0x3f ; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z4.d @@ -1098,17 +1098,16 @@ define @fshl_i64( %a, %b define @fshl_illegal_i64( %a, %b, %c){ ; CHECK-LABEL: fshl_illegal_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z6.d, #63 // =0x3f -; CHECK-NEXT: mov z7.d, z4.d ; CHECK-NEXT: lsr z2.d, z2.d, #1 ; CHECK-NEXT: lsr z3.d, z3.d, #1 -; CHECK-NEXT: bic z4.d, z6.d, z4.d -; CHECK-NEXT: and z7.d, z7.d, #0x3f +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: bic z7.d, z6.d, z4.d +; CHECK-NEXT: and z4.d, z4.d, #0x3f ; CHECK-NEXT: bic z6.d, z6.d, z5.d ; CHECK-NEXT: and z5.d, z5.d, #0x3f -; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z7.d -; CHECK-NEXT: lsr z2.d, p0/m, z2.d, z4.d +; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z4.d +; CHECK-NEXT: lsr z2.d, p0/m, z2.d, z7.d ; CHECK-NEXT: lsr z3.d, p0/m, z3.d, z6.d ; CHECK-NEXT: lsl z1.d, p0/m, z1.d, z5.d ; CHECK-NEXT: orr z0.d, z0.d, z2.d @@ -1121,9 +1120,9 @@ define @fshl_illegal_i64( %a, @fshl_rot_i64( %a, %b){ ; CHECK-LABEL: fshl_rot_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: subr z1.d, z1.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z2.d, z2.d, #0x3f ; CHECK-NEXT: and z1.d, z1.d, #0x3f ; CHECK-NEXT: lslr z2.d, p0/m, z2.d, z0.d @@ -1138,11 +1137,11 @@ define @fshl_rot_i64( %a, @fshl_rot_illegal_i64( %a, %b){ ; CHECK-LABEL: fshl_rot_illegal_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z4.d, z2.d ; CHECK-NEXT: subr z2.d, z2.d, #0 // =0x0 ; CHECK-NEXT: mov z5.d, z3.d ; CHECK-NEXT: subr z3.d, z3.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z4.d, z4.d, #0x3f ; CHECK-NEXT: and z2.d, z2.d, #0x3f ; CHECK-NEXT: and z5.d, z5.d, #0x3f @@ -1175,10 +1174,10 @@ define @fshl_rot_const_i64( %a){ define @fshr_i64( %a, %b, %c){ ; CHECK-LABEL: fshr_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z3.d, #63 // =0x3f ; CHECK-NEXT: mov z4.d, z2.d ; CHECK-NEXT: lsl z0.d, z0.d, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: bic z2.d, z3.d, z2.d ; CHECK-NEXT: and z4.d, z4.d, #0x3f ; CHECK-NEXT: lsr z1.d, p0/m, z1.d, z4.d @@ -1192,9 +1191,9 @@ define @fshr_i64( %a, %b define @fshr_rot_i64( %a, %b){ ; CHECK-LABEL: fshr_rot_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: subr z1.d, z1.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z2.d, z2.d, #0x3f ; CHECK-NEXT: and z1.d, z1.d, #0x3f ; CHECK-NEXT: lsrr z2.d, p0/m, z2.d, z0.d diff --git a/llvm/test/CodeGen/AArch64/load-insert-zero.ll b/llvm/test/CodeGen/AArch64/load-insert-zero.ll index 993af08a66dd..23d545459295 100644 --- a/llvm/test/CodeGen/AArch64/load-insert-zero.ll +++ b/llvm/test/CodeGen/AArch64/load-insert-zero.ll @@ -469,18 +469,18 @@ define void @predictor_4x4_neon(ptr nocapture noundef writeonly %0, i64 noundef ; CHECK-NEXT: lsr w8, w8, #24 ; CHECK-NEXT: uaddl v0.8h, v0.8b, v1.8b ; CHECK-NEXT: urhadd v1.8b, v1.8b, v2.8b -; CHECK-NEXT: str s1, [x0] ; CHECK-NEXT: add v0.8h, v0.8h, v3.8h ; CHECK-NEXT: dup v3.8b, w8 +; CHECK-NEXT: str s1, [x0] ; CHECK-NEXT: lsl x8, x1, #1 ; CHECK-NEXT: rshrn v0.8b, v0.8h, #2 ; CHECK-NEXT: zip1 v2.2s, v1.2s, v3.2s ; CHECK-NEXT: str s0, [x0, x1] ; CHECK-NEXT: zip1 v3.2s, v0.2s, v3.2s ; CHECK-NEXT: ext v2.8b, v2.8b, v0.8b, #1 +; CHECK-NEXT: ext v1.8b, v3.8b, v0.8b, #1 ; CHECK-NEXT: str s2, [x0, x8] ; CHECK-NEXT: add x8, x8, x1 -; CHECK-NEXT: ext v1.8b, v3.8b, v0.8b, #1 ; CHECK-NEXT: str s1, [x0, x8] ; CHECK-NEXT: ret %5 = load i32, ptr %2, align 4 @@ -608,9 +608,9 @@ define void @predictor_4x4_neon_new(ptr nocapture noundef writeonly %0, i64 noun define @loadnxv8i8(ptr %p) { ; CHECK-LABEL: loadnxv8i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl1 ; CHECK-NEXT: mov z0.h, #0 // =0x0 ; CHECK-NEXT: ldrb w8, [x0] +; CHECK-NEXT: ptrue p0.h, vl1 ; CHECK-NEXT: mov z0.h, p0/m, w8 ; CHECK-NEXT: ret %l = load i8, ptr %p @@ -631,9 +631,9 @@ define @loadnxv16i8(ptr %p) { define @loadnxv4i16(ptr %p) { ; CHECK-LABEL: loadnxv4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl1 ; CHECK-NEXT: mov z0.s, #0 // =0x0 ; CHECK-NEXT: ldrh w8, [x0] +; CHECK-NEXT: ptrue p0.s, vl1 ; CHECK-NEXT: mov z0.s, p0/m, w8 ; CHECK-NEXT: ret %l = load i16, ptr %p @@ -654,9 +654,9 @@ define @loadnxv8i16(ptr %p) { define @loadnxv2i32(ptr %p) { ; CHECK-LABEL: loadnxv2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: mov z0.d, #0 // =0x0 ; CHECK-NEXT: ldr w8, [x0] +; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: mov z0.d, p0/m, x8 ; CHECK-NEXT: ret %l = load i32, ptr %p @@ -688,9 +688,9 @@ define @loadnxv2i64(ptr %p) { define @loadnxv4f16(ptr %p) { ; CHECK-LABEL: loadnxv4f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, wzr ; CHECK-NEXT: index z0.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s ; CHECK-NEXT: mov z0.h, #0 // =0x0 @@ -715,9 +715,9 @@ define @loadnxv8f16(ptr %p) { define @loadnxv4bf16(ptr %p) { ; CHECK-LABEL: loadnxv4bf16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, wzr ; CHECK-NEXT: index z0.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s ; CHECK-NEXT: mov z0.h, #0 // =0x0 @@ -742,9 +742,9 @@ define @loadnxv8bf16(ptr %p) { define @loadnxv2f32(ptr %p) { ; CHECK-LABEL: loadnxv2f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: index z0.d, #0, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d ; CHECK-NEXT: mov z0.s, #0 // =0x0 @@ -782,9 +782,9 @@ define @loadnxv2f64(ptr %p) { define @loadnxv8i8_offset(ptr %p) { ; CHECK-LABEL: loadnxv8i8_offset: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl1 ; CHECK-NEXT: mov z0.h, #0 // =0x0 ; CHECK-NEXT: ldrb w8, [x0, #1] +; CHECK-NEXT: ptrue p0.h, vl1 ; CHECK-NEXT: mov z0.h, p0/m, w8 ; CHECK-NEXT: ret %g = getelementptr inbounds i8, ptr %p, i64 1 @@ -807,9 +807,9 @@ define @loadnxv16i8_offset(ptr %p) { define @loadnxv4i16_offset(ptr %p) { ; CHECK-LABEL: loadnxv4i16_offset: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl1 ; CHECK-NEXT: mov z0.s, #0 // =0x0 ; CHECK-NEXT: ldurh w8, [x0, #1] +; CHECK-NEXT: ptrue p0.s, vl1 ; CHECK-NEXT: mov z0.s, p0/m, w8 ; CHECK-NEXT: ret %g = getelementptr inbounds i8, ptr %p, i64 1 @@ -832,9 +832,9 @@ define @loadnxv8i16_offset(ptr %p) { define @loadnxv2i32_offset(ptr %p) { ; CHECK-LABEL: loadnxv2i32_offset: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: mov z0.d, #0 // =0x0 ; CHECK-NEXT: ldur w8, [x0, #1] +; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: mov z0.d, p0/m, x8 ; CHECK-NEXT: ret %g = getelementptr inbounds i8, ptr %p, i64 1 @@ -869,9 +869,9 @@ define @loadnxv2i64_offset(ptr %p) { define @loadnxv4f16_offset(ptr %p) { ; CHECK-LABEL: loadnxv4f16_offset: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, wzr ; CHECK-NEXT: index z0.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s ; CHECK-NEXT: mov z0.h, #0 // =0x0 @@ -898,9 +898,9 @@ define @loadnxv8f16_offset(ptr %p) { define @loadnxv4bf16_offset(ptr %p) { ; CHECK-LABEL: loadnxv4bf16_offset: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, wzr ; CHECK-NEXT: index z0.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s ; CHECK-NEXT: mov z0.h, #0 // =0x0 @@ -927,9 +927,9 @@ define @loadnxv8bf16_offset(ptr %p) { define @loadnxv2f32_offset(ptr %p) { ; CHECK-LABEL: loadnxv2f32_offset: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: index z0.d, #0, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d ; CHECK-NEXT: mov z0.s, #0 // =0x0 diff --git a/llvm/test/CodeGen/AArch64/logic-shift.ll b/llvm/test/CodeGen/AArch64/logic-shift.ll index 39f82dd4593f..31047954401c 100644 --- a/llvm/test/CodeGen/AArch64/logic-shift.ll +++ b/llvm/test/CodeGen/AArch64/logic-shift.ll @@ -34,9 +34,9 @@ define i32 @or_lshr_commute1(i32 %x0, i32 %x1, i32 %y, i32 %z) { define <8 x i16> @or_lshr_commute2(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %y, <8 x i16> %z) { ; CHECK-LABEL: or_lshr_commute2: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.8h, v2.8h ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b -; CHECK-NEXT: ushl v0.8h, v0.8h, v2.8h +; CHECK-NEXT: neg v1.8h, v2.8h +; CHECK-NEXT: ushl v0.8h, v0.8h, v1.8h ; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = lshr <8 x i16> %x0, %y @@ -49,9 +49,9 @@ define <8 x i16> @or_lshr_commute2(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %y, < define <2 x i64> @or_lshr_commute3(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %y, <2 x i64> %z) { ; CHECK-LABEL: or_lshr_commute3: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.2d, v2.2d ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b -; CHECK-NEXT: ushl v0.2d, v0.2d, v2.2d +; CHECK-NEXT: neg v1.2d, v2.2d +; CHECK-NEXT: ushl v0.2d, v0.2d, v1.2d ; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = lshr <2 x i64> %x0, %y @@ -94,9 +94,9 @@ define i64 @or_ashr_commute1(i64 %x0, i64 %x1, i64 %y, i64 %z) { define <4 x i32> @or_ashr_commute2(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %y, <4 x i32> %z) { ; CHECK-LABEL: or_ashr_commute2: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.4s, v2.4s ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b -; CHECK-NEXT: sshl v0.4s, v0.4s, v2.4s +; CHECK-NEXT: neg v1.4s, v2.4s +; CHECK-NEXT: sshl v0.4s, v0.4s, v1.4s ; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = ashr <4 x i32> %x0, %y @@ -109,9 +109,9 @@ define <4 x i32> @or_ashr_commute2(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %y, < define <16 x i8> @or_ashr_commute3(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %y, <16 x i8> %z) { ; CHECK-LABEL: or_ashr_commute3: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.16b, v2.16b ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b -; CHECK-NEXT: sshl v0.16b, v0.16b, v2.16b +; CHECK-NEXT: neg v1.16b, v2.16b +; CHECK-NEXT: sshl v0.16b, v0.16b, v1.16b ; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = ashr <16 x i8> %x0, %y @@ -262,9 +262,9 @@ define i32 @xor_lshr_commute1(i32 %x0, i32 %x1, i32 %y, i32 %z) { define <8 x i16> @xor_lshr_commute2(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %y, <8 x i16> %z) { ; CHECK-LABEL: xor_lshr_commute2: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.8h, v2.8h ; CHECK-NEXT: eor v0.16b, v0.16b, v1.16b -; CHECK-NEXT: ushl v0.8h, v0.8h, v2.8h +; CHECK-NEXT: neg v1.8h, v2.8h +; CHECK-NEXT: ushl v0.8h, v0.8h, v1.8h ; CHECK-NEXT: eor v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = lshr <8 x i16> %x0, %y @@ -277,9 +277,9 @@ define <8 x i16> @xor_lshr_commute2(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %y, define <2 x i64> @xor_lshr_commute3(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %y, <2 x i64> %z) { ; CHECK-LABEL: xor_lshr_commute3: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.2d, v2.2d ; CHECK-NEXT: eor v0.16b, v0.16b, v1.16b -; CHECK-NEXT: ushl v0.2d, v0.2d, v2.2d +; CHECK-NEXT: neg v1.2d, v2.2d +; CHECK-NEXT: ushl v0.2d, v0.2d, v1.2d ; CHECK-NEXT: eor v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = lshr <2 x i64> %x0, %y @@ -322,9 +322,9 @@ define i64 @xor_ashr_commute1(i64 %x0, i64 %x1, i64 %y, i64 %z) { define <4 x i32> @xor_ashr_commute2(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %y, <4 x i32> %z) { ; CHECK-LABEL: xor_ashr_commute2: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.4s, v2.4s ; CHECK-NEXT: eor v0.16b, v0.16b, v1.16b -; CHECK-NEXT: sshl v0.4s, v0.4s, v2.4s +; CHECK-NEXT: neg v1.4s, v2.4s +; CHECK-NEXT: sshl v0.4s, v0.4s, v1.4s ; CHECK-NEXT: eor v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = ashr <4 x i32> %x0, %y @@ -337,9 +337,9 @@ define <4 x i32> @xor_ashr_commute2(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %y, define <16 x i8> @xor_ashr_commute3(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %y, <16 x i8> %z) { ; CHECK-LABEL: xor_ashr_commute3: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.16b, v2.16b ; CHECK-NEXT: eor v0.16b, v0.16b, v1.16b -; CHECK-NEXT: sshl v0.16b, v0.16b, v2.16b +; CHECK-NEXT: neg v1.16b, v2.16b +; CHECK-NEXT: sshl v0.16b, v0.16b, v1.16b ; CHECK-NEXT: eor v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = ashr <16 x i8> %x0, %y @@ -490,9 +490,9 @@ define i32 @and_lshr_commute1(i32 %x0, i32 %x1, i32 %y, i32 %z) { define <8 x i16> @and_lshr_commute2(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %y, <8 x i16> %z) { ; CHECK-LABEL: and_lshr_commute2: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.8h, v2.8h ; CHECK-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-NEXT: ushl v0.8h, v0.8h, v2.8h +; CHECK-NEXT: neg v1.8h, v2.8h +; CHECK-NEXT: ushl v0.8h, v0.8h, v1.8h ; CHECK-NEXT: and v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = lshr <8 x i16> %x0, %y @@ -505,9 +505,9 @@ define <8 x i16> @and_lshr_commute2(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %y, define <2 x i64> @and_lshr_commute3(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %y, <2 x i64> %z) { ; CHECK-LABEL: and_lshr_commute3: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.2d, v2.2d ; CHECK-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-NEXT: ushl v0.2d, v0.2d, v2.2d +; CHECK-NEXT: neg v1.2d, v2.2d +; CHECK-NEXT: ushl v0.2d, v0.2d, v1.2d ; CHECK-NEXT: and v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = lshr <2 x i64> %x0, %y @@ -550,9 +550,9 @@ define i64 @and_ashr_commute1(i64 %x0, i64 %x1, i64 %y, i64 %z) { define <4 x i32> @and_ashr_commute2(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %y, <4 x i32> %z) { ; CHECK-LABEL: and_ashr_commute2: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.4s, v2.4s ; CHECK-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-NEXT: sshl v0.4s, v0.4s, v2.4s +; CHECK-NEXT: neg v1.4s, v2.4s +; CHECK-NEXT: sshl v0.4s, v0.4s, v1.4s ; CHECK-NEXT: and v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = ashr <4 x i32> %x0, %y @@ -565,9 +565,9 @@ define <4 x i32> @and_ashr_commute2(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %y, define <16 x i8> @and_ashr_commute3(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %y, <16 x i8> %z) { ; CHECK-LABEL: and_ashr_commute3: ; CHECK: // %bb.0: -; CHECK-NEXT: neg v2.16b, v2.16b ; CHECK-NEXT: and v0.16b, v0.16b, v1.16b -; CHECK-NEXT: sshl v0.16b, v0.16b, v2.16b +; CHECK-NEXT: neg v1.16b, v2.16b +; CHECK-NEXT: sshl v0.16b, v0.16b, v1.16b ; CHECK-NEXT: and v0.16b, v0.16b, v3.16b ; CHECK-NEXT: ret %sh1 = ashr <16 x i8> %x0, %y diff --git a/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll b/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll index 06570b4539cc..fac96e07de54 100644 --- a/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll +++ b/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll @@ -258,10 +258,10 @@ define @splice_nxv2i1_idx( %a, @llvm.experimental.vector.splice.nxv2i1( %a, %b, i32 1) ret %res @@ -273,10 +273,10 @@ define @splice_nxv4i1_idx( %a, @llvm.experimental.vector.splice.nxv4i1( %a, %b, i32 2) ret %res @@ -288,10 +288,10 @@ define @splice_nxv8i1_idx( %a, @llvm.experimental.vector.splice.nxv8i1( %a, %b, i32 4) ret %res @@ -303,10 +303,10 @@ define @splice_nxv16i1_idx( %a, @llvm.experimental.vector.splice.nxv16i1( %a, %b, i32 8) ret %res @@ -350,16 +350,16 @@ define @splice_nxv16f32_16( %a, @splice_nxv16i8_neg17( %a, @splice_nxv8i16_neg9( %a, @splice_nxv8f16_neg9( %a, @splice_nxv2i1( %a, ; CHECK-NEXT: ptrue p2.d, vl1 ; CHECK-NEXT: mov z0.d, p1/z, #1 // =0x1 ; CHECK-NEXT: mov z1.d, p0/z, #1 // =0x1 +; CHECK-NEXT: rev p0.d, p2.d +; CHECK-NEXT: splice z1.d, p0, z1.d, z0.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: rev p2.d, p2.d -; CHECK-NEXT: splice z1.d, p2, z1.d, z0.d ; CHECK-NEXT: and z1.d, z1.d, #0x1 ; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 ; CHECK-NEXT: ret @@ -716,9 +716,9 @@ define @splice_nxv4i1( %a, ; CHECK-NEXT: ptrue p2.s, vl1 ; CHECK-NEXT: mov z0.s, p1/z, #1 // =0x1 ; CHECK-NEXT: mov z1.s, p0/z, #1 // =0x1 +; CHECK-NEXT: rev p0.s, p2.s +; CHECK-NEXT: splice z1.s, p0, z1.s, z0.s ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: rev p2.s, p2.s -; CHECK-NEXT: splice z1.s, p2, z1.s, z0.s ; CHECK-NEXT: and z1.s, z1.s, #0x1 ; CHECK-NEXT: cmpne p0.s, p0/z, z1.s, #0 ; CHECK-NEXT: ret @@ -733,9 +733,9 @@ define @splice_nxv8i1( %a, ; CHECK-NEXT: ptrue p2.h, vl1 ; CHECK-NEXT: mov z0.h, p1/z, #1 // =0x1 ; CHECK-NEXT: mov z1.h, p0/z, #1 // =0x1 +; CHECK-NEXT: rev p0.h, p2.h +; CHECK-NEXT: splice z1.h, p0, z1.h, z0.h ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: rev p2.h, p2.h -; CHECK-NEXT: splice z1.h, p2, z1.h, z0.h ; CHECK-NEXT: and z1.h, z1.h, #0x1 ; CHECK-NEXT: cmpne p0.h, p0/z, z1.h, #0 ; CHECK-NEXT: ret @@ -750,9 +750,9 @@ define @splice_nxv16i1( %a, @splice_nxv8i32( %a, @splice_nxv16f32_neg17( %a, %a, <24 x i8> %b, <24 x i8> %c, <24 ; CHECK-NEXT: ld1 { v5.b }[15], [x8] ; CHECK-NEXT: ld1 { v7.b }[7], [x10] ; CHECK-NEXT: addp v1.2s, v19.2s, v19.2s +; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: sdot v16.4s, v5.16b, v4.16b ; CHECK-NEXT: sdot v18.2s, v7.8b, v6.8b -; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: fmov w9, s1 ; CHECK-NEXT: addv s2, v16.4s ; CHECK-NEXT: addp v3.2s, v18.2s, v18.2s @@ -975,8 +975,8 @@ define i32 @test_sdot_v24i8_double_nomla(<24 x i8> %a, <24 x i8> %b, <24 x i8> % ; CHECK-NEXT: addv s3, v5.4s ; CHECK-NEXT: addp v1.2s, v17.2s, v17.2s ; CHECK-NEXT: addp v2.2s, v7.2s, v7.2s -; CHECK-NEXT: addv s0, v6.4s ; CHECK-NEXT: fmov w10, s3 +; CHECK-NEXT: addv s0, v6.4s ; CHECK-NEXT: fmov w9, s1 ; CHECK-NEXT: fmov w11, s2 ; CHECK-NEXT: fmov w8, s0 @@ -998,26 +998,26 @@ entry: define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { ; CHECK-LABEL: test_udot_v25i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldp q4, q0, [x0] +; CHECK-NEXT: ldp q2, q0, [x0] ; CHECK-NEXT: ldp q5, q1, [x1] -; CHECK-NEXT: ushll2 v2.8h, v0.16b, #0 -; CHECK-NEXT: ushll v6.8h, v4.8b, #0 +; CHECK-NEXT: ushll2 v3.8h, v0.16b, #0 +; CHECK-NEXT: ushll v6.8h, v2.8b, #0 ; CHECK-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-NEXT: ushll2 v3.8h, v1.16b, #0 +; CHECK-NEXT: ushll2 v4.8h, v1.16b, #0 ; CHECK-NEXT: ushll v7.8h, v5.8b, #0 ; CHECK-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-NEXT: umull v2.4s, v3.4h, v2.4h -; CHECK-NEXT: movi v3.2d, #0000000000000000 +; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 +; CHECK-NEXT: umull v3.4s, v4.4h, v3.4h +; CHECK-NEXT: movi v4.2d, #0000000000000000 ; CHECK-NEXT: umull2 v16.4s, v7.8h, v6.8h ; CHECK-NEXT: umull v6.4s, v7.4h, v6.4h -; CHECK-NEXT: mov v3.s[0], v2.s[0] -; CHECK-NEXT: ushll2 v2.8h, v4.16b, #0 -; CHECK-NEXT: ushll2 v4.8h, v5.16b, #0 -; CHECK-NEXT: umlal v6.4s, v1.4h, v0.4h +; CHECK-NEXT: mov v4.s[0], v3.s[0] +; CHECK-NEXT: ushll2 v3.8h, v5.16b, #0 ; CHECK-NEXT: umlal2 v16.4s, v1.8h, v0.8h -; CHECK-NEXT: umlal v3.4s, v4.4h, v2.4h -; CHECK-NEXT: umlal2 v16.4s, v4.8h, v2.8h -; CHECK-NEXT: add v0.4s, v6.4s, v3.4s +; CHECK-NEXT: umlal v6.4s, v1.4h, v0.4h +; CHECK-NEXT: umlal v4.4s, v3.4h, v2.4h +; CHECK-NEXT: umlal2 v16.4s, v3.8h, v2.8h +; CHECK-NEXT: add v0.4s, v6.4s, v4.4s ; CHECK-NEXT: add v0.4s, v0.4s, v16.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 @@ -1039,17 +1039,17 @@ define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) { ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ldp q2, q1, [x0] ; CHECK-NEXT: movi v0.2d, #0000000000000000 -; CHECK-NEXT: ushll v3.8h, v1.8b, #0 +; CHECK-NEXT: ushll2 v3.8h, v1.16b, #0 +; CHECK-NEXT: ushll v1.8h, v1.8b, #0 ; CHECK-NEXT: ushll v4.8h, v2.8b, #0 -; CHECK-NEXT: ushll2 v1.8h, v1.16b, #0 ; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 -; CHECK-NEXT: uaddl2 v5.4s, v4.8h, v3.8h -; CHECK-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-NEXT: uaddl v3.4s, v4.4h, v3.4h -; CHECK-NEXT: mov v0.s[0], v1.s[0] -; CHECK-NEXT: uaddw2 v1.4s, v5.4s, v2.8h +; CHECK-NEXT: ushll v3.4s, v3.4h, #0 +; CHECK-NEXT: uaddl2 v5.4s, v4.8h, v1.8h +; CHECK-NEXT: uaddl v1.4s, v4.4h, v1.4h +; CHECK-NEXT: mov v0.s[0], v3.s[0] +; CHECK-NEXT: uaddw2 v3.4s, v5.4s, v2.8h +; CHECK-NEXT: add v1.4s, v1.4s, v3.4s ; CHECK-NEXT: uaddw v0.4s, v0.4s, v2.4h -; CHECK-NEXT: add v1.4s, v3.4s, v1.4s ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w0, s0 @@ -1063,26 +1063,26 @@ entry: define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { ; CHECK-LABEL: test_sdot_v25i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldp q4, q0, [x0] +; CHECK-NEXT: ldp q2, q0, [x0] ; CHECK-NEXT: ldp q5, q1, [x1] -; CHECK-NEXT: sshll2 v2.8h, v0.16b, #0 -; CHECK-NEXT: sshll v6.8h, v4.8b, #0 +; CHECK-NEXT: sshll2 v3.8h, v0.16b, #0 +; CHECK-NEXT: sshll v6.8h, v2.8b, #0 ; CHECK-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-NEXT: sshll2 v3.8h, v1.16b, #0 +; CHECK-NEXT: sshll2 v4.8h, v1.16b, #0 ; CHECK-NEXT: sshll v7.8h, v5.8b, #0 ; CHECK-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-NEXT: smull v2.4s, v3.4h, v2.4h -; CHECK-NEXT: movi v3.2d, #0000000000000000 +; CHECK-NEXT: sshll2 v2.8h, v2.16b, #0 +; CHECK-NEXT: smull v3.4s, v4.4h, v3.4h +; CHECK-NEXT: movi v4.2d, #0000000000000000 ; CHECK-NEXT: smull2 v16.4s, v7.8h, v6.8h ; CHECK-NEXT: smull v6.4s, v7.4h, v6.4h -; CHECK-NEXT: mov v3.s[0], v2.s[0] -; CHECK-NEXT: sshll2 v2.8h, v4.16b, #0 -; CHECK-NEXT: sshll2 v4.8h, v5.16b, #0 -; CHECK-NEXT: smlal v6.4s, v1.4h, v0.4h +; CHECK-NEXT: mov v4.s[0], v3.s[0] +; CHECK-NEXT: sshll2 v3.8h, v5.16b, #0 ; CHECK-NEXT: smlal2 v16.4s, v1.8h, v0.8h -; CHECK-NEXT: smlal v3.4s, v4.4h, v2.4h -; CHECK-NEXT: smlal2 v16.4s, v4.8h, v2.8h -; CHECK-NEXT: add v0.4s, v6.4s, v3.4s +; CHECK-NEXT: smlal v6.4s, v1.4h, v0.4h +; CHECK-NEXT: smlal v4.4s, v3.4h, v2.4h +; CHECK-NEXT: smlal2 v16.4s, v3.8h, v2.8h +; CHECK-NEXT: add v0.4s, v6.4s, v4.4s ; CHECK-NEXT: add v0.4s, v0.4s, v16.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 @@ -1105,222 +1105,222 @@ define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: fmov s4, w0 -; CHECK-NEXT: ldr b0, [sp, #80] -; CHECK-NEXT: add x8, sp, #88 ; CHECK-NEXT: ldr b1, [sp, #16] -; CHECK-NEXT: add x10, sp, #24 -; CHECK-NEXT: ldr b2, [sp, #280] -; CHECK-NEXT: ld1 { v0.b }[1], [x8] +; CHECK-NEXT: ldr b0, [sp, #80] +; CHECK-NEXT: add x11, sp, #24 ; CHECK-NEXT: ldr b3, [sp, #216] +; CHECK-NEXT: add x10, sp, #88 +; CHECK-NEXT: ldr b2, [sp, #280] +; CHECK-NEXT: ld1 { v1.b }[1], [x11] ; CHECK-NEXT: add x11, sp, #224 -; CHECK-NEXT: mov v4.b[1], w1 -; CHECK-NEXT: ld1 { v1.b }[1], [x10] +; CHECK-NEXT: ldr b4, [sp, #152] +; CHECK-NEXT: ldr b6, [sp, #480] +; CHECK-NEXT: ld1 { v0.b }[1], [x10] ; CHECK-NEXT: add x10, sp, #288 -; CHECK-NEXT: ldr b5, [sp, #152] -; CHECK-NEXT: add x9, sp, #96 -; CHECK-NEXT: ld1 { v2.b }[1], [x10] +; CHECK-NEXT: add x12, sp, #160 ; CHECK-NEXT: ld1 { v3.b }[1], [x11] -; CHECK-NEXT: add x10, sp, #160 -; CHECK-NEXT: ld1 { v0.b }[2], [x9] -; CHECK-NEXT: ld1 { v5.b }[1], [x10] -; CHECK-NEXT: add x10, sp, #32 -; CHECK-NEXT: add x11, sp, #296 -; CHECK-NEXT: mov v4.b[2], w2 -; CHECK-NEXT: ld1 { v1.b }[2], [x10] -; CHECK-NEXT: add x10, sp, #232 +; CHECK-NEXT: add x11, sp, #488 +; CHECK-NEXT: ld1 { v2.b }[1], [x10] +; CHECK-NEXT: ld1 { v4.b }[1], [x12] +; CHECK-NEXT: ld1 { v6.b }[1], [x11] +; CHECK-NEXT: add x11, sp, #32 +; CHECK-NEXT: add x9, sp, #96 ; CHECK-NEXT: add x8, sp, #104 -; CHECK-NEXT: ld1 { v2.b }[2], [x11] -; CHECK-NEXT: ld1 { v3.b }[2], [x10] +; CHECK-NEXT: ld1 { v1.b }[2], [x11] +; CHECK-NEXT: add x11, sp, #232 +; CHECK-NEXT: ld1 { v0.b }[2], [x9] +; CHECK-NEXT: add x9, sp, #296 +; CHECK-NEXT: ld1 { v3.b }[2], [x11] ; CHECK-NEXT: add x11, sp, #168 +; CHECK-NEXT: ld1 { v2.b }[2], [x9] +; CHECK-NEXT: ld1 { v4.b }[2], [x11] +; CHECK-NEXT: add x11, sp, #40 +; CHECK-NEXT: ld1 { v1.b }[3], [x11] ; CHECK-NEXT: ld1 { v0.b }[3], [x8] -; CHECK-NEXT: add x8, sp, #40 -; CHECK-NEXT: ld1 { v5.b }[2], [x11] -; CHECK-NEXT: ld1 { v1.b }[3], [x8] -; CHECK-NEXT: add x8, sp, #240 -; CHECK-NEXT: mov v4.b[3], w3 -; CHECK-NEXT: ld1 { v3.b }[3], [x8] -; CHECK-NEXT: add x8, sp, #176 -; CHECK-NEXT: add x12, sp, #112 -; CHECK-NEXT: add x13, sp, #48 -; CHECK-NEXT: add x9, sp, #120 -; CHECK-NEXT: ld1 { v5.b }[3], [x8] -; CHECK-NEXT: ld1 { v0.b }[4], [x12] -; CHECK-NEXT: add x12, sp, #184 -; CHECK-NEXT: ld1 { v1.b }[4], [x13] -; CHECK-NEXT: add x15, sp, #56 -; CHECK-NEXT: add x14, sp, #128 -; CHECK-NEXT: mov v4.b[4], w4 -; CHECK-NEXT: add x11, sp, #304 -; CHECK-NEXT: add x13, sp, #256 -; CHECK-NEXT: ld1 { v5.b }[4], [x12] -; CHECK-NEXT: ld1 { v0.b }[5], [x9] -; CHECK-NEXT: add x9, sp, #192 -; CHECK-NEXT: add x12, sp, #248 -; CHECK-NEXT: ld1 { v1.b }[5], [x15] -; CHECK-NEXT: add x15, sp, #200 -; CHECK-NEXT: ld1 { v3.b }[4], [x12] -; CHECK-NEXT: ld1 { v2.b }[3], [x11] +; CHECK-NEXT: add x8, sp, #304 +; CHECK-NEXT: add x10, sp, #112 +; CHECK-NEXT: add x11, sp, #240 +; CHECK-NEXT: add x13, sp, #56 +; CHECK-NEXT: ld1 { v2.b }[3], [x8] +; CHECK-NEXT: add x8, sp, #48 +; CHECK-NEXT: ld1 { v3.b }[3], [x11] +; CHECK-NEXT: ld1 { v1.b }[4], [x8] +; CHECK-NEXT: ld1 { v0.b }[4], [x10] +; CHECK-NEXT: add x15, sp, #312 +; CHECK-NEXT: add x12, sp, #120 +; CHECK-NEXT: add x8, sp, #248 ; CHECK-NEXT: add x11, sp, #64 -; CHECK-NEXT: mov v4.b[5], w5 -; CHECK-NEXT: ld1 { v5.b }[5], [x9] -; CHECK-NEXT: ld1 { v0.b }[6], [x14] -; CHECK-NEXT: ldr b6, [sp, #352] -; CHECK-NEXT: add x10, sp, #136 -; CHECK-NEXT: ld1 { v1.b }[6], [x11] -; CHECK-NEXT: add x11, sp, #360 -; CHECK-NEXT: ld1 { v3.b }[5], [x13] +; CHECK-NEXT: ld1 { v2.b }[4], [x15] +; CHECK-NEXT: ld1 { v3.b }[4], [x8] +; CHECK-NEXT: add x15, sp, #320 +; CHECK-NEXT: ld1 { v1.b }[5], [x13] +; CHECK-NEXT: ld1 { v0.b }[5], [x12] ; CHECK-NEXT: ldr b18, [sp, #552] -; CHECK-NEXT: ld1 { v5.b }[6], [x15] -; CHECK-NEXT: add x14, sp, #208 -; CHECK-NEXT: ld1 { v6.b }[1], [x11] -; CHECK-NEXT: mov v4.b[6], w6 -; CHECK-NEXT: ld1 { v0.b }[7], [x10] -; CHECK-NEXT: add x10, sp, #560 -; CHECK-NEXT: add x9, sp, #264 -; CHECK-NEXT: ld1 { v18.b }[1], [x10] +; CHECK-NEXT: add x14, sp, #128 +; CHECK-NEXT: add x16, sp, #256 +; CHECK-NEXT: ldr b16, [sp, #352] +; CHECK-NEXT: ld1 { v2.b }[5], [x15] +; CHECK-NEXT: add x15, sp, #176 +; CHECK-NEXT: ld1 { v3.b }[5], [x16] +; CHECK-NEXT: ld1 { v1.b }[6], [x11] +; CHECK-NEXT: add x11, sp, #560 +; CHECK-NEXT: ld1 { v0.b }[6], [x14] +; CHECK-NEXT: add x16, sp, #360 +; CHECK-NEXT: ld1 { v4.b }[3], [x15] +; CHECK-NEXT: ld1 { v18.b }[1], [x11] +; CHECK-NEXT: add x10, sp, #72 +; CHECK-NEXT: ld1 { v16.b }[1], [x16] +; CHECK-NEXT: add x9, sp, #136 +; CHECK-NEXT: add x14, sp, #184 +; CHECK-NEXT: ld1 { v1.b }[7], [x10] ; CHECK-NEXT: add x10, sp, #568 -; CHECK-NEXT: ld1 { v5.b }[7], [x14] -; CHECK-NEXT: ld1 { v3.b }[6], [x9] +; CHECK-NEXT: ld1 { v0.b }[7], [x9] +; CHECK-NEXT: ld1 { v4.b }[4], [x14] ; CHECK-NEXT: add x9, sp, #368 -; CHECK-NEXT: ld1 { v6.b }[2], [x9] -; CHECK-NEXT: add x11, sp, #488 -; CHECK-NEXT: ldr b7, [sp, #144] -; CHECK-NEXT: mov v4.b[7], w7 ; CHECK-NEXT: ld1 { v18.b }[2], [x10] -; CHECK-NEXT: add x10, sp, #376 -; CHECK-NEXT: sshll v17.8h, v5.8b, #0 -; CHECK-NEXT: ldr b5, [sp, #480] -; CHECK-NEXT: sshll v7.8h, v7.8b, #0 -; CHECK-NEXT: ld1 { v6.b }[3], [x10] +; CHECK-NEXT: add x11, sp, #496 +; CHECK-NEXT: ld1 { v16.b }[2], [x9] +; CHECK-NEXT: fmov s5, w0 +; CHECK-NEXT: add x9, sp, #192 +; CHECK-NEXT: ld1 { v6.b }[2], [x11] ; CHECK-NEXT: add x10, sp, #576 -; CHECK-NEXT: add x8, sp, #312 -; CHECK-NEXT: ld1 { v5.b }[1], [x11] +; CHECK-NEXT: ld1 { v4.b }[5], [x9] +; CHECK-NEXT: add x9, sp, #376 ; CHECK-NEXT: ld1 { v18.b }[3], [x10] -; CHECK-NEXT: add x11, sp, #496 -; CHECK-NEXT: sshll v16.8h, v4.8b, #0 -; CHECK-NEXT: ldr b4, [sp, #344] -; CHECK-NEXT: add x10, sp, #384 -; CHECK-NEXT: ld1 { v6.b }[4], [x10] +; CHECK-NEXT: add x11, sp, #504 +; CHECK-NEXT: ld1 { v16.b }[3], [x9] +; CHECK-NEXT: mov v5.b[1], w1 +; CHECK-NEXT: ldr b7, [sp, #144] +; CHECK-NEXT: ldr b17, [sp, #344] +; CHECK-NEXT: add x9, sp, #200 +; CHECK-NEXT: ld1 { v6.b }[3], [x11] ; CHECK-NEXT: add x10, sp, #584 -; CHECK-NEXT: ld1 { v2.b }[4], [x8] -; CHECK-NEXT: sshll v19.8h, v4.8b, #0 -; CHECK-NEXT: ld1 { v5.b }[2], [x11] +; CHECK-NEXT: ld1 { v4.b }[6], [x9] +; CHECK-NEXT: add x9, sp, #384 ; CHECK-NEXT: ld1 { v18.b }[4], [x10] -; CHECK-NEXT: smull2 v4.4s, v16.8h, v17.8h -; CHECK-NEXT: smull v16.4s, v16.4h, v17.4h -; CHECK-NEXT: ldr b17, [sp, #416] -; CHECK-NEXT: add x11, sp, #504 -; CHECK-NEXT: add x10, sp, #424 -; CHECK-NEXT: add x16, sp, #320 -; CHECK-NEXT: smull v19.4s, v7.4h, v19.4h -; CHECK-NEXT: movi v7.2d, #0000000000000000 -; CHECK-NEXT: ld1 { v5.b }[3], [x11] -; CHECK-NEXT: add x11, sp, #392 -; CHECK-NEXT: ld1 { v17.b }[1], [x10] -; CHECK-NEXT: add x10, sp, #592 -; CHECK-NEXT: ld1 { v2.b }[5], [x16] -; CHECK-NEXT: ld1 { v6.b }[5], [x11] -; CHECK-NEXT: ld1 { v18.b }[5], [x10] +; CHECK-NEXT: sshll v7.8h, v7.8b, #0 +; CHECK-NEXT: sshll v17.8h, v17.8b, #0 ; CHECK-NEXT: add x11, sp, #512 -; CHECK-NEXT: add x10, sp, #432 +; CHECK-NEXT: ld1 { v16.b }[4], [x9] +; CHECK-NEXT: ld1 { v6.b }[4], [x11] +; CHECK-NEXT: add x11, sp, #592 +; CHECK-NEXT: mov v5.b[2], w2 +; CHECK-NEXT: add x10, sp, #392 +; CHECK-NEXT: ldr b19, [sp, #680] +; CHECK-NEXT: ld1 { v18.b }[5], [x11] +; CHECK-NEXT: smull v7.4s, v7.4h, v17.4h +; CHECK-NEXT: ldr b17, [sp, #416] +; CHECK-NEXT: ld1 { v16.b }[5], [x10] +; CHECK-NEXT: add x10, sp, #688 ; CHECK-NEXT: add x12, sp, #328 -; CHECK-NEXT: mov v7.s[0], v19.s[0] -; CHECK-NEXT: ld1 { v5.b }[4], [x11] -; CHECK-NEXT: add x11, sp, #400 -; CHECK-NEXT: ld1 { v17.b }[2], [x10] +; CHECK-NEXT: add x9, sp, #424 +; CHECK-NEXT: ld1 { v19.b }[1], [x10] ; CHECK-NEXT: add x10, sp, #600 -; CHECK-NEXT: ldr b19, [sp, #680] ; CHECK-NEXT: ldr b20, [sp, #616] ; CHECK-NEXT: ld1 { v2.b }[6], [x12] -; CHECK-NEXT: ld1 { v6.b }[6], [x11] +; CHECK-NEXT: ld1 { v17.b }[1], [x9] +; CHECK-NEXT: add x11, sp, #400 ; CHECK-NEXT: ld1 { v18.b }[6], [x10] -; CHECK-NEXT: add x11, sp, #688 ; CHECK-NEXT: add x12, sp, #624 -; CHECK-NEXT: ld1 { v19.b }[1], [x11] +; CHECK-NEXT: mov v5.b[3], w3 +; CHECK-NEXT: ld1 { v16.b }[6], [x11] +; CHECK-NEXT: add x11, sp, #696 ; CHECK-NEXT: ld1 { v20.b }[1], [x12] -; CHECK-NEXT: add x10, sp, #408 +; CHECK-NEXT: add x9, sp, #432 +; CHECK-NEXT: ld1 { v19.b }[2], [x11] ; CHECK-NEXT: add x11, sp, #608 -; CHECK-NEXT: add x12, sp, #440 -; CHECK-NEXT: ld1 { v6.b }[7], [x10] +; CHECK-NEXT: ld1 { v17.b }[2], [x9] +; CHECK-NEXT: add x10, sp, #408 ; CHECK-NEXT: ld1 { v18.b }[7], [x11] -; CHECK-NEXT: ld1 { v17.b }[3], [x12] -; CHECK-NEXT: add x10, sp, #696 ; CHECK-NEXT: add x11, sp, #632 -; CHECK-NEXT: ld1 { v19.b }[2], [x10] -; CHECK-NEXT: add x10, sp, #448 +; CHECK-NEXT: ld1 { v16.b }[7], [x10] ; CHECK-NEXT: ld1 { v20.b }[2], [x11] -; CHECK-NEXT: add x11, sp, #640 -; CHECK-NEXT: sshll v6.8h, v6.8b, #0 -; CHECK-NEXT: ld1 { v17.b }[4], [x10] +; CHECK-NEXT: mov v5.b[4], w4 ; CHECK-NEXT: add x10, sp, #704 -; CHECK-NEXT: sshll v18.8h, v18.8b, #0 +; CHECK-NEXT: add x12, sp, #440 ; CHECK-NEXT: ld1 { v19.b }[3], [x10] -; CHECK-NEXT: add x10, sp, #712 -; CHECK-NEXT: add x12, sp, #520 -; CHECK-NEXT: ld1 { v20.b }[3], [x11] -; CHECK-NEXT: add x11, sp, #648 -; CHECK-NEXT: ldr b21, [sp, #544] -; CHECK-NEXT: smull2 v22.4s, v6.8h, v18.8h -; CHECK-NEXT: smull v6.4s, v6.4h, v18.4h -; CHECK-NEXT: ldr b18, [sp, #744] -; CHECK-NEXT: ld1 { v19.b }[4], [x10] -; CHECK-NEXT: ld1 { v5.b }[5], [x12] -; CHECK-NEXT: add x12, sp, #656 -; CHECK-NEXT: ld1 { v20.b }[4], [x11] -; CHECK-NEXT: add x11, sp, #456 -; CHECK-NEXT: sshll v21.8h, v21.8b, #0 -; CHECK-NEXT: ld1 { v17.b }[5], [x11] -; CHECK-NEXT: add x11, sp, #720 +; CHECK-NEXT: add x10, sp, #448 +; CHECK-NEXT: ld1 { v17.b }[3], [x12] +; CHECK-NEXT: add x12, sp, #640 +; CHECK-NEXT: sshll v21.8h, v16.8b, #0 +; CHECK-NEXT: ld1 { v20.b }[3], [x12] ; CHECK-NEXT: sshll v18.8h, v18.8b, #0 +; CHECK-NEXT: add x11, sp, #712 +; CHECK-NEXT: mov v5.b[5], w5 +; CHECK-NEXT: ld1 { v19.b }[4], [x11] +; CHECK-NEXT: add x9, sp, #520 +; CHECK-NEXT: ld1 { v17.b }[4], [x10] +; CHECK-NEXT: add x10, sp, #648 +; CHECK-NEXT: ldr b22, [sp, #544] +; CHECK-NEXT: ld1 { v20.b }[4], [x10] +; CHECK-NEXT: smull2 v16.4s, v21.8h, v18.8h +; CHECK-NEXT: smull v18.4s, v21.4h, v18.4h +; CHECK-NEXT: ldr b21, [sp, #744] +; CHECK-NEXT: add x11, sp, #720 +; CHECK-NEXT: ld1 { v6.b }[5], [x9] +; CHECK-NEXT: add x9, sp, #456 ; CHECK-NEXT: ld1 { v19.b }[5], [x11] +; CHECK-NEXT: mov v5.b[6], w6 +; CHECK-NEXT: ld1 { v17.b }[5], [x9] +; CHECK-NEXT: add x9, sp, #656 +; CHECK-NEXT: sshll v22.8h, v22.8b, #0 +; CHECK-NEXT: sshll v21.8h, v21.8b, #0 +; CHECK-NEXT: ld1 { v20.b }[5], [x9] ; CHECK-NEXT: add x10, sp, #528 -; CHECK-NEXT: add x11, sp, #464 -; CHECK-NEXT: ld1 { v20.b }[5], [x12] -; CHECK-NEXT: ld1 { v5.b }[6], [x10] -; CHECK-NEXT: add x12, sp, #728 -; CHECK-NEXT: add x13, sp, #664 -; CHECK-NEXT: add x8, sp, #72 -; CHECK-NEXT: ld1 { v17.b }[6], [x11] -; CHECK-NEXT: ld1 { v19.b }[6], [x12] -; CHECK-NEXT: ld1 { v1.b }[7], [x8] +; CHECK-NEXT: add x11, sp, #728 +; CHECK-NEXT: ld1 { v6.b }[6], [x10] +; CHECK-NEXT: add x10, sp, #464 +; CHECK-NEXT: ld1 { v19.b }[6], [x11] +; CHECK-NEXT: add x11, sp, #664 +; CHECK-NEXT: ld1 { v17.b }[6], [x10] +; CHECK-NEXT: smull v21.4s, v22.4h, v21.4h +; CHECK-NEXT: movi v22.2d, #0000000000000000 +; CHECK-NEXT: ld1 { v20.b }[6], [x11] +; CHECK-NEXT: mov v5.b[7], w7 +; CHECK-NEXT: add x9, sp, #536 +; CHECK-NEXT: add x10, sp, #736 +; CHECK-NEXT: add x11, sp, #208 +; CHECK-NEXT: add x13, sp, #264 +; CHECK-NEXT: ld1 { v6.b }[7], [x9] +; CHECK-NEXT: ld1 { v19.b }[7], [x10] +; CHECK-NEXT: ld1 { v4.b }[7], [x11] +; CHECK-NEXT: add x9, sp, #472 +; CHECK-NEXT: add x10, sp, #672 +; CHECK-NEXT: ld1 { v3.b }[6], [x13] +; CHECK-NEXT: ld1 { v17.b }[7], [x9] +; CHECK-NEXT: ld1 { v20.b }[7], [x10] ; CHECK-NEXT: add x8, sp, #336 -; CHECK-NEXT: ld1 { v20.b }[6], [x13] -; CHECK-NEXT: add x9, sp, #272 -; CHECK-NEXT: smull v18.4s, v21.4h, v18.4h +; CHECK-NEXT: mov v22.s[0], v21.s[0] ; CHECK-NEXT: movi v21.2d, #0000000000000000 -; CHECK-NEXT: add x10, sp, #536 +; CHECK-NEXT: sshll v5.8h, v5.8b, #0 +; CHECK-NEXT: sshll v6.8h, v6.8b, #0 +; CHECK-NEXT: sshll v19.8h, v19.8b, #0 ; CHECK-NEXT: ld1 { v2.b }[7], [x8] -; CHECK-NEXT: ld1 { v3.b }[7], [x9] -; CHECK-NEXT: ld1 { v5.b }[7], [x10] -; CHECK-NEXT: add x8, sp, #472 -; CHECK-NEXT: add x9, sp, #736 -; CHECK-NEXT: add x10, sp, #672 -; CHECK-NEXT: ld1 { v17.b }[7], [x8] -; CHECK-NEXT: ld1 { v19.b }[7], [x9] -; CHECK-NEXT: ld1 { v20.b }[7], [x10] +; CHECK-NEXT: add x8, sp, #272 +; CHECK-NEXT: sshll v4.8h, v4.8b, #0 +; CHECK-NEXT: ld1 { v3.b }[7], [x8] +; CHECK-NEXT: sshll v17.8h, v17.8b, #0 +; CHECK-NEXT: sshll v20.8h, v20.8b, #0 ; CHECK-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-NEXT: mov v21.s[0], v18.s[0] ; CHECK-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-NEXT: smlal v18.4s, v6.4h, v19.4h +; CHECK-NEXT: smlal2 v16.4s, v6.8h, v19.8h +; CHECK-NEXT: mov v21.s[0], v7.s[0] +; CHECK-NEXT: smull v6.4s, v5.4h, v4.4h ; CHECK-NEXT: sshll v2.8h, v2.8b, #0 ; CHECK-NEXT: sshll v3.8h, v3.8b, #0 -; CHECK-NEXT: sshll v5.8h, v5.8b, #0 -; CHECK-NEXT: sshll v17.8h, v17.8b, #0 -; CHECK-NEXT: sshll v18.8h, v19.8b, #0 -; CHECK-NEXT: sshll v19.8h, v20.8b, #0 -; CHECK-NEXT: smlal v16.4s, v0.4h, v2.4h +; CHECK-NEXT: smlal v22.4s, v17.4h, v20.4h +; CHECK-NEXT: smull2 v4.4s, v5.8h, v4.8h +; CHECK-NEXT: smlal v21.4s, v1.4h, v3.4h +; CHECK-NEXT: smlal2 v16.4s, v17.8h, v20.8h +; CHECK-NEXT: smlal v6.4s, v0.4h, v2.4h +; CHECK-NEXT: add v5.4s, v18.4s, v22.4s ; CHECK-NEXT: smlal2 v4.4s, v0.8h, v2.8h -; CHECK-NEXT: smlal v7.4s, v1.4h, v3.4h -; CHECK-NEXT: smlal v6.4s, v5.4h, v18.4h -; CHECK-NEXT: smlal2 v22.4s, v5.8h, v18.8h -; CHECK-NEXT: smlal v21.4s, v17.4h, v19.4h +; CHECK-NEXT: add v0.4s, v6.4s, v21.4s +; CHECK-NEXT: add v2.4s, v5.4s, v16.4s ; CHECK-NEXT: smlal2 v4.4s, v1.8h, v3.8h -; CHECK-NEXT: add v0.4s, v16.4s, v7.4s -; CHECK-NEXT: add v1.4s, v6.4s, v21.4s -; CHECK-NEXT: smlal2 v22.4s, v17.8h, v19.8h +; CHECK-NEXT: add v0.4s, v0.4s, v2.4s ; CHECK-NEXT: add v0.4s, v0.4s, v4.4s -; CHECK-NEXT: add v1.4s, v1.4s, v22.4s -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w0, s0 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -1584,34 +1584,34 @@ entry: define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { ; CHECK-LABEL: test_udot_v33i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldp q2, q3, [x0] -; CHECK-NEXT: movi v18.2d, #0000000000000000 -; CHECK-NEXT: ldp q4, q5, [x1] ; CHECK-NEXT: ldr b0, [x0, #32] ; CHECK-NEXT: ldr b1, [x1, #32] +; CHECK-NEXT: ldp q2, q4, [x0] +; CHECK-NEXT: ldp q3, q6, [x1] ; CHECK-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-NEXT: ushll v6.8h, v2.8b, #0 -; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 ; CHECK-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-NEXT: ushll v7.8h, v4.8b, #0 -; CHECK-NEXT: ushll2 v4.8h, v4.16b, #0 -; CHECK-NEXT: ushll2 v16.8h, v3.16b, #0 -; CHECK-NEXT: ushll v3.8h, v3.8b, #0 -; CHECK-NEXT: ushll2 v19.8h, v5.16b, #0 -; CHECK-NEXT: ushll v5.8h, v5.8b, #0 +; CHECK-NEXT: ushll v5.8h, v2.8b, #0 +; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 +; CHECK-NEXT: ushll2 v16.8h, v4.16b, #0 +; CHECK-NEXT: ushll v7.8h, v3.8b, #0 +; CHECK-NEXT: ushll2 v3.8h, v3.16b, #0 +; CHECK-NEXT: ushll v4.8h, v4.8b, #0 ; CHECK-NEXT: umull v0.4s, v1.4h, v0.4h -; CHECK-NEXT: umull2 v1.4s, v7.8h, v6.8h -; CHECK-NEXT: umull2 v17.4s, v4.8h, v2.8h -; CHECK-NEXT: umull v2.4s, v4.4h, v2.4h -; CHECK-NEXT: umlal2 v17.4s, v19.8h, v16.8h -; CHECK-NEXT: umlal2 v1.4s, v5.8h, v3.8h -; CHECK-NEXT: mov v18.s[0], v0.s[0] -; CHECK-NEXT: umlal v2.4s, v19.4h, v16.4h -; CHECK-NEXT: add v0.4s, v1.4s, v17.4s -; CHECK-NEXT: umlal v18.4s, v7.4h, v6.4h -; CHECK-NEXT: umlal v18.4s, v5.4h, v3.4h -; CHECK-NEXT: add v0.4s, v2.4s, v0.4s -; CHECK-NEXT: add v0.4s, v18.4s, v0.4s +; CHECK-NEXT: movi v1.2d, #0000000000000000 +; CHECK-NEXT: ushll2 v19.8h, v6.16b, #0 +; CHECK-NEXT: ushll v6.8h, v6.8b, #0 +; CHECK-NEXT: umull2 v17.4s, v7.8h, v5.8h +; CHECK-NEXT: umull2 v18.4s, v3.8h, v2.8h +; CHECK-NEXT: mov v1.s[0], v0.s[0] +; CHECK-NEXT: umull v0.4s, v3.4h, v2.4h +; CHECK-NEXT: umlal2 v18.4s, v19.8h, v16.8h +; CHECK-NEXT: umlal2 v17.4s, v6.8h, v4.8h +; CHECK-NEXT: umlal v1.4s, v7.4h, v5.4h +; CHECK-NEXT: umlal v0.4s, v19.4h, v16.4h +; CHECK-NEXT: add v2.4s, v17.4s, v18.4s +; CHECK-NEXT: umlal v1.4s, v6.4h, v4.4h +; CHECK-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: add w0, w8, w2 @@ -1639,14 +1639,14 @@ define i32 @test_udot_v33i8_nomla(ptr nocapture readonly %a1) { ; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 ; CHECK-NEXT: ushll2 v3.8h, v3.16b, #0 ; CHECK-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-NEXT: uaddl2 v6.4s, v3.8h, v2.8h -; CHECK-NEXT: uaddl v2.4s, v3.4h, v2.4h +; CHECK-NEXT: uaddl2 v6.4s, v5.8h, v4.8h ; CHECK-NEXT: mov v0.s[0], v1.s[0] -; CHECK-NEXT: uaddl2 v1.4s, v5.8h, v4.8h -; CHECK-NEXT: add v1.4s, v1.4s, v6.4s +; CHECK-NEXT: uaddl2 v1.4s, v3.8h, v2.8h +; CHECK-NEXT: uaddl v2.4s, v3.4h, v2.4h +; CHECK-NEXT: add v1.4s, v6.4s, v1.4s ; CHECK-NEXT: uaddw v0.4s, v0.4s, v5.4h -; CHECK-NEXT: uaddw v0.4s, v0.4s, v4.4h ; CHECK-NEXT: add v1.4s, v2.4s, v1.4s +; CHECK-NEXT: uaddw v0.4s, v0.4s, v4.4h ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w0, s0 @@ -1660,34 +1660,34 @@ entry: define i32 @test_sdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { ; CHECK-LABEL: test_sdot_v33i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldp q2, q3, [x0] -; CHECK-NEXT: movi v18.2d, #0000000000000000 -; CHECK-NEXT: ldp q4, q5, [x1] ; CHECK-NEXT: ldr b0, [x0, #32] ; CHECK-NEXT: ldr b1, [x1, #32] +; CHECK-NEXT: ldp q2, q4, [x0] +; CHECK-NEXT: ldp q3, q6, [x1] ; CHECK-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-NEXT: sshll v6.8h, v2.8b, #0 -; CHECK-NEXT: sshll2 v2.8h, v2.16b, #0 ; CHECK-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-NEXT: sshll v7.8h, v4.8b, #0 -; CHECK-NEXT: sshll2 v4.8h, v4.16b, #0 -; CHECK-NEXT: sshll2 v16.8h, v3.16b, #0 -; CHECK-NEXT: sshll v3.8h, v3.8b, #0 -; CHECK-NEXT: sshll2 v19.8h, v5.16b, #0 -; CHECK-NEXT: sshll v5.8h, v5.8b, #0 +; CHECK-NEXT: sshll v5.8h, v2.8b, #0 +; CHECK-NEXT: sshll2 v2.8h, v2.16b, #0 +; CHECK-NEXT: sshll2 v16.8h, v4.16b, #0 +; CHECK-NEXT: sshll v7.8h, v3.8b, #0 +; CHECK-NEXT: sshll2 v3.8h, v3.16b, #0 +; CHECK-NEXT: sshll v4.8h, v4.8b, #0 ; CHECK-NEXT: smull v0.4s, v1.4h, v0.4h -; CHECK-NEXT: smull2 v1.4s, v7.8h, v6.8h -; CHECK-NEXT: smull2 v17.4s, v4.8h, v2.8h -; CHECK-NEXT: smull v2.4s, v4.4h, v2.4h -; CHECK-NEXT: smlal2 v17.4s, v19.8h, v16.8h -; CHECK-NEXT: smlal2 v1.4s, v5.8h, v3.8h -; CHECK-NEXT: mov v18.s[0], v0.s[0] -; CHECK-NEXT: smlal v2.4s, v19.4h, v16.4h -; CHECK-NEXT: add v0.4s, v1.4s, v17.4s -; CHECK-NEXT: smlal v18.4s, v7.4h, v6.4h -; CHECK-NEXT: smlal v18.4s, v5.4h, v3.4h -; CHECK-NEXT: add v0.4s, v2.4s, v0.4s -; CHECK-NEXT: add v0.4s, v18.4s, v0.4s +; CHECK-NEXT: movi v1.2d, #0000000000000000 +; CHECK-NEXT: sshll2 v19.8h, v6.16b, #0 +; CHECK-NEXT: sshll v6.8h, v6.8b, #0 +; CHECK-NEXT: smull2 v17.4s, v7.8h, v5.8h +; CHECK-NEXT: smull2 v18.4s, v3.8h, v2.8h +; CHECK-NEXT: mov v1.s[0], v0.s[0] +; CHECK-NEXT: smull v0.4s, v3.4h, v2.4h +; CHECK-NEXT: smlal2 v18.4s, v19.8h, v16.8h +; CHECK-NEXT: smlal2 v17.4s, v6.8h, v4.8h +; CHECK-NEXT: smlal v1.4s, v7.4h, v5.4h +; CHECK-NEXT: smlal v0.4s, v19.4h, v16.4h +; CHECK-NEXT: add v2.4s, v17.4s, v18.4s +; CHECK-NEXT: smlal v1.4s, v6.4h, v4.4h +; CHECK-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: add w0, w8, w2 @@ -2018,151 +2018,151 @@ define i32 @test_sdot_v33i8_double_nomla(<33 x i8> %a, <33 x i8> %b, <33 x i8> % ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: .cfi_def_cfa_offset 16 ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ldr b1, [sp, #80] +; CHECK-NEXT: ldr b0, [sp, #80] ; CHECK-NEXT: add x8, sp, #88 ; CHECK-NEXT: ldr b2, [sp, #144] -; CHECK-NEXT: add x9, sp, #152 -; CHECK-NEXT: ldr b3, [sp, #16] -; CHECK-NEXT: add x12, sp, #32 -; CHECK-NEXT: ld1 { v1.b }[1], [x8] -; CHECK-NEXT: ld1 { v2.b }[1], [x9] ; CHECK-NEXT: add x9, sp, #96 -; CHECK-NEXT: add x8, sp, #24 -; CHECK-NEXT: add x11, sp, #112 -; CHECK-NEXT: fmov s0, w0 -; CHECK-NEXT: ld1 { v3.b }[1], [x8] -; CHECK-NEXT: add x8, sp, #160 -; CHECK-NEXT: ldr b4, [sp, #480] -; CHECK-NEXT: ld1 { v1.b }[2], [x9] -; CHECK-NEXT: add x9, sp, #104 -; CHECK-NEXT: ld1 { v2.b }[2], [x8] +; CHECK-NEXT: add x10, sp, #152 +; CHECK-NEXT: add x11, sp, #160 +; CHECK-NEXT: ld1 { v0.b }[1], [x8] +; CHECK-NEXT: ld1 { v2.b }[1], [x10] +; CHECK-NEXT: add x8, sp, #104 +; CHECK-NEXT: ldr b3, [sp, #16] +; CHECK-NEXT: ldr b5, [sp, #480] +; CHECK-NEXT: fmov s1, w0 +; CHECK-NEXT: add x10, sp, #24 +; CHECK-NEXT: add x13, sp, #488 +; CHECK-NEXT: ldr b4, [sp, #608] +; CHECK-NEXT: ld1 { v0.b }[2], [x9] +; CHECK-NEXT: ld1 { v2.b }[2], [x11] +; CHECK-NEXT: add x12, sp, #112 +; CHECK-NEXT: ld1 { v3.b }[1], [x10] +; CHECK-NEXT: ld1 { v5.b }[1], [x13] +; CHECK-NEXT: add x10, sp, #616 +; CHECK-NEXT: mov v1.b[1], w1 +; CHECK-NEXT: ld1 { v4.b }[1], [x10] +; CHECK-NEXT: add x10, sp, #32 +; CHECK-NEXT: ld1 { v0.b }[3], [x8] ; CHECK-NEXT: add x8, sp, #168 -; CHECK-NEXT: add x10, sp, #120 -; CHECK-NEXT: add x13, sp, #48 -; CHECK-NEXT: ld1 { v3.b }[2], [x12] -; CHECK-NEXT: add x12, sp, #40 -; CHECK-NEXT: ldr b5, [sp, #608] -; CHECK-NEXT: ld1 { v1.b }[3], [x9] +; CHECK-NEXT: add x9, sp, #120 ; CHECK-NEXT: ld1 { v2.b }[3], [x8] -; CHECK-NEXT: mov v0.b[1], w1 -; CHECK-NEXT: add x9, sp, #128 -; CHECK-NEXT: add x14, sp, #184 -; CHECK-NEXT: ldr b16, [sp, #544] -; CHECK-NEXT: ld1 { v3.b }[3], [x12] -; CHECK-NEXT: add x12, sp, #176 -; CHECK-NEXT: ldr b17, [sp, #672] -; CHECK-NEXT: ld1 { v1.b }[4], [x11] -; CHECK-NEXT: add x11, sp, #488 -; CHECK-NEXT: ld1 { v2.b }[4], [x12] -; CHECK-NEXT: ld1 { v4.b }[1], [x11] -; CHECK-NEXT: mov v0.b[2], w2 -; CHECK-NEXT: add x11, sp, #192 -; CHECK-NEXT: ld1 { v3.b }[4], [x13] -; CHECK-NEXT: add x13, sp, #616 -; CHECK-NEXT: add x12, sp, #56 -; CHECK-NEXT: ld1 { v1.b }[5], [x10] -; CHECK-NEXT: ld1 { v5.b }[1], [x13] -; CHECK-NEXT: add x13, sp, #496 +; CHECK-NEXT: add x8, sp, #496 +; CHECK-NEXT: ld1 { v3.b }[2], [x10] +; CHECK-NEXT: ld1 { v5.b }[2], [x8] +; CHECK-NEXT: add x8, sp, #176 +; CHECK-NEXT: ldr b6, [sp, #544] +; CHECK-NEXT: ld1 { v0.b }[4], [x12] +; CHECK-NEXT: add x14, sp, #552 +; CHECK-NEXT: ldr b7, [sp, #672] +; CHECK-NEXT: ld1 { v2.b }[4], [x8] +; CHECK-NEXT: add x13, sp, #40 +; CHECK-NEXT: ld1 { v6.b }[1], [x14] +; CHECK-NEXT: mov v1.b[2], w2 +; CHECK-NEXT: add x11, sp, #128 +; CHECK-NEXT: ld1 { v3.b }[3], [x13] +; CHECK-NEXT: ld1 { v0.b }[5], [x9] +; CHECK-NEXT: add x9, sp, #680 +; CHECK-NEXT: add x13, sp, #184 +; CHECK-NEXT: ld1 { v7.b }[1], [x9] +; CHECK-NEXT: ld1 { v2.b }[5], [x13] +; CHECK-NEXT: add x13, sp, #624 +; CHECK-NEXT: add x15, sp, #504 ; CHECK-NEXT: ld1 { v4.b }[2], [x13] -; CHECK-NEXT: ld1 { v2.b }[5], [x14] -; CHECK-NEXT: add x14, sp, #680 -; CHECK-NEXT: ld1 { v17.b }[1], [x14] -; CHECK-NEXT: add x13, sp, #504 -; CHECK-NEXT: ld1 { v3.b }[5], [x12] -; CHECK-NEXT: ld1 { v1.b }[6], [x9] -; CHECK-NEXT: add x9, sp, #552 -; CHECK-NEXT: add x12, sp, #688 -; CHECK-NEXT: ld1 { v16.b }[1], [x9] -; CHECK-NEXT: add x9, sp, #624 -; CHECK-NEXT: ld1 { v4.b }[3], [x13] -; CHECK-NEXT: ld1 { v2.b }[6], [x11] +; CHECK-NEXT: add x10, sp, #136 +; CHECK-NEXT: ld1 { v0.b }[6], [x11] ; CHECK-NEXT: add x11, sp, #560 -; CHECK-NEXT: add x8, sp, #136 -; CHECK-NEXT: ld1 { v17.b }[2], [x12] -; CHECK-NEXT: ld1 { v5.b }[2], [x9] -; CHECK-NEXT: ld1 { v1.b }[7], [x8] -; CHECK-NEXT: ld1 { v16.b }[2], [x11] -; CHECK-NEXT: add x8, sp, #512 -; CHECK-NEXT: mov v0.b[3], w3 -; CHECK-NEXT: ld1 { v4.b }[4], [x8] -; CHECK-NEXT: add x8, sp, #568 -; CHECK-NEXT: add x9, sp, #696 -; CHECK-NEXT: add x11, sp, #632 -; CHECK-NEXT: ld1 { v17.b }[3], [x9] -; CHECK-NEXT: add x9, sp, #520 -; CHECK-NEXT: ld1 { v16.b }[3], [x8] -; CHECK-NEXT: ld1 { v5.b }[3], [x11] -; CHECK-NEXT: add x8, sp, #640 -; CHECK-NEXT: ld1 { v4.b }[5], [x9] +; CHECK-NEXT: ld1 { v5.b }[3], [x15] +; CHECK-NEXT: ld1 { v6.b }[2], [x11] +; CHECK-NEXT: add x11, sp, #688 +; CHECK-NEXT: mov v1.b[3], w3 +; CHECK-NEXT: ld1 { v7.b }[2], [x11] +; CHECK-NEXT: add x9, sp, #632 +; CHECK-NEXT: add x11, sp, #512 +; CHECK-NEXT: ld1 { v0.b }[7], [x10] +; CHECK-NEXT: ld1 { v4.b }[3], [x9] +; CHECK-NEXT: add x9, sp, #568 +; CHECK-NEXT: add x10, sp, #696 +; CHECK-NEXT: ld1 { v6.b }[3], [x9] +; CHECK-NEXT: ld1 { v5.b }[4], [x11] +; CHECK-NEXT: ld1 { v7.b }[3], [x10] +; CHECK-NEXT: add x9, sp, #640 +; CHECK-NEXT: mov v1.b[4], w4 +; CHECK-NEXT: ld1 { v4.b }[4], [x9] ; CHECK-NEXT: add x9, sp, #576 -; CHECK-NEXT: add x11, sp, #704 +; CHECK-NEXT: add x10, sp, #704 +; CHECK-NEXT: add x11, sp, #520 +; CHECK-NEXT: ld1 { v6.b }[4], [x9] ; CHECK-NEXT: ldr b18, [sp, #736] -; CHECK-NEXT: mov v0.b[4], w4 -; CHECK-NEXT: ld1 { v17.b }[4], [x11] -; CHECK-NEXT: ld1 { v16.b }[4], [x9] -; CHECK-NEXT: ld1 { v5.b }[4], [x8] -; CHECK-NEXT: add x9, sp, #528 -; CHECK-NEXT: sshll v18.8h, v18.8b, #0 -; CHECK-NEXT: add x8, sp, #648 +; CHECK-NEXT: ld1 { v7.b }[4], [x10] +; CHECK-NEXT: ld1 { v5.b }[5], [x11] +; CHECK-NEXT: add x12, sp, #192 +; CHECK-NEXT: add x8, sp, #48 +; CHECK-NEXT: ld1 { v2.b }[6], [x12] +; CHECK-NEXT: add x9, sp, #648 +; CHECK-NEXT: ld1 { v3.b }[4], [x8] +; CHECK-NEXT: add x10, sp, #528 ; CHECK-NEXT: add x11, sp, #584 ; CHECK-NEXT: add x12, sp, #712 -; CHECK-NEXT: ld1 { v4.b }[6], [x9] -; CHECK-NEXT: movi v7.2d, #0000000000000000 -; CHECK-NEXT: ld1 { v16.b }[5], [x11] -; CHECK-NEXT: ld1 { v17.b }[5], [x12] -; CHECK-NEXT: ld1 { v5.b }[5], [x8] -; CHECK-NEXT: mov v0.b[5], w5 -; CHECK-NEXT: add x9, sp, #536 -; CHECK-NEXT: sshll v18.4s, v18.4h, #0 -; CHECK-NEXT: add x8, sp, #656 +; CHECK-NEXT: sshll v18.8h, v18.8b, #0 +; CHECK-NEXT: mov v1.b[5], w5 +; CHECK-NEXT: ld1 { v6.b }[5], [x11] +; CHECK-NEXT: ld1 { v7.b }[5], [x12] +; CHECK-NEXT: ld1 { v4.b }[5], [x9] +; CHECK-NEXT: ld1 { v5.b }[6], [x10] +; CHECK-NEXT: add x14, sp, #56 +; CHECK-NEXT: movi v17.2d, #0000000000000000 +; CHECK-NEXT: ld1 { v3.b }[5], [x14] +; CHECK-NEXT: add x9, sp, #656 +; CHECK-NEXT: add x10, sp, #536 ; CHECK-NEXT: add x11, sp, #592 ; CHECK-NEXT: add x12, sp, #720 -; CHECK-NEXT: ld1 { v4.b }[7], [x9] -; CHECK-NEXT: ld1 { v16.b }[6], [x11] -; CHECK-NEXT: ld1 { v17.b }[6], [x12] -; CHECK-NEXT: ld1 { v5.b }[6], [x8] -; CHECK-NEXT: ldr b6, [sp, #208] -; CHECK-NEXT: add x10, sp, #64 -; CHECK-NEXT: mov v7.s[0], v18.s[0] -; CHECK-NEXT: mov v0.b[6], w6 -; CHECK-NEXT: ld1 { v3.b }[6], [x10] +; CHECK-NEXT: sshll v18.4s, v18.4h, #0 +; CHECK-NEXT: ldr b16, [sp, #208] +; CHECK-NEXT: ld1 { v6.b }[6], [x11] +; CHECK-NEXT: ld1 { v7.b }[6], [x12] +; CHECK-NEXT: ld1 { v4.b }[6], [x9] +; CHECK-NEXT: ld1 { v5.b }[7], [x10] +; CHECK-NEXT: add x8, sp, #64 +; CHECK-NEXT: mov v1.b[6], w6 +; CHECK-NEXT: sshll v16.8h, v16.8b, #0 +; CHECK-NEXT: ld1 { v3.b }[6], [x8] ; CHECK-NEXT: add x8, sp, #664 ; CHECK-NEXT: add x9, sp, #600 ; CHECK-NEXT: add x10, sp, #728 -; CHECK-NEXT: sshll v4.8h, v4.8b, #0 -; CHECK-NEXT: sshll v6.8h, v6.8b, #0 -; CHECK-NEXT: ld1 { v16.b }[7], [x9] -; CHECK-NEXT: ld1 { v17.b }[7], [x10] -; CHECK-NEXT: ld1 { v5.b }[7], [x8] +; CHECK-NEXT: mov v17.s[0], v18.s[0] +; CHECK-NEXT: ld1 { v6.b }[7], [x9] +; CHECK-NEXT: ld1 { v7.b }[7], [x10] +; CHECK-NEXT: ld1 { v4.b }[7], [x8] +; CHECK-NEXT: sshll v5.8h, v5.8b, #0 ; CHECK-NEXT: movi v18.2d, #0000000000000000 -; CHECK-NEXT: mov v0.b[7], w7 +; CHECK-NEXT: sshll v16.4s, v16.4h, #0 +; CHECK-NEXT: mov v1.b[7], w7 ; CHECK-NEXT: add x9, sp, #200 -; CHECK-NEXT: add x10, sp, #72 -; CHECK-NEXT: saddw v7.4s, v7.4s, v4.4h -; CHECK-NEXT: sshll v6.4s, v6.4h, #0 -; CHECK-NEXT: sshll v16.8h, v16.8b, #0 -; CHECK-NEXT: sshll v17.8h, v17.8b, #0 -; CHECK-NEXT: sshll v5.8h, v5.8b, #0 +; CHECK-NEXT: add x8, sp, #72 ; CHECK-NEXT: ld1 { v2.b }[7], [x9] -; CHECK-NEXT: ld1 { v3.b }[7], [x10] -; CHECK-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-NEXT: mov v18.s[0], v6.s[0] +; CHECK-NEXT: sshll v6.8h, v6.8b, #0 +; CHECK-NEXT: ld1 { v3.b }[7], [x8] +; CHECK-NEXT: sshll v7.8h, v7.8b, #0 +; CHECK-NEXT: sshll v4.8h, v4.8b, #0 +; CHECK-NEXT: saddw v17.4s, v17.4s, v5.4h +; CHECK-NEXT: mov v18.s[0], v16.s[0] ; CHECK-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-NEXT: saddl2 v6.4s, v17.8h, v16.8h -; CHECK-NEXT: saddl2 v4.4s, v5.8h, v4.8h -; CHECK-NEXT: saddl v16.4s, v17.4h, v16.4h -; CHECK-NEXT: saddw v5.4s, v7.4s, v5.4h +; CHECK-NEXT: sshll v1.8h, v1.8b, #0 ; CHECK-NEXT: sshll v2.8h, v2.8b, #0 ; CHECK-NEXT: sshll v3.8h, v3.8b, #0 -; CHECK-NEXT: saddl2 v17.4s, v0.8h, v1.8h -; CHECK-NEXT: saddw v0.4s, v18.4s, v0.4h +; CHECK-NEXT: saddl2 v16.4s, v7.8h, v6.8h +; CHECK-NEXT: saddl2 v5.4s, v4.8h, v5.8h +; CHECK-NEXT: saddl v6.4s, v7.4h, v6.4h +; CHECK-NEXT: saddw v4.4s, v17.4s, v4.4h +; CHECK-NEXT: saddl2 v17.4s, v1.8h, v0.8h +; CHECK-NEXT: saddw v1.4s, v18.4s, v1.4h ; CHECK-NEXT: saddl2 v7.4s, v3.8h, v2.8h -; CHECK-NEXT: add v4.4s, v4.4s, v6.4s -; CHECK-NEXT: saddl v2.4s, v3.4h, v2.4h ; CHECK-NEXT: add v5.4s, v5.4s, v16.4s -; CHECK-NEXT: saddw v0.4s, v0.4s, v1.4h +; CHECK-NEXT: saddl v2.4s, v3.4h, v2.4h +; CHECK-NEXT: add v4.4s, v4.4s, v6.4s +; CHECK-NEXT: saddw v0.4s, v1.4s, v0.4h ; CHECK-NEXT: add v6.4s, v17.4s, v7.4s -; CHECK-NEXT: add v1.4s, v5.4s, v4.4s +; CHECK-NEXT: add v1.4s, v4.4s, v5.4s ; CHECK-NEXT: add v0.4s, v0.4s, v2.4s ; CHECK-NEXT: add v1.4s, v6.4s, v1.4s ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s diff --git a/llvm/test/CodeGen/AArch64/neon-extadd.ll b/llvm/test/CodeGen/AArch64/neon-extadd.ll index 913205f32753..16200435c5c3 100644 --- a/llvm/test/CodeGen/AArch64/neon-extadd.ll +++ b/llvm/test/CodeGen/AArch64/neon-extadd.ll @@ -515,15 +515,15 @@ define <20 x i32> @v20(<20 x i8> %s0, <20 x i8> %s1) { ; CHECK-NEXT: mov v0.b[6], w6 ; CHECK-NEXT: ld1 { v1.b }[7], [x9] ; CHECK-NEXT: uaddl v2.8h, v3.8b, v2.8b +; CHECK-NEXT: ushll v3.4s, v4.4h, #0 ; CHECK-NEXT: mov v0.b[7], w7 -; CHECK-NEXT: ushll2 v3.4s, v2.8h, #0 -; CHECK-NEXT: ushll v2.4s, v2.4h, #0 ; CHECK-NEXT: uaddl v0.8h, v0.8b, v1.8b -; CHECK-NEXT: ushll v1.4s, v4.4h, #0 -; CHECK-NEXT: stp q3, q1, [x8, #48] -; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0 +; CHECK-NEXT: ushll2 v1.4s, v2.8h, #0 +; CHECK-NEXT: ushll v2.4s, v2.4h, #0 +; CHECK-NEXT: stp q1, q3, [x8, #48] +; CHECK-NEXT: ushll2 v3.4s, v0.8h, #0 ; CHECK-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-NEXT: stp q1, q2, [x8, #16] +; CHECK-NEXT: stp q3, q2, [x8, #16] ; CHECK-NEXT: str q0, [x8] ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/neon-shift-neg.ll b/llvm/test/CodeGen/AArch64/neon-shift-neg.ll index 881bbf315e8e..45272143e859 100644 --- a/llvm/test/CodeGen/AArch64/neon-shift-neg.ll +++ b/llvm/test/CodeGen/AArch64/neon-shift-neg.ll @@ -375,8 +375,8 @@ entry: define @shrn64x2( %a, i64 %b) { ; CHECK-LABEL: shrn64x2: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: neg x8, x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: asr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -391,8 +391,8 @@ entry: define @shrn32x4( %a, i32 %b) { ; CHECK-LABEL: shrn32x4: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: neg w8, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: asr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -407,8 +407,8 @@ entry: define @shrn16x8( %a, i16 %b) { ; CHECK-LABEL: shrn16x8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: neg w8, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: asr z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret @@ -423,8 +423,8 @@ entry: define @shrn8x16( %a, i8 %b) { ; CHECK-LABEL: shrn8x16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: neg w8, w0 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z1.b, w8 ; CHECK-NEXT: asr z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: ret @@ -439,8 +439,8 @@ entry: define @lshrn64x2( %a, i64 %b) { ; CHECK-LABEL: lshrn64x2: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: neg x8, x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: lsr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -455,8 +455,8 @@ entry: define @lshrn32x4( %a, i32 %b) { ; CHECK-LABEL: lshrn32x4: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: neg w8, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: lsr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -471,8 +471,8 @@ entry: define @lshrn16x8( %a, i16 %b) { ; CHECK-LABEL: lshrn16x8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: neg w8, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: lsr z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret @@ -487,8 +487,8 @@ entry: define @lshrn8x16( %a, i8 %b) { ; CHECK-LABEL: lshrn8x16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: neg w8, w0 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z1.b, w8 ; CHECK-NEXT: lsr z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: ret @@ -503,8 +503,8 @@ entry: define @shln64x2( %a, i64 %b) { ; CHECK-LABEL: shln64x2: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: neg x8, x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret @@ -519,8 +519,8 @@ entry: define @shln32x4( %a, i32 %b) { ; CHECK-LABEL: shln32x4: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: neg w8, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: lsl z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -535,8 +535,8 @@ entry: define @shln16x8( %a, i16 %b) { ; CHECK-LABEL: shln16x8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: neg w8, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: lsl z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret @@ -551,8 +551,8 @@ entry: define @shln8x16( %a, i8 %b) { ; CHECK-LABEL: shln8x16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: neg w8, w0 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z1.b, w8 ; CHECK-NEXT: lsl z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/predicated-add-sub.ll b/llvm/test/CodeGen/AArch64/predicated-add-sub.ll index 6b3cfc040cb3..20088354bdb7 100644 --- a/llvm/test/CodeGen/AArch64/predicated-add-sub.ll +++ b/llvm/test/CodeGen/AArch64/predicated-add-sub.ll @@ -83,11 +83,11 @@ define @zext.add.2xi64( %a, @zext.add.8xi32( %a, %v) #0 { ; CHECK-LABEL: zext.add.8xi32: ; CHECK: // %bb.0: -; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: mov z2.s, #1 // =0x1 +; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: punpklo p0.h, p0.b -; CHECK-NEXT: add z1.s, p1/m, z1.s, z2.s ; CHECK-NEXT: add z0.s, p0/m, z0.s, z2.s +; CHECK-NEXT: add z1.s, p1/m, z1.s, z2.s ; CHECK-NEXT: ret %extend = zext %v to %result = add %a, %extend @@ -103,8 +103,8 @@ define @zext.add.16xi32( %a, @zext.sub.2xi64( %a, @zext.sub.8xi32( %a, %v) #0 { ; CHECK-LABEL: zext.sub.8xi32: ; CHECK: // %bb.0: -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: mov z2.s, #-1 // =0xffffffffffffffff +; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: add z0.s, p1/m, z0.s, z2.s ; CHECK-NEXT: add z1.s, p0/m, z1.s, z2.s @@ -214,8 +214,8 @@ define @zext.sub.16xi32( %a, @sext.add.2xi64( %a, @sext.add.8xi32( %a, %v) #0 { ; CHECK-LABEL: sext.add.8xi32: ; CHECK: // %bb.0: -; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: mov z2.s, #-1 // =0xffffffffffffffff +; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: punpklo p0.h, p0.b -; CHECK-NEXT: add z1.s, p1/m, z1.s, z2.s ; CHECK-NEXT: add z0.s, p0/m, z0.s, z2.s +; CHECK-NEXT: add z1.s, p1/m, z1.s, z2.s ; CHECK-NEXT: ret %extend = sext %v to %result = add %a, %extend @@ -325,8 +325,8 @@ define @sext.add.16xi32( %a, @sext.sub.2xi64( %a, @sext.sub.8xi32( %a, %v) #0 { ; CHECK-LABEL: sext.sub.8xi32: ; CHECK: // %bb.0: -; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: mov z2.s, #-1 // =0xffffffffffffffff +; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: punpklo p0.h, p0.b -; CHECK-NEXT: sub z1.s, p1/m, z1.s, z2.s ; CHECK-NEXT: sub z0.s, p0/m, z0.s, z2.s +; CHECK-NEXT: sub z1.s, p1/m, z1.s, z2.s ; CHECK-NEXT: ret %extend = sext %v to %result = sub %a, %extend @@ -436,8 +436,8 @@ define @sext.sub.16xi32( %a, This Inner Loop Header: Depth=1 +; CHECK-NEXT: str q14, [sp, #32] // 16-byte Folded Spill ; CHECK-NEXT: ldr q14, [x8] ; CHECK-NEXT: mov x12, xzr -; CHECK-NEXT: str q18, [sp, #32] // 16-byte Folded Spill ; CHECK-NEXT: ldr x14, [x12] -; CHECK-NEXT: ldr q15, [x12] -; CHECK-NEXT: add x7, x11, x8 +; CHECK-NEXT: stp q29, q15, [sp] // 32-byte Folded Spill +; CHECK-NEXT: add x19, x11, x8 ; CHECK-NEXT: fmov x15, d14 ; CHECK-NEXT: mov x16, v14.d[1] -; CHECK-NEXT: ldr q18, [sp, #64] // 16-byte Folded Reload -; CHECK-NEXT: fmov x18, d15 -; CHECK-NEXT: mov x13, v15.d[1] -; CHECK-NEXT: ldr x5, [x8] +; CHECK-NEXT: ldr q15, [x12] ; CHECK-NEXT: ldr q14, [x10], #64 -; CHECK-NEXT: ldr x7, [x7, #128] +; CHECK-NEXT: mov v8.16b, v28.16b +; CHECK-NEXT: fmov x13, d15 +; CHECK-NEXT: mov x18, v15.d[1] +; CHECK-NEXT: mov v28.16b, v24.16b ; CHECK-NEXT: mul x17, x15, x14 -; CHECK-NEXT: mov v6.16b, v0.16b -; CHECK-NEXT: mov v9.16b, v27.16b ; CHECK-NEXT: mov x12, v14.d[1] ; CHECK-NEXT: fmov x4, d14 -; CHECK-NEXT: mov v27.16b, v23.16b +; CHECK-NEXT: mov v24.16b, v20.16b +; CHECK-NEXT: mov v20.16b, v17.16b +; CHECK-NEXT: mov v17.16b, v5.16b ; CHECK-NEXT: mul x1, x16, x14 -; CHECK-NEXT: mov v23.16b, v19.16b -; CHECK-NEXT: mov v19.16b, v7.16b -; CHECK-NEXT: mov v7.16b, v2.16b -; CHECK-NEXT: stp q26, q31, [sp] // 32-byte Folded Spill -; CHECK-NEXT: mov v31.16b, v22.16b -; CHECK-NEXT: mul x0, x18, x14 -; CHECK-NEXT: mov v26.16b, v10.16b -; CHECK-NEXT: mov v22.16b, v5.16b +; CHECK-NEXT: ldr q5, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: ldr x5, [x8] +; CHECK-NEXT: ldr x19, [x19, #128] +; CHECK-NEXT: mov v29.16b, v21.16b +; CHECK-NEXT: mov v21.16b, v0.16b +; CHECK-NEXT: mul x0, x13, x14 +; CHECK-NEXT: mov v25.16b, v6.16b +; CHECK-NEXT: mov v6.16b, v2.16b ; CHECK-NEXT: fmov d15, x17 -; CHECK-NEXT: mov v5.16b, v1.16b -; CHECK-NEXT: mov v8.16b, v20.16b -; CHECK-NEXT: mul x2, x13, x14 -; CHECK-NEXT: mov v20.16b, v16.16b -; CHECK-NEXT: mov v16.16b, v3.16b -; CHECK-NEXT: mov v10.16b, v21.16b -; CHECK-NEXT: mov v21.16b, v17.16b -; CHECK-NEXT: mov v17.16b, v4.16b +; CHECK-NEXT: mov v26.16b, v22.16b +; CHECK-NEXT: mov v22.16b, v18.16b +; CHECK-NEXT: mul x2, x18, x14 +; CHECK-NEXT: mov v18.16b, v7.16b +; CHECK-NEXT: mov v7.16b, v3.16b +; CHECK-NEXT: mov v16.16b, v4.16b +; CHECK-NEXT: add x8, x8, #8 +; CHECK-NEXT: add x9, x9, #1 ; CHECK-NEXT: mov v15.d[1], x1 ; CHECK-NEXT: mul x3, x12, x14 -; CHECK-NEXT: add x8, x8, #8 -; CHECK-NEXT: fmov d14, x0 ; CHECK-NEXT: cmp x8, #64 -; CHECK-NEXT: add x9, x9, #1 +; CHECK-NEXT: fmov d14, x0 ; CHECK-NEXT: mul x14, x4, x14 -; CHECK-NEXT: add v18.2d, v18.2d, v15.2d -; CHECK-NEXT: mul x19, x15, x5 +; CHECK-NEXT: add v5.2d, v5.2d, v15.2d +; CHECK-NEXT: mul x20, x15, x5 ; CHECK-NEXT: mov v14.d[1], x2 -; CHECK-NEXT: mul x15, x15, x7 +; CHECK-NEXT: mul x15, x15, x19 ; CHECK-NEXT: fmov d0, x14 -; CHECK-NEXT: str q18, [sp, #64] // 16-byte Folded Spill -; CHECK-NEXT: ldp q18, q15, [sp, #32] // 32-byte Folded Reload -; CHECK-NEXT: mul x6, x16, x5 -; CHECK-NEXT: fmov d1, x19 +; CHECK-NEXT: str q5, [sp, #64] // 16-byte Folded Spill +; CHECK-NEXT: ldr q5, [sp, #48] // 16-byte Folded Reload +; CHECK-NEXT: mul x21, x13, x19 +; CHECK-NEXT: add v5.2d, v5.2d, v14.2d +; CHECK-NEXT: fmov d3, x20 +; CHECK-NEXT: mul x7, x16, x5 ; CHECK-NEXT: mov v0.d[1], x3 -; CHECK-NEXT: mul x16, x16, x7 -; CHECK-NEXT: fmov d2, x15 -; CHECK-NEXT: add v15.2d, v15.2d, v14.2d -; CHECK-NEXT: mul x21, x18, x7 -; CHECK-NEXT: mov v1.d[1], x6 -; CHECK-NEXT: mul x0, x4, x7 -; CHECK-NEXT: str q15, [sp, #48] // 16-byte Folded Spill -; CHECK-NEXT: add v15.2d, v11.2d, v14.2d -; CHECK-NEXT: mov v2.d[1], x16 -; CHECK-NEXT: ldr q11, [sp, #80] // 16-byte Folded Reload -; CHECK-NEXT: mul x20, x13, x7 -; CHECK-NEXT: fmov d3, x21 -; CHECK-NEXT: add v11.2d, v11.2d, v0.2d -; CHECK-NEXT: add v12.2d, v12.2d, v1.2d -; CHECK-NEXT: mul x22, x12, x7 -; CHECK-NEXT: fmov d4, x0 -; CHECK-NEXT: add v18.2d, v18.2d, v2.2d -; CHECK-NEXT: mov v2.16b, v7.16b -; CHECK-NEXT: mul x14, x18, x5 -; CHECK-NEXT: mov v7.16b, v19.16b -; CHECK-NEXT: mov v19.16b, v23.16b -; CHECK-NEXT: mov v3.d[1], x20 -; CHECK-NEXT: mov v23.16b, v27.16b -; CHECK-NEXT: add v27.2d, v9.2d, v1.2d -; CHECK-NEXT: mul x15, x4, x5 -; CHECK-NEXT: str q11, [sp, #80] // 16-byte Folded Spill -; CHECK-NEXT: mov v11.16b, v15.16b -; CHECK-NEXT: mov v4.d[1], x22 -; CHECK-NEXT: add v19.2d, v19.2d, v1.2d -; CHECK-NEXT: add v7.2d, v7.2d, v1.2d +; CHECK-NEXT: fmov d1, x15 +; CHECK-NEXT: mul x16, x16, x19 +; CHECK-NEXT: str q5, [sp, #48] // 16-byte Folded Spill +; CHECK-NEXT: add v5.2d, v13.2d, v14.2d +; CHECK-NEXT: fmov d2, x21 +; CHECK-NEXT: ldr q13, [sp, #80] // 16-byte Folded Reload +; CHECK-NEXT: mul x6, x18, x5 +; CHECK-NEXT: ldp q15, q14, [sp, #16] // 32-byte Folded Reload +; CHECK-NEXT: mov v3.d[1], x7 +; CHECK-NEXT: add v13.2d, v13.2d, v0.2d +; CHECK-NEXT: mul x18, x18, x19 +; CHECK-NEXT: mov v1.d[1], x16 +; CHECK-NEXT: mul x22, x4, x19 +; CHECK-NEXT: str q13, [sp, #80] // 16-byte Folded Spill +; CHECK-NEXT: mov v13.16b, v5.16b +; CHECK-NEXT: mov v5.16b, v17.16b +; CHECK-NEXT: mov v17.16b, v20.16b +; CHECK-NEXT: mov v20.16b, v24.16b ; CHECK-NEXT: mul x13, x13, x5 -; CHECK-NEXT: add v23.2d, v23.2d, v1.2d -; CHECK-NEXT: add v1.2d, v5.2d, v1.2d -; CHECK-NEXT: fmov d14, x14 -; CHECK-NEXT: add v30.2d, v30.2d, v3.2d -; CHECK-NEXT: mov v3.16b, v16.16b +; CHECK-NEXT: mov v24.16b, v28.16b +; CHECK-NEXT: add v11.2d, v11.2d, v3.2d +; CHECK-NEXT: mov v2.d[1], x18 +; CHECK-NEXT: add v15.2d, v15.2d, v1.2d +; CHECK-NEXT: add v27.2d, v27.2d, v3.2d +; CHECK-NEXT: mul x17, x12, x19 +; CHECK-NEXT: add v23.2d, v23.2d, v3.2d +; CHECK-NEXT: add v19.2d, v19.2d, v3.2d +; CHECK-NEXT: fmov d4, x22 +; CHECK-NEXT: add v10.2d, v10.2d, v3.2d +; CHECK-NEXT: mul x14, x4, x5 +; CHECK-NEXT: fmov d0, x13 +; CHECK-NEXT: add v14.2d, v14.2d, v2.2d +; CHECK-NEXT: add v2.2d, v6.2d, v3.2d ; CHECK-NEXT: mul x12, x12, x5 -; CHECK-NEXT: mov v16.16b, v20.16b -; CHECK-NEXT: mov v5.16b, v22.16b -; CHECK-NEXT: fmov d0, x15 -; CHECK-NEXT: add v28.2d, v28.2d, v4.2d -; CHECK-NEXT: mov v4.16b, v17.16b -; CHECK-NEXT: mov v17.16b, v21.16b -; CHECK-NEXT: mov v21.16b, v10.16b -; CHECK-NEXT: mov v10.16b, v26.16b -; CHECK-NEXT: mov v14.d[1], x13 -; CHECK-NEXT: mov v22.16b, v31.16b -; CHECK-NEXT: ldp q26, q31, [sp] // 32-byte Folded Reload -; CHECK-NEXT: mov v0.d[1], x12 -; CHECK-NEXT: add v13.2d, v13.2d, v14.2d -; CHECK-NEXT: add v31.2d, v31.2d, v14.2d -; CHECK-NEXT: add v26.2d, v26.2d, v14.2d -; CHECK-NEXT: add v24.2d, v24.2d, v14.2d -; CHECK-NEXT: add v22.2d, v22.2d, v14.2d -; CHECK-NEXT: add v20.2d, v8.2d, v14.2d -; CHECK-NEXT: add v10.2d, v10.2d, v14.2d -; CHECK-NEXT: add v16.2d, v16.2d, v14.2d -; CHECK-NEXT: add v5.2d, v5.2d, v14.2d -; CHECK-NEXT: add v3.2d, v3.2d, v14.2d -; CHECK-NEXT: add v2.2d, v2.2d, v14.2d -; CHECK-NEXT: add v29.2d, v29.2d, v0.2d -; CHECK-NEXT: add v25.2d, v25.2d, v0.2d -; CHECK-NEXT: add v21.2d, v21.2d, v0.2d +; CHECK-NEXT: mov v3.16b, v7.16b +; CHECK-NEXT: mov v7.16b, v18.16b +; CHECK-NEXT: mov v4.d[1], x17 +; CHECK-NEXT: mov v18.16b, v22.16b +; CHECK-NEXT: mov v0.d[1], x6 +; CHECK-NEXT: fmov d1, x14 +; CHECK-NEXT: add v28.2d, v8.2d, v4.2d +; CHECK-NEXT: mov v1.d[1], x12 +; CHECK-NEXT: add v31.2d, v31.2d, v0.2d +; CHECK-NEXT: add v30.2d, v30.2d, v0.2d +; CHECK-NEXT: add v12.2d, v12.2d, v0.2d +; CHECK-NEXT: add v24.2d, v24.2d, v0.2d +; CHECK-NEXT: add v22.2d, v26.2d, v0.2d +; CHECK-NEXT: add v20.2d, v20.2d, v0.2d +; CHECK-NEXT: add v18.2d, v18.2d, v0.2d ; CHECK-NEXT: add v17.2d, v17.2d, v0.2d -; CHECK-NEXT: add v4.2d, v4.2d, v0.2d -; CHECK-NEXT: add v0.2d, v6.2d, v0.2d +; CHECK-NEXT: add v7.2d, v7.2d, v0.2d +; CHECK-NEXT: add v4.2d, v16.2d, v0.2d +; CHECK-NEXT: add v3.2d, v3.2d, v0.2d +; CHECK-NEXT: mov v0.16b, v21.16b +; CHECK-NEXT: mov v21.16b, v29.16b +; CHECK-NEXT: ldr q29, [sp] // 16-byte Folded Reload +; CHECK-NEXT: add v9.2d, v9.2d, v1.2d +; CHECK-NEXT: add v6.2d, v25.2d, v1.2d +; CHECK-NEXT: add v5.2d, v5.2d, v1.2d +; CHECK-NEXT: add v29.2d, v29.2d, v1.2d +; CHECK-NEXT: add v21.2d, v21.2d, v1.2d +; CHECK-NEXT: add v0.2d, v0.2d, v1.2d ; CHECK-NEXT: b.ne .LBB0_1 ; CHECK-NEXT: // %bb.2: // %for.cond.cleanup -; CHECK-NEXT: ldr q6, [sp, #48] // 16-byte Folded Reload +; CHECK-NEXT: ldr q1, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: adrp x8, C ; CHECK-NEXT: add x8, x8, :lo12:C -; CHECK-NEXT: stp q12, q31, [x8, #80] +; CHECK-NEXT: stp q11, q30, [x8, #80] ; CHECK-NEXT: ldp x20, x19, [sp, #176] // 16-byte Folded Reload -; CHECK-NEXT: str q6, [x8] -; CHECK-NEXT: ldr q6, [sp, #64] // 16-byte Folded Reload -; CHECK-NEXT: str q29, [x8, #112] +; CHECK-NEXT: str q1, [x8] +; CHECK-NEXT: ldr q1, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: stp q15, q14, [x8, #144] ; CHECK-NEXT: ldp x22, x21, [sp, #160] // 16-byte Folded Reload -; CHECK-NEXT: stp q6, q11, [x8, #16] -; CHECK-NEXT: ldr q6, [sp, #80] // 16-byte Folded Reload -; CHECK-NEXT: stp q18, q30, [x8, #144] -; CHECK-NEXT: ldp d9, d8, [sp, #144] // 16-byte Folded Reload -; CHECK-NEXT: stp q6, q13, [x8, #48] +; CHECK-NEXT: stp q1, q13, [x8, #16] +; CHECK-NEXT: ldr q1, [sp, #80] // 16-byte Folded Reload +; CHECK-NEXT: stp q28, q12, [x8, #176] ; CHECK-NEXT: ldp d13, d12, [sp, #112] // 16-byte Folded Reload -; CHECK-NEXT: stp q28, q26, [x8, #176] +; CHECK-NEXT: stp q1, q31, [x8, #48] ; CHECK-NEXT: ldp d15, d14, [sp, #96] // 16-byte Folded Reload -; CHECK-NEXT: stp q19, q10, [x8, #336] +; CHECK-NEXT: stp q9, q24, [x8, #240] +; CHECK-NEXT: ldp d9, d8, [sp, #144] // 16-byte Folded Reload +; CHECK-NEXT: stp q19, q18, [x8, #336] +; CHECK-NEXT: stp q10, q7, [x8, #400] ; CHECK-NEXT: ldp d11, d10, [sp, #128] // 16-byte Folded Reload +; CHECK-NEXT: str q29, [x8, #112] ; CHECK-NEXT: str q27, [x8, #208] -; CHECK-NEXT: stp q25, q24, [x8, #240] ; CHECK-NEXT: stp q23, q22, [x8, #272] ; CHECK-NEXT: stp q21, q20, [x8, #304] -; CHECK-NEXT: stp q17, q16, [x8, #368] -; CHECK-NEXT: stp q7, q5, [x8, #400] -; CHECK-NEXT: stp q4, q3, [x8, #432] -; CHECK-NEXT: stp q1, q2, [x8, #464] +; CHECK-NEXT: stp q6, q17, [x8, #368] +; CHECK-NEXT: stp q5, q4, [x8, #432] +; CHECK-NEXT: stp q2, q3, [x8, #464] ; CHECK-NEXT: str q0, [x8, #496] ; CHECK-NEXT: add sp, sp, #192 ; CHECK-NEXT: .cfi_def_cfa_offset 0 diff --git a/llvm/test/CodeGen/AArch64/rcpc3-sve.ll b/llvm/test/CodeGen/AArch64/rcpc3-sve.ll index 47e338151749..6b03e5d12bfd 100644 --- a/llvm/test/CodeGen/AArch64/rcpc3-sve.ll +++ b/llvm/test/CodeGen/AArch64/rcpc3-sve.ll @@ -8,8 +8,8 @@ define hidden @test_load_sve_lane0(ptr nocapture noundef readonly %a, noundef %b) local_unnamed_addr { ; CHECK-LABEL: test_load_sve_lane0: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: ldapr x8, [x0] +; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: mov z0.d, p0/m, x8 ; CHECK-NEXT: ret %1 = load atomic i64, ptr %a acquire, align 8 @@ -20,9 +20,9 @@ define hidden @test_load_sve_lane0(ptr nocapture noundef read define hidden @test_load_sve_lane1(ptr nocapture noundef readonly %a, noundef %b) local_unnamed_addr { ; CHECK-LABEL: test_load_sve_lane1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z1.d, #0, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: ldapr x8, [x0] ; CHECK-NEXT: cmpeq p0.d, p0/z, z1.d, z2.d diff --git a/llvm/test/CodeGen/AArch64/reassocmls.ll b/llvm/test/CodeGen/AArch64/reassocmls.ll index 381caffba92e..acbf9fc584a2 100644 --- a/llvm/test/CodeGen/AArch64/reassocmls.ll +++ b/llvm/test/CodeGen/AArch64/reassocmls.ll @@ -79,7 +79,7 @@ define i64 @mls_i64_C(i64 %a, i64 %b, i64 %c, i64 %d, i64 %e) { ; CHECK-LABEL: mls_i64_C: ; CHECK: // %bb.0: ; CHECK-NEXT: mul x8, x2, x1 -; CHECK-NEXT: mov w9, #10 +; CHECK-NEXT: mov w9, #10 // =0xa ; CHECK-NEXT: madd x8, x4, x3, x8 ; CHECK-NEXT: sub x0, x9, x8 ; CHECK-NEXT: ret @@ -290,9 +290,9 @@ define @smlsl_nxv8i16( %a, @umlsl_nxv8i16( %a, %b, %c, %d, %e) { ; CHECK-LABEL: umlsl_nxv8i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and z3.h, z3.h, #0xff ; CHECK-NEXT: and z4.h, z4.h, #0xff +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and z1.h, z1.h, #0xff ; CHECK-NEXT: and z2.h, z2.h, #0xff ; CHECK-NEXT: mls z0.h, p0/m, z4.h, z3.h @@ -326,8 +326,8 @@ define @mls_nxv8i16( %a, define @mla_nxv8i16( %a, %b, %c, %d, %e) { ; CHECK-LABEL: mla_nxv8i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mul z1.h, z2.h, z1.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mla z1.h, p0/m, z4.h, z3.h ; CHECK-NEXT: add z0.h, z1.h, z0.h ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/reduce-shuffle.ll b/llvm/test/CodeGen/AArch64/reduce-shuffle.ll index a080a7403811..325ab444205b 100644 --- a/llvm/test/CodeGen/AArch64/reduce-shuffle.ll +++ b/llvm/test/CodeGen/AArch64/reduce-shuffle.ll @@ -20,111 +20,111 @@ define i32 @v1(ptr nocapture noundef readonly %p1, i32 noundef %i1, ptr nocaptur ; CHECK-NEXT: ldr d6, [x10, x8] ; CHECK-NEXT: ldr d5, [x11] ; CHECK-NEXT: ldr d7, [x11, x9] -; CHECK-NEXT: usubl v2.8h, v2.8b, v3.8b ; CHECK-NEXT: usubl v0.8h, v0.8b, v1.8b -; CHECK-NEXT: usubl v1.8h, v4.8b, v5.8b +; CHECK-NEXT: usubl v1.8h, v2.8b, v3.8b +; CHECK-NEXT: usubl v2.8h, v4.8b, v5.8b ; CHECK-NEXT: usubl v3.8h, v6.8b, v7.8b -; CHECK-NEXT: shll2 v4.4s, v2.8h, #16 -; CHECK-NEXT: shll2 v5.4s, v0.8h, #16 +; CHECK-NEXT: shll2 v4.4s, v0.8h, #16 +; CHECK-NEXT: shll2 v5.4s, v1.8h, #16 ; CHECK-NEXT: shll2 v6.4s, v3.8h, #16 -; CHECK-NEXT: shll2 v7.4s, v1.8h, #16 -; CHECK-NEXT: saddw v2.4s, v4.4s, v2.4h -; CHECK-NEXT: saddw v0.4s, v5.4s, v0.4h +; CHECK-NEXT: shll2 v7.4s, v2.8h, #16 +; CHECK-NEXT: saddw v0.4s, v4.4s, v0.4h +; CHECK-NEXT: saddw v1.4s, v5.4s, v1.4h ; CHECK-NEXT: saddw v3.4s, v6.4s, v3.4h -; CHECK-NEXT: saddw v1.4s, v7.4s, v1.4h +; CHECK-NEXT: saddw v2.4s, v7.4s, v2.4h +; CHECK-NEXT: zip1 v4.4s, v1.4s, v0.4s +; CHECK-NEXT: zip2 v6.4s, v1.4s, v0.4s +; CHECK-NEXT: uzp2 v5.4s, v3.4s, v2.4s ; CHECK-NEXT: mov v7.16b, v2.16b -; CHECK-NEXT: zip1 v4.4s, v2.4s, v0.4s -; CHECK-NEXT: zip2 v6.4s, v2.4s, v0.4s -; CHECK-NEXT: uzp2 v5.4s, v3.4s, v1.4s -; CHECK-NEXT: mov v17.16b, v1.16b -; CHECK-NEXT: zip2 v16.4s, v1.4s, v3.4s -; CHECK-NEXT: mov v7.s[3], v0.s[2] -; CHECK-NEXT: ext v18.16b, v3.16b, v3.16b, #12 -; CHECK-NEXT: ext v2.16b, v2.16b, v4.16b, #8 -; CHECK-NEXT: mov v17.s[1], v3.s[0] +; CHECK-NEXT: ext v17.16b, v3.16b, v3.16b, #12 +; CHECK-NEXT: zip2 v18.4s, v3.4s, v2.4s +; CHECK-NEXT: ext v16.16b, v1.16b, v4.16b, #8 +; CHECK-NEXT: mov v1.s[3], v0.s[2] +; CHECK-NEXT: mov v7.s[1], v3.s[0] ; CHECK-NEXT: uzp2 v0.4s, v5.4s, v3.4s -; CHECK-NEXT: zip2 v5.4s, v3.4s, v1.4s -; CHECK-NEXT: mov v3.s[0], v1.s[1] -; CHECK-NEXT: ext v1.16b, v1.16b, v18.16b, #12 -; CHECK-NEXT: mov v16.d[1], v7.d[1] -; CHECK-NEXT: mov v17.d[1], v2.d[1] +; CHECK-NEXT: zip2 v5.4s, v2.4s, v3.4s +; CHECK-NEXT: mov v3.s[0], v2.s[1] +; CHECK-NEXT: ext v2.16b, v2.16b, v17.16b, #12 +; CHECK-NEXT: mov v18.d[1], v1.d[1] +; CHECK-NEXT: mov v7.d[1], v16.d[1] ; CHECK-NEXT: mov v0.d[1], v6.d[1] -; CHECK-NEXT: mov v5.d[1], v7.d[1] ; CHECK-NEXT: mov v3.d[1], v4.d[1] -; CHECK-NEXT: mov v1.d[1], v6.d[1] -; CHECK-NEXT: add v0.4s, v0.4s, v5.4s -; CHECK-NEXT: add v2.4s, v3.4s, v17.4s -; CHECK-NEXT: sub v3.4s, v17.4s, v3.4s -; CHECK-NEXT: sub v1.4s, v16.4s, v1.4s +; CHECK-NEXT: mov v5.d[1], v1.d[1] +; CHECK-NEXT: mov v2.d[1], v6.d[1] +; CHECK-NEXT: add v0.4s, v0.4s, v18.4s +; CHECK-NEXT: add v1.4s, v3.4s, v7.4s +; CHECK-NEXT: sub v3.4s, v7.4s, v3.4s +; CHECK-NEXT: sub v2.4s, v5.4s, v2.4s ; CHECK-NEXT: rev64 v4.4s, v0.4s -; CHECK-NEXT: rev64 v5.4s, v2.4s -; CHECK-NEXT: add v6.4s, v1.4s, v3.4s -; CHECK-NEXT: sub v1.4s, v3.4s, v1.4s +; CHECK-NEXT: rev64 v6.4s, v1.4s +; CHECK-NEXT: sub v5.4s, v3.4s, v2.4s +; CHECK-NEXT: add v2.4s, v2.4s, v3.4s ; CHECK-NEXT: mov v4.d[1], v0.d[1] -; CHECK-NEXT: mov v5.d[1], v2.d[1] -; CHECK-NEXT: rev64 v3.4s, v1.4s -; CHECK-NEXT: sub v2.4s, v2.4s, v4.4s -; CHECK-NEXT: add v0.4s, v0.4s, v5.4s -; CHECK-NEXT: rev64 v4.4s, v6.4s -; CHECK-NEXT: rev64 v5.4s, v2.4s -; CHECK-NEXT: rev64 v7.4s, v0.4s -; CHECK-NEXT: addp v16.4s, v0.4s, v6.4s -; CHECK-NEXT: addp v17.4s, v2.4s, v1.4s -; CHECK-NEXT: sub v4.4s, v6.4s, v4.4s -; CHECK-NEXT: sub v1.4s, v1.4s, v3.4s -; CHECK-NEXT: sub v2.4s, v2.4s, v5.4s -; CHECK-NEXT: sub v0.4s, v0.4s, v7.4s -; CHECK-NEXT: zip1 v21.4s, v16.4s, v16.4s -; CHECK-NEXT: ext v5.16b, v17.16b, v1.16b, #4 -; CHECK-NEXT: ext v6.16b, v16.16b, v4.16b, #4 -; CHECK-NEXT: mov v18.16b, v1.16b -; CHECK-NEXT: mov v19.16b, v4.16b -; CHECK-NEXT: ext v3.16b, v2.16b, v17.16b, #8 -; CHECK-NEXT: ext v7.16b, v0.16b, v16.16b, #4 -; CHECK-NEXT: mov v18.s[2], v17.s[3] -; CHECK-NEXT: zip2 v5.4s, v5.4s, v17.4s -; CHECK-NEXT: zip2 v6.4s, v6.4s, v16.4s -; CHECK-NEXT: mov v19.s[2], v16.s[3] -; CHECK-NEXT: trn2 v0.4s, v21.4s, v0.4s -; CHECK-NEXT: ext v20.16b, v3.16b, v2.16b, #4 -; CHECK-NEXT: ext v7.16b, v7.16b, v7.16b, #4 -; CHECK-NEXT: mov v2.s[2], v17.s[1] -; CHECK-NEXT: ext v1.16b, v1.16b, v5.16b, #12 -; CHECK-NEXT: ext v4.16b, v4.16b, v6.16b, #12 -; CHECK-NEXT: mov v5.16b, v18.16b -; CHECK-NEXT: uzp2 v3.4s, v3.4s, v20.4s -; CHECK-NEXT: mov v6.16b, v7.16b -; CHECK-NEXT: mov v20.16b, v19.16b -; CHECK-NEXT: mov v21.16b, v2.16b -; CHECK-NEXT: mov v5.s[1], v17.s[2] -; CHECK-NEXT: sub v7.4s, v0.4s, v7.4s -; CHECK-NEXT: mov v6.s[0], v16.s[1] -; CHECK-NEXT: mov v20.s[1], v16.s[2] -; CHECK-NEXT: sub v16.4s, v19.4s, v4.4s -; CHECK-NEXT: mov v21.s[1], v17.s[0] -; CHECK-NEXT: sub v2.4s, v2.4s, v3.4s -; CHECK-NEXT: sub v17.4s, v18.4s, v1.4s -; CHECK-NEXT: add v1.4s, v5.4s, v1.4s +; CHECK-NEXT: mov v6.d[1], v1.d[1] +; CHECK-NEXT: rev64 v3.4s, v5.4s +; CHECK-NEXT: rev64 v7.4s, v2.4s +; CHECK-NEXT: sub v1.4s, v1.4s, v4.4s ; CHECK-NEXT: add v0.4s, v0.4s, v6.4s -; CHECK-NEXT: add v4.4s, v20.4s, v4.4s -; CHECK-NEXT: add v3.4s, v21.4s, v3.4s -; CHECK-NEXT: mov v1.d[1], v17.d[1] -; CHECK-NEXT: mov v0.d[1], v7.d[1] -; CHECK-NEXT: mov v4.d[1], v16.d[1] -; CHECK-NEXT: mov v3.d[1], v2.d[1] +; CHECK-NEXT: sub v3.4s, v5.4s, v3.4s +; CHECK-NEXT: addp v4.4s, v1.4s, v5.4s +; CHECK-NEXT: sub v5.4s, v2.4s, v7.4s +; CHECK-NEXT: addp v2.4s, v0.4s, v2.4s +; CHECK-NEXT: rev64 v6.4s, v0.4s +; CHECK-NEXT: rev64 v7.4s, v1.4s +; CHECK-NEXT: ext v16.16b, v4.16b, v3.16b, #4 +; CHECK-NEXT: ext v17.16b, v2.16b, v5.16b, #4 +; CHECK-NEXT: sub v0.4s, v0.4s, v6.4s +; CHECK-NEXT: sub v1.4s, v1.4s, v7.4s +; CHECK-NEXT: mov v7.16b, v3.16b +; CHECK-NEXT: zip2 v6.4s, v16.4s, v4.4s +; CHECK-NEXT: mov v16.16b, v5.16b +; CHECK-NEXT: zip2 v17.4s, v17.4s, v2.4s +; CHECK-NEXT: ext v18.16b, v0.16b, v2.16b, #4 +; CHECK-NEXT: mov v7.s[2], v4.s[3] +; CHECK-NEXT: mov v21.16b, v1.16b +; CHECK-NEXT: mov v16.s[2], v2.s[3] +; CHECK-NEXT: ext v5.16b, v5.16b, v17.16b, #12 +; CHECK-NEXT: zip1 v17.4s, v2.4s, v2.4s +; CHECK-NEXT: ext v3.16b, v3.16b, v6.16b, #12 +; CHECK-NEXT: ext v18.16b, v18.16b, v18.16b, #4 +; CHECK-NEXT: mov v19.16b, v7.16b +; CHECK-NEXT: ext v6.16b, v1.16b, v4.16b, #8 +; CHECK-NEXT: mov v21.s[2], v4.s[1] +; CHECK-NEXT: mov v20.16b, v16.16b +; CHECK-NEXT: mov v19.s[1], v4.s[2] +; CHECK-NEXT: trn2 v0.4s, v17.4s, v0.4s +; CHECK-NEXT: sub v16.4s, v16.4s, v5.4s +; CHECK-NEXT: mov v17.16b, v18.16b +; CHECK-NEXT: ext v1.16b, v6.16b, v1.16b, #4 +; CHECK-NEXT: sub v7.4s, v7.4s, v3.4s +; CHECK-NEXT: mov v20.s[1], v2.s[2] +; CHECK-NEXT: mov v17.s[0], v2.s[1] +; CHECK-NEXT: mov v2.16b, v21.16b +; CHECK-NEXT: add v3.4s, v19.4s, v3.4s +; CHECK-NEXT: uzp2 v1.4s, v6.4s, v1.4s +; CHECK-NEXT: add v5.4s, v20.4s, v5.4s +; CHECK-NEXT: mov v2.s[1], v4.s[0] +; CHECK-NEXT: sub v4.4s, v0.4s, v18.4s +; CHECK-NEXT: mov v3.d[1], v7.d[1] +; CHECK-NEXT: add v0.4s, v0.4s, v17.4s +; CHECK-NEXT: mov v5.d[1], v16.d[1] +; CHECK-NEXT: sub v6.4s, v21.4s, v1.4s +; CHECK-NEXT: add v1.4s, v2.4s, v1.4s +; CHECK-NEXT: mov v0.d[1], v4.d[1] +; CHECK-NEXT: cmlt v4.8h, v3.8h, #0 +; CHECK-NEXT: cmlt v2.8h, v5.8h, #0 +; CHECK-NEXT: mov v1.d[1], v6.d[1] +; CHECK-NEXT: add v3.4s, v4.4s, v3.4s +; CHECK-NEXT: cmlt v6.8h, v0.8h, #0 +; CHECK-NEXT: add v5.4s, v2.4s, v5.4s +; CHECK-NEXT: eor v3.16b, v3.16b, v4.16b ; CHECK-NEXT: cmlt v7.8h, v1.8h, #0 -; CHECK-NEXT: cmlt v2.8h, v0.8h, #0 -; CHECK-NEXT: cmlt v6.8h, v4.8h, #0 -; CHECK-NEXT: cmlt v5.8h, v3.8h, #0 +; CHECK-NEXT: add v0.4s, v6.4s, v0.4s +; CHECK-NEXT: eor v2.16b, v5.16b, v2.16b ; CHECK-NEXT: add v1.4s, v7.4s, v1.4s -; CHECK-NEXT: add v0.4s, v2.4s, v0.4s -; CHECK-NEXT: add v4.4s, v6.4s, v4.4s -; CHECK-NEXT: add v3.4s, v5.4s, v3.4s +; CHECK-NEXT: eor v0.16b, v0.16b, v6.16b +; CHECK-NEXT: add v2.4s, v2.4s, v3.4s ; CHECK-NEXT: eor v1.16b, v1.16b, v7.16b -; CHECK-NEXT: eor v0.16b, v0.16b, v2.16b -; CHECK-NEXT: eor v2.16b, v3.16b, v5.16b -; CHECK-NEXT: eor v3.16b, v4.16b, v6.16b -; CHECK-NEXT: add v1.4s, v3.4s, v1.4s ; CHECK-NEXT: add v0.4s, v0.4s, v2.4s ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: addv s0, v0.4s @@ -278,13 +278,13 @@ define i32 @v2(ptr nocapture noundef readonly %p1, i32 noundef %i1, ptr nocaptur ; CHECK-NEXT: mov v1.d[1], v6.d[1] ; CHECK-NEXT: add v2.4s, v2.4s, v16.4s ; CHECK-NEXT: add v3.4s, v4.4s, v17.4s +; CHECK-NEXT: rev64 v5.4s, v2.4s ; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: sub v1.4s, v17.4s, v4.4s -; CHECK-NEXT: rev64 v5.4s, v2.4s ; CHECK-NEXT: rev64 v6.4s, v3.4s +; CHECK-NEXT: mov v5.d[1], v2.d[1] ; CHECK-NEXT: sub v4.4s, v1.4s, v0.4s ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-NEXT: mov v5.d[1], v2.d[1] ; CHECK-NEXT: mov v6.d[1], v3.d[1] ; CHECK-NEXT: sub v3.4s, v3.4s, v5.4s ; CHECK-NEXT: add v1.4s, v2.4s, v6.4s @@ -304,40 +304,40 @@ define i32 @v2(ptr nocapture noundef readonly %p1, i32 noundef %i1, ptr nocaptur ; CHECK-NEXT: mov v4.d[1], v16.d[1] ; CHECK-NEXT: mov v1.d[1], v7.d[1] ; CHECK-NEXT: add v0.4s, v17.4s, v1.4s -; CHECK-NEXT: sub v1.4s, v1.4s, v17.4s ; CHECK-NEXT: add v2.4s, v18.4s, v4.4s +; CHECK-NEXT: sub v1.4s, v1.4s, v17.4s ; CHECK-NEXT: sub v3.4s, v4.4s, v18.4s -; CHECK-NEXT: zip2 v4.4s, v0.4s, v1.4s -; CHECK-NEXT: ext v5.16b, v0.16b, v0.16b, #4 -; CHECK-NEXT: ext v6.16b, v2.16b, v2.16b, #4 +; CHECK-NEXT: ext v4.16b, v0.16b, v0.16b, #4 +; CHECK-NEXT: ext v5.16b, v2.16b, v2.16b, #4 +; CHECK-NEXT: zip2 v6.4s, v0.4s, v1.4s ; CHECK-NEXT: zip2 v7.4s, v1.4s, v0.4s ; CHECK-NEXT: zip2 v16.4s, v3.4s, v2.4s ; CHECK-NEXT: zip2 v17.4s, v2.4s, v3.4s ; CHECK-NEXT: zip1 v0.4s, v0.4s, v1.4s -; CHECK-NEXT: zip1 v2.4s, v2.4s, v3.4s -; CHECK-NEXT: ext v1.16b, v5.16b, v1.16b, #8 -; CHECK-NEXT: ext v18.16b, v6.16b, v3.16b, #8 -; CHECK-NEXT: add v3.4s, v16.4s, v7.4s -; CHECK-NEXT: sub v4.4s, v4.4s, v17.4s -; CHECK-NEXT: sub v0.4s, v0.4s, v2.4s -; CHECK-NEXT: ext v1.16b, v1.16b, v5.16b, #4 -; CHECK-NEXT: ext v5.16b, v18.16b, v6.16b, #4 -; CHECK-NEXT: cmlt v2.8h, v4.8h, #0 -; CHECK-NEXT: cmlt v6.8h, v3.8h, #0 -; CHECK-NEXT: add v3.4s, v6.4s, v3.4s -; CHECK-NEXT: add v4.4s, v2.4s, v4.4s -; CHECK-NEXT: add v1.4s, v5.4s, v1.4s +; CHECK-NEXT: ext v18.16b, v4.16b, v1.16b, #8 +; CHECK-NEXT: ext v19.16b, v5.16b, v3.16b, #8 +; CHECK-NEXT: zip1 v1.4s, v2.4s, v3.4s +; CHECK-NEXT: add v2.4s, v16.4s, v7.4s +; CHECK-NEXT: sub v3.4s, v6.4s, v17.4s +; CHECK-NEXT: ext v4.16b, v18.16b, v4.16b, #4 +; CHECK-NEXT: ext v5.16b, v19.16b, v5.16b, #4 +; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s +; CHECK-NEXT: cmlt v1.8h, v3.8h, #0 +; CHECK-NEXT: cmlt v6.8h, v2.8h, #0 +; CHECK-NEXT: add v4.4s, v5.4s, v4.4s ; CHECK-NEXT: cmlt v5.8h, v0.8h, #0 +; CHECK-NEXT: add v2.4s, v6.4s, v2.4s +; CHECK-NEXT: add v3.4s, v1.4s, v3.4s +; CHECK-NEXT: cmlt v7.8h, v4.8h, #0 ; CHECK-NEXT: add v0.4s, v5.4s, v0.4s -; CHECK-NEXT: eor v2.16b, v4.16b, v2.16b -; CHECK-NEXT: eor v3.16b, v3.16b, v6.16b -; CHECK-NEXT: cmlt v4.8h, v1.8h, #0 -; CHECK-NEXT: add v2.4s, v3.4s, v2.4s -; CHECK-NEXT: add v1.4s, v4.4s, v1.4s +; CHECK-NEXT: eor v2.16b, v2.16b, v6.16b +; CHECK-NEXT: eor v1.16b, v3.16b, v1.16b +; CHECK-NEXT: add v3.4s, v7.4s, v4.4s ; CHECK-NEXT: eor v0.16b, v0.16b, v5.16b -; CHECK-NEXT: add v0.4s, v0.4s, v2.4s -; CHECK-NEXT: eor v1.16b, v1.16b, v4.16b -; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: add v1.4s, v2.4s, v1.4s +; CHECK-NEXT: eor v2.16b, v3.16b, v7.16b +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-NEXT: add v0.4s, v2.4s, v0.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: lsr w9, w8, #16 @@ -461,93 +461,93 @@ define i32 @v3(ptr nocapture noundef readonly %p1, i32 noundef %i1, ptr nocaptur ; CHECK-NEXT: ldr d3, [x11, x9] ; CHECK-NEXT: ldr d4, [x10] ; CHECK-NEXT: ldr d5, [x11] +; CHECK-NEXT: shll2 v6.4s, v0.8h, #16 ; CHECK-NEXT: usubl v2.8h, v2.8b, v3.8b -; CHECK-NEXT: usubl v3.8h, v4.8b, v5.8b -; CHECK-NEXT: shll2 v4.4s, v0.8h, #16 -; CHECK-NEXT: shll2 v5.4s, v1.8h, #16 -; CHECK-NEXT: saddw v0.4s, v4.4s, v0.4h -; CHECK-NEXT: shll2 v4.4s, v2.8h, #16 -; CHECK-NEXT: saddw v1.4s, v5.4s, v1.4h -; CHECK-NEXT: shll2 v5.4s, v3.8h, #16 -; CHECK-NEXT: saddw v2.4s, v4.4s, v2.4h -; CHECK-NEXT: saddw v3.4s, v5.4s, v3.4h -; CHECK-NEXT: rev64 v4.4s, v0.4s +; CHECK-NEXT: shll2 v3.4s, v1.8h, #16 +; CHECK-NEXT: usubl v4.8h, v4.8b, v5.8b +; CHECK-NEXT: saddw v0.4s, v6.4s, v0.4h +; CHECK-NEXT: shll2 v5.4s, v2.8h, #16 +; CHECK-NEXT: saddw v1.4s, v3.4s, v1.4h +; CHECK-NEXT: shll2 v3.4s, v4.8h, #16 +; CHECK-NEXT: rev64 v6.4s, v0.4s +; CHECK-NEXT: saddw v2.4s, v5.4s, v2.4h ; CHECK-NEXT: rev64 v5.4s, v1.4s -; CHECK-NEXT: rev64 v6.4s, v2.4s -; CHECK-NEXT: rev64 v7.4s, v3.4s -; CHECK-NEXT: sub v4.4s, v0.4s, v4.4s +; CHECK-NEXT: saddw v3.4s, v3.4s, v4.4h +; CHECK-NEXT: rev64 v4.4s, v2.4s +; CHECK-NEXT: sub v6.4s, v0.4s, v6.4s ; CHECK-NEXT: addp v0.4s, v1.4s, v0.4s +; CHECK-NEXT: rev64 v7.4s, v3.4s ; CHECK-NEXT: sub v5.4s, v1.4s, v5.4s -; CHECK-NEXT: sub v6.4s, v2.4s, v6.4s +; CHECK-NEXT: sub v4.4s, v2.4s, v4.4s ; CHECK-NEXT: addp v2.4s, v2.4s, v3.4s -; CHECK-NEXT: sub v1.4s, v3.4s, v7.4s +; CHECK-NEXT: ext v1.16b, v5.16b, v6.16b, #4 +; CHECK-NEXT: sub v7.4s, v3.4s, v7.4s ; CHECK-NEXT: ext v3.16b, v0.16b, v0.16b, #8 -; CHECK-NEXT: ext v7.16b, v5.16b, v4.16b, #4 -; CHECK-NEXT: mov v4.s[3], v5.s[2] -; CHECK-NEXT: zip2 v16.4s, v6.4s, v1.4s -; CHECK-NEXT: zip1 v1.4s, v6.4s, v1.4s -; CHECK-NEXT: uzp2 v6.4s, v2.4s, v0.4s -; CHECK-NEXT: ext v5.16b, v7.16b, v5.16b, #4 +; CHECK-NEXT: mov v6.s[3], v5.s[2] +; CHECK-NEXT: zip2 v16.4s, v4.4s, v7.4s +; CHECK-NEXT: zip1 v4.4s, v4.4s, v7.4s +; CHECK-NEXT: ext v1.16b, v1.16b, v5.16b, #4 +; CHECK-NEXT: uzp2 v5.4s, v2.4s, v0.4s ; CHECK-NEXT: uzp1 v0.4s, v2.4s, v0.4s ; CHECK-NEXT: uzp1 v7.4s, v2.4s, v3.4s ; CHECK-NEXT: uzp2 v2.4s, v2.4s, v3.4s -; CHECK-NEXT: mov v16.d[1], v4.d[1] -; CHECK-NEXT: rev64 v3.4s, v6.4s -; CHECK-NEXT: mov v1.d[1], v5.d[1] +; CHECK-NEXT: mov v16.d[1], v6.d[1] +; CHECK-NEXT: mov v4.d[1], v1.d[1] +; CHECK-NEXT: rev64 v1.4s, v5.4s ; CHECK-NEXT: rev64 v0.4s, v0.4s ; CHECK-NEXT: sub v2.4s, v7.4s, v2.4s -; CHECK-NEXT: sub v4.4s, v1.4s, v16.4s -; CHECK-NEXT: add v0.4s, v3.4s, v0.4s -; CHECK-NEXT: add v1.4s, v16.4s, v1.4s -; CHECK-NEXT: zip1 v3.4s, v2.4s, v4.4s +; CHECK-NEXT: sub v3.4s, v4.4s, v16.4s +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: add v1.4s, v16.4s, v4.4s +; CHECK-NEXT: zip1 v4.4s, v2.4s, v3.4s ; CHECK-NEXT: zip1 v5.4s, v0.4s, v1.4s ; CHECK-NEXT: uzp2 v6.4s, v0.4s, v1.4s ; CHECK-NEXT: zip2 v7.4s, v0.4s, v1.4s -; CHECK-NEXT: zip2 v17.4s, v2.4s, v4.4s -; CHECK-NEXT: ext v16.16b, v2.16b, v3.16b, #8 +; CHECK-NEXT: zip2 v17.4s, v2.4s, v3.4s +; CHECK-NEXT: ext v16.16b, v2.16b, v4.16b, #8 ; CHECK-NEXT: trn2 v5.4s, v0.4s, v5.4s ; CHECK-NEXT: uzp2 v6.4s, v6.4s, v0.4s -; CHECK-NEXT: mov v2.s[3], v4.s[2] +; CHECK-NEXT: mov v2.s[3], v3.s[2] ; CHECK-NEXT: mov v0.s[1], v1.s[1] ; CHECK-NEXT: mov v5.d[1], v16.d[1] ; CHECK-NEXT: mov v6.d[1], v17.d[1] ; CHECK-NEXT: mov v7.d[1], v2.d[1] -; CHECK-NEXT: mov v0.d[1], v3.d[1] +; CHECK-NEXT: mov v0.d[1], v4.d[1] ; CHECK-NEXT: add v1.4s, v6.4s, v7.4s -; CHECK-NEXT: sub v2.4s, v7.4s, v6.4s -; CHECK-NEXT: add v3.4s, v5.4s, v0.4s +; CHECK-NEXT: add v2.4s, v5.4s, v0.4s +; CHECK-NEXT: sub v3.4s, v7.4s, v6.4s ; CHECK-NEXT: sub v0.4s, v0.4s, v5.4s -; CHECK-NEXT: zip2 v4.4s, v1.4s, v2.4s -; CHECK-NEXT: ext v5.16b, v1.16b, v1.16b, #4 -; CHECK-NEXT: ext v6.16b, v3.16b, v3.16b, #4 -; CHECK-NEXT: zip2 v7.4s, v2.4s, v1.4s -; CHECK-NEXT: zip2 v16.4s, v0.4s, v3.4s -; CHECK-NEXT: zip2 v17.4s, v3.4s, v0.4s -; CHECK-NEXT: zip1 v1.4s, v1.4s, v2.4s -; CHECK-NEXT: ext v2.16b, v5.16b, v2.16b, #8 -; CHECK-NEXT: ext v18.16b, v6.16b, v0.16b, #8 -; CHECK-NEXT: zip1 v0.4s, v3.4s, v0.4s -; CHECK-NEXT: add v3.4s, v16.4s, v7.4s -; CHECK-NEXT: sub v4.4s, v4.4s, v17.4s -; CHECK-NEXT: ext v2.16b, v2.16b, v5.16b, #4 -; CHECK-NEXT: ext v5.16b, v18.16b, v6.16b, #4 +; CHECK-NEXT: ext v4.16b, v1.16b, v1.16b, #4 +; CHECK-NEXT: ext v5.16b, v2.16b, v2.16b, #4 +; CHECK-NEXT: zip2 v6.4s, v1.4s, v3.4s +; CHECK-NEXT: zip2 v7.4s, v3.4s, v1.4s +; CHECK-NEXT: zip2 v16.4s, v0.4s, v2.4s +; CHECK-NEXT: zip2 v17.4s, v2.4s, v0.4s +; CHECK-NEXT: zip1 v1.4s, v1.4s, v3.4s +; CHECK-NEXT: ext v18.16b, v4.16b, v3.16b, #8 +; CHECK-NEXT: ext v19.16b, v5.16b, v0.16b, #8 +; CHECK-NEXT: zip1 v0.4s, v2.4s, v0.4s +; CHECK-NEXT: add v2.4s, v16.4s, v7.4s +; CHECK-NEXT: sub v3.4s, v6.4s, v17.4s +; CHECK-NEXT: ext v4.16b, v18.16b, v4.16b, #4 +; CHECK-NEXT: ext v5.16b, v19.16b, v5.16b, #4 ; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s -; CHECK-NEXT: cmlt v1.8h, v4.8h, #0 -; CHECK-NEXT: cmlt v6.8h, v3.8h, #0 -; CHECK-NEXT: add v3.4s, v6.4s, v3.4s -; CHECK-NEXT: add v4.4s, v1.4s, v4.4s -; CHECK-NEXT: add v2.4s, v5.4s, v2.4s +; CHECK-NEXT: cmlt v1.8h, v3.8h, #0 +; CHECK-NEXT: cmlt v6.8h, v2.8h, #0 +; CHECK-NEXT: add v4.4s, v5.4s, v4.4s ; CHECK-NEXT: cmlt v5.8h, v0.8h, #0 +; CHECK-NEXT: add v2.4s, v6.4s, v2.4s +; CHECK-NEXT: add v3.4s, v1.4s, v3.4s +; CHECK-NEXT: cmlt v7.8h, v4.8h, #0 ; CHECK-NEXT: add v0.4s, v5.4s, v0.4s -; CHECK-NEXT: eor v1.16b, v4.16b, v1.16b -; CHECK-NEXT: eor v3.16b, v3.16b, v6.16b -; CHECK-NEXT: cmlt v4.8h, v2.8h, #0 -; CHECK-NEXT: add v1.4s, v3.4s, v1.4s -; CHECK-NEXT: add v2.4s, v4.4s, v2.4s +; CHECK-NEXT: eor v2.16b, v2.16b, v6.16b +; CHECK-NEXT: eor v1.16b, v3.16b, v1.16b +; CHECK-NEXT: add v3.4s, v7.4s, v4.4s ; CHECK-NEXT: eor v0.16b, v0.16b, v5.16b +; CHECK-NEXT: add v1.4s, v2.4s, v1.4s +; CHECK-NEXT: eor v2.16b, v3.16b, v7.16b ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-NEXT: eor v1.16b, v2.16b, v4.16b -; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: add v0.4s, v2.4s, v0.4s ; CHECK-NEXT: addv s0, v0.4s ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: lsr w9, w8, #16 diff --git a/llvm/test/CodeGen/AArch64/sat-add.ll b/llvm/test/CodeGen/AArch64/sat-add.ll index 86c224bee990..2deb19be2482 100644 --- a/llvm/test/CodeGen/AArch64/sat-add.ll +++ b/llvm/test/CodeGen/AArch64/sat-add.ll @@ -346,9 +346,9 @@ define <16 x i8> @unsigned_sat_constant_v16i8_using_min(<16 x i8> %x) { ; CHECK-LABEL: unsigned_sat_constant_v16i8_using_min: ; CHECK: // %bb.0: ; CHECK-NEXT: movi v1.16b, #213 +; CHECK-NEXT: movi v2.16b, #42 ; CHECK-NEXT: umin v0.16b, v0.16b, v1.16b -; CHECK-NEXT: movi v1.16b, #42 -; CHECK-NEXT: add v0.16b, v0.16b, v1.16b +; CHECK-NEXT: add v0.16b, v0.16b, v2.16b ; CHECK-NEXT: ret %c = icmp ult <16 x i8> %x, %s = select <16 x i1> %c, <16 x i8> %x, <16 x i8> @@ -384,9 +384,9 @@ define <8 x i16> @unsigned_sat_constant_v8i16_using_min(<8 x i16> %x) { ; CHECK-LABEL: unsigned_sat_constant_v8i16_using_min: ; CHECK: // %bb.0: ; CHECK-NEXT: mvni v1.8h, #42 +; CHECK-NEXT: movi v2.8h, #42 ; CHECK-NEXT: umin v0.8h, v0.8h, v1.8h -; CHECK-NEXT: movi v1.8h, #42 -; CHECK-NEXT: add v0.8h, v0.8h, v1.8h +; CHECK-NEXT: add v0.8h, v0.8h, v2.8h ; CHECK-NEXT: ret %c = icmp ult <8 x i16> %x, %s = select <8 x i1> %c, <8 x i16> %x, <8 x i16> diff --git a/llvm/test/CodeGen/AArch64/sext.ll b/llvm/test/CodeGen/AArch64/sext.ll index 3e0d5dd87509..5237a3491de9 100644 --- a/llvm/test/CodeGen/AArch64/sext.ll +++ b/llvm/test/CodeGen/AArch64/sext.ll @@ -245,15 +245,15 @@ define <3 x i32> @sext_v3i8_v3i32(<3 x i8> %a) { ; CHECK-GI-LABEL: sext_v3i8_v3i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov w8, #24 // =0x18 -; CHECK-GI-NEXT: fmov s1, w0 -; CHECK-GI-NEXT: fmov s0, w8 -; CHECK-GI-NEXT: mov v1.s[1], w1 -; CHECK-GI-NEXT: mov v0.s[1], w8 -; CHECK-GI-NEXT: mov v1.s[2], w2 -; CHECK-GI-NEXT: mov v0.s[2], w8 -; CHECK-GI-NEXT: neg v2.4s, v0.4s -; CHECK-GI-NEXT: ushl v0.4s, v1.4s, v0.4s -; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v2.4s +; CHECK-GI-NEXT: fmov s0, w0 +; CHECK-GI-NEXT: fmov s1, w8 +; CHECK-GI-NEXT: mov v0.s[1], w1 +; CHECK-GI-NEXT: mov v1.s[1], w8 +; CHECK-GI-NEXT: mov v0.s[2], w2 +; CHECK-GI-NEXT: mov v1.s[2], w8 +; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: neg v1.4s, v1.4s +; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v1.4s ; CHECK-GI-NEXT: ret entry: %c = sext <3 x i8> %a to <3 x i32> @@ -408,15 +408,15 @@ define <3 x i32> @sext_v3i10_v3i32(<3 x i10> %a) { ; CHECK-GI-LABEL: sext_v3i10_v3i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov w8, #22 // =0x16 -; CHECK-GI-NEXT: fmov s1, w0 -; CHECK-GI-NEXT: fmov s0, w8 -; CHECK-GI-NEXT: mov v1.s[1], w1 -; CHECK-GI-NEXT: mov v0.s[1], w8 -; CHECK-GI-NEXT: mov v1.s[2], w2 -; CHECK-GI-NEXT: mov v0.s[2], w8 -; CHECK-GI-NEXT: neg v2.4s, v0.4s -; CHECK-GI-NEXT: ushl v0.4s, v1.4s, v0.4s -; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v2.4s +; CHECK-GI-NEXT: fmov s0, w0 +; CHECK-GI-NEXT: fmov s1, w8 +; CHECK-GI-NEXT: mov v0.s[1], w1 +; CHECK-GI-NEXT: mov v1.s[1], w8 +; CHECK-GI-NEXT: mov v0.s[2], w2 +; CHECK-GI-NEXT: mov v1.s[2], w8 +; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: neg v1.4s, v1.4s +; CHECK-GI-NEXT: sshl v0.4s, v0.4s, v1.4s ; CHECK-GI-NEXT: ret entry: %c = sext <3 x i10> %a to <3 x i32> diff --git a/llvm/test/CodeGen/AArch64/sink-addsub-of-const.ll b/llvm/test/CodeGen/AArch64/sink-addsub-of-const.ll index f4f75bb9c782..88e062d2c999 100644 --- a/llvm/test/CodeGen/AArch64/sink-addsub-of-const.ll +++ b/llvm/test/CodeGen/AArch64/sink-addsub-of-const.ll @@ -158,8 +158,8 @@ define i32 @sink_sub_from_const_to_sub2(i32 %a, i32 %b) { define <4 x i32> @vec_sink_add_of_const_to_add0(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_add_of_const_to_add0: ; CHECK: // %bb.0: -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: adrp x8, .LCPI12_0 +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI12_0] ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -170,8 +170,8 @@ define <4 x i32> @vec_sink_add_of_const_to_add0(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_add_of_const_to_add1(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_add_of_const_to_add1: ; CHECK: // %bb.0: -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: adrp x8, .LCPI13_0 +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI13_0] ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -186,8 +186,8 @@ define <4 x i32> @vec_sink_add_of_const_to_add1(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_sub_of_const_to_add0(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_sub_of_const_to_add0: ; CHECK: // %bb.0: -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: adrp x8, .LCPI14_0 +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI14_0] ; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -198,8 +198,8 @@ define <4 x i32> @vec_sink_sub_of_const_to_add0(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_sub_of_const_to_add1(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_sub_of_const_to_add1: ; CHECK: // %bb.0: -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: adrp x8, .LCPI15_0 +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI15_0] ; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -214,8 +214,8 @@ define <4 x i32> @vec_sink_sub_of_const_to_add1(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_sub_from_const_to_add0(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_sub_from_const_to_add0: ; CHECK: // %bb.0: -; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: adrp x8, .LCPI16_0 +; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI16_0] ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -226,8 +226,8 @@ define <4 x i32> @vec_sink_sub_from_const_to_add0(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_sub_from_const_to_add1(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_sub_from_const_to_add1: ; CHECK: // %bb.0: -; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: adrp x8, .LCPI17_0 +; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI17_0] ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -242,8 +242,8 @@ define <4 x i32> @vec_sink_sub_from_const_to_add1(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_add_of_const_to_sub(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_add_of_const_to_sub: ; CHECK: // %bb.0: -; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: adrp x8, .LCPI18_0 +; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI18_0] ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -254,8 +254,8 @@ define <4 x i32> @vec_sink_add_of_const_to_sub(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_add_of_const_to_sub2(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_add_of_const_to_sub2: ; CHECK: // %bb.0: -; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: adrp x8, .LCPI19_0 +; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI19_0] ; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -270,8 +270,8 @@ define <4 x i32> @vec_sink_add_of_const_to_sub2(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_sub_of_const_to_sub(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_sub_of_const_to_sub: ; CHECK: // %bb.0: -; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: adrp x8, .LCPI20_0 +; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI20_0] ; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -282,8 +282,8 @@ define <4 x i32> @vec_sink_sub_of_const_to_sub(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_sub_of_const_to_sub2(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_sub_of_const_to_sub2: ; CHECK: // %bb.0: -; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: adrp x8, .LCPI21_0 +; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI21_0] ; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret @@ -298,8 +298,8 @@ define <4 x i32> @vec_sink_sub_of_const_to_sub2(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_sub_from_const_to_sub(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_sub_from_const_to_sub: ; CHECK: // %bb.0: -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: adrp x8, .LCPI22_0 +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI22_0] ; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ret @@ -310,8 +310,8 @@ define <4 x i32> @vec_sink_sub_from_const_to_sub(<4 x i32> %a, <4 x i32> %b) { define <4 x i32> @vec_sink_sub_from_const_to_sub2(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: vec_sink_sub_from_const_to_sub2: ; CHECK: // %bb.0: -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: adrp x8, .LCPI23_0 +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI23_0] ; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll b/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll index 0c674c5685e0..1d1bae42c9e3 100644 --- a/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll +++ b/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll @@ -28,8 +28,8 @@ define void @dont_coalesce_arg_i8(i8 %arg, ptr %ptr) #0 { ; CHECK-NEXT: smstop sm ; CHECK-NEXT: bl use_i8 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: ldr z0, [sp] // 16-byte Folded Reload +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: st1b { z0.b }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload @@ -61,8 +61,8 @@ define void @dont_coalesce_arg_i16(i16 %arg, ptr %ptr) #0 { ; CHECK-NEXT: smstop sm ; CHECK-NEXT: bl use_i16 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldr z0, [sp] // 16-byte Folded Reload +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: st1h { z0.h }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload @@ -94,8 +94,8 @@ define void @dont_coalesce_arg_i32(i32 %arg, ptr %ptr) #0 { ; CHECK-NEXT: smstop sm ; CHECK-NEXT: bl use_i32 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr z0, [sp] // 16-byte Folded Reload +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: st1w { z0.s }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload @@ -127,8 +127,8 @@ define void @dont_coalesce_arg_i64(i64 %arg, ptr %ptr) #0 { ; CHECK-NEXT: smstop sm ; CHECK-NEXT: bl use_i64 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr z0, [sp] // 16-byte Folded Reload +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: st1d { z0.d }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload @@ -165,8 +165,8 @@ define void @dont_coalesce_arg_f16(half %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr h0, [sp, #14] // 2-byte Folded Reload ; CHECK-NEXT: bl use_f16 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1h { z0.h }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -205,8 +205,8 @@ define void @dont_coalesce_arg_f32(float %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr s0, [sp, #12] // 4-byte Folded Reload ; CHECK-NEXT: bl use_f32 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1w { z0.s }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -245,8 +245,8 @@ define void @dont_coalesce_arg_f64(double %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: bl use_f64 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1d { z0.d }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -290,8 +290,8 @@ define void @dont_coalesce_arg_v1i8(<1 x i8> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: bl use_v16i8 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1b { z0.b }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -331,8 +331,8 @@ define void @dont_coalesce_arg_v1i16(<1 x i16> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: bl use_v8i16 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1h { z0.h }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -372,8 +372,8 @@ define void @dont_coalesce_arg_v1i32(<1 x i32> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: bl use_v4i32 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1w { z0.s }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -413,8 +413,8 @@ define void @dont_coalesce_arg_v1i64(<1 x i64> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: bl use_v2i64 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1d { z0.d }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -454,8 +454,8 @@ define void @dont_coalesce_arg_v1f16(<1 x half> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr h0, [sp, #14] // 2-byte Folded Reload ; CHECK-NEXT: bl use_v8f16 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1h { z0.h }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -495,8 +495,8 @@ define void @dont_coalesce_arg_v1f32(<1 x float> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: bl use_v4f32 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1w { z0.s }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -536,8 +536,8 @@ define void @dont_coalesce_arg_v1f64(<1 x double> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: bl use_v2f64 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1d { z0.d }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -581,8 +581,8 @@ define void @dont_coalesce_arg_v16i8(<16 x i8> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: bl use_v16i8 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1b { z0.b }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -621,8 +621,8 @@ define void @dont_coalesce_arg_v8i16(<8 x i16> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: bl use_v8i16 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1h { z0.h }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -661,8 +661,8 @@ define void @dont_coalesce_arg_v4i32(<4 x i32> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: bl use_v4i32 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1w { z0.s }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -701,8 +701,8 @@ define void @dont_coalesce_arg_v2i64(<2 x i64> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: bl use_v2i64 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1d { z0.d }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -741,8 +741,8 @@ define void @dont_coalesce_arg_v8f16(<8 x half> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: bl use_v8f16 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1h { z0.h }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -781,8 +781,8 @@ define void @dont_coalesce_arg_v8bf16(<8 x bfloat> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: bl use_v8bf16 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1h { z0.h }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -821,8 +821,8 @@ define void @dont_coalesce_arg_v4f32(<4 x float> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: bl use_v4f32 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1d { z0.d }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -861,8 +861,8 @@ define void @dont_coalesce_arg_v2f64(<2 x double> %arg, ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: bl use_v2f64 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr z0, [x8] // 16-byte Folded Reload ; CHECK-NEXT: st1d { z0.d }, p0, [x19] ; CHECK-NEXT: addvl sp, sp, #1 @@ -894,9 +894,9 @@ define void @dont_coalesce_arg_v8i1(<8 x i1> %arg, ptr %ptr) #0 { ; CHECK-NEXT: stp x30, x19, [sp, #80] // 16-byte Folded Spill ; CHECK-NEXT: sub sp, sp, #16 ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: mov z1.d, z0.d +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: add x8, sp, #16 ; CHECK-NEXT: mov x19, x0 ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -942,13 +942,13 @@ define void @dont_coalesce_res_i8(ptr %ptr) #0 { ; CHECK-NEXT: smstop sm ; CHECK-NEXT: bl get_i8 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: fmov s0, w0 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload -; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: st1b { z0.b }, p0, [x19] ; CHECK-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload ; CHECK-NEXT: ret %res = call i8 @get_i8() @@ -969,13 +969,13 @@ define void @dont_coalesce_res_i16(ptr %ptr) #0 { ; CHECK-NEXT: smstop sm ; CHECK-NEXT: bl get_i16 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: fmov s0, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload -; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: st1h { z0.h }, p0, [x19] ; CHECK-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload ; CHECK-NEXT: ret %res = call i16 @get_i16() @@ -996,13 +996,13 @@ define void @dont_coalesce_res_i32(ptr %ptr) #0 { ; CHECK-NEXT: smstop sm ; CHECK-NEXT: bl get_i32 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: fmov s0, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload -; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: st1w { z0.s }, p0, [x19] ; CHECK-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload ; CHECK-NEXT: ret %res = call i32 @get_i32() @@ -1023,13 +1023,13 @@ define void @dont_coalesce_res_i64(ptr %ptr) #0 { ; CHECK-NEXT: smstop sm ; CHECK-NEXT: bl get_i64 ; CHECK-NEXT: smstart sm -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: fmov d0, x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload -; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: st1d { z0.d }, p0, [x19] ; CHECK-NEXT: ldp x30, x19, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp], #80 // 16-byte Folded Reload ; CHECK-NEXT: ret %res = call i64 @get_i64() @@ -1056,11 +1056,11 @@ define void @dont_coalesce_res_f16(ptr %ptr) #0 { ; CHECK-NEXT: ldr h0, [sp, #14] // 2-byte Folded Reload ; CHECK-NEXT: // kill: def $h0 killed $h0 def $z0 ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1h { z0.h }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1h { z0.h }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call half @get_f16() @@ -1086,11 +1086,11 @@ define void @dont_coalesce_res_f32(ptr %ptr) #0 { ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr s0, [sp, #12] // 4-byte Folded Reload ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1w { z0.s }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1w { z0.s }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call float @get_f32() @@ -1116,11 +1116,11 @@ define void @dont_coalesce_res_f64(ptr %ptr) #0 { ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1d { z0.d }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1d { z0.d }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call double @get_f64() @@ -1150,11 +1150,11 @@ define void @dont_coalesce_res_v1i8(ptr %ptr) #0 { ; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1b { z0.b }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1b { z0.b }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <1 x i8> @get_v1i8() @@ -1181,11 +1181,11 @@ define void @dont_coalesce_res_v1i16(ptr %ptr) #0 { ; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1h { z0.h }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1h { z0.h }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <1 x i16> @get_v1i16() @@ -1212,11 +1212,11 @@ define void @dont_coalesce_res_v1i32(ptr %ptr) #0 { ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1w { z0.s }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1w { z0.s }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <1 x i32> @get_v1i32() @@ -1243,11 +1243,11 @@ define void @dont_coalesce_res_v1i64(ptr %ptr) #0 { ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1d { z0.d }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1d { z0.d }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <1 x i64> @get_v1i64() @@ -1275,11 +1275,11 @@ define void @dont_coalesce_res_v1f16(ptr %ptr) #0 { ; CHECK-NEXT: ldr h0, [sp, #14] // 2-byte Folded Reload ; CHECK-NEXT: // kill: def $h0 killed $h0 def $z0 ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1h { z0.h }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1h { z0.h }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <1 x half> @get_v1f16() @@ -1306,11 +1306,11 @@ define void @dont_coalesce_res_v1f32(ptr %ptr) #0 { ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1w { z0.s }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1w { z0.s }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <1 x float> @get_v1f32() @@ -1337,11 +1337,11 @@ define void @dont_coalesce_res_v1f64(ptr %ptr) #0 { ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr d0, [sp, #8] // 8-byte Folded Reload ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1d { z0.d }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1d { z0.d }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <1 x double> @get_v1f64() @@ -1373,11 +1373,11 @@ define void @dont_coalesce_res_v16i8(ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1b { z0.b }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1b { z0.b }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <16 x i8> @get_v16i8() @@ -1404,11 +1404,11 @@ define void @dont_coalesce_res_v8i16(ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1h { z0.h }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1h { z0.h }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <8 x i16> @get_v8i16() @@ -1435,11 +1435,11 @@ define void @dont_coalesce_res_v4i32(ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1w { z0.s }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1w { z0.s }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <4 x i32> @get_v4i32() @@ -1466,11 +1466,11 @@ define void @dont_coalesce_res_v2i64(ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1d { z0.d }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1d { z0.d }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <2 x i64> @get_v2i64() @@ -1497,11 +1497,11 @@ define void @dont_coalesce_res_v8f16(ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1h { z0.h }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1h { z0.h }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <8 x half> @get_v8f16() @@ -1528,11 +1528,11 @@ define void @dont_coalesce_res_v4f32(ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1w { z0.s }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1w { z0.s }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <4 x float> @get_v4f32() @@ -1559,11 +1559,11 @@ define void @dont_coalesce_res_v2f64(ptr %ptr) #0 { ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: ldp d9, d8, [sp, #64] // 16-byte Folded Reload +; CHECK-NEXT: st1d { z0.d }, p0, [x19] +; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: ldp d11, d10, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: ldp d13, d12, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: ldp d15, d14, [sp, #16] // 16-byte Folded Reload -; CHECK-NEXT: st1d { z0.d }, p0, [x19] -; CHECK-NEXT: ldp x30, x19, [sp, #80] // 16-byte Folded Reload ; CHECK-NEXT: add sp, sp, #96 ; CHECK-NEXT: ret %res = call <2 x double> @get_v2f64() diff --git a/llvm/test/CodeGen/AArch64/sme-streaming-compatible-interface.ll b/llvm/test/CodeGen/AArch64/sme-streaming-compatible-interface.ll index 47b24290d3c8..1e16f140676b 100644 --- a/llvm/test/CodeGen/AArch64/sme-streaming-compatible-interface.ll +++ b/llvm/test/CodeGen/AArch64/sme-streaming-compatible-interface.ll @@ -151,8 +151,8 @@ define <2 x double> @streaming_compatible_with_neon_vectors(<2 x double> %arg) " ; CHECK-NEXT: // %bb.3: ; CHECK-NEXT: smstart sm ; CHECK-NEXT: .LBB4_4: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: add x8, sp, #16 +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: ldr z1, [x8] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-fp-dots.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-fp-dots.ll index 0097968b1171..b4fd5a2272e7 100644 --- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-fp-dots.ll +++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-fp-dots.ll @@ -26,16 +26,16 @@ define void @fdot_multi_za32_f16_vg1x2(i32 %slice, %unused, < define void @fdot_multi_za32_f16_vg1x4(i32 %slice, %unused, %zn0, %zn1, %zn2, %zn3, ; CHECK-LABEL: fdot_multi_za32_f16_vg1x4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: fdot za.s[w8, 0, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: fdot za.s[w8, 7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret @@ -71,16 +71,16 @@ define void @bfdot_multi_za32_bf16_vg1x2(i32 %slice, %unused, define void @fdot_multi_za32_bf16_vg1x4(i32 %slice, %unused, %zn0, %zn1, %zn2, %zn3, ; CHECK-LABEL: fdot_multi_za32_bf16_vg1x4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: bfdot za.s[w8, 0, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: bfdot za.s[w8, 7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-int-dots.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-int-dots.ll index 6d9860483711..e154a4df86ef 100644 --- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-int-dots.ll +++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-int-dots.ll @@ -26,16 +26,16 @@ define void @udot_multi_za32_u16_vg1x2(i32 %slice, %unused, < define void @udot_multi_za32_u16_vg1x4(i32 %slice, %unused, %zn0, %zn1, %zn2, %zn3, ; CHECK-LABEL: udot_multi_za32_u16_vg1x4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: udot za.s[w8, 7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret @@ -68,16 +68,16 @@ define void @udot_multi_za32_u8_vg1x2(i32 %slice, %unused, %unused, %zn0, %zn1, %zn2, %zn3, ; CHECK-LABEL: udot_multi_za32_u8_vg1x4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1b { z27.b }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: udot za.s[w8, 7, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: ret @@ -110,16 +110,16 @@ define void @udot_multi_za64_u16_vg1x2(i32 %slice, %unused, < define void @udot_multi_za64_u16_vg1x4(i32 %slice, %unused, %zn0, %zn1, %zn2, %zn3, ; CHECK-LABEL: udot_multi_za64_u16_vg1x4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: udot za.d[w8, 0, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: udot za.d[w8, 7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret @@ -152,16 +152,16 @@ define void @usdot_multi_za32_u8_vg1x2(i32 %slice, %unused, < define void @usdot_multi_za32_u8_vg1x4(i32 %slice, %unused, %zn0, %zn1, %zn2, %zn3, ; CHECK-LABEL: usdot_multi_za32_u8_vg1x4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1b { z27.b }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: usdot za.s[w8, 0, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: usdot za.s[w8, 7, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: ret @@ -197,16 +197,16 @@ define void @sdot_multi_za32_u16_vg1x2(i32 %slice, %unused, < define void @sdot_multi_za32_u16_vg1x4(i32 %slice, %unused, %zn0, %zn1, %zn2, %zn3, ; CHECK-LABEL: sdot_multi_za32_u16_vg1x4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: sdot za.s[w8, 0, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: sdot za.s[w8, 7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret @@ -239,16 +239,16 @@ define void @sdot_multi_za32_u8_vg1x2(i32 %slice, %unused, %unused, %zn0, %zn1, %zn2, %zn3, ; CHECK-LABEL: sdot_multi_za32_u8_vg1x4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1b { z27.b }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: sdot za.s[w8, 0, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: sdot za.s[w8, 7, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: ret @@ -281,16 +281,16 @@ define void @sdot_multi_za64_u16_vg1x2(i32 %slice, %unused, < define void @sdot_multi_za64_u16_vg1x4(i32 %slice, %unused, %zn0, %zn1, %zn2, %zn3, ; CHECK-LABEL: sdot_multi_za64_u16_vg1x4: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: sdot za.d[w8, 0, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: sdot za.d[w8, 7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-max.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-max.ll index e95d29f65e55..92e8877927ea 100644 --- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-max.ll +++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-max.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -verify-machineinstrs < %s | FileCheck %s ; SMAX (Single, x2) @@ -151,8 +152,7 @@ define { , } @multi_vec_max_single_x2 ; SMAX (Single, x4) -define { , , , } -@multi_vec_max_single_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_s8: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -170,8 +170,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_max_single_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_s16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -189,8 +188,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_max_single_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_s32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -208,8 +206,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_max_single_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_s64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -229,8 +226,7 @@ define { , , , , , , } -@multi_vec_max_single_x4_u8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_u8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_u8: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -248,8 +244,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_max_single_x4_u16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_u16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_u16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -267,8 +262,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_max_single_x4_u32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_u32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_u32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -286,8 +280,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_max_single_x4_u64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_u64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_u64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -307,8 +300,7 @@ define { , , , , , , } -@multi_vec_max_single_x4_f16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_f16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_f16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -326,8 +318,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_max_single_x4_f32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_f32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_f32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -345,8 +336,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_max_single_x4_f64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_max_single_x4_f64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_max_single_x4_f64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -537,104 +527,100 @@ define { , } @multi_vec_max_multi_x2_ ; SMAX (Multi, x4) -define { , , , } -@multi_vec_max_multi_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_s8: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1b { z31.b }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: smax { z24.b - z27.b }, { z24.b - z27.b }, { z28.b - z31.b } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.smax.x4.nxv16i8( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_max_multi_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_s16: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: smax { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.smax.x4.nxv8i16( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_max_multi_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_s32: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: smax { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.smax.x4.nxv4i32( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_max_multi_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_s64: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: smax { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.smax.x4.nxv2i64( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) @@ -643,104 +629,100 @@ define { , , , , , , } -@multi_vec_max_multi_x4_u8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_u8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_u8: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1b { z31.b }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: umax { z24.b - z27.b }, { z24.b - z27.b }, { z28.b - z31.b } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.umax.x4.nxv16i8( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_max_multi_x4_u16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_u16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_u16: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: umax { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.umax.x4.nxv8i16( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_max_multi_x4_u32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_u32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_u32: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: umax { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.umax.x4.nxv4i32( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_max_multi_x4_u64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_u64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_u64: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: umax { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.umax.x4.nxv2i64( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) @@ -749,78 +731,75 @@ define { , , , , , , } -@multi_vec_max_multi_x4_f16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_f16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_f16: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fmax { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.fmax.x4.nxv8f16( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_max_multi_x4_f32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_f32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_f32: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fmax { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.fmax.x4.nxv4f32( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_max_multi_x4_f64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_max_multi_x4_f64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_max_multi_x4_f64: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fmax { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.fmax.x4.nxv2f64( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) @@ -870,8 +849,7 @@ define { , } @multi_vec_maxnm_single ; FMAXNM (Single, x4) -define { , , , } -@multi_vec_maxnm_single_x4_f16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_maxnm_single_x4_f16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_maxnm_single_x4_f16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -889,8 +867,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_maxnm_single_x4_f32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_maxnm_single_x4_f32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_maxnm_single_x4_f32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -908,8 +885,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_maxnm_single_x4_f64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_maxnm_single_x4_f64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_maxnm_single_x4_f64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -976,19 +952,18 @@ define { , } @multi_vec_maxnm_x2_f64( ; FMAXNM (Multi, x4) -define { , , , } -@multi_vec_maxnm_x4_f16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_maxnm_x4_f16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_maxnm_x4_f16: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fmaxnm { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -996,24 +971,23 @@ define { , , , , , , } - @llvm.aarch64.sve.fmaxnm.x4.nxv8f16( %zdn1, %zdn2, %zdn3, %zdn4, + @llvm.aarch64.sve.fmaxnm.x4.nxv8f16( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_maxnm_x4_f32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_maxnm_x4_f32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_maxnm_x4_f32: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fmaxnm { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -1026,19 +1000,18 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_maxnm_x4_f64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_maxnm_x4_f64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_maxnm_x4_f64: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fmaxnm { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-min.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-min.ll index 21a55c6436ac..363f9ba5d353 100644 --- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-min.ll +++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-min.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -verify-machineinstrs < %s | FileCheck %s ; SMIN (Single, x2) @@ -151,8 +152,7 @@ define { , } @multi_vec_min_single_x2 ; SMIN (Single, x4) -define { , , , } -@multi_vec_min_single_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_s8: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -170,8 +170,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_min_single_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_s16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -189,8 +188,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_min_single_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_s32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -208,8 +206,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_min_single_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_s64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -229,8 +226,7 @@ define { , , , , , , } -@multi_vec_min_single_x4_u8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_u8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_u8: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -248,8 +244,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_min_single_x4_u16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_u16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_u16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -267,8 +262,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_min_single_x4_u32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_u32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_u32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -286,8 +280,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_min_single_x4_u64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_u64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_u64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -307,8 +300,7 @@ define { , , , , , , } -@multi_vec_min_single_x4_f16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_f16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_f16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -326,8 +318,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_min_single_x4_f32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_f32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_f32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -345,8 +336,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_min_single_x4_f64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_min_single_x4_f64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_min_single_x4_f64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -537,104 +527,100 @@ define { , } @multi_vec_min_multi_x2_ ; SMIN (Multi, x4) -define { , , , } -@multi_vec_min_multi_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_s8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1b { z31.b }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: smin { z24.b - z27.b }, { z24.b - z27.b }, { z28.b - z31.b } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.smin.x4.nxv16i8( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_min_multi_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_s16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: smin { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.smin.x4.nxv8i16( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_min_multi_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_s32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: smin { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.smin.x4.nxv4i32( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_min_multi_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_s64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: smin { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.smin.x4.nxv2i64( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) @@ -643,104 +629,100 @@ define { , , , , , , } -@multi_vec_min_multi_x4_u8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_u8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_u8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1b { z31.b }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: umin { z24.b - z27.b }, { z24.b - z27.b }, { z28.b - z31.b } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.umin.x4.nxv16i8( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_min_multi_x4_u16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_u16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_u16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: umin { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.umin.x4.nxv8i16( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_min_multi_x4_u32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_u32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_u32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: umin { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.umin.x4.nxv4i32( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_min_multi_x4_u64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_u64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_u64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: umin { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.umin.x4.nxv2i64( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) @@ -749,78 +731,75 @@ define { , , , , , , } -@multi_vec_min_multi_x4_f16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_f16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fmin { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.fmin.x4.nxv8f16( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_min_multi_x4_f32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_f32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fmin { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.fmin.x4.nxv4f32( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_min_multi_x4_f64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_min_multi_x4_f64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_min_multi_x4_f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fmin { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.fmin.x4.nxv2f64( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) @@ -870,8 +849,7 @@ define { , } @multi_vec_minnm_single ; FMINNM (Single, x4) -define { , , , } -@multi_vec_minnm_single_x4_f16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_minnm_single_x4_f16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_minnm_single_x4_f16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -889,8 +867,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_minnm_single_x4_f32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_minnm_single_x4_f32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_minnm_single_x4_f32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -908,8 +885,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_minnm_single_x4_f64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_minnm_single_x4_f64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_minnm_single_x4_f64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -976,19 +952,18 @@ define { , } @multi_vec_minnm_x2_f64( ; FMINNM (Multi, x4) -define { , , , } -@multi_vec_minnm_x4_f16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_minnm_x4_f16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_minnm_x4_f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fminnm { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -1001,19 +976,18 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_minnm_x4_f32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_minnm_x4_f32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_minnm_x4_f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fminnm { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -1026,19 +1000,18 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_minnm_x4_f64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_minnm_x4_f64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_minnm_x4_f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z30.d, z7.d ; CHECK-NEXT: mov z27.d, z4.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z29.d, z6.d ; CHECK-NEXT: mov z26.d, z3.d ; CHECK-NEXT: mov z28.d, z5.d ; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: fminnm { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-mlall.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-mlall.ll index f766bfcff4d1..346afc611eb7 100644 --- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-mlall.ll +++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-mlall.ll @@ -142,16 +142,16 @@ define void @multi_vector_mul_add_multi_long_vg4x2_s16(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm0, %zm1, %zm2, %zm3) { ; CHECK-LABEL: multi_vector_mul_add_multi_long_vg4x4_s8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1b { z27.b }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: smlall za.s[w8, 0:3, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: smlall za.s[w8, 4:7, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: ret @@ -164,16 +164,16 @@ define void @multi_vector_mul_add_multi_long_vg4x4_s8(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm0, %zm1, %zm2, %zm3) { ; CHECK-LABEL: multi_vector_mul_add_multi_long_vg4x4_s16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: smlall za.d[w8, 0:3, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: smlall za.d[w8, 4:7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret @@ -418,16 +418,16 @@ define void @multi_vector_mul_add_multi_long_vg4x2_u16(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm0, %zm1, %zm2, %zm3) { ; CHECK-LABEL: multi_vector_mul_add_multi_long_vg4x4_u8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1b { z27.b }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: umlall za.s[w8, 0:3, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: umlall za.s[w8, 4:7, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: ret @@ -440,16 +440,16 @@ define void @multi_vector_mul_add_multi_long_vg4x4_u8(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm0, %zm1, %zm2, %zm3) { ; CHECK-LABEL: multi_vector_mul_add_multi_long_vg4x4_u16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: umlall za.d[w8, 0:3, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: umlall za.d[w8, 4:7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret @@ -694,16 +694,16 @@ define void @multi_vector_mul_sub_multi_long_vg4x2_s16(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm0, %zm1, %zm2, %zm3) { ; CHECK-LABEL: multi_vector_mul_sub_multi_long_vg4x4_s8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1b { z27.b }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: smlsll za.s[w8, 0:3, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: smlsll za.s[w8, 4:7, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: ret @@ -716,16 +716,16 @@ define void @multi_vector_mul_sub_multi_long_vg4x4_s8(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm0, %zm1, %zm2, %zm3) { ; CHECK-LABEL: multi_vector_mul_sub_multi_long_vg4x4_s16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: smlsll za.d[w8, 0:3, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: smlsll za.d[w8, 4:7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret @@ -970,16 +970,16 @@ define void @multi_vector_mul_sub_multi_long_vg4x2_u16(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm0, %zm1, %zm2, %zm3) { ; CHECK-LABEL: multi_vector_mul_sub_multi_long_vg4x4_u8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1b { z27.b }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: umlsll za.s[w8, 0:3, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: umlsll za.s[w8, 4:7, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: ret @@ -992,16 +992,16 @@ define void @multi_vector_mul_sub_multi_long_vg4x4_u8(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm0, %zm1, %zm2, %zm3) { ; CHECK-LABEL: multi_vector_mul_sub_multi_long_vg4x4_u16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1h { z27.h }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: umlsll za.d[w8, 0:3, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: umlsll za.d[w8, 4:7, vgx4], { z28.h - z31.h }, { z24.h - z27.h } ; CHECK-NEXT: ret @@ -1275,16 +1275,16 @@ define void @multi_vector_mul_add_multi_unsigned_long_vg4x2_u8(i32 %slice, %dummy, %zn0, %zn1, %zn2, %zn3, %zm0, %zm1, %zm2, %zm3) { ; CHECK-LABEL: multi_vector_mul_add_multi_unsigned_long_vg4x4_u8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z26.d, z7.d ; CHECK-NEXT: mov z31.d, z4.d -; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov z25.d, z6.d ; CHECK-NEXT: mov z30.d, z3.d ; CHECK-NEXT: mov z24.d, z5.d ; CHECK-NEXT: mov z29.d, z2.d -; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: ld1b { z27.b }, p0/z, [x1] +; CHECK-NEXT: mov z28.d, z1.d ; CHECK-NEXT: usmlall za.s[w8, 0:3, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: usmlall za.s[w8, 4:7, vgx4], { z28.b - z31.b }, { z24.b - z27.b } ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-rshl.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-rshl.ll index d138a3af4385..12a940ff03e2 100644 --- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-rshl.ll +++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-rshl.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -verify-machineinstrs < %s | FileCheck %s ; SRSHL (Single, x2) @@ -56,8 +57,7 @@ define { , } @multi_vec_rounding_shl_single ; SRSHL (Single, x4) -define { , , , } -@multi_vec_rounding_shl_single_x4_s8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_rounding_shl_single_x4_s8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_rounding_shl_single_x4_s8: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -75,8 +75,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_single_x4_s16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_rounding_shl_single_x4_s16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_rounding_shl_single_x4_s16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -94,8 +93,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_single_x4_s32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_rounding_shl_single_x4_s32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_rounding_shl_single_x4_s32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -113,8 +111,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_single_x4_s64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_rounding_shl_single_x4_s64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_rounding_shl_single_x4_s64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -188,8 +185,7 @@ define { , } @multi_vec_rounding_shl_single ; URSHL (Single, x4) -define { , , , } -@multi_vec_rounding_shl_single_x4_u8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_rounding_shl_single_x4_u8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_rounding_shl_single_x4_u8: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -207,8 +203,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_single_x4_u16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_rounding_shl_single_x4_u16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_rounding_shl_single_x4_u16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -226,8 +221,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_single_x4_u32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_rounding_shl_single_x4_u32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_rounding_shl_single_x4_u32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -245,8 +239,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_single_x4_u64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_rounding_shl_single_x4_u64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_rounding_shl_single_x4_u64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -328,19 +321,18 @@ define { , } @multi_vec_rounding_shl_x2_s64 ; SRSHL (Multi, x4) -define { , , , } -@multi_vec_rounding_shl_x4_s8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_rounding_shl_x4_s8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_rounding_shl_x4_s8: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1b { z31.b }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: srshl { z24.b - z27.b }, { z24.b - z27.b }, { z28.b - z31.b } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -353,19 +345,18 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_x4_s16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_rounding_shl_x4_s16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_rounding_shl_x4_s16: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: srshl { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -378,19 +369,18 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_x4_s32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_rounding_shl_x4_s32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_rounding_shl_x4_s32: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: srshl { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -403,19 +393,18 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_x4_s64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_rounding_shl_x4_s64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_rounding_shl_x4_s64: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: srshl { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -492,19 +481,18 @@ define { , } @multi_vec_rounding_uhl_x2_u64 ; URSHL (Multi, x4) -define { , , , } -@multi_vec_rounding_shl_x4_u8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_rounding_shl_x4_u8( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_rounding_shl_x4_u8: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1b { z31.b }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: urshl { z24.b - z27.b }, { z24.b - z27.b }, { z28.b - z31.b } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -517,19 +505,18 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_x4_u16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_rounding_shl_x4_u16( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_rounding_shl_x4_u16: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: urshl { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -542,19 +529,18 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_x4_u32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_rounding_shl_x4_u32( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_rounding_shl_x4_u32: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: urshl { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d @@ -567,19 +553,18 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_rounding_shl_x4_u64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_rounding_shl_x4_u64( %dummy, %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) { ; CHECK-LABEL: multi_vec_rounding_shl_x4_u64: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: urshl { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-sqdmulh.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-sqdmulh.ll index 9c5dff6c3bf6..e71afe213d8a 100644 --- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-sqdmulh.ll +++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-sqdmulh.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -verify-machineinstrs < %s | FileCheck %s ; SQDMULH (Single, x2) @@ -56,8 +57,7 @@ define { , } @multi_vec_sat_double_mulh_sin ; SQDMULH (Single, x4) -define { , , , } -@multi_vec_sat_double_mulh_single_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_sat_double_mulh_single_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_sat_double_mulh_single_x4_s8: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -75,8 +75,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_sat_double_mulh_single_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_sat_double_mulh_single_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_sat_double_mulh_single_x4_s16: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -94,8 +93,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_sat_double_mulh_single_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_sat_double_mulh_single_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_sat_double_mulh_single_x4_s32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -113,8 +111,7 @@ define { , , , , , , } %res } -define { , , , } -@multi_vec_sat_double_mulh_single_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { +define { , , , } @multi_vec_sat_double_mulh_single_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, %zm) { ; CHECK-LABEL: multi_vec_sat_double_mulh_single_x4_s64: ; CHECK: // %bb.0: ; CHECK-NEXT: mov z27.d, z4.d @@ -196,104 +193,100 @@ define { , } @multi_vec_sat_double_mulh_mul ; SQDMULH (x4, Multi) -define { , , , } -@multi_vec_sat_double_mulh_multi_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_sat_double_mulh_multi_x4_s8( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_sat_double_mulh_multi_x4_s8: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1b { z31.b }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: sqdmulh { z24.b - z27.b }, { z24.b - z27.b }, { z28.b - z31.b } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.sqdmulh.vgx4.nxv16i8( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_sat_double_mulh_multi_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_sat_double_mulh_multi_x4_s16( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_sat_double_mulh_multi_x4_s16: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1h { z31.h }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: sqdmulh { z24.h - z27.h }, { z24.h - z27.h }, { z28.h - z31.h } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.sqdmulh.vgx4.nxv8i16( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_sat_double_mulh_multi_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_sat_double_mulh_multi_x4_s32( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_sat_double_mulh_multi_x4_s32: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1w { z31.s }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: sqdmulh { z24.s - z27.s }, { z24.s - z27.s }, { z28.s - z31.s } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.sqdmulh.vgx4.nxv4i32( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) ret { , , , } %res } -define { , , , } -@multi_vec_sat_double_mulh_multi_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, - %zm1, %zm2, %zm3, %zm4) { +define { , , , } @multi_vec_sat_double_mulh_multi_x4_s64( %unused, %zdn1, %zdn2, %zdn3, %zdn4, ; CHECK-LABEL: multi_vec_sat_double_mulh_multi_x4_s64: ; CHECK: // %bb.0: +; CHECK-NEXT: mov z30.d, z7.d +; CHECK-NEXT: mov z27.d, z4.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov z30.d, z7.d -; CHECK-NEXT: mov z27.d, z4.d -; CHECK-NEXT: mov z29.d, z6.d -; CHECK-NEXT: mov z26.d, z3.d -; CHECK-NEXT: mov z28.d, z5.d -; CHECK-NEXT: mov z25.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d +; CHECK-NEXT: mov z29.d, z6.d +; CHECK-NEXT: mov z26.d, z3.d +; CHECK-NEXT: mov z28.d, z5.d +; CHECK-NEXT: mov z25.d, z2.d ; CHECK-NEXT: ld1d { z31.d }, p0/z, [x0] +; CHECK-NEXT: mov z24.d, z1.d ; CHECK-NEXT: sqdmulh { z24.d - z27.d }, { z24.d - z27.d }, { z28.d - z31.d } ; CHECK-NEXT: mov z0.d, z24.d ; CHECK-NEXT: mov z1.d, z25.d ; CHECK-NEXT: mov z2.d, z26.d ; CHECK-NEXT: mov z3.d, z27.d ; CHECK-NEXT: ret + %zm1, %zm2, %zm3, %zm4) { %res = call { , , , } @llvm.aarch64.sve.sqdmulh.vgx4.nxv2i64( %zdn1, %zdn2, %zdn3, %zdn4, %zm1, %zm2, %zm3, %zm4) diff --git a/llvm/test/CodeGen/AArch64/split-vector-insert.ll b/llvm/test/CodeGen/AArch64/split-vector-insert.ll index a507296338f9..b8ab9a00c698 100644 --- a/llvm/test/CodeGen/AArch64/split-vector-insert.ll +++ b/llvm/test/CodeGen/AArch64/split-vector-insert.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc < %s -debug-only=legalize-types 2>&1 | FileCheck %s --check-prefix=CHECK-LEGALIZATION ; RUN: llc < %s | FileCheck %s ; REQUIRES: asserts @@ -9,54 +10,94 @@ declare @llvm.vector.insert.nxv2i64.v8i64(, declare @llvm.vector.insert.nxv2f64.v8f64(, <8 x double>, i64) define @test_nxv2i64_v8i64( %a, <8 x i64> %b) #0 { -; CHECK-LEGALIZATION: Legally typed node: [[T1:t[0-9]+]]: nxv2i64 = insert_subvector {{t[0-9]+}}, {{t[0-9]+}}, Constant:i64<0> -; CHECK-LEGALIZATION: Legally typed node: [[T2:t[0-9]+]]: nxv2i64 = insert_subvector [[T1]], {{t[0-9]+}}, Constant:i64<2> -; CHECK-LEGALIZATION: Legally typed node: [[T3:t[0-9]+]]: nxv2i64 = insert_subvector [[T2]], {{t[0-9]+}}, Constant:i64<4> -; CHECK-LEGALIZATION: Legally typed node: [[T4:t[0-9]+]]: nxv2i64 = insert_subvector [[T3]], {{t[0-9]+}}, Constant:i64<6> - +; CHECK-LEGALIZATION-LABEL: test_nxv2i64_v8i64: +; CHECK-LEGALIZATION: // %bb.0: +; CHECK-LEGALIZATION-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-LEGALIZATION-NEXT: .cfi_def_cfa_offset 16 +; CHECK-LEGALIZATION-NEXT: .cfi_offset w29, -16 +; CHECK-LEGALIZATION-NEXT: addvl sp, sp, #-3 +; CHECK-LEGALIZATION-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG +; CHECK-LEGALIZATION-NEXT: cntd x8 +; CHECK-LEGALIZATION-NEXT: ptrue p0.d, vl2 +; CHECK-LEGALIZATION-NEXT: mov w9, #2 // =0x2 +; CHECK-LEGALIZATION-NEXT: sub x8, x8, #2 +; CHECK-LEGALIZATION-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-LEGALIZATION-NEXT: mov x10, sp +; CHECK-LEGALIZATION-NEXT: cmp x8, #2 +; CHECK-LEGALIZATION-NEXT: mov z0.d, p0/m, z1.d +; CHECK-LEGALIZATION-NEXT: ptrue p0.d +; CHECK-LEGALIZATION-NEXT: csel x9, x8, x9, lo +; CHECK-LEGALIZATION-NEXT: cmp x8, #4 +; CHECK-LEGALIZATION-NEXT: lsl x9, x9, #3 +; CHECK-LEGALIZATION-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-LEGALIZATION-NEXT: str q2, [x10, x9] +; CHECK-LEGALIZATION-NEXT: mov w9, #4 // =0x4 +; CHECK-LEGALIZATION-NEXT: addvl x10, sp, #1 +; CHECK-LEGALIZATION-NEXT: ld1d { z0.d }, p0/z, [sp] +; CHECK-LEGALIZATION-NEXT: csel x9, x8, x9, lo +; CHECK-LEGALIZATION-NEXT: cmp x8, #6 +; CHECK-LEGALIZATION-NEXT: lsl x9, x9, #3 +; CHECK-LEGALIZATION-NEXT: st1d { z0.d }, p0, [sp, #1, mul vl] +; CHECK-LEGALIZATION-NEXT: str q3, [x10, x9] +; CHECK-LEGALIZATION-NEXT: mov w9, #6 // =0x6 +; CHECK-LEGALIZATION-NEXT: ld1d { z0.d }, p0/z, [sp, #1, mul vl] +; CHECK-LEGALIZATION-NEXT: csel x8, x8, x9, lo +; CHECK-LEGALIZATION-NEXT: addvl x9, sp, #2 +; CHECK-LEGALIZATION-NEXT: lsl x8, x8, #3 +; CHECK-LEGALIZATION-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl] +; CHECK-LEGALIZATION-NEXT: str q4, [x9, x8] +; CHECK-LEGALIZATION-NEXT: ld1d { z0.d }, p0/z, [sp, #2, mul vl] +; CHECK-LEGALIZATION-NEXT: addvl sp, sp, #3 +; CHECK-LEGALIZATION-NEXT: .cfi_def_cfa wsp, 16 +; CHECK-LEGALIZATION-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-LEGALIZATION-NEXT: .cfi_def_cfa_offset 0 +; CHECK-LEGALIZATION-NEXT: .cfi_restore w29 +; CHECK-LEGALIZATION-NEXT: ret +; ; CHECK-LABEL: test_nxv2i64_v8i64: ; CHECK: // %bb.0: -; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: addvl sp, sp, #-3 -; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG -; CHECK-NEXT: ptrue p1.d, vl2 -; CHECK-NEXT: cntd x8 -; CHECK-NEXT: mov w9, #2 // =0x2 -; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: sub x8, x8, #2 -; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 -; CHECK-NEXT: mov x10, sp -; CHECK-NEXT: cmp x8, #2 -; CHECK-NEXT: csel x9, x8, x9, lo -; CHECK-NEXT: cmp x8, #4 -; CHECK-NEXT: lsl x9, x9, #3 -; CHECK-NEXT: mov z0.d, p1/m, z1.d -; CHECK-NEXT: st1d { z0.d }, p0, [sp] -; CHECK-NEXT: str q2, [x10, x9] -; CHECK-NEXT: mov w9, #4 // =0x4 -; CHECK-NEXT: addvl x10, sp, #1 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp] -; CHECK-NEXT: csel x9, x8, x9, lo -; CHECK-NEXT: cmp x8, #6 -; CHECK-NEXT: lsl x9, x9, #3 -; CHECK-NEXT: st1d { z0.d }, p0, [sp, #1, mul vl] -; CHECK-NEXT: str q3, [x10, x9] -; CHECK-NEXT: mov w9, #6 // =0x6 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #1, mul vl] -; CHECK-NEXT: csel x8, x8, x9, lo -; CHECK-NEXT: addvl x9, sp, #2 -; CHECK-NEXT: lsl x8, x8, #3 -; CHECK-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl] -; CHECK-NEXT: str q4, [x9, x8] -; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #2, mul vl] -; CHECK-NEXT: addvl sp, sp, #3 -; CHECK-NEXT: .cfi_def_cfa wsp, 16 -; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload -; CHECK-NEXT: .cfi_def_cfa_offset 0 -; CHECK-NEXT: .cfi_restore w29 -; CHECK-NEXT: ret +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: addvl sp, sp, #-3 +; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG +; CHECK-NEXT: cntd x8 +; CHECK-NEXT: ptrue p0.d, vl2 +; CHECK-NEXT: mov w9, #2 // =0x2 +; CHECK-NEXT: sub x8, x8, #2 +; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-NEXT: mov x10, sp +; CHECK-NEXT: cmp x8, #2 +; CHECK-NEXT: mov z0.d, p0/m, z1.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: csel x9, x8, x9, lo +; CHECK-NEXT: cmp x8, #4 +; CHECK-NEXT: lsl x9, x9, #3 +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: str q2, [x10, x9] +; CHECK-NEXT: mov w9, #4 // =0x4 +; CHECK-NEXT: addvl x10, sp, #1 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp] +; CHECK-NEXT: csel x9, x8, x9, lo +; CHECK-NEXT: cmp x8, #6 +; CHECK-NEXT: lsl x9, x9, #3 +; CHECK-NEXT: st1d { z0.d }, p0, [sp, #1, mul vl] +; CHECK-NEXT: str q3, [x10, x9] +; CHECK-NEXT: mov w9, #6 // =0x6 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #1, mul vl] +; CHECK-NEXT: csel x8, x8, x9, lo +; CHECK-NEXT: addvl x9, sp, #2 +; CHECK-NEXT: lsl x8, x8, #3 +; CHECK-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl] +; CHECK-NEXT: str q4, [x9, x8] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #2, mul vl] +; CHECK-NEXT: addvl sp, sp, #3 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w29 +; CHECK-NEXT: ret + @@ -66,54 +107,94 @@ define @test_nxv2i64_v8i64( %a, <8 x i64> % } define @test_nxv2f64_v8f64( %a, <8 x double> %b) #0 { -; CHECK-LEGALIZATION: Legally typed node: [[T1:t[0-9]+]]: nxv2f64 = insert_subvector {{t[0-9]+}}, {{t[0-9]+}}, Constant:i64<0> -; CHECK-LEGALIZATION: Legally typed node: [[T2:t[0-9]+]]: nxv2f64 = insert_subvector [[T1]], {{t[0-9]+}}, Constant:i64<2> -; CHECK-LEGALIZATION: Legally typed node: [[T3:t[0-9]+]]: nxv2f64 = insert_subvector [[T2]], {{t[0-9]+}}, Constant:i64<4> -; CHECK-LEGALIZATION: Legally typed node: [[T4:t[0-9]+]]: nxv2f64 = insert_subvector [[T3]], {{t[0-9]+}}, Constant:i64<6> - +; CHECK-LEGALIZATION-LABEL: test_nxv2f64_v8f64: +; CHECK-LEGALIZATION: // %bb.0: +; CHECK-LEGALIZATION-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-LEGALIZATION-NEXT: .cfi_def_cfa_offset 16 +; CHECK-LEGALIZATION-NEXT: .cfi_offset w29, -16 +; CHECK-LEGALIZATION-NEXT: addvl sp, sp, #-3 +; CHECK-LEGALIZATION-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG +; CHECK-LEGALIZATION-NEXT: cntd x8 +; CHECK-LEGALIZATION-NEXT: ptrue p0.d, vl2 +; CHECK-LEGALIZATION-NEXT: mov w9, #2 // =0x2 +; CHECK-LEGALIZATION-NEXT: sub x8, x8, #2 +; CHECK-LEGALIZATION-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-LEGALIZATION-NEXT: mov x10, sp +; CHECK-LEGALIZATION-NEXT: cmp x8, #2 +; CHECK-LEGALIZATION-NEXT: mov z0.d, p0/m, z1.d +; CHECK-LEGALIZATION-NEXT: ptrue p0.d +; CHECK-LEGALIZATION-NEXT: csel x9, x8, x9, lo +; CHECK-LEGALIZATION-NEXT: cmp x8, #4 +; CHECK-LEGALIZATION-NEXT: lsl x9, x9, #3 +; CHECK-LEGALIZATION-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-LEGALIZATION-NEXT: str q2, [x10, x9] +; CHECK-LEGALIZATION-NEXT: mov w9, #4 // =0x4 +; CHECK-LEGALIZATION-NEXT: addvl x10, sp, #1 +; CHECK-LEGALIZATION-NEXT: ld1d { z0.d }, p0/z, [sp] +; CHECK-LEGALIZATION-NEXT: csel x9, x8, x9, lo +; CHECK-LEGALIZATION-NEXT: cmp x8, #6 +; CHECK-LEGALIZATION-NEXT: lsl x9, x9, #3 +; CHECK-LEGALIZATION-NEXT: st1d { z0.d }, p0, [sp, #1, mul vl] +; CHECK-LEGALIZATION-NEXT: str q3, [x10, x9] +; CHECK-LEGALIZATION-NEXT: mov w9, #6 // =0x6 +; CHECK-LEGALIZATION-NEXT: ld1d { z0.d }, p0/z, [sp, #1, mul vl] +; CHECK-LEGALIZATION-NEXT: csel x8, x8, x9, lo +; CHECK-LEGALIZATION-NEXT: addvl x9, sp, #2 +; CHECK-LEGALIZATION-NEXT: lsl x8, x8, #3 +; CHECK-LEGALIZATION-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl] +; CHECK-LEGALIZATION-NEXT: str q4, [x9, x8] +; CHECK-LEGALIZATION-NEXT: ld1d { z0.d }, p0/z, [sp, #2, mul vl] +; CHECK-LEGALIZATION-NEXT: addvl sp, sp, #3 +; CHECK-LEGALIZATION-NEXT: .cfi_def_cfa wsp, 16 +; CHECK-LEGALIZATION-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-LEGALIZATION-NEXT: .cfi_def_cfa_offset 0 +; CHECK-LEGALIZATION-NEXT: .cfi_restore w29 +; CHECK-LEGALIZATION-NEXT: ret +; ; CHECK-LABEL: test_nxv2f64_v8f64: ; CHECK: // %bb.0: -; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: addvl sp, sp, #-3 -; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG -; CHECK-NEXT: ptrue p1.d, vl2 -; CHECK-NEXT: cntd x8 -; CHECK-NEXT: mov w9, #2 // =0x2 -; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: sub x8, x8, #2 -; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 -; CHECK-NEXT: mov x10, sp -; CHECK-NEXT: cmp x8, #2 -; CHECK-NEXT: csel x9, x8, x9, lo -; CHECK-NEXT: cmp x8, #4 -; CHECK-NEXT: lsl x9, x9, #3 -; CHECK-NEXT: mov z0.d, p1/m, z1.d -; CHECK-NEXT: st1d { z0.d }, p0, [sp] -; CHECK-NEXT: str q2, [x10, x9] -; CHECK-NEXT: mov w9, #4 // =0x4 -; CHECK-NEXT: addvl x10, sp, #1 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp] -; CHECK-NEXT: csel x9, x8, x9, lo -; CHECK-NEXT: cmp x8, #6 -; CHECK-NEXT: lsl x9, x9, #3 -; CHECK-NEXT: st1d { z0.d }, p0, [sp, #1, mul vl] -; CHECK-NEXT: str q3, [x10, x9] -; CHECK-NEXT: mov w9, #6 // =0x6 -; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #1, mul vl] -; CHECK-NEXT: csel x8, x8, x9, lo -; CHECK-NEXT: addvl x9, sp, #2 -; CHECK-NEXT: lsl x8, x8, #3 -; CHECK-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl] -; CHECK-NEXT: str q4, [x9, x8] -; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #2, mul vl] -; CHECK-NEXT: addvl sp, sp, #3 -; CHECK-NEXT: .cfi_def_cfa wsp, 16 -; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload -; CHECK-NEXT: .cfi_def_cfa_offset 0 -; CHECK-NEXT: .cfi_restore w29 -; CHECK-NEXT: ret +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: addvl sp, sp, #-3 +; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x18, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 24 * VG +; CHECK-NEXT: cntd x8 +; CHECK-NEXT: ptrue p0.d, vl2 +; CHECK-NEXT: mov w9, #2 // =0x2 +; CHECK-NEXT: sub x8, x8, #2 +; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 +; CHECK-NEXT: mov x10, sp +; CHECK-NEXT: cmp x8, #2 +; CHECK-NEXT: mov z0.d, p0/m, z1.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: csel x9, x8, x9, lo +; CHECK-NEXT: cmp x8, #4 +; CHECK-NEXT: lsl x9, x9, #3 +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: str q2, [x10, x9] +; CHECK-NEXT: mov w9, #4 // =0x4 +; CHECK-NEXT: addvl x10, sp, #1 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp] +; CHECK-NEXT: csel x9, x8, x9, lo +; CHECK-NEXT: cmp x8, #6 +; CHECK-NEXT: lsl x9, x9, #3 +; CHECK-NEXT: st1d { z0.d }, p0, [sp, #1, mul vl] +; CHECK-NEXT: str q3, [x10, x9] +; CHECK-NEXT: mov w9, #6 // =0x6 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #1, mul vl] +; CHECK-NEXT: csel x8, x8, x9, lo +; CHECK-NEXT: addvl x9, sp, #2 +; CHECK-NEXT: lsl x8, x8, #3 +; CHECK-NEXT: st1d { z0.d }, p0, [sp, #2, mul vl] +; CHECK-NEXT: str q4, [x9, x8] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #2, mul vl] +; CHECK-NEXT: addvl sp, sp, #3 +; CHECK-NEXT: .cfi_def_cfa wsp, 16 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: .cfi_def_cfa_offset 0 +; CHECK-NEXT: .cfi_restore w29 +; CHECK-NEXT: ret + diff --git a/llvm/test/CodeGen/AArch64/srem-seteq-vec-splat.ll b/llvm/test/CodeGen/AArch64/srem-seteq-vec-splat.ll index 1d9cb88260b6..c0c0ae5c9d1f 100644 --- a/llvm/test/CodeGen/AArch64/srem-seteq-vec-splat.ll +++ b/llvm/test/CodeGen/AArch64/srem-seteq-vec-splat.ll @@ -111,6 +111,7 @@ define <4 x i32> @test_srem_odd_undef1(<4 x i32> %X) nounwind { ; CHECK-LABEL: test_srem_odd_undef1: ; CHECK: // %bb.0: ; CHECK-NEXT: mov w8, #34079 // =0x851f +; CHECK-NEXT: movi v3.4s, #25 ; CHECK-NEXT: movk w8, #20971, lsl #16 ; CHECK-NEXT: dup v1.4s, w8 ; CHECK-NEXT: smull2 v2.2d, v0.4s, v1.4s @@ -118,9 +119,8 @@ define <4 x i32> @test_srem_odd_undef1(<4 x i32> %X) nounwind { ; CHECK-NEXT: uzp2 v1.4s, v1.4s, v2.4s ; CHECK-NEXT: sshr v2.4s, v1.4s, #3 ; CHECK-NEXT: usra v2.4s, v1.4s, #31 -; CHECK-NEXT: movi v1.4s, #25 -; CHECK-NEXT: mls v0.4s, v2.4s, v1.4s ; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: mls v0.4s, v2.4s, v3.4s ; CHECK-NEXT: cmeq v0.4s, v0.4s, #0 ; CHECK-NEXT: and v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret @@ -134,6 +134,7 @@ define <4 x i32> @test_srem_even_undef1(<4 x i32> %X) nounwind { ; CHECK-LABEL: test_srem_even_undef1: ; CHECK: // %bb.0: ; CHECK-NEXT: mov w8, #34079 // =0x851f +; CHECK-NEXT: movi v3.4s, #100 ; CHECK-NEXT: movk w8, #20971, lsl #16 ; CHECK-NEXT: dup v1.4s, w8 ; CHECK-NEXT: smull2 v2.2d, v0.4s, v1.4s @@ -141,9 +142,8 @@ define <4 x i32> @test_srem_even_undef1(<4 x i32> %X) nounwind { ; CHECK-NEXT: uzp2 v1.4s, v1.4s, v2.4s ; CHECK-NEXT: sshr v2.4s, v1.4s, #5 ; CHECK-NEXT: usra v2.4s, v1.4s, #31 -; CHECK-NEXT: movi v1.4s, #100 -; CHECK-NEXT: mls v0.4s, v2.4s, v1.4s ; CHECK-NEXT: movi v1.4s, #1 +; CHECK-NEXT: mls v0.4s, v2.4s, v3.4s ; CHECK-NEXT: cmeq v0.4s, v0.4s, #0 ; CHECK-NEXT: and v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret @@ -201,11 +201,11 @@ define <4 x i32> @test_srem_pow2(<4 x i32> %X) nounwind { define <4 x i32> @test_srem_int_min(<4 x i32> %X) nounwind { ; CHECK-LABEL: test_srem_int_min: ; CHECK: // %bb.0: -; CHECK-NEXT: cmlt v1.4s, v0.4s, #0 -; CHECK-NEXT: mov v2.16b, v0.16b -; CHECK-NEXT: usra v2.4s, v1.4s, #1 +; CHECK-NEXT: cmlt v2.4s, v0.4s, #0 +; CHECK-NEXT: mov v3.16b, v0.16b ; CHECK-NEXT: movi v1.4s, #128, lsl #24 -; CHECK-NEXT: and v1.16b, v2.16b, v1.16b +; CHECK-NEXT: usra v3.4s, v2.4s, #1 +; CHECK-NEXT: and v1.16b, v3.16b, v1.16b ; CHECK-NEXT: add v0.4s, v1.4s, v0.4s ; CHECK-NEXT: movi v1.4s, #1 ; CHECK-NEXT: cmeq v0.4s, v0.4s, #0 diff --git a/llvm/test/CodeGen/AArch64/srem-vector-lkk.ll b/llvm/test/CodeGen/AArch64/srem-vector-lkk.ll index 0598af7c9806..a74f0c86fe18 100644 --- a/llvm/test/CodeGen/AArch64/srem-vector-lkk.ll +++ b/llvm/test/CodeGen/AArch64/srem-vector-lkk.ll @@ -245,6 +245,7 @@ define <4 x i32> @fold_srem_v4i32(<4 x i32> %x) { ; CHECK-LABEL: fold_srem_v4i32: ; CHECK: // %bb.0: ; CHECK-NEXT: mov w8, #26215 // =0x6667 +; CHECK-NEXT: movi v3.4s, #10 ; CHECK-NEXT: movk w8, #26214, lsl #16 ; CHECK-NEXT: dup v1.4s, w8 ; CHECK-NEXT: smull2 v2.2d, v0.4s, v1.4s @@ -252,8 +253,7 @@ define <4 x i32> @fold_srem_v4i32(<4 x i32> %x) { ; CHECK-NEXT: uzp2 v1.4s, v1.4s, v2.4s ; CHECK-NEXT: sshr v2.4s, v1.4s, #2 ; CHECK-NEXT: usra v2.4s, v1.4s, #31 -; CHECK-NEXT: movi v1.4s, #10 -; CHECK-NEXT: mls v0.4s, v2.4s, v1.4s +; CHECK-NEXT: mls v0.4s, v2.4s, v3.4s ; CHECK-NEXT: ret %1 = srem <4 x i32> %x, ret <4 x i32> %1 diff --git a/llvm/test/CodeGen/AArch64/sve-abd.ll b/llvm/test/CodeGen/AArch64/sve-abd.ll index 95aec0a49261..7b492229e3d2 100644 --- a/llvm/test/CodeGen/AArch64/sve-abd.ll +++ b/llvm/test/CodeGen/AArch64/sve-abd.ll @@ -24,10 +24,10 @@ define @sabd_b( %a, %b) define @sabd_b_promoted_ops( %a, %b) #0 { ; CHECK-LABEL: sabd_b_promoted_ops: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p2.b ; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z1.b, p1/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: sabd z0.b, p2/m, z0.b, z1.b +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: sabd z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: ret %a.sext = sext %a to %b.sext = sext %b to @@ -144,10 +144,10 @@ define @uabd_b( %a, %b) define @uabd_b_promoted_ops( %a, %b) #0 { ; CHECK-LABEL: uabd_b_promoted_ops: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p2.b ; CHECK-NEXT: mov z0.b, p0/z, #1 // =0x1 ; CHECK-NEXT: mov z1.b, p1/z, #1 // =0x1 -; CHECK-NEXT: uabd z0.b, p2/m, z0.b, z1.b +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: uabd z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: ret %a.zext = zext %a to %b.zext = zext %b to @@ -173,9 +173,9 @@ define @uabd_h( %a, %b) define @uabd_h_promoted_ops( %a, %b) #0 { ; CHECK-LABEL: uabd_h_promoted_ops: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and z0.h, z0.h, #0xff ; CHECK-NEXT: and z1.h, z1.h, #0xff +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: uabd z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %a.zext = zext %a to @@ -202,9 +202,9 @@ define @uabd_s( %a, %b) define @uabd_s_promoted_ops( %a, %b) #0 { ; CHECK-LABEL: uabd_s_promoted_ops: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z0.s, z0.s, #0xffff ; CHECK-NEXT: and z1.s, z1.s, #0xffff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uabd z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %a.zext = zext %a to @@ -231,9 +231,9 @@ define @uabd_d( %a, %b) define @uabd_d_promoted_ops( %a, %b) #0 { ; CHECK-LABEL: uabd_d_promoted_ops: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z0.d, z0.d, #0xffffffff ; CHECK-NEXT: and z1.d, z1.d, #0xffffffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uabd z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %a.zext = zext %a to @@ -248,8 +248,8 @@ define @uabd_d_promoted_ops( %a, @uabd_non_matching_extension( %a, %b) #0 { ; CHECK-LABEL: uabd_non_matching_extension: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z1.s, z1.s, #0xff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uabd z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %a.zext = zext %a to @@ -265,9 +265,9 @@ define @uabd_non_matching_extension( %a, @uabd_non_matching_promoted_ops( %a, %b) #0 { ; CHECK-LABEL: uabd_non_matching_promoted_ops: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z0.s, z0.s, #0xff ; CHECK-NEXT: and z1.s, z1.s, #0xffff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uabd z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %a.zext = zext %a to diff --git a/llvm/test/CodeGen/AArch64/sve-bitcast.ll b/llvm/test/CodeGen/AArch64/sve-bitcast.ll index 7dd568fc837a..95f43ba51263 100644 --- a/llvm/test/CodeGen/AArch64/sve-bitcast.ll +++ b/llvm/test/CodeGen/AArch64/sve-bitcast.ll @@ -1698,8 +1698,8 @@ define @bitcast_nxv8i8_to_nxv1i64( %v) #0 { ; CHECK_BE: // %bb.0: ; CHECK_BE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK_BE-NEXT: addvl sp, sp, #-1 -; CHECK_BE-NEXT: ptrue p0.b ; CHECK_BE-NEXT: uzp1 z0.b, z0.b, z0.b +; CHECK_BE-NEXT: ptrue p0.b ; CHECK_BE-NEXT: ptrue p1.d ; CHECK_BE-NEXT: st1b { z0.b }, p0, [sp] ; CHECK_BE-NEXT: ld1d { z0.d }, p1/z, [sp] @@ -1720,8 +1720,8 @@ define @bitcast_nxv4i16_to_nxv1i64( %v) #0 ; CHECK_BE: // %bb.0: ; CHECK_BE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK_BE-NEXT: addvl sp, sp, #-1 -; CHECK_BE-NEXT: ptrue p0.h ; CHECK_BE-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK_BE-NEXT: ptrue p0.h ; CHECK_BE-NEXT: ptrue p1.d ; CHECK_BE-NEXT: st1h { z0.h }, p0, [sp] ; CHECK_BE-NEXT: ld1d { z0.d }, p1/z, [sp] @@ -1742,8 +1742,8 @@ define @bitcast_nxv2i32_to_nxv1i64( %v) #0 ; CHECK_BE: // %bb.0: ; CHECK_BE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK_BE-NEXT: addvl sp, sp, #-1 -; CHECK_BE-NEXT: ptrue p0.s ; CHECK_BE-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK_BE-NEXT: ptrue p0.s ; CHECK_BE-NEXT: ptrue p1.d ; CHECK_BE-NEXT: st1w { z0.s }, p0, [sp] ; CHECK_BE-NEXT: ld1d { z0.d }, p1/z, [sp] @@ -2218,8 +2218,8 @@ define @bitcast_nxv8i8_to_nxv1f64( %v) #0 ; CHECK_BE: // %bb.0: ; CHECK_BE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK_BE-NEXT: addvl sp, sp, #-1 -; CHECK_BE-NEXT: ptrue p0.b ; CHECK_BE-NEXT: uzp1 z0.b, z0.b, z0.b +; CHECK_BE-NEXT: ptrue p0.b ; CHECK_BE-NEXT: ptrue p1.d ; CHECK_BE-NEXT: st1b { z0.b }, p0, [sp] ; CHECK_BE-NEXT: ld1d { z0.d }, p1/z, [sp] @@ -2240,8 +2240,8 @@ define @bitcast_nxv4i16_to_nxv1f64( %v) ; CHECK_BE: // %bb.0: ; CHECK_BE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK_BE-NEXT: addvl sp, sp, #-1 -; CHECK_BE-NEXT: ptrue p0.h ; CHECK_BE-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK_BE-NEXT: ptrue p0.h ; CHECK_BE-NEXT: ptrue p1.d ; CHECK_BE-NEXT: st1h { z0.h }, p0, [sp] ; CHECK_BE-NEXT: ld1d { z0.d }, p1/z, [sp] @@ -2262,8 +2262,8 @@ define @bitcast_nxv2i32_to_nxv1f64( %v) ; CHECK_BE: // %bb.0: ; CHECK_BE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK_BE-NEXT: addvl sp, sp, #-1 -; CHECK_BE-NEXT: ptrue p0.s ; CHECK_BE-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK_BE-NEXT: ptrue p0.s ; CHECK_BE-NEXT: ptrue p1.d ; CHECK_BE-NEXT: st1w { z0.s }, p0, [sp] ; CHECK_BE-NEXT: ld1d { z0.d }, p1/z, [sp] @@ -2827,11 +2827,11 @@ define @bitcast_nxv2f16_to_nxv1i32( %v) #0 ; CHECK_BE-NEXT: addvl sp, sp, #-2 ; CHECK_BE-NEXT: ptrue p0.d ; CHECK_BE-NEXT: ptrue p1.h -; CHECK_BE-NEXT: ptrue p2.s ; CHECK_BE-NEXT: st1h { z0.d }, p0, [sp] +; CHECK_BE-NEXT: ptrue p0.s ; CHECK_BE-NEXT: ld1h { z0.h }, p1/z, [sp] ; CHECK_BE-NEXT: st1h { z0.h }, p1, [sp, #1, mul vl] -; CHECK_BE-NEXT: ld1w { z0.s }, p2/z, [sp, #1, mul vl] +; CHECK_BE-NEXT: ld1w { z0.s }, p0/z, [sp, #1, mul vl] ; CHECK_BE-NEXT: addvl sp, sp, #2 ; CHECK_BE-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK_BE-NEXT: ret @@ -2860,11 +2860,11 @@ define @bitcast_nxv2bf16_to_nxv1i32( %v) ; CHECK_BE-NEXT: addvl sp, sp, #-2 ; CHECK_BE-NEXT: ptrue p0.d ; CHECK_BE-NEXT: ptrue p1.h -; CHECK_BE-NEXT: ptrue p2.s ; CHECK_BE-NEXT: st1h { z0.d }, p0, [sp] +; CHECK_BE-NEXT: ptrue p0.s ; CHECK_BE-NEXT: ld1h { z0.h }, p1/z, [sp] ; CHECK_BE-NEXT: st1h { z0.h }, p1, [sp, #1, mul vl] -; CHECK_BE-NEXT: ld1w { z0.s }, p2/z, [sp, #1, mul vl] +; CHECK_BE-NEXT: ld1w { z0.s }, p0/z, [sp, #1, mul vl] ; CHECK_BE-NEXT: addvl sp, sp, #2 ; CHECK_BE-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK_BE-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-calling-convention-mixed.ll b/llvm/test/CodeGen/AArch64/sve-calling-convention-mixed.ll index 56b023086ea2..3b7b03e6ef61 100644 --- a/llvm/test/CodeGen/AArch64/sve-calling-convention-mixed.ll +++ b/llvm/test/CodeGen/AArch64/sve-calling-convention-mixed.ll @@ -64,12 +64,12 @@ define float @foo2(ptr %x0, ptr %x1) nounwind { ; CHECK-NEXT: add x9, sp, #16 ; CHECK-NEXT: mov w2, #2 // =0x2 ; CHECK-NEXT: mov w3, #3 // =0x3 +; CHECK-NEXT: ld4d { z1.d - z4.d }, p0/z, [x0] +; CHECK-NEXT: mov w0, wzr ; CHECK-NEXT: mov w4, #4 // =0x4 ; CHECK-NEXT: mov w5, #5 // =0x5 ; CHECK-NEXT: mov w6, #6 // =0x6 ; CHECK-NEXT: mov w7, #7 // =0x7 -; CHECK-NEXT: ld4d { z1.d - z4.d }, p0/z, [x0] -; CHECK-NEXT: mov w0, wzr ; CHECK-NEXT: ld4d { z16.d - z19.d }, p0/z, [x1] ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w1, #1 // =0x1 @@ -182,8 +182,8 @@ entry: define double @foo5(i32 %i0, i32 %i1, i32 %i2, i32 %i3, i32 %i4, i32 %i5, ptr %ptr1, ptr %ptr2, double %x0, %x1, %x2) nounwind { ; CHECK-LABEL: foo5: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr x8, [sp] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ld1d { z5.d }, p0/z, [x8, #1, mul vl] ; CHECK-NEXT: ld1d { z6.d }, p0/z, [x8] ; CHECK-NEXT: ld1d { z7.d }, p0/z, [x8, #3, mul vl] @@ -229,10 +229,10 @@ entry: define void @aavpcs1(i32 %s0, i32 %s1, i32 %s2, i32 %s3, i32 %s4, i32 %s5, i32 %s6, %s7, %s8, %s9, %s10, %s11, %s12, %s13, %s14, %s15, %s16, ptr %ptr) nounwind { ; CHECK-LABEL: aavpcs1: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldp x8, x9, [sp] -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x8] +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ld1w { z24.s }, p0/z, [x7] +; CHECK-NEXT: ld1w { z3.s }, p0/z, [x8] ; CHECK-NEXT: st1w { z0.s }, p0, [x9] ; CHECK-NEXT: st1w { z1.s }, p0, [x9] ; CHECK-NEXT: st1w { z2.s }, p0, [x9] @@ -261,12 +261,12 @@ entry: define void @aavpcs2(float %s0, float %s1, float %s2, float %s3, float %s4, float %s5, float %s6, %s7, %s8, %s9, %s10, %s11, %s12, %s13, %s14, %s15, %s16,ptr %ptr) nounwind { ; CHECK-LABEL: aavpcs2: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldp x8, x9, [sp] -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8] +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x7] ; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z3.s }, p0/z, [x6] +; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8] ; CHECK-NEXT: ld1w { z4.s }, p0/z, [x5] ; CHECK-NEXT: ld1w { z5.s }, p0/z, [x1] ; CHECK-NEXT: ld1w { z6.s }, p0/z, [x4] @@ -299,11 +299,10 @@ entry: define void @aavpcs3(float %s0, float %s1, float %s2, float %s3, float %s4, float %s5, float %s6, float %s7, %s8, %s9, %s10, %s11, %s12, %s13, %s14, %s15, %s16, %s17, %p0, ptr %ptr) nounwind { ; CHECK-LABEL: aavpcs3: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr x8, [sp] -; CHECK-NEXT: ldr x9, [sp, #16] -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x8] +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8] +; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z2.s }, p0/z, [x7] ; CHECK-NEXT: ld1w { z3.s }, p0/z, [x1] ; CHECK-NEXT: ld1w { z4.s }, p0/z, [x6] @@ -311,15 +310,16 @@ define void @aavpcs3(float %s0, float %s1, float %s2, float %s3, float %s4, floa ; CHECK-NEXT: ld1w { z6.s }, p0/z, [x2] ; CHECK-NEXT: ld1w { z7.s }, p0/z, [x4] ; CHECK-NEXT: ld1w { z24.s }, p0/z, [x3] -; CHECK-NEXT: st1w { z0.s }, p0, [x9] -; CHECK-NEXT: st1w { z3.s }, p0, [x9] -; CHECK-NEXT: st1w { z6.s }, p0, [x9] -; CHECK-NEXT: st1w { z24.s }, p0, [x9] -; CHECK-NEXT: st1w { z7.s }, p0, [x9] -; CHECK-NEXT: st1w { z5.s }, p0, [x9] -; CHECK-NEXT: st1w { z4.s }, p0, [x9] -; CHECK-NEXT: st1w { z2.s }, p0, [x9] -; CHECK-NEXT: st1w { z1.s }, p0, [x9] +; CHECK-NEXT: ldr x8, [sp, #16] +; CHECK-NEXT: st1w { z1.s }, p0, [x8] +; CHECK-NEXT: st1w { z3.s }, p0, [x8] +; CHECK-NEXT: st1w { z6.s }, p0, [x8] +; CHECK-NEXT: st1w { z24.s }, p0, [x8] +; CHECK-NEXT: st1w { z7.s }, p0, [x8] +; CHECK-NEXT: st1w { z5.s }, p0, [x8] +; CHECK-NEXT: st1w { z4.s }, p0, [x8] +; CHECK-NEXT: st1w { z2.s }, p0, [x8] +; CHECK-NEXT: st1w { z0.s }, p0, [x8] ; CHECK-NEXT: ret entry: store volatile %s8, ptr %ptr @@ -339,8 +339,8 @@ entry: define void @aavpcs4(i32 %s0, i32 %s1, i32 %s2, i32 %s3, i32 %s4, i32 %s5, i32 %s6, i32 %s7, %s8, %s9, %s10, %s11, %s12, %s13, %s14, %s15, %s16, %s17, ptr %ptr) nounwind { ; CHECK-LABEL: aavpcs4: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr x8, [sp] +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr x9, [sp, #16] ; CHECK-NEXT: ld1w { z24.s }, p0/z, [x8] ; CHECK-NEXT: st1w { z0.s }, p0, [x9] @@ -371,11 +371,10 @@ entry: define @aavpcs5(float %s0, float %s1, float %s2, float %s3, float %s4, float %s5, float %s6, float %s7, %s8, %s9, %s10, %s11, %s12, %s13, %s14, %s15, %s16, %s17, ptr %ptr) nounwind { ; CHECK-LABEL: aavpcs5: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr x8, [sp] -; CHECK-NEXT: ldr x9, [sp, #16] -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x8] +; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z2.s }, p0/z, [x7] ; CHECK-NEXT: ld1w { z3.s }, p0/z, [x1] ; CHECK-NEXT: ld1w { z4.s }, p0/z, [x6] @@ -383,15 +382,16 @@ define @aavpcs5(float %s0, float %s1, float %s2, float %s3, ; CHECK-NEXT: ld1w { z6.s }, p0/z, [x2] ; CHECK-NEXT: ld1w { z7.s }, p0/z, [x4] ; CHECK-NEXT: ld1w { z24.s }, p0/z, [x3] -; CHECK-NEXT: st1w { z0.s }, p0, [x9] -; CHECK-NEXT: st1w { z3.s }, p0, [x9] -; CHECK-NEXT: st1w { z6.s }, p0, [x9] -; CHECK-NEXT: st1w { z24.s }, p0, [x9] -; CHECK-NEXT: st1w { z7.s }, p0, [x9] -; CHECK-NEXT: st1w { z5.s }, p0, [x9] -; CHECK-NEXT: st1w { z4.s }, p0, [x9] -; CHECK-NEXT: st1w { z2.s }, p0, [x9] -; CHECK-NEXT: st1w { z1.s }, p0, [x9] +; CHECK-NEXT: ldr x8, [sp, #16] +; CHECK-NEXT: st1w { z0.s }, p0, [x8] +; CHECK-NEXT: st1w { z3.s }, p0, [x8] +; CHECK-NEXT: st1w { z6.s }, p0, [x8] +; CHECK-NEXT: st1w { z24.s }, p0, [x8] +; CHECK-NEXT: st1w { z7.s }, p0, [x8] +; CHECK-NEXT: st1w { z5.s }, p0, [x8] +; CHECK-NEXT: st1w { z4.s }, p0, [x8] +; CHECK-NEXT: st1w { z2.s }, p0, [x8] +; CHECK-NEXT: st1w { z1.s }, p0, [x8] ; CHECK-NEXT: ret entry: store volatile %s8, ptr %ptr @@ -409,11 +409,10 @@ entry: define void @aapcs1(float %s0, float %s1, float %s2, float %s3, float %s4, float %s5, float %s6, float %s7, %s8, %s9, %s10, %s11, %s12, %s13, %s14, %s15, %s16, %s17, ptr %ptr) nounwind { ; CHECK-LABEL: aapcs1: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldr x8, [sp] -; CHECK-NEXT: ldr x9, [sp, #16] -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x8] +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ld1w { z0.s }, p0/z, [x8] +; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z2.s }, p0/z, [x7] ; CHECK-NEXT: ld1w { z3.s }, p0/z, [x1] ; CHECK-NEXT: ld1w { z4.s }, p0/z, [x6] @@ -421,15 +420,16 @@ define void @aapcs1(float %s0, float %s1, float %s2, float %s3, float %s4, float ; CHECK-NEXT: ld1w { z6.s }, p0/z, [x2] ; CHECK-NEXT: ld1w { z7.s }, p0/z, [x4] ; CHECK-NEXT: ld1w { z16.s }, p0/z, [x3] -; CHECK-NEXT: st1w { z0.s }, p0, [x9] -; CHECK-NEXT: st1w { z3.s }, p0, [x9] -; CHECK-NEXT: st1w { z6.s }, p0, [x9] -; CHECK-NEXT: st1w { z16.s }, p0, [x9] -; CHECK-NEXT: st1w { z7.s }, p0, [x9] -; CHECK-NEXT: st1w { z5.s }, p0, [x9] -; CHECK-NEXT: st1w { z4.s }, p0, [x9] -; CHECK-NEXT: st1w { z2.s }, p0, [x9] -; CHECK-NEXT: st1w { z1.s }, p0, [x9] +; CHECK-NEXT: ldr x8, [sp, #16] +; CHECK-NEXT: st1w { z1.s }, p0, [x8] +; CHECK-NEXT: st1w { z3.s }, p0, [x8] +; CHECK-NEXT: st1w { z6.s }, p0, [x8] +; CHECK-NEXT: st1w { z16.s }, p0, [x8] +; CHECK-NEXT: st1w { z7.s }, p0, [x8] +; CHECK-NEXT: st1w { z5.s }, p0, [x8] +; CHECK-NEXT: st1w { z4.s }, p0, [x8] +; CHECK-NEXT: st1w { z2.s }, p0, [x8] +; CHECK-NEXT: st1w { z0.s }, p0, [x8] ; CHECK-NEXT: ret entry: store volatile %s8, ptr %ptr @@ -486,13 +486,13 @@ define void @non_sve_caller_high_range_non_sve_callee_high_range(float %f0, floa ; CHECK-NEXT: fmov s2, #2.00000000 ; CHECK-NEXT: fmov s3, #3.00000000 ; CHECK-NEXT: fmov s4, #4.00000000 -; CHECK-NEXT: fmov s5, #5.00000000 -; CHECK-NEXT: fmov s6, #6.00000000 -; CHECK-NEXT: fmov s7, #7.00000000 ; CHECK-NEXT: ld1w { z16.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z17.s }, p0/z, [x1] ; CHECK-NEXT: addvl x0, sp, #1 +; CHECK-NEXT: fmov s5, #5.00000000 +; CHECK-NEXT: fmov s6, #6.00000000 ; CHECK-NEXT: mov x1, sp +; CHECK-NEXT: fmov s7, #7.00000000 ; CHECK-NEXT: st1w { z17.s }, p0, [sp] ; CHECK-NEXT: st1w { z16.s }, p0, [sp, #1, mul vl] ; CHECK-NEXT: bl non_sve_callee_high_range @@ -548,20 +548,20 @@ define @sve_caller_non_sve_callee_high_range( %a, %b) { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov x8, #-1 // =0xffffffffffffffff -; CHECK-NEXT: whilels p1.s, xzr, x8 ; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s ; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1 -; CHECK-NEXT: lastb w8, p1, z0.s +; CHECK-NEXT: whilels p0.s, xzr, x8 +; CHECK-NEXT: lastb w8, p0, z0.s ; CHECK-NEXT: and w0, w8, #0x1 ; CHECK-NEXT: ret %vcond = fcmp oeq %a, %b diff --git a/llvm/test/CodeGen/AArch64/sve-doublereduct.ll b/llvm/test/CodeGen/AArch64/sve-doublereduct.ll index f5721cd0fd79..7bc31d44bb65 100644 --- a/llvm/test/CodeGen/AArch64/sve-doublereduct.ll +++ b/llvm/test/CodeGen/AArch64/sve-doublereduct.ll @@ -87,8 +87,8 @@ define float @fmaximum_f32( %a, %b) { define i32 @add_i32( %a, %b) { ; CHECK-LABEL: add_i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: add z0.s, z0.s, z1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: add z0.s, z0.s, z2.s ; CHECK-NEXT: uaddv d0, p0, z0.s ; CHECK-NEXT: fmov x0, d0 @@ -160,8 +160,8 @@ define i16 @add_ext_v32i16( %a, %b) { define i32 @and_i32( %a, %b) { ; CHECK-LABEL: and_i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z0.d, z0.d, z1.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z0.d, z0.d, z2.d ; CHECK-NEXT: andv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 @@ -175,8 +175,8 @@ define i32 @and_i32( %a, %b) { define i32 @or_i32( %a, %b) { ; CHECK-LABEL: or_i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: orr z0.d, z0.d, z1.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: orr z0.d, z0.d, z2.d ; CHECK-NEXT: orv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 @@ -190,8 +190,8 @@ define i32 @or_i32( %a, %b) { define i32 @xor_i32( %a, %b) { ; CHECK-LABEL: xor_i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: eor3 z0.d, z0.d, z1.d, z2.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: eorv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-expand-div.ll b/llvm/test/CodeGen/AArch64/sve-expand-div.ll index fe5cdc938772..180c64e0a7de 100644 --- a/llvm/test/CodeGen/AArch64/sve-expand-div.ll +++ b/llvm/test/CodeGen/AArch64/sve-expand-div.ll @@ -10,8 +10,8 @@ define @sdiv_i8( %a) #0 { ; CHECK-LABEL: sdiv_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z1.b, #86 // =0x56 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: smulh z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: lsr z1.b, z0.b, #7 ; CHECK-NEXT: add z0.b, z0.b, z1.b @@ -23,8 +23,8 @@ define @sdiv_i8( %a) #0 { define @sdiv_i16( %a) #0 { ; CHECK-LABEL: sdiv_i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #21846 // =0x5556 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: smulh z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: lsr z1.h, z0.h, #15 @@ -37,8 +37,8 @@ define @sdiv_i16( %a) #0 { define @sdiv_i32( %a) #0 { ; CHECK-LABEL: sdiv_i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #21846 // =0x5556 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: movk w8, #21845, lsl #16 ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: smulh z0.s, p0/m, z0.s, z1.s @@ -52,8 +52,8 @@ define @sdiv_i32( %a) #0 { define @sdiv_i64( %a) #0 { ; CHECK-LABEL: sdiv_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #6148914691236517205 // =0x5555555555555555 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movk x8, #21846 ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: smulh z0.d, p0/m, z0.d, z1.d @@ -71,8 +71,8 @@ define @sdiv_i64( %a) #0 { define @udiv_i8( %a) #0 { ; CHECK-LABEL: udiv_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z1.b, #-85 // =0xffffffffffffffab +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: umulh z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: lsr z0.b, z0.b, #1 ; CHECK-NEXT: ret @@ -83,8 +83,8 @@ define @udiv_i8( %a) #0 { define @udiv_i16( %a) #0 { ; CHECK-LABEL: udiv_i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #-21845 // =0xffffaaab +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: umulh z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: lsr z0.h, z0.h, #1 @@ -96,8 +96,8 @@ define @udiv_i16( %a) #0 { define @udiv_i32( %a) #0 { ; CHECK-LABEL: udiv_i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #43691 // =0xaaab +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: movk w8, #43690, lsl #16 ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: umulh z0.s, p0/m, z0.s, z1.s @@ -110,8 +110,8 @@ define @udiv_i32( %a) #0 { define @udiv_i64( %a) #0 { ; CHECK-LABEL: udiv_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #-6148914691236517206 // =0xaaaaaaaaaaaaaaaa +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movk x8, #43691 ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: umulh z0.d, p0/m, z0.d, z1.d diff --git a/llvm/test/CodeGen/AArch64/sve-extract-element.ll b/llvm/test/CodeGen/AArch64/sve-extract-element.ll index a3c34b53baa0..6d4f5963881e 100644 --- a/llvm/test/CodeGen/AArch64/sve-extract-element.ll +++ b/llvm/test/CodeGen/AArch64/sve-extract-element.ll @@ -616,8 +616,8 @@ define i1 @test_last_8xi1( %a) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: mov x8, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z0.h, p0/z, #1 // =0x1 -; CHECK-NEXT: whilels p1.h, xzr, x8 -; CHECK-NEXT: lastb w8, p1, z0.h +; CHECK-NEXT: whilels p0.h, xzr, x8 +; CHECK-NEXT: lastb w8, p0, z0.h ; CHECK-NEXT: and w0, w8, #0x1 ; CHECK-NEXT: ret %vscale = call i64 @llvm.vscale.i64() @@ -630,10 +630,10 @@ define i1 @test_last_8xi1( %a) #0 { define i1 @test_lanex_4xi1( %a, i32 %x) #0 { ; CHECK-LABEL: test_lanex_4xi1: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, w0 ; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1 -; CHECK-NEXT: whilels p1.s, xzr, x8 -; CHECK-NEXT: lastb w8, p1, z0.s +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: whilels p0.s, xzr, x8 +; CHECK-NEXT: lastb w8, p0, z0.s ; CHECK-NEXT: and w0, w8, #0x1 ; CHECK-NEXT: ret %b = extractelement %a, i32 %x diff --git a/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll index bc1c563810f3..b9c531fe3352 100644 --- a/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll +++ b/llvm/test/CodeGen/AArch64/sve-extract-fixed-from-scalable-vector.ll @@ -100,16 +100,16 @@ define <2 x i64> @extract_v2i64_nxv8i64_8( %arg) { ; CHECK-NEXT: addvl sp, sp, #-4 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cnth x8 ; CHECK-NEXT: mov w9, #8 // =0x8 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: sub x8, x8, #2 ; CHECK-NEXT: cmp x8, #8 +; CHECK-NEXT: st1d { z3.d }, p0, [sp, #3, mul vl] ; CHECK-NEXT: csel x8, x8, x9, lo +; CHECK-NEXT: st1d { z2.d }, p0, [sp, #2, mul vl] ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #3 -; CHECK-NEXT: st1d { z3.d }, p0, [sp, #3, mul vl] -; CHECK-NEXT: st1d { z2.d }, p0, [sp, #2, mul vl] ; CHECK-NEXT: st1d { z1.d }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1d { z0.d }, p0, [sp] ; CHECK-NEXT: ldr q0, [x9, x8] @@ -183,10 +183,10 @@ define <4 x i1> @extract_v4i1_nxv32i1_16( %arg) { ; CHECK-NEXT: addvl sp, sp, #-8 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0xc0, 0x00, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 64 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p2.b ; CHECK-NEXT: mov z0.b, p1/z, #1 // =0x1 ; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1 ; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: ptrue p2.b ; CHECK-NEXT: add x8, x8, #16 ; CHECK-NEXT: st1b { z0.b }, p2, [sp, #1, mul vl] ; CHECK-NEXT: st1b { z1.b }, p2, [sp] diff --git a/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll index e2f8dad03ef6..88268104889f 100644 --- a/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll +++ b/llvm/test/CodeGen/AArch64/sve-extract-fixed-vector.ll @@ -17,15 +17,15 @@ define <2 x i64> @extract_v2i64_nxv2i64_idx2( %vec) nounwind { ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cntd x8 ; CHECK-NEXT: mov w9, #2 // =0x2 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: sub x8, x8, #2 ; CHECK-NEXT: cmp x8, #2 +; CHECK-NEXT: st1d { z0.d }, p0, [sp] ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #3 -; CHECK-NEXT: st1d { z0.d }, p0, [sp] ; CHECK-NEXT: ldr q0, [x9, x8] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -50,15 +50,15 @@ define <4 x i32> @extract_v4i32_nxv4i32_idx4( %vec) nounwind { ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cntw x8 ; CHECK-NEXT: mov w9, #4 // =0x4 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sub x8, x8, #4 ; CHECK-NEXT: cmp x8, #4 +; CHECK-NEXT: st1w { z0.s }, p0, [sp] ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #2 -; CHECK-NEXT: st1w { z0.s }, p0, [sp] ; CHECK-NEXT: ldr q0, [x9, x8] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -115,15 +115,15 @@ define <8 x i16> @extract_v8i16_nxv8i16_idx8( %vec) nounwind { ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cnth x8 ; CHECK-NEXT: mov w9, #8 // =0x8 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: sub x8, x8, #8 ; CHECK-NEXT: cmp x8, #8 +; CHECK-NEXT: st1h { z0.h }, p0, [sp] ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #1 -; CHECK-NEXT: st1h { z0.h }, p0, [sp] ; CHECK-NEXT: ldr q0, [x9, x8] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -214,14 +214,14 @@ define <16 x i8> @extract_v16i8_nxv16i8_idx16( %vec) nounwind ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: rdvl x8, #1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov w9, #16 // =0x10 ; CHECK-NEXT: sub x8, x8, #16 ; CHECK-NEXT: cmp x8, #16 +; CHECK-NEXT: st1b { z0.b }, p0, [sp] ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp -; CHECK-NEXT: st1b { z0.b }, p0, [sp] ; CHECK-NEXT: ldr q0, [x9, x8] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload diff --git a/llvm/test/CodeGen/AArch64/sve-extract-scalable-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-scalable-vector.ll index e60a2f142922..3c0bd501f45d 100644 --- a/llvm/test/CodeGen/AArch64/sve-extract-scalable-vector.ll +++ b/llvm/test/CodeGen/AArch64/sve-extract-scalable-vector.ll @@ -65,27 +65,25 @@ define @extract_nxv14i1_nxv28i1_14( %in) uw ; CHECK-NEXT: addvl sp, sp, #-1 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: punpkhi p2.h, p1.b -; CHECK-NEXT: str p6, [sp, #5, mul vl] // 2-byte Folded Spill +; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: punpklo p1.h, p1.b ; CHECK-NEXT: str p5, [sp, #6, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: punpkhi p0.h, p0.b -; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: punpklo p2.h, p2.b ; CHECK-NEXT: punpkhi p3.h, p1.b -; CHECK-NEXT: punpklo p1.h, p1.b -; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: punpkhi p4.h, p2.b ; CHECK-NEXT: punpklo p2.h, p2.b +; CHECK-NEXT: uzp1 p4.s, p4.s, p0.s +; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: punpkhi p5.h, p3.b -; CHECK-NEXT: punpklo p3.h, p3.b -; CHECK-NEXT: punpkhi p6.h, p1.b ; CHECK-NEXT: punpklo p1.h, p1.b ; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: uzp1 p2.s, p5.s, p2.s +; CHECK-NEXT: punpklo p3.h, p3.b +; CHECK-NEXT: punpkhi p5.h, p1.b +; CHECK-NEXT: punpklo p1.h, p1.b +; CHECK-NEXT: punpkhi p0.h, p0.b +; CHECK-NEXT: uzp1 p3.s, p5.s, p3.s ; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p3.s, p6.s, p3.s -; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p4.s, p4.s, p0.s ; CHECK-NEXT: uzp1 p0.s, p0.s, p1.s ; CHECK-NEXT: uzp1 p1.h, p2.h, p4.h ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload diff --git a/llvm/test/CodeGen/AArch64/sve-fcmp.ll b/llvm/test/CodeGen/AArch64/sve-fcmp.ll index f7e3b6d0171a..35cbe65c6a8b 100644 --- a/llvm/test/CodeGen/AArch64/sve-fcmp.ll +++ b/llvm/test/CodeGen/AArch64/sve-fcmp.ll @@ -374,8 +374,8 @@ define @one_zero( %x) { define @ueq_zero( %x) { ; CHECK-LABEL: ueq_zero: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, #0 // =0x0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: fcmuo p1.s, p0/z, z0.s, z1.s ; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0 ; CHECK-NEXT: sel p0.b, p0, p0.b, p1.b diff --git a/llvm/test/CodeGen/AArch64/sve-fcopysign.ll b/llvm/test/CodeGen/AArch64/sve-fcopysign.ll index f15807597ac2..78843e392e53 100644 --- a/llvm/test/CodeGen/AArch64/sve-fcopysign.ll +++ b/llvm/test/CodeGen/AArch64/sve-fcopysign.ll @@ -63,10 +63,10 @@ define @test_copysign_v4f32_v4f64( %a, ; CHECK-EXTEND-ROUND-NEXT: ptrue p0.d ; CHECK-EXTEND-ROUND-NEXT: uunpkhi z3.d, z0.s ; CHECK-EXTEND-ROUND-NEXT: uunpklo z0.d, z0.s -; CHECK-EXTEND-ROUND-NEXT: and z3.s, z3.s, #0x7fffffff -; CHECK-EXTEND-ROUND-NEXT: and z0.s, z0.s, #0x7fffffff ; CHECK-EXTEND-ROUND-NEXT: fcvt z2.s, p0/m, z2.d ; CHECK-EXTEND-ROUND-NEXT: fcvt z1.s, p0/m, z1.d +; CHECK-EXTEND-ROUND-NEXT: and z3.s, z3.s, #0x7fffffff +; CHECK-EXTEND-ROUND-NEXT: and z0.s, z0.s, #0x7fffffff ; CHECK-EXTEND-ROUND-NEXT: and z2.s, z2.s, #0x80000000 ; CHECK-EXTEND-ROUND-NEXT: and z1.s, z1.s, #0x80000000 ; CHECK-EXTEND-ROUND-NEXT: orr z2.d, z3.d, z2.d @@ -115,9 +115,9 @@ declare @llvm.copysign.v2f64( %a, @test_copysign_v4f64_v4f32( %a, %b) #0 { ; CHECK-LABEL: test_copysign_v4f64_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z3.d, z2.s ; CHECK-NEXT: uunpkhi z2.d, z2.s +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z0.d, z0.d, #0x7fffffffffffffff ; CHECK-NEXT: and z1.d, z1.d, #0x7fffffffffffffff ; CHECK-NEXT: fcvt z3.d, p0/m, z3.s @@ -193,10 +193,10 @@ define @test_copysign_v4f16_v4f64( %a, @test_copysign_v8f16_v8f32( %a, @test_copysign_v8f16_v8f32( %a, @test_copysign_nxv4f32_nxv4f16( %a, %b) #0 { ; CHECK-NO-EXTEND-ROUND-LABEL: test_copysign_nxv4f32_nxv4f16: ; CHECK-NO-EXTEND-ROUND: // %bb.0: -; CHECK-NO-EXTEND-ROUND-NEXT: ptrue p0.s ; CHECK-NO-EXTEND-ROUND-NEXT: and z1.s, z1.s, #0x80000000 ; CHECK-NO-EXTEND-ROUND-NEXT: and z0.s, z0.s, #0x7fffffff +; CHECK-NO-EXTEND-ROUND-NEXT: ptrue p0.s ; CHECK-NO-EXTEND-ROUND-NEXT: orr z0.d, z0.d, z1.d ; CHECK-NO-EXTEND-ROUND-NEXT: fcvt z0.h, p0/m, z0.s ; CHECK-NO-EXTEND-ROUND-NEXT: ret @@ -285,9 +285,9 @@ define @test_copysign_nxv4f32_nxv4f16( % define @test_copysign_nxv2f64_nxv2f32( %a, %b) #0 { ; CHECK-NO-EXTEND-ROUND-LABEL: test_copysign_nxv2f64_nxv2f32: ; CHECK-NO-EXTEND-ROUND: // %bb.0: -; CHECK-NO-EXTEND-ROUND-NEXT: ptrue p0.d ; CHECK-NO-EXTEND-ROUND-NEXT: and z1.d, z1.d, #0x8000000000000000 ; CHECK-NO-EXTEND-ROUND-NEXT: and z0.d, z0.d, #0x7fffffffffffffff +; CHECK-NO-EXTEND-ROUND-NEXT: ptrue p0.d ; CHECK-NO-EXTEND-ROUND-NEXT: orr z0.d, z0.d, z1.d ; CHECK-NO-EXTEND-ROUND-NEXT: fcvt z0.s, p0/m, z0.d ; CHECK-NO-EXTEND-ROUND-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-fcvt.ll b/llvm/test/CodeGen/AArch64/sve-fcvt.ll index 0fe38bf9ae71..fc5128fffad3 100644 --- a/llvm/test/CodeGen/AArch64/sve-fcvt.ll +++ b/llvm/test/CodeGen/AArch64/sve-fcvt.ll @@ -454,9 +454,9 @@ define @fcvtzu_d_nxv2f64( %a) { define @scvtf_h_nxv2i1( %a) { ; CHECK-LABEL: scvtf_h_nxv2i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z0.d, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: scvtf z0.h, p1/m, z0.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: scvtf z0.h, p0/m, z0.d ; CHECK-NEXT: ret %res = sitofp %a to ret %res @@ -495,9 +495,9 @@ define @scvtf_h_nxv2i64( %a) { define @scvtf_h_nxv3i1( %a) { ; CHECK-LABEL: scvtf_h_nxv3i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: scvtf z0.h, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: scvtf z0.h, p0/m, z0.s ; CHECK-NEXT: ret %res = sitofp %a to ret %res @@ -516,9 +516,9 @@ define @scvtf_h_nxv3i16( %a) { define @scvtf_h_nxv4i1( %a) { ; CHECK-LABEL: scvtf_h_nxv4i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: scvtf z0.h, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: scvtf z0.h, p0/m, z0.s ; CHECK-NEXT: ret %res = sitofp %a to ret %res @@ -547,9 +547,9 @@ define @scvtf_h_nxv4i32( %a) { define @scvtf_h_nxv7i1( %a) { ; CHECK-LABEL: scvtf_h_nxv7i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.h ; CHECK-NEXT: mov z0.h, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: scvtf z0.h, p1/m, z0.h +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: scvtf z0.h, p0/m, z0.h ; CHECK-NEXT: ret %res = sitofp %a to ret %res @@ -568,9 +568,9 @@ define @scvtf_h_nxv7i16( %a) { define @scvtf_h_nxv8i1( %a) { ; CHECK-LABEL: scvtf_h_nxv8i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.h ; CHECK-NEXT: mov z0.h, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: scvtf z0.h, p1/m, z0.h +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: scvtf z0.h, p0/m, z0.h ; CHECK-NEXT: ret %res = sitofp %a to ret %res @@ -589,9 +589,9 @@ define @scvtf_h_nxv8i16( %a) { define @scvtf_s_nxv2i1( %a) { ; CHECK-LABEL: scvtf_s_nxv2i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z0.d, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: scvtf z0.s, p1/m, z0.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: scvtf z0.s, p0/m, z0.d ; CHECK-NEXT: ret %res = sitofp %a to ret %res @@ -620,9 +620,9 @@ define @scvtf_s_nxv2i64( %a) { define @scvtf_s_nxv3i1( %a) { ; CHECK-LABEL: scvtf_s_nxv3i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: scvtf z0.s, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: scvtf z0.s, p0/m, z0.s ; CHECK-NEXT: ret %res = sitofp %a to ret %res @@ -641,9 +641,9 @@ define @scvtf_s_nxv3i32( %a) { define @scvtf_s_nxv4i1( %a) { ; CHECK-LABEL: scvtf_s_nxv4i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: scvtf z0.s, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: scvtf z0.s, p0/m, z0.s ; CHECK-NEXT: ret %res = sitofp %a to ret %res @@ -662,9 +662,9 @@ define @scvtf_s_nxv4i32( %a) { define @scvtf_d_nxv2i1( %a) { ; CHECK-LABEL: scvtf_d_nxv2i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z0.d, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: scvtf z0.d, p1/m, z0.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: scvtf z0.d, p0/m, z0.d ; CHECK-NEXT: ret %res = sitofp %a to ret %res @@ -695,9 +695,9 @@ define @scvtf_d_nxv2i64( %a) { define @ucvtf_h_nxv2i1( %a) { ; CHECK-LABEL: ucvtf_h_nxv2i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z0.d, p0/z, #1 // =0x1 -; CHECK-NEXT: ucvtf z0.h, p1/m, z0.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: ucvtf z0.h, p0/m, z0.d ; CHECK-NEXT: ret %res = uitofp %a to ret %res @@ -736,9 +736,9 @@ define @ucvtf_h_nxv2i64( %a) { define @ucvtf_h_nxv3i1( %a) { ; CHECK-LABEL: ucvtf_h_nxv3i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1 -; CHECK-NEXT: ucvtf z0.h, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ucvtf z0.h, p0/m, z0.s ; CHECK-NEXT: ret %res = uitofp %a to ret %res @@ -767,9 +767,9 @@ define @ucvtf_h_nxv3i32( %a) { define @ucvtf_h_nxv4i1( %a) { ; CHECK-LABEL: ucvtf_h_nxv4i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1 -; CHECK-NEXT: ucvtf z0.h, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ucvtf z0.h, p0/m, z0.s ; CHECK-NEXT: ret %res = uitofp %a to ret %res @@ -798,9 +798,9 @@ define @ucvtf_h_nxv4i32( %a) { define @ucvtf_h_nxv8i1( %a) { ; CHECK-LABEL: ucvtf_h_nxv8i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.h ; CHECK-NEXT: mov z0.h, p0/z, #1 // =0x1 -; CHECK-NEXT: ucvtf z0.h, p1/m, z0.h +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: ucvtf z0.h, p0/m, z0.h ; CHECK-NEXT: ret %res = uitofp %a to ret %res @@ -819,9 +819,9 @@ define @ucvtf_h_nxv8i16( %a) { define @ucvtf_s_nxv2i1( %a) { ; CHECK-LABEL: ucvtf_s_nxv2i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z0.d, p0/z, #1 // =0x1 -; CHECK-NEXT: ucvtf z0.s, p1/m, z0.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: ucvtf z0.s, p0/m, z0.d ; CHECK-NEXT: ret %res = uitofp %a to ret %res @@ -850,9 +850,9 @@ define @ucvtf_s_nxv2i64( %a) { define @ucvtf_s_nxv4i1( %a) { ; CHECK-LABEL: ucvtf_s_nxv4i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1 -; CHECK-NEXT: ucvtf z0.s, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ucvtf z0.s, p0/m, z0.s ; CHECK-NEXT: ret %res = uitofp %a to ret %res @@ -871,9 +871,9 @@ define @ucvtf_s_nxv4i32( %a) { define @ucvtf_d_nxv2i1( %a) { ; CHECK-LABEL: ucvtf_d_nxv2i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z0.d, p0/z, #1 // =0x1 -; CHECK-NEXT: ucvtf z0.d, p1/m, z0.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: ucvtf z0.d, p0/m, z0.d ; CHECK-NEXT: ret %res = uitofp %a to ret %res diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-addressing-modes.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-addressing-modes.ll index ed7ea657874a..28e1412c524a 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-addressing-modes.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-addressing-modes.ll @@ -7,8 +7,8 @@ target triple = "aarch64-unknown-linux-gnu" define void @masked_gather_base_plus_stride_v8f32(ptr %dst, ptr %src) #0 { ; CHECK-LABEL: masked_gather_base_plus_stride_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: index z0.s, #0, #7 +; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1, z0.s, sxtw #2] ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret @@ -21,8 +21,8 @@ define void @masked_gather_base_plus_stride_v8f32(ptr %dst, ptr %src) #0 { define void @masked_gather_base_plus_stride_v4f64(ptr %dst, ptr %src) #0 { ; CHECK-LABEL: masked_gather_base_plus_stride_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: mov x8, #-32 // =0xffffffffffffffe0 +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: index z0.d, #-2, x8 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1, z0.d, lsl #3] ; CHECK-NEXT: st1d { z0.d }, p0, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-build-vector.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-build-vector.ll index ad482118ec0b..47fda39d8400 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-build-vector.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-build-vector.ll @@ -7,8 +7,8 @@ target triple = "aarch64-unknown-linux-gnu" define void @build_vector_7_inc1_v32i8(ptr %a) #0 { ; VBITS_GE_256-LABEL: build_vector_7_inc1_v32i8: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.b, vl32 ; VBITS_GE_256-NEXT: index z0.b, #7, #1 +; VBITS_GE_256-NEXT: ptrue p0.b, vl32 ; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0] ; VBITS_GE_256-NEXT: ret store <32 x i8> , ptr %a, align 1 @@ -18,8 +18,8 @@ define void @build_vector_7_inc1_v32i8(ptr %a) #0 { define void @build_vector_0_inc2_v16i16(ptr %a) #0 { ; VBITS_GE_256-LABEL: build_vector_0_inc2_v16i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: index z0.h, #0, #2 +; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0] ; VBITS_GE_256-NEXT: ret store <16 x i16> , ptr %a, align 2 @@ -30,8 +30,8 @@ define void @build_vector_0_inc2_v16i16(ptr %a) #0 { define void @build_vector_0_dec3_v8i32(ptr %a) #0 { ; VBITS_GE_256-LABEL: build_vector_0_dec3_v8i32: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: index z0.s, #0, #-3 +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_256-NEXT: ret store <8 x i32> , ptr %a, align 4 @@ -42,8 +42,8 @@ define void @build_vector_0_dec3_v8i32(ptr %a) #0 { define void @build_vector_minus2_dec32_v4i64(ptr %a) #0 { ; VBITS_GE_256-LABEL: build_vector_minus2_dec32_v4i64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #-32 // =0xffffffffffffffe0 +; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: index z0.d, #-2, x8 ; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0] ; VBITS_GE_256-NEXT: ret @@ -53,11 +53,6 @@ define void @build_vector_minus2_dec32_v4i64(ptr %a) #0 { ; Constant but not a sequence. define void @build_vector_no_stride_v4i64(ptr %a) #0 { -; VBITS_GE_256-LABEL: .LCPI4_0: -; VBITS_GE_256: .xword 0 -; VBITS_GE_256-NEXT: .xword 4 -; VBITS_GE_256-NEXT: .xword 1 -; VBITS_GE_256-NEXT: .xword 8 ; VBITS_GE_256-LABEL: build_vector_no_stride_v4i64: ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-concat.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-concat.ll index 65cb448cac11..f7751131005e 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-concat.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-concat.ll @@ -38,9 +38,9 @@ define void @concat_v32i8(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] -; CHECK-NEXT: ptrue p1.b, vl32 ; CHECK-NEXT: splice z0.b, p0, z0.b, z1.b -; CHECK-NEXT: st1b { z0.b }, p1, [x2] +; CHECK-NEXT: ptrue p0.b, vl32 +; CHECK-NEXT: st1b { z0.b }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <16 x i8>, ptr %a %op2 = load <16 x i8>, ptr %b @@ -66,11 +66,11 @@ define void @concat_v64i8(ptr %a, ptr %b, ptr %c) #0 { ; VBITS_GE_512-LABEL: concat_v64i8: ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: ptrue p0.b, vl32 -; VBITS_GE_512-NEXT: ptrue p1.b, vl64 ; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x1] ; VBITS_GE_512-NEXT: splice z0.b, p0, z0.b, z1.b -; VBITS_GE_512-NEXT: st1b { z0.b }, p1, [x2] +; VBITS_GE_512-NEXT: ptrue p0.b, vl64 +; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x2] ; VBITS_GE_512-NEXT: ret %op1 = load <32 x i8>, ptr %a %op2 = load <32 x i8>, ptr %b @@ -90,11 +90,11 @@ define void @concat_v128i8(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 { ; CHECK-LABEL: concat_v128i8: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.b, vl64 -; CHECK-NEXT: ptrue p1.b, vl128 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0] ; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1] ; CHECK-NEXT: splice z0.b, p0, z0.b, z1.b -; CHECK-NEXT: st1b { z0.b }, p1, [x2] +; CHECK-NEXT: ptrue p0.b, vl128 +; CHECK-NEXT: st1b { z0.b }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <64 x i8>, ptr %a %op2 = load <64 x i8>, ptr %b @@ -122,11 +122,11 @@ define void @concat_v256i8(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 { ; CHECK-LABEL: concat_v256i8: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.b, vl128 -; CHECK-NEXT: ptrue p1.b, vl256 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0] ; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1] ; CHECK-NEXT: splice z0.b, p0, z0.b, z1.b -; CHECK-NEXT: st1b { z0.b }, p1, [x2] +; CHECK-NEXT: ptrue p0.b, vl256 +; CHECK-NEXT: st1b { z0.b }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <128 x i8>, ptr %a %op2 = load <128 x i8>, ptr %b @@ -198,9 +198,9 @@ define void @concat_v16i16(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] -; CHECK-NEXT: ptrue p1.h, vl16 ; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h -; CHECK-NEXT: st1h { z0.h }, p1, [x2] +; CHECK-NEXT: ptrue p0.h, vl16 +; CHECK-NEXT: st1h { z0.h }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <8 x i16>, ptr %a %op2 = load <8 x i16>, ptr %b @@ -224,11 +224,11 @@ define void @concat_v32i16(ptr %a, ptr %b, ptr %c) #0 { ; VBITS_GE_512-LABEL: concat_v32i16: ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: ptrue p0.h, vl16 -; VBITS_GE_512-NEXT: ptrue p1.h, vl32 ; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1] ; VBITS_GE_512-NEXT: splice z0.h, p0, z0.h, z1.h -; VBITS_GE_512-NEXT: st1h { z0.h }, p1, [x2] +; VBITS_GE_512-NEXT: ptrue p0.h, vl32 +; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x2] ; VBITS_GE_512-NEXT: ret %op1 = load <16 x i16>, ptr %a %op2 = load <16 x i16>, ptr %b @@ -244,11 +244,11 @@ define void @concat_v64i16(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 { ; CHECK-LABEL: concat_v64i16: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h, vl32 -; CHECK-NEXT: ptrue p1.h, vl64 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0] ; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1] ; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h -; CHECK-NEXT: st1h { z0.h }, p1, [x2] +; CHECK-NEXT: ptrue p0.h, vl64 +; CHECK-NEXT: st1h { z0.h }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <32 x i16>, ptr %a %op2 = load <32 x i16>, ptr %b @@ -268,11 +268,11 @@ define void @concat_v128i16(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 { ; CHECK-LABEL: concat_v128i16: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h, vl64 -; CHECK-NEXT: ptrue p1.h, vl128 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0] ; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1] ; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h -; CHECK-NEXT: st1h { z0.h }, p1, [x2] +; CHECK-NEXT: ptrue p0.h, vl128 +; CHECK-NEXT: st1h { z0.h }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <64 x i16>, ptr %a %op2 = load <64 x i16>, ptr %b @@ -328,9 +328,9 @@ define void @concat_v8i32(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] -; CHECK-NEXT: ptrue p1.s, vl8 ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s -; CHECK-NEXT: st1w { z0.s }, p1, [x2] +; CHECK-NEXT: ptrue p0.s, vl8 +; CHECK-NEXT: st1w { z0.s }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <4 x i32>, ptr %a %op2 = load <4 x i32>, ptr %b @@ -353,11 +353,11 @@ define void @concat_v16i32(ptr %a, ptr %b, ptr %c) #0 { ; VBITS_GE_512-LABEL: concat_v16i32: ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: ptrue p0.s, vl8 -; VBITS_GE_512-NEXT: ptrue p1.s, vl16 ; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1] ; VBITS_GE_512-NEXT: splice z0.s, p0, z0.s, z1.s -; VBITS_GE_512-NEXT: st1w { z0.s }, p1, [x2] +; VBITS_GE_512-NEXT: ptrue p0.s, vl16 +; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2] ; VBITS_GE_512-NEXT: ret %op1 = load <8 x i32>, ptr %a %op2 = load <8 x i32>, ptr %b @@ -371,11 +371,11 @@ define void @concat_v32i32(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 { ; CHECK-LABEL: concat_v32i32: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s, vl16 -; CHECK-NEXT: ptrue p1.s, vl32 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1] ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s -; CHECK-NEXT: st1w { z0.s }, p1, [x2] +; CHECK-NEXT: ptrue p0.s, vl32 +; CHECK-NEXT: st1w { z0.s }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <16 x i32>, ptr %a %op2 = load <16 x i32>, ptr %b @@ -391,11 +391,11 @@ define void @concat_v64i32(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 { ; CHECK-LABEL: concat_v64i32: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s, vl32 -; CHECK-NEXT: ptrue p1.s, vl64 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1] ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s -; CHECK-NEXT: st1w { z0.s }, p1, [x2] +; CHECK-NEXT: ptrue p0.s, vl64 +; CHECK-NEXT: st1w { z0.s }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <32 x i32>, ptr %a %op2 = load <32 x i32>, ptr %b @@ -433,9 +433,9 @@ define void @concat_v4i64(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] -; CHECK-NEXT: ptrue p1.d, vl4 ; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d -; CHECK-NEXT: st1d { z0.d }, p1, [x2] +; CHECK-NEXT: ptrue p0.d, vl4 +; CHECK-NEXT: st1d { z0.d }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <2 x i64>, ptr %a %op2 = load <2 x i64>, ptr %b @@ -458,11 +458,11 @@ define void @concat_v8i64(ptr %a, ptr %b, ptr %c) #0 { ; VBITS_GE_512-LABEL: concat_v8i64: ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: ptrue p0.d, vl4 -; VBITS_GE_512-NEXT: ptrue p1.d, vl8 ; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: splice z0.d, p0, z0.d, z1.d -; VBITS_GE_512-NEXT: st1d { z0.d }, p1, [x2] +; VBITS_GE_512-NEXT: ptrue p0.d, vl8 +; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2] ; VBITS_GE_512-NEXT: ret %op1 = load <4 x i64>, ptr %a %op2 = load <4 x i64>, ptr %b @@ -475,11 +475,11 @@ define void @concat_v16i64(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 { ; CHECK-LABEL: concat_v16i64: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl8 -; CHECK-NEXT: ptrue p1.d, vl16 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d -; CHECK-NEXT: st1d { z0.d }, p1, [x2] +; CHECK-NEXT: ptrue p0.d, vl16 +; CHECK-NEXT: st1d { z0.d }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <8 x i64>, ptr %a %op2 = load <8 x i64>, ptr %b @@ -493,11 +493,11 @@ define void @concat_v32i64(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 { ; CHECK-LABEL: concat_v32i64: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl16 -; CHECK-NEXT: ptrue p1.d, vl32 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d -; CHECK-NEXT: st1d { z0.d }, p1, [x2] +; CHECK-NEXT: ptrue p0.d, vl32 +; CHECK-NEXT: st1d { z0.d }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <16 x i64>, ptr %a %op2 = load <16 x i64>, ptr %b @@ -541,9 +541,9 @@ define void @concat_v16f16(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] -; CHECK-NEXT: ptrue p1.h, vl16 ; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h -; CHECK-NEXT: st1h { z0.h }, p1, [x2] +; CHECK-NEXT: ptrue p0.h, vl16 +; CHECK-NEXT: st1h { z0.h }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <8 x half>, ptr %a %op2 = load <8 x half>, ptr %b @@ -567,11 +567,11 @@ define void @concat_v32f16(ptr %a, ptr %b, ptr %c) #0 { ; VBITS_GE_512-LABEL: concat_v32f16: ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: ptrue p0.h, vl16 -; VBITS_GE_512-NEXT: ptrue p1.h, vl32 ; VBITS_GE_512-NEXT: ld1h { z0.h }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x1] ; VBITS_GE_512-NEXT: splice z0.h, p0, z0.h, z1.h -; VBITS_GE_512-NEXT: st1h { z0.h }, p1, [x2] +; VBITS_GE_512-NEXT: ptrue p0.h, vl32 +; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x2] ; VBITS_GE_512-NEXT: ret %op1 = load <16 x half>, ptr %a %op2 = load <16 x half>, ptr %b @@ -587,11 +587,11 @@ define void @concat_v64f16(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 { ; CHECK-LABEL: concat_v64f16: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h, vl32 -; CHECK-NEXT: ptrue p1.h, vl64 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0] ; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1] ; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h -; CHECK-NEXT: st1h { z0.h }, p1, [x2] +; CHECK-NEXT: ptrue p0.h, vl64 +; CHECK-NEXT: st1h { z0.h }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <32 x half>, ptr %a %op2 = load <32 x half>, ptr %b @@ -611,11 +611,11 @@ define void @concat_v128f16(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 { ; CHECK-LABEL: concat_v128f16: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h, vl64 -; CHECK-NEXT: ptrue p1.h, vl128 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0] ; CHECK-NEXT: ld1h { z1.h }, p0/z, [x1] ; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h -; CHECK-NEXT: st1h { z0.h }, p1, [x2] +; CHECK-NEXT: ptrue p0.h, vl128 +; CHECK-NEXT: st1h { z0.h }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <64 x half>, ptr %a %op2 = load <64 x half>, ptr %b @@ -671,9 +671,9 @@ define void @concat_v8f32(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] -; CHECK-NEXT: ptrue p1.s, vl8 ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s -; CHECK-NEXT: st1w { z0.s }, p1, [x2] +; CHECK-NEXT: ptrue p0.s, vl8 +; CHECK-NEXT: st1w { z0.s }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <4 x float>, ptr %a %op2 = load <4 x float>, ptr %b @@ -696,11 +696,11 @@ define void @concat_v16f32(ptr %a, ptr %b, ptr %c) #0 { ; VBITS_GE_512-LABEL: concat_v16f32: ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: ptrue p0.s, vl8 -; VBITS_GE_512-NEXT: ptrue p1.s, vl16 ; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x1] ; VBITS_GE_512-NEXT: splice z0.s, p0, z0.s, z1.s -; VBITS_GE_512-NEXT: st1w { z0.s }, p1, [x2] +; VBITS_GE_512-NEXT: ptrue p0.s, vl16 +; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x2] ; VBITS_GE_512-NEXT: ret %op1 = load <8 x float>, ptr %a %op2 = load <8 x float>, ptr %b @@ -714,11 +714,11 @@ define void @concat_v32f32(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 { ; CHECK-LABEL: concat_v32f32: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s, vl16 -; CHECK-NEXT: ptrue p1.s, vl32 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1] ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s -; CHECK-NEXT: st1w { z0.s }, p1, [x2] +; CHECK-NEXT: ptrue p0.s, vl32 +; CHECK-NEXT: st1w { z0.s }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <16 x float>, ptr %a %op2 = load <16 x float>, ptr %b @@ -734,11 +734,11 @@ define void @concat_v64f32(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 { ; CHECK-LABEL: concat_v64f32: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s, vl32 -; CHECK-NEXT: ptrue p1.s, vl64 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x1] ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s -; CHECK-NEXT: st1w { z0.s }, p1, [x2] +; CHECK-NEXT: ptrue p0.s, vl64 +; CHECK-NEXT: st1w { z0.s }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <32 x float>, ptr %a %op2 = load <32 x float>, ptr %b @@ -776,9 +776,9 @@ define void @concat_v4f64(ptr %a, ptr %b, ptr %c) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] -; CHECK-NEXT: ptrue p1.d, vl4 ; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d -; CHECK-NEXT: st1d { z0.d }, p1, [x2] +; CHECK-NEXT: ptrue p0.d, vl4 +; CHECK-NEXT: st1d { z0.d }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <2 x double>, ptr %a %op2 = load <2 x double>, ptr %b @@ -801,11 +801,11 @@ define void @concat_v8f64(ptr %a, ptr %b, ptr %c) #0 { ; VBITS_GE_512-LABEL: concat_v8f64: ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: ptrue p0.d, vl4 -; VBITS_GE_512-NEXT: ptrue p1.d, vl8 ; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: splice z0.d, p0, z0.d, z1.d -; VBITS_GE_512-NEXT: st1d { z0.d }, p1, [x2] +; VBITS_GE_512-NEXT: ptrue p0.d, vl8 +; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x2] ; VBITS_GE_512-NEXT: ret %op1 = load <4 x double>, ptr %a %op2 = load <4 x double>, ptr %b @@ -818,11 +818,11 @@ define void @concat_v16f64(ptr %a, ptr %b, ptr %c) vscale_range(8,0) #0 { ; CHECK-LABEL: concat_v16f64: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl8 -; CHECK-NEXT: ptrue p1.d, vl16 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d -; CHECK-NEXT: st1d { z0.d }, p1, [x2] +; CHECK-NEXT: ptrue p0.d, vl16 +; CHECK-NEXT: st1d { z0.d }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <8 x double>, ptr %a %op2 = load <8 x double>, ptr %b @@ -836,11 +836,11 @@ define void @concat_v32f64(ptr %a, ptr %b, ptr %c) vscale_range(16,0) #0 { ; CHECK-LABEL: concat_v32f64: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl16 -; CHECK-NEXT: ptrue p1.d, vl32 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d -; CHECK-NEXT: st1d { z0.d }, p1, [x2] +; CHECK-NEXT: ptrue p0.d, vl32 +; CHECK-NEXT: st1d { z0.d }, p0, [x2] ; CHECK-NEXT: ret %op1 = load <16 x double>, ptr %a %op2 = load <16 x double>, ptr %b diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-vector-elt.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-vector-elt.ll index 485124c1d59e..ad4efeaf3924 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-vector-elt.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-vector-elt.ll @@ -70,9 +70,9 @@ define half @extractelement_v64f16(ptr %a) vscale_range(8,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: mov w8, #63 // =0x3f -; CHECK-NEXT: whilels p1.h, xzr, x8 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0] -; CHECK-NEXT: lastb h0, p1, z0.h +; CHECK-NEXT: whilels p0.h, xzr, x8 +; CHECK-NEXT: lastb h0, p0, z0.h ; CHECK-NEXT: ret %op1 = load <64 x half>, ptr %a %r = extractelement <64 x half> %op1, i64 63 @@ -84,9 +84,9 @@ define half @extractelement_v128f16(ptr %a) vscale_range(16,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.h, vl128 ; CHECK-NEXT: mov w8, #127 // =0x7f -; CHECK-NEXT: whilels p1.h, xzr, x8 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0] -; CHECK-NEXT: lastb h0, p1, z0.h +; CHECK-NEXT: whilels p0.h, xzr, x8 +; CHECK-NEXT: lastb h0, p0, z0.h ; CHECK-NEXT: ret %op1 = load <128 x half>, ptr %a %r = extractelement <128 x half> %op1, i64 127 @@ -154,9 +154,9 @@ define float @extractelement_v32f32(ptr %a) vscale_range(8,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: mov w8, #31 // =0x1f -; CHECK-NEXT: whilels p1.s, xzr, x8 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] -; CHECK-NEXT: lastb s0, p1, z0.s +; CHECK-NEXT: whilels p0.s, xzr, x8 +; CHECK-NEXT: lastb s0, p0, z0.s ; CHECK-NEXT: ret %op1 = load <32 x float>, ptr %a %r = extractelement <32 x float> %op1, i64 31 @@ -168,9 +168,9 @@ define float @extractelement_v64f32(ptr %a) vscale_range(16,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.s, vl64 ; CHECK-NEXT: mov w8, #63 // =0x3f -; CHECK-NEXT: whilels p1.s, xzr, x8 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] -; CHECK-NEXT: lastb s0, p1, z0.s +; CHECK-NEXT: whilels p0.s, xzr, x8 +; CHECK-NEXT: lastb s0, p0, z0.s ; CHECK-NEXT: ret %op1 = load <64 x float>, ptr %a %r = extractelement <64 x float> %op1, i64 63 @@ -236,9 +236,9 @@ define double @extractelement_v16f64(ptr %a) vscale_range(8,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: mov w8, #15 // =0xf -; CHECK-NEXT: whilels p1.d, xzr, x8 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] -; CHECK-NEXT: lastb d0, p1, z0.d +; CHECK-NEXT: whilels p0.d, xzr, x8 +; CHECK-NEXT: lastb d0, p0, z0.d ; CHECK-NEXT: ret %op1 = load <16 x double>, ptr %a %r = extractelement <16 x double> %op1, i64 15 @@ -250,9 +250,9 @@ define double @extractelement_v32f64(ptr %a) vscale_range(16,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: mov w8, #31 // =0x1f -; CHECK-NEXT: whilels p1.d, xzr, x8 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] -; CHECK-NEXT: lastb d0, p1, z0.d +; CHECK-NEXT: whilels p0.d, xzr, x8 +; CHECK-NEXT: lastb d0, p0, z0.d ; CHECK-NEXT: ret %op1 = load <32 x double>, ptr %a %r = extractelement <32 x double> %op1, i64 31 diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fcopysign.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fcopysign.ll index bca3dfe5717e..e77cd9ef55ea 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fcopysign.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fcopysign.ll @@ -396,9 +396,9 @@ define void @test_copysign_v4f32_v4f64(ptr %ap, ptr %bp) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: mvni v1.4s, #128, lsl #24 ; CHECK-NEXT: ldr q2, [x0] -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] -; CHECK-NEXT: fcvt z0.s, p1/m, z0.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: fcvt z0.s, p0/m, z0.d ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: bit v0.16b, v2.16b, v1.16b ; CHECK-NEXT: str q0, [x0] @@ -450,12 +450,12 @@ define void @test_copysign_v4f64_v4f32(ptr %ap, ptr %bp) vscale_range(2,0) #0 { ; ; CHECK_EXTEND_ROUND-LABEL: test_copysign_v4f64_v4f32: ; CHECK_EXTEND_ROUND: // %bb.0: -; CHECK_EXTEND_ROUND-NEXT: ptrue p0.d, vl4 ; CHECK_EXTEND_ROUND-NEXT: ldr q0, [x1] +; CHECK_EXTEND_ROUND-NEXT: ptrue p0.d, vl4 ; CHECK_EXTEND_ROUND-NEXT: uunpklo z0.d, z0.s -; CHECK_EXTEND_ROUND-NEXT: fcvt z0.d, p0/m, z0.s ; CHECK_EXTEND_ROUND-NEXT: ld1d { z1.d }, p0/z, [x0] ; CHECK_EXTEND_ROUND-NEXT: and z1.d, z1.d, #0x7fffffffffffffff +; CHECK_EXTEND_ROUND-NEXT: fcvt z0.d, p0/m, z0.s ; CHECK_EXTEND_ROUND-NEXT: and z0.d, z0.d, #0x8000000000000000 ; CHECK_EXTEND_ROUND-NEXT: orr z0.d, z1.d, z0.d ; CHECK_EXTEND_ROUND-NEXT: st1d { z0.d }, p0, [x0] @@ -494,9 +494,9 @@ define void @test_copysign_v4f16_v4f64(ptr %ap, ptr %bp) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: mvni v1.4h, #128, lsl #8 ; CHECK-NEXT: ldr d2, [x0] -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] -; CHECK-NEXT: fcvt z0.h, p1/m, z0.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: fcvt z0.h, p0/m, z0.d ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: bit v0.8b, v2.8b, v1.8b @@ -521,9 +521,9 @@ define void @test_copysign_v8f16_v8f32(ptr %ap, ptr %bp) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: mvni v1.8h, #128, lsl #8 ; CHECK-NEXT: ldr q2, [x0] -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x1] -; CHECK-NEXT: fcvt z0.h, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: fcvt z0.h, p0/m, z0.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: bit v0.16b, v2.16b, v1.16b ; CHECK-NEXT: str q0, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc.ll index 6da07b855a5c..b60988be1e76 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc.ll @@ -462,11 +462,11 @@ define void @fcvt_v8f64_v8f16(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ptrue p1.d ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] -; VBITS_GE_256-NEXT: fcvt z0.h, p1/m, z0.d -; VBITS_GE_256-NEXT: fcvt z1.h, p1/m, z1.d +; VBITS_GE_256-NEXT: ptrue p0.d +; VBITS_GE_256-NEXT: fcvt z0.h, p0/m, z0.d +; VBITS_GE_256-NEXT: fcvt z1.h, p0/m, z1.d ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll index 13ebda1df7f9..d1e9dc13f50e 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll @@ -34,15 +34,15 @@ define <8 x half> @select_v8f16(<8 x half> %op1, <8 x half> %op2, i1 %mask) vsca define void @select_v16f16(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { ; CHECK-LABEL: select_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: mov z0.h, w2 -; CHECK-NEXT: ptrue p1.h +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: ptrue p1.h, vl16 ; CHECK-NEXT: and z0.h, z0.h, #0x1 -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; CHECK-NEXT: sel z0.h, p1, z1.h, z2.h -; CHECK-NEXT: st1h { z0.h }, p0, [x0] +; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: ld1h { z1.h }, p1/z, [x1] +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h +; CHECK-NEXT: st1h { z0.h }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <16 x half>, ptr %a %op2 = load volatile <16 x half>, ptr %b @@ -54,33 +54,33 @@ define void @select_v16f16(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { define void @select_v32f16(ptr %a, ptr %b, i1 %mask) #0 { ; VBITS_GE_256-LABEL: select_v32f16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: mov z0.h, w2 +; VBITS_GE_256-NEXT: ptrue p0.h ; VBITS_GE_256-NEXT: mov x8, #16 // =0x10 -; VBITS_GE_256-NEXT: ptrue p1.h +; VBITS_GE_256-NEXT: ptrue p1.h, vl16 ; VBITS_GE_256-NEXT: and z0.h, z0.h, #0x1 -; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0, x8, lsl #1] -; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0] -; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x1, x8, lsl #1] -; VBITS_GE_256-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x1] -; VBITS_GE_256-NEXT: sel z1.h, p1, z1.h, z3.h -; VBITS_GE_256-NEXT: mov z0.h, p1/m, z2.h -; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0, x8, lsl #1] -; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0] +; VBITS_GE_256-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; VBITS_GE_256-NEXT: ld1h { z0.h }, p1/z, [x0, x8, lsl #1] +; VBITS_GE_256-NEXT: ld1h { z1.h }, p1/z, [x0] +; VBITS_GE_256-NEXT: ld1h { z2.h }, p1/z, [x1, x8, lsl #1] +; VBITS_GE_256-NEXT: ld1h { z3.h }, p1/z, [x1] +; VBITS_GE_256-NEXT: sel z0.h, p0, z0.h, z2.h +; VBITS_GE_256-NEXT: sel z1.h, p0, z1.h, z3.h +; VBITS_GE_256-NEXT: st1h { z0.h }, p1, [x0, x8, lsl #1] +; VBITS_GE_256-NEXT: st1h { z1.h }, p1, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: select_v32f16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.h, vl32 ; VBITS_GE_512-NEXT: mov z0.h, w2 -; VBITS_GE_512-NEXT: ptrue p1.h +; VBITS_GE_512-NEXT: ptrue p0.h +; VBITS_GE_512-NEXT: ptrue p1.h, vl32 ; VBITS_GE_512-NEXT: and z0.h, z0.h, #0x1 -; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x0] -; VBITS_GE_512-NEXT: ld1h { z2.h }, p0/z, [x1] -; VBITS_GE_512-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; VBITS_GE_512-NEXT: sel z0.h, p1, z1.h, z2.h -; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0] +; VBITS_GE_512-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; VBITS_GE_512-NEXT: ld1h { z0.h }, p1/z, [x0] +; VBITS_GE_512-NEXT: ld1h { z1.h }, p1/z, [x1] +; VBITS_GE_512-NEXT: sel z0.h, p0, z0.h, z1.h +; VBITS_GE_512-NEXT: st1h { z0.h }, p1, [x0] ; VBITS_GE_512-NEXT: ret %op1 = load volatile <32 x half>, ptr %a %op2 = load volatile <32 x half>, ptr %b @@ -92,15 +92,15 @@ define void @select_v32f16(ptr %a, ptr %b, i1 %mask) #0 { define void @select_v64f16(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { ; CHECK-LABEL: select_v64f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: mov z0.h, w2 -; CHECK-NEXT: ptrue p1.h +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: ptrue p1.h, vl64 ; CHECK-NEXT: and z0.h, z0.h, #0x1 -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; CHECK-NEXT: sel z0.h, p1, z1.h, z2.h -; CHECK-NEXT: st1h { z0.h }, p0, [x0] +; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: ld1h { z1.h }, p1/z, [x1] +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h +; CHECK-NEXT: st1h { z0.h }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <64 x half>, ptr %a %op2 = load volatile <64 x half>, ptr %b @@ -112,15 +112,15 @@ define void @select_v64f16(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { define void @select_v128f16(ptr %a, ptr %b, i1 %mask) vscale_range(16,0) #0 { ; CHECK-LABEL: select_v128f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl128 ; CHECK-NEXT: mov z0.h, w2 -; CHECK-NEXT: ptrue p1.h +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: ptrue p1.h, vl128 ; CHECK-NEXT: and z0.h, z0.h, #0x1 -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; CHECK-NEXT: sel z0.h, p1, z1.h, z2.h -; CHECK-NEXT: st1h { z0.h }, p0, [x0] +; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: ld1h { z1.h }, p1/z, [x1] +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h +; CHECK-NEXT: st1h { z0.h }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <128 x half>, ptr %a %op2 = load volatile <128 x half>, ptr %b @@ -158,15 +158,15 @@ define <4 x float> @select_v4f32(<4 x float> %op1, <4 x float> %op2, i1 %mask) v define void @select_v8f32(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { ; CHECK-LABEL: select_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: and w8, w2, #0x1 -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z0.s, w8 -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; CHECK-NEXT: sel z0.s, p1, z1.s, z2.s -; CHECK-NEXT: st1w { z0.s }, p0, [x0] +; CHECK-NEXT: ptrue p1.s, vl8 +; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0] +; CHECK-NEXT: ld1w { z1.s }, p1/z, [x1] +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: st1w { z0.s }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <8 x float>, ptr %a %op2 = load volatile <8 x float>, ptr %b @@ -178,33 +178,33 @@ define void @select_v8f32(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { define void @select_v16f32(ptr %a, ptr %b, i1 %mask) #0 { ; VBITS_GE_256-LABEL: select_v16f32: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: and w8, w2, #0x1 -; VBITS_GE_256-NEXT: ptrue p1.s +; VBITS_GE_256-NEXT: ptrue p0.s ; VBITS_GE_256-NEXT: mov z0.s, w8 +; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 -; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0, x8, lsl #2] -; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0] -; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1, x8, lsl #2] -; VBITS_GE_256-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1] -; VBITS_GE_256-NEXT: sel z1.s, p1, z1.s, z3.s -; VBITS_GE_256-NEXT: mov z0.s, p1/m, z2.s -; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0, x8, lsl #2] -; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0] +; VBITS_GE_256-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x0, x8, lsl #2] +; VBITS_GE_256-NEXT: ld1w { z1.s }, p1/z, [x0] +; VBITS_GE_256-NEXT: ld1w { z2.s }, p1/z, [x1, x8, lsl #2] +; VBITS_GE_256-NEXT: ld1w { z3.s }, p1/z, [x1] +; VBITS_GE_256-NEXT: sel z0.s, p0, z0.s, z2.s +; VBITS_GE_256-NEXT: sel z1.s, p0, z1.s, z3.s +; VBITS_GE_256-NEXT: st1w { z0.s }, p1, [x0, x8, lsl #2] +; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: select_v16f32: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl16 ; VBITS_GE_512-NEXT: and w8, w2, #0x1 -; VBITS_GE_512-NEXT: ptrue p1.s +; VBITS_GE_512-NEXT: ptrue p0.s ; VBITS_GE_512-NEXT: mov z0.s, w8 -; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x0] -; VBITS_GE_512-NEXT: ld1w { z2.s }, p0/z, [x1] -; VBITS_GE_512-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; VBITS_GE_512-NEXT: sel z0.s, p1, z1.s, z2.s -; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0] +; VBITS_GE_512-NEXT: ptrue p1.s, vl16 +; VBITS_GE_512-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; VBITS_GE_512-NEXT: ld1w { z0.s }, p1/z, [x0] +; VBITS_GE_512-NEXT: ld1w { z1.s }, p1/z, [x1] +; VBITS_GE_512-NEXT: sel z0.s, p0, z0.s, z1.s +; VBITS_GE_512-NEXT: st1w { z0.s }, p1, [x0] ; VBITS_GE_512-NEXT: ret %op1 = load volatile <16 x float>, ptr %a %op2 = load volatile <16 x float>, ptr %b @@ -216,15 +216,15 @@ define void @select_v16f32(ptr %a, ptr %b, i1 %mask) #0 { define void @select_v32f32(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { ; CHECK-LABEL: select_v32f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: and w8, w2, #0x1 -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z0.s, w8 -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; CHECK-NEXT: sel z0.s, p1, z1.s, z2.s -; CHECK-NEXT: st1w { z0.s }, p0, [x0] +; CHECK-NEXT: ptrue p1.s, vl32 +; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0] +; CHECK-NEXT: ld1w { z1.s }, p1/z, [x1] +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: st1w { z0.s }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <32 x float>, ptr %a %op2 = load volatile <32 x float>, ptr %b @@ -236,15 +236,15 @@ define void @select_v32f32(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { define void @select_v64f32(ptr %a, ptr %b, i1 %mask) vscale_range(16,0) #0 { ; CHECK-LABEL: select_v64f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl64 ; CHECK-NEXT: and w8, w2, #0x1 -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z0.s, w8 -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; CHECK-NEXT: sel z0.s, p1, z1.s, z2.s -; CHECK-NEXT: st1w { z0.s }, p0, [x0] +; CHECK-NEXT: ptrue p1.s, vl64 +; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0] +; CHECK-NEXT: ld1w { z1.s }, p1/z, [x1] +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: st1w { z0.s }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <64 x float>, ptr %a %op2 = load volatile <64 x float>, ptr %b @@ -282,16 +282,16 @@ define <2 x double> @select_v2f64(<2 x double> %op1, <2 x double> %op2, i1 %mask define void @select_v4f64(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { ; CHECK-LABEL: select_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2 ; CHECK-NEXT: and x8, x2, #0x1 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, x8 -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; CHECK-NEXT: sel z0.d, p1, z1.d, z2.d -; CHECK-NEXT: st1d { z0.d }, p0, [x0] +; CHECK-NEXT: ptrue p1.d, vl4 +; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: st1d { z0.d }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <4 x double>, ptr %a %op2 = load volatile <4 x double>, ptr %b @@ -303,35 +303,35 @@ define void @select_v4f64(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { define void @select_v8f64(ptr %a, ptr %b, i1 %mask) #0 { ; VBITS_GE_256-LABEL: select_v8f64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: // kill: def $w2 killed $w2 def $x2 ; VBITS_GE_256-NEXT: and x8, x2, #0x1 -; VBITS_GE_256-NEXT: ptrue p1.d +; VBITS_GE_256-NEXT: ptrue p0.d ; VBITS_GE_256-NEXT: mov z0.d, x8 +; VBITS_GE_256-NEXT: ptrue p1.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0, x8, lsl #3] -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0] -; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1, x8, lsl #3] -; VBITS_GE_256-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1] -; VBITS_GE_256-NEXT: sel z1.d, p1, z1.d, z3.d -; VBITS_GE_256-NEXT: mov z0.d, p1/m, z2.d -; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0, x8, lsl #3] -; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0] +; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p1/z, [x0, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1d { z1.d }, p1/z, [x0] +; VBITS_GE_256-NEXT: ld1d { z2.d }, p1/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1d { z3.d }, p1/z, [x1] +; VBITS_GE_256-NEXT: sel z0.d, p0, z0.d, z2.d +; VBITS_GE_256-NEXT: sel z1.d, p0, z1.d, z3.d +; VBITS_GE_256-NEXT: st1d { z0.d }, p1, [x0, x8, lsl #3] +; VBITS_GE_256-NEXT: st1d { z1.d }, p1, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: select_v8f64: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: // kill: def $w2 killed $w2 def $x2 ; VBITS_GE_512-NEXT: and x8, x2, #0x1 -; VBITS_GE_512-NEXT: ptrue p1.d +; VBITS_GE_512-NEXT: ptrue p0.d ; VBITS_GE_512-NEXT: mov z0.d, x8 -; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x0] -; VBITS_GE_512-NEXT: ld1d { z2.d }, p0/z, [x1] -; VBITS_GE_512-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; VBITS_GE_512-NEXT: sel z0.d, p1, z1.d, z2.d -; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0] +; VBITS_GE_512-NEXT: ptrue p1.d, vl8 +; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; VBITS_GE_512-NEXT: ld1d { z0.d }, p1/z, [x0] +; VBITS_GE_512-NEXT: ld1d { z1.d }, p1/z, [x1] +; VBITS_GE_512-NEXT: sel z0.d, p0, z0.d, z1.d +; VBITS_GE_512-NEXT: st1d { z0.d }, p1, [x0] ; VBITS_GE_512-NEXT: ret %op1 = load volatile <8 x double>, ptr %a %op2 = load volatile <8 x double>, ptr %b @@ -343,16 +343,16 @@ define void @select_v8f64(ptr %a, ptr %b, i1 %mask) #0 { define void @select_v16f64(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { ; CHECK-LABEL: select_v16f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2 ; CHECK-NEXT: and x8, x2, #0x1 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, x8 -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; CHECK-NEXT: sel z0.d, p1, z1.d, z2.d -; CHECK-NEXT: st1d { z0.d }, p0, [x0] +; CHECK-NEXT: ptrue p1.d, vl16 +; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: st1d { z0.d }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <16 x double>, ptr %a %op2 = load volatile <16 x double>, ptr %b @@ -364,16 +364,16 @@ define void @select_v16f64(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { define void @select_v32f64(ptr %a, ptr %b, i1 %mask) vscale_range(16,0) #0 { ; CHECK-LABEL: select_v32f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2 ; CHECK-NEXT: and x8, x2, #0x1 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, x8 -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; CHECK-NEXT: sel z0.d, p1, z1.d, z2.d -; CHECK-NEXT: st1d { z0.d }, p0, [x0] +; CHECK-NEXT: ptrue p1.d, vl32 +; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: st1d { z0.d }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <32 x double>, ptr %a %op2 = load volatile <32 x double>, ptr %b diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-to-int.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-to-int.ll index da0cf927d74d..af54b146c5b6 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-to-int.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-to-int.ll @@ -131,8 +131,8 @@ define <4 x i32> @fcvtzu_v4f16_v4i32(<4 x half> %op1) vscale_range(2,0) #0 { define void @fcvtzu_v8f16_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: fcvtzu_v8f16_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: fcvtzu z0.s, p0/m, z0.h ; CHECK-NEXT: st1w { z0.s }, p0, [x1] @@ -357,7 +357,6 @@ define void @fcvtzu_v16f32_v16i16(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 -; VBITS_GE_256-NEXT: ptrue p1.h, vl16 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] ; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0] ; VBITS_GE_256-NEXT: fcvtzu z0.s, p0/m, z0.s @@ -366,7 +365,8 @@ define void @fcvtzu_v16f32_v16i16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: splice z1.h, p0, z1.h, z0.h -; VBITS_GE_256-NEXT: st1h { z1.h }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.h, vl16 +; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: fcvtzu_v16f32_v16i16: @@ -532,8 +532,8 @@ define <2 x i64> @fcvtzu_v2f32_v2i64(<2 x float> %op1) vscale_range(2,0) #0 { define void @fcvtzu_v4f32_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: fcvtzu_v4f32_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: fcvtzu z0.d, p0/m, z0.s ; CHECK-NEXT: st1d { z0.d }, p0, [x1] @@ -752,7 +752,6 @@ define void @fcvtzu_v8f64_v8i32(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] ; VBITS_GE_256-NEXT: fcvtzu z0.d, p0/m, z0.d @@ -761,7 +760,8 @@ define void @fcvtzu_v8f64_v8i32(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s -; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 +; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: fcvtzu_v8f64_v8i32: @@ -1024,8 +1024,8 @@ define <4 x i32> @fcvtzs_v4f16_v4i32(<4 x half> %op1) vscale_range(2,0) #0 { define void @fcvtzs_v8f16_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: fcvtzs_v8f16_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.h ; CHECK-NEXT: st1w { z0.s }, p0, [x1] @@ -1250,7 +1250,6 @@ define void @fcvtzs_v16f32_v16i16(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 -; VBITS_GE_256-NEXT: ptrue p1.h, vl16 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] ; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0] ; VBITS_GE_256-NEXT: fcvtzs z0.s, p0/m, z0.s @@ -1259,7 +1258,8 @@ define void @fcvtzs_v16f32_v16i16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: splice z1.h, p0, z1.h, z0.h -; VBITS_GE_256-NEXT: st1h { z1.h }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.h, vl16 +; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: fcvtzs_v16f32_v16i16: @@ -1425,8 +1425,8 @@ define <2 x i64> @fcvtzs_v2f32_v2i64(<2 x float> %op1) vscale_range(2,0) #0 { define void @fcvtzs_v4f32_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: fcvtzs_v4f32_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.s ; CHECK-NEXT: st1d { z0.d }, p0, [x1] @@ -1645,7 +1645,6 @@ define void @fcvtzs_v8f64_v8i32(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] ; VBITS_GE_256-NEXT: fcvtzs z0.d, p0/m, z0.d @@ -1654,7 +1653,8 @@ define void @fcvtzs_v8f64_v8i32(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s -; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 +; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: fcvtzs_v8f64_v8i32: diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll index 115f722986b5..61e04682fa0b 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp128.ll @@ -85,10 +85,10 @@ define void @fcvt_v4f128_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: str q0, [sp, #32] // 16-byte Folded Spill ; CHECK-NEXT: ldr q0, [sp, #48] // 16-byte Folded Reload ; CHECK-NEXT: bl __trunctfdf2 -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldr q1, [sp, #32] // 16-byte Folded Reload ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: add x8, sp, #128 +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: mov v0.d[1], v1.d[0] ; CHECK-NEXT: ldr z1, [x8, #1, mul vl] // 16-byte Folded Reload ; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d @@ -111,14 +111,14 @@ define void @fcvt_v4f128_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: ldr q0, [sp, #112] // 16-byte Folded Reload ; CHECK-NEXT: bl __trunctfdf2 ; CHECK-NEXT: ldr q1, [sp, #96] // 16-byte Folded Reload -; CHECK-NEXT: ptrue p1.d, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: add x8, sp, #128 -; CHECK-NEXT: ptrue p0.d, vl4 +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: mov v0.d[1], v1.d[0] ; CHECK-NEXT: ldr z1, [x8] // 16-byte Folded Reload ; CHECK-NEXT: mov x8, #4 // =0x4 -; CHECK-NEXT: splice z0.d, p1, z0.d, z1.d +; CHECK-NEXT: splice z0.d, p0, z0.d, z1.d +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: st1d { z0.d }, p0, [x19, x8, lsl #3] ; CHECK-NEXT: add x8, sp, #128 ; CHECK-NEXT: ldr z0, [x8, #1, mul vl] // 16-byte Folded Reload diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-frame-offests-crash.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-frame-offests-crash.ll index a4b5ccd69fdb..1bd688d23050 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-frame-offests-crash.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-frame-offests-crash.ll @@ -21,24 +21,24 @@ define dso_local void @func1(ptr %v1, ptr %v2, ptr %v3, ptr %v4, ptr %v5, ptr %v ; CHECK-NEXT: .cfi_offset w22, -32 ; CHECK-NEXT: .cfi_offset w29, -48 ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: add x11, sp, #176 -; CHECK-NEXT: add x10, sp, #144 -; CHECK-NEXT: add x9, sp, #112 +; CHECK-NEXT: add x10, sp, #176 ; CHECK-NEXT: add x8, sp, #48 +; CHECK-NEXT: add x9, sp, #144 ; CHECK-NEXT: add x20, sp, #176 -; CHECK-NEXT: ldp x13, x12, [sp, #328] ; CHECK-NEXT: ldr x15, [sp, #104] +; CHECK-NEXT: ld1d { z3.d }, p0/z, [x10] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x8] +; CHECK-NEXT: add x8, sp, #112 +; CHECK-NEXT: ld1d { z2.d }, p0/z, [x9] +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x8] ; CHECK-NEXT: ldur q4, [sp, #88] -; CHECK-NEXT: ldp x16, x17, [sp, #208] +; CHECK-NEXT: ldp x9, x8, [sp, #328] ; CHECK-NEXT: ldr x19, [sp, #272] +; CHECK-NEXT: ldp x11, x10, [sp, #312] +; CHECK-NEXT: ldp x13, x12, [sp, #296] +; CHECK-NEXT: ldp x18, x14, [sp, #280] +; CHECK-NEXT: ldp x16, x17, [sp, #208] ; CHECK-NEXT: ldp x21, x22, [sp, #352] -; CHECK-NEXT: ld1d { z3.d }, p0/z, [x11] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x10] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x9] -; CHECK-NEXT: ld1d { z0.d }, p0/z, [x8] -; CHECK-NEXT: ldp x8, x14, [sp, #312] -; CHECK-NEXT: ldp x10, x9, [sp, #296] -; CHECK-NEXT: ldp x18, x11, [sp, #280] ; CHECK-NEXT: st1d { z3.d }, p0, [x20] ; CHECK-NEXT: add x20, sp, #144 ; CHECK-NEXT: st1d { z2.d }, p0, [x20] @@ -53,10 +53,10 @@ define dso_local void @func1(ptr %v1, ptr %v2, ptr %v3, ptr %v4, ptr %v5, ptr %v ; CHECK-NEXT: stp x16, x17, [sp, #208] ; CHECK-NEXT: stur q4, [sp, #88] ; CHECK-NEXT: str x15, [sp, #104] -; CHECK-NEXT: stp x11, x10, [sp, #288] -; CHECK-NEXT: stp x9, x8, [sp, #304] -; CHECK-NEXT: stp x14, x13, [sp, #320] -; CHECK-NEXT: str x12, [sp, #336] +; CHECK-NEXT: stp x14, x13, [sp, #288] +; CHECK-NEXT: stp x12, x11, [sp, #304] +; CHECK-NEXT: stp x10, x9, [sp, #320] +; CHECK-NEXT: str x8, [sp, #336] ; CHECK-NEXT: ldr x29, [sp], #48 // 8-byte Folded Reload ; CHECK-NEXT: b func2 ptr %v9, ptr %v10, ptr %v11, ptr %v12, ptr %v13, ptr %v14, ptr %v15, ptr %v16, diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-insert-vector-elt.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-insert-vector-elt.ll index 977c528e2583..6f4d257039bc 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-insert-vector-elt.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-insert-vector-elt.ll @@ -36,16 +36,16 @@ define <8 x half> @insertelement_v8f16(<8 x half> %op1) vscale_range(2,0) #0 { define void @insertelement_v16f16(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: insertelement_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: mov w8, #15 // =0xf ; CHECK-NEXT: index z0.h, #0, #1 -; CHECK-NEXT: ptrue p1.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x0] -; CHECK-NEXT: cmpeq p1.h, p1/z, z0.h, z1.h -; CHECK-NEXT: fmov h0, #5.00000000 -; CHECK-NEXT: mov z2.h, p1/m, h0 -; CHECK-NEXT: st1h { z2.h }, p0, [x1] +; CHECK-NEXT: ptrue p1.h, vl16 +; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, z1.h +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: fmov h1, #5.00000000 +; CHECK-NEXT: mov z0.h, p0/m, h1 +; CHECK-NEXT: st1h { z0.h }, p1, [x1] ; CHECK-NEXT: ret %op1 = load <16 x half>, ptr %a %r = insertelement <16 x half> %op1, half 5.0, i64 15 @@ -56,33 +56,33 @@ define void @insertelement_v16f16(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @insertelement_v32f16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: insertelement_v32f16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: mov w8, #15 // =0xf ; VBITS_GE_256-NEXT: index z0.h, #0, #1 -; VBITS_GE_256-NEXT: ptrue p1.h +; VBITS_GE_256-NEXT: ptrue p0.h ; VBITS_GE_256-NEXT: mov z1.h, w8 +; VBITS_GE_256-NEXT: ptrue p1.h, vl16 ; VBITS_GE_256-NEXT: mov x8, #16 // =0x10 -; VBITS_GE_256-NEXT: fmov h2, #5.00000000 -; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x0, x8, lsl #1] -; VBITS_GE_256-NEXT: cmpeq p1.h, p1/z, z0.h, z1.h -; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0] -; VBITS_GE_256-NEXT: mov z3.h, p1/m, h2 -; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x1] -; VBITS_GE_256-NEXT: st1h { z3.h }, p0, [x1, x8, lsl #1] +; VBITS_GE_256-NEXT: cmpeq p0.h, p0/z, z0.h, z1.h +; VBITS_GE_256-NEXT: fmov h0, #5.00000000 +; VBITS_GE_256-NEXT: ld1h { z1.h }, p1/z, [x0, x8, lsl #1] +; VBITS_GE_256-NEXT: mov z1.h, p0/m, h0 +; VBITS_GE_256-NEXT: ld1h { z0.h }, p1/z, [x0] +; VBITS_GE_256-NEXT: st1h { z1.h }, p1, [x1, x8, lsl #1] +; VBITS_GE_256-NEXT: st1h { z0.h }, p1, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: insertelement_v32f16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.h, vl32 ; VBITS_GE_512-NEXT: mov w8, #31 // =0x1f ; VBITS_GE_512-NEXT: index z0.h, #0, #1 -; VBITS_GE_512-NEXT: ptrue p1.h +; VBITS_GE_512-NEXT: ptrue p0.h ; VBITS_GE_512-NEXT: mov z1.h, w8 -; VBITS_GE_512-NEXT: ld1h { z2.h }, p0/z, [x0] -; VBITS_GE_512-NEXT: cmpeq p1.h, p1/z, z0.h, z1.h -; VBITS_GE_512-NEXT: fmov h0, #5.00000000 -; VBITS_GE_512-NEXT: mov z2.h, p1/m, h0 -; VBITS_GE_512-NEXT: st1h { z2.h }, p0, [x1] +; VBITS_GE_512-NEXT: ptrue p1.h, vl32 +; VBITS_GE_512-NEXT: cmpeq p0.h, p0/z, z0.h, z1.h +; VBITS_GE_512-NEXT: ld1h { z0.h }, p1/z, [x0] +; VBITS_GE_512-NEXT: fmov h1, #5.00000000 +; VBITS_GE_512-NEXT: mov z0.h, p0/m, h1 +; VBITS_GE_512-NEXT: st1h { z0.h }, p1, [x1] ; VBITS_GE_512-NEXT: ret %op1 = load <32 x half>, ptr %a %r = insertelement <32 x half> %op1, half 5.0, i64 31 @@ -93,16 +93,16 @@ define void @insertelement_v32f16(ptr %a, ptr %b) #0 { define void @insertelement_v64f16(ptr %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: insertelement_v64f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: mov w8, #63 // =0x3f ; CHECK-NEXT: index z0.h, #0, #1 -; CHECK-NEXT: ptrue p1.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x0] -; CHECK-NEXT: cmpeq p1.h, p1/z, z0.h, z1.h -; CHECK-NEXT: fmov h0, #5.00000000 -; CHECK-NEXT: mov z2.h, p1/m, h0 -; CHECK-NEXT: st1h { z2.h }, p0, [x1] +; CHECK-NEXT: ptrue p1.h, vl64 +; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, z1.h +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: fmov h1, #5.00000000 +; CHECK-NEXT: mov z0.h, p0/m, h1 +; CHECK-NEXT: st1h { z0.h }, p1, [x1] ; CHECK-NEXT: ret %op1 = load <64 x half>, ptr %a %r = insertelement <64 x half> %op1, half 5.0, i64 63 @@ -113,16 +113,16 @@ define void @insertelement_v64f16(ptr %a, ptr %b) vscale_range(8,0) #0 { define void @insertelement_v128f16(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: insertelement_v128f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl128 ; CHECK-NEXT: mov w8, #127 // =0x7f ; CHECK-NEXT: index z0.h, #0, #1 -; CHECK-NEXT: ptrue p1.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x0] -; CHECK-NEXT: cmpeq p1.h, p1/z, z0.h, z1.h -; CHECK-NEXT: fmov h0, #5.00000000 -; CHECK-NEXT: mov z2.h, p1/m, h0 -; CHECK-NEXT: st1h { z2.h }, p0, [x1] +; CHECK-NEXT: ptrue p1.h, vl128 +; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, z1.h +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: fmov h1, #5.00000000 +; CHECK-NEXT: mov z0.h, p0/m, h1 +; CHECK-NEXT: st1h { z0.h }, p1, [x1] ; CHECK-NEXT: ret %op1 = load <128 x half>, ptr %a %r = insertelement <128 x half> %op1, half 5.0, i64 127 @@ -157,16 +157,16 @@ define <4 x float> @insertelement_v4f32(<4 x float> %op1) vscale_range(2,0) #0 { define void @insertelement_v8f32(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: insertelement_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: mov w8, #7 // =0x7 ; CHECK-NEXT: index z0.s, #0, #1 -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0] -; CHECK-NEXT: cmpeq p1.s, p1/z, z0.s, z1.s -; CHECK-NEXT: fmov s0, #5.00000000 -; CHECK-NEXT: mov z2.s, p1/m, s0 -; CHECK-NEXT: st1w { z2.s }, p0, [x1] +; CHECK-NEXT: ptrue p1.s, vl8 +; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s +; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0] +; CHECK-NEXT: fmov s1, #5.00000000 +; CHECK-NEXT: mov z0.s, p0/m, s1 +; CHECK-NEXT: st1w { z0.s }, p1, [x1] ; CHECK-NEXT: ret %op1 = load <8 x float>, ptr %a %r = insertelement <8 x float> %op1, float 5.0, i64 7 @@ -177,33 +177,33 @@ define void @insertelement_v8f32(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @insertelement_v16f32(ptr %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: insertelement_v16f32: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov w8, #7 // =0x7 ; VBITS_GE_256-NEXT: index z0.s, #0, #1 -; VBITS_GE_256-NEXT: ptrue p1.s +; VBITS_GE_256-NEXT: ptrue p0.s ; VBITS_GE_256-NEXT: mov z1.s, w8 +; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 -; VBITS_GE_256-NEXT: fmov s2, #5.00000000 -; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x0, x8, lsl #2] -; VBITS_GE_256-NEXT: cmpeq p1.s, p1/z, z0.s, z1.s -; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0] -; VBITS_GE_256-NEXT: mov z3.s, p1/m, s2 -; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x1] -; VBITS_GE_256-NEXT: st1w { z3.s }, p0, [x1, x8, lsl #2] +; VBITS_GE_256-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s +; VBITS_GE_256-NEXT: fmov s0, #5.00000000 +; VBITS_GE_256-NEXT: ld1w { z1.s }, p1/z, [x0, x8, lsl #2] +; VBITS_GE_256-NEXT: mov z1.s, p0/m, s0 +; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x0] +; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x1, x8, lsl #2] +; VBITS_GE_256-NEXT: st1w { z0.s }, p1, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: insertelement_v16f32: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl16 ; VBITS_GE_512-NEXT: mov w8, #15 // =0xf ; VBITS_GE_512-NEXT: index z0.s, #0, #1 -; VBITS_GE_512-NEXT: ptrue p1.s +; VBITS_GE_512-NEXT: ptrue p0.s ; VBITS_GE_512-NEXT: mov z1.s, w8 -; VBITS_GE_512-NEXT: ld1w { z2.s }, p0/z, [x0] -; VBITS_GE_512-NEXT: cmpeq p1.s, p1/z, z0.s, z1.s -; VBITS_GE_512-NEXT: fmov s0, #5.00000000 -; VBITS_GE_512-NEXT: mov z2.s, p1/m, s0 -; VBITS_GE_512-NEXT: st1w { z2.s }, p0, [x1] +; VBITS_GE_512-NEXT: ptrue p1.s, vl16 +; VBITS_GE_512-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s +; VBITS_GE_512-NEXT: ld1w { z0.s }, p1/z, [x0] +; VBITS_GE_512-NEXT: fmov s1, #5.00000000 +; VBITS_GE_512-NEXT: mov z0.s, p0/m, s1 +; VBITS_GE_512-NEXT: st1w { z0.s }, p1, [x1] ; VBITS_GE_512-NEXT: ret %op1 = load <16 x float>, ptr %a %r = insertelement <16 x float> %op1, float 5.0, i64 15 @@ -214,16 +214,16 @@ define void @insertelement_v16f32(ptr %a, ptr %b) #0 { define void @insertelement_v32f32(ptr %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: insertelement_v32f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: mov w8, #31 // =0x1f ; CHECK-NEXT: index z0.s, #0, #1 -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0] -; CHECK-NEXT: cmpeq p1.s, p1/z, z0.s, z1.s -; CHECK-NEXT: fmov s0, #5.00000000 -; CHECK-NEXT: mov z2.s, p1/m, s0 -; CHECK-NEXT: st1w { z2.s }, p0, [x1] +; CHECK-NEXT: ptrue p1.s, vl32 +; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s +; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0] +; CHECK-NEXT: fmov s1, #5.00000000 +; CHECK-NEXT: mov z0.s, p0/m, s1 +; CHECK-NEXT: st1w { z0.s }, p1, [x1] ; CHECK-NEXT: ret %op1 = load <32 x float>, ptr %a %r = insertelement <32 x float> %op1, float 5.0, i64 31 @@ -234,16 +234,16 @@ define void @insertelement_v32f32(ptr %a, ptr %b) vscale_range(8,0) #0 { define void @insertelement_v64f32(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: insertelement_v64f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl64 ; CHECK-NEXT: mov w8, #63 // =0x3f ; CHECK-NEXT: index z0.s, #0, #1 -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0] -; CHECK-NEXT: cmpeq p1.s, p1/z, z0.s, z1.s -; CHECK-NEXT: fmov s0, #5.00000000 -; CHECK-NEXT: mov z2.s, p1/m, s0 -; CHECK-NEXT: st1w { z2.s }, p0, [x1] +; CHECK-NEXT: ptrue p1.s, vl64 +; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s +; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0] +; CHECK-NEXT: fmov s1, #5.00000000 +; CHECK-NEXT: mov z0.s, p0/m, s1 +; CHECK-NEXT: st1w { z0.s }, p1, [x1] ; CHECK-NEXT: ret %op1 = load <64 x float>, ptr %a %r = insertelement <64 x float> %op1, float 5.0, i64 63 @@ -276,16 +276,16 @@ define <2 x double> @insertelement_v2f64(<2 x double> %op1) vscale_range(2,0) #0 define void @insertelement_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: insertelement_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z0.d, #0, #1 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0] -; CHECK-NEXT: cmpeq p1.d, p1/z, z0.d, z1.d -; CHECK-NEXT: fmov d0, #5.00000000 -; CHECK-NEXT: mov z2.d, p1/m, d0 -; CHECK-NEXT: st1d { z2.d }, p0, [x1] +; CHECK-NEXT: ptrue p1.d, vl4 +; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0] +; CHECK-NEXT: fmov d1, #5.00000000 +; CHECK-NEXT: mov z0.d, p0/m, d1 +; CHECK-NEXT: st1d { z0.d }, p1, [x1] ; CHECK-NEXT: ret %op1 = load <4 x double>, ptr %a %r = insertelement <4 x double> %op1, double 5.0, i64 3 @@ -296,33 +296,33 @@ define void @insertelement_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @insertelement_v8f64(ptr %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: insertelement_v8f64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov w8, #3 // =0x3 ; VBITS_GE_256-NEXT: index z0.d, #0, #1 -; VBITS_GE_256-NEXT: ptrue p1.d +; VBITS_GE_256-NEXT: ptrue p0.d ; VBITS_GE_256-NEXT: mov z1.d, x8 +; VBITS_GE_256-NEXT: ptrue p1.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: fmov d2, #5.00000000 -; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x0, x8, lsl #3] -; VBITS_GE_256-NEXT: cmpeq p1.d, p1/z, z0.d, z1.d -; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0] -; VBITS_GE_256-NEXT: mov z3.d, p1/m, d2 -; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x1] -; VBITS_GE_256-NEXT: st1d { z3.d }, p0, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d +; VBITS_GE_256-NEXT: fmov d0, #5.00000000 +; VBITS_GE_256-NEXT: ld1d { z1.d }, p1/z, [x0, x8, lsl #3] +; VBITS_GE_256-NEXT: mov z1.d, p0/m, d0 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p1/z, [x0] +; VBITS_GE_256-NEXT: st1d { z1.d }, p1, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: st1d { z0.d }, p1, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: insertelement_v8f64: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: mov w8, #7 // =0x7 ; VBITS_GE_512-NEXT: index z0.d, #0, #1 -; VBITS_GE_512-NEXT: ptrue p1.d +; VBITS_GE_512-NEXT: ptrue p0.d ; VBITS_GE_512-NEXT: mov z1.d, x8 -; VBITS_GE_512-NEXT: ld1d { z2.d }, p0/z, [x0] -; VBITS_GE_512-NEXT: cmpeq p1.d, p1/z, z0.d, z1.d -; VBITS_GE_512-NEXT: fmov d0, #5.00000000 -; VBITS_GE_512-NEXT: mov z2.d, p1/m, d0 -; VBITS_GE_512-NEXT: st1d { z2.d }, p0, [x1] +; VBITS_GE_512-NEXT: ptrue p1.d, vl8 +; VBITS_GE_512-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d +; VBITS_GE_512-NEXT: ld1d { z0.d }, p1/z, [x0] +; VBITS_GE_512-NEXT: fmov d1, #5.00000000 +; VBITS_GE_512-NEXT: mov z0.d, p0/m, d1 +; VBITS_GE_512-NEXT: st1d { z0.d }, p1, [x1] ; VBITS_GE_512-NEXT: ret %op1 = load <8 x double>, ptr %a %r = insertelement <8 x double> %op1, double 5.0, i64 7 @@ -333,16 +333,16 @@ define void @insertelement_v8f64(ptr %a, ptr %b) #0 { define void @insertelement_v16f64(ptr %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: insertelement_v16f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: mov w8, #15 // =0xf ; CHECK-NEXT: index z0.d, #0, #1 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0] -; CHECK-NEXT: cmpeq p1.d, p1/z, z0.d, z1.d -; CHECK-NEXT: fmov d0, #5.00000000 -; CHECK-NEXT: mov z2.d, p1/m, d0 -; CHECK-NEXT: st1d { z2.d }, p0, [x1] +; CHECK-NEXT: ptrue p1.d, vl16 +; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0] +; CHECK-NEXT: fmov d1, #5.00000000 +; CHECK-NEXT: mov z0.d, p0/m, d1 +; CHECK-NEXT: st1d { z0.d }, p1, [x1] ; CHECK-NEXT: ret %op1 = load <16 x double>, ptr %a %r = insertelement <16 x double> %op1, double 5.0, i64 15 @@ -353,16 +353,16 @@ define void @insertelement_v16f64(ptr %a, ptr %b) vscale_range(8,0) #0 { define void @insertelement_v32f64(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: insertelement_v32f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: mov w8, #31 // =0x1f ; CHECK-NEXT: index z0.d, #0, #1 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0] -; CHECK-NEXT: cmpeq p1.d, p1/z, z0.d, z1.d -; CHECK-NEXT: fmov d0, #5.00000000 -; CHECK-NEXT: mov z2.d, p1/m, d0 -; CHECK-NEXT: st1d { z2.d }, p0, [x1] +; CHECK-NEXT: ptrue p1.d, vl32 +; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0] +; CHECK-NEXT: fmov d1, #5.00000000 +; CHECK-NEXT: mov z0.d, p0/m, d1 +; CHECK-NEXT: st1d { z0.d }, p1, [x1] ; CHECK-NEXT: ret %op1 = load <32 x double>, ptr %a %r = insertelement <32 x double> %op1, double 5.0, i64 31 diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll index b45a62f4e758..58fca3a2cf8b 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll @@ -1388,11 +1388,11 @@ define void @abs_v128i16(ptr %a) vscale_range(2,0) #0 { ; CHECK-NEXT: mov x10, #64 // =0x40 ; CHECK-NEXT: mov x11, #80 // =0x50 ; CHECK-NEXT: mov x12, #32 // =0x20 -; CHECK-NEXT: mov x13, #48 // =0x30 -; CHECK-NEXT: mov x14, #16 // =0x10 ; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] ; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0, x9, lsl #1] ; CHECK-NEXT: ld1h { z2.h }, p0/z, [x0, x10, lsl #1] +; CHECK-NEXT: mov x13, #48 // =0x30 +; CHECK-NEXT: mov x14, #16 // =0x10 ; CHECK-NEXT: ld1h { z3.h }, p0/z, [x0, x11, lsl #1] ; CHECK-NEXT: ld1h { z4.h }, p0/z, [x0, x12, lsl #1] ; CHECK-NEXT: ld1h { z5.h }, p0/z, [x0, x13, lsl #1] diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-div.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-div.ll index 11ed69513917..0ddf434eff93 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-div.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-div.ll @@ -15,9 +15,9 @@ target triple = "aarch64-unknown-linux-gnu" define <8 x i8> @sdiv_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { ; VBITS_GE_128-LABEL: sdiv_v8i8: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll v1.8h, v1.8b, #0 ; VBITS_GE_128-NEXT: sshll v0.8h, v0.8b, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0 ; VBITS_GE_128-NEXT: sshll v1.4s, v1.4h, #0 @@ -94,11 +94,11 @@ define <8 x i8> @sdiv_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { define <16 x i8> @sdiv_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 { ; VBITS_GE_128-LABEL: sdiv_v16i8: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll2 v2.8h, v1.16b, #0 ; VBITS_GE_128-NEXT: sshll2 v3.8h, v0.16b, #0 ; VBITS_GE_128-NEXT: sshll v1.8h, v1.8b, #0 ; VBITS_GE_128-NEXT: sshll v0.8h, v0.8b, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll2 v4.4s, v2.8h, #0 ; VBITS_GE_128-NEXT: sshll2 v5.4s, v3.8h, #0 ; VBITS_GE_128-NEXT: sshll v2.4s, v2.4h, #0 @@ -203,7 +203,6 @@ define void @sdiv_v128i8(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: sdiv_v128i8: ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.b, vl128 -; CHECK-NEXT: ptrue p1.h, vl128 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0] ; CHECK-NEXT: ld1b { z1.b }, p0/z, [x1] ; CHECK-NEXT: ptrue p0.s, vl64 @@ -221,7 +220,8 @@ define void @sdiv_v128i8(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-NEXT: uzp1 z1.h, z2.h, z2.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: splice z1.h, p0, z1.h, z0.h -; CHECK-NEXT: st1b { z1.h }, p1, [x0] +; CHECK-NEXT: ptrue p0.h, vl128 +; CHECK-NEXT: st1b { z1.h }, p0, [x0] ; CHECK-NEXT: ret %op1 = load <128 x i8>, ptr %a %op2 = load <128 x i8>, ptr %b @@ -260,14 +260,14 @@ define void @sdiv_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-NEXT: sdivr z3.s, p1/m, z3.s, z5.s ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: sdiv z0.s, p1/m, z0.s, z1.s -; CHECK-NEXT: ptrue p1.h, vl64 ; CHECK-NEXT: uzp1 z1.h, z4.h, z4.h +; CHECK-NEXT: ptrue p1.h, vl64 ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z1.h, p1, z1.h, z2.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: splice z3.h, p1, z3.h, z0.h -; CHECK-NEXT: ptrue p1.b, vl128 ; CHECK-NEXT: uzp1 z0.b, z1.b, z1.b +; CHECK-NEXT: ptrue p1.b, vl128 ; CHECK-NEXT: uzp1 z1.b, z3.b, z3.b ; CHECK-NEXT: splice z0.b, p1, z0.b, z1.b ; CHECK-NEXT: st1b { z0.b }, p0, [x0] @@ -284,18 +284,18 @@ define void @sdiv_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 { define <4 x i16> @sdiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; VBITS_GE_128-LABEL: sdiv_v4i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll v1.4s, v1.4h, #0 ; VBITS_GE_128-NEXT: sshll v0.4s, v0.4h, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sdiv z0.s, p0/m, z0.s, z1.s ; VBITS_GE_128-NEXT: xtn v0.4h, v0.4s ; VBITS_GE_128-NEXT: ret ; ; VBITS_GE_256-LABEL: sdiv_v4i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: sshll v1.4s, v1.4h, #0 ; VBITS_GE_256-NEXT: sshll v0.4s, v0.4h, #0 +; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: sdivr z1.s, p0/m, z1.s, z0.s ; VBITS_GE_256-NEXT: mov w8, v1.s[1] ; VBITS_GE_256-NEXT: mov v0.16b, v1.16b @@ -309,9 +309,9 @@ define <4 x i16> @sdiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; ; VBITS_GE_512-LABEL: sdiv_v4i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl4 ; VBITS_GE_512-NEXT: sshll v1.4s, v1.4h, #0 ; VBITS_GE_512-NEXT: sshll v0.4s, v0.4h, #0 +; VBITS_GE_512-NEXT: ptrue p0.s, vl4 ; VBITS_GE_512-NEXT: sdivr z1.s, p0/m, z1.s, z0.s ; VBITS_GE_512-NEXT: mov w8, v1.s[1] ; VBITS_GE_512-NEXT: mov v0.16b, v1.16b @@ -329,11 +329,11 @@ define <4 x i16> @sdiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { define <8 x i16> @sdiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; VBITS_GE_128-LABEL: sdiv_v8i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0 ; VBITS_GE_128-NEXT: sshll v1.4s, v1.4h, #0 ; VBITS_GE_128-NEXT: sshll v0.4s, v0.4h, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_128-NEXT: sdiv z0.s, p0/m, z0.s, z1.s ; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v2.8h @@ -341,9 +341,9 @@ define <8 x i16> @sdiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; ; VBITS_GE_256-LABEL: sdiv_v8i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z1 ; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0 +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h ; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h ; VBITS_GE_256-NEXT: sdiv z0.s, p0/m, z0.s, z1.s @@ -353,9 +353,9 @@ define <8 x i16> @sdiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; ; VBITS_GE_512-LABEL: sdiv_v8i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z1 ; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0 +; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: sunpklo z1.s, z1.h ; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h ; VBITS_GE_512-NEXT: sdiv z0.s, p0/m, z0.s, z1.s @@ -369,8 +369,8 @@ define <8 x i16> @sdiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { define void @sdiv_v16i16(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: sdiv_v16i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q4, q1, [x1] +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldr q0, [x0, #16] ; VBITS_GE_128-NEXT: sshll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0 @@ -542,8 +542,8 @@ define void @sdiv_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @sdiv_v16i32(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: sdiv_v16i32: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1] +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0] ; VBITS_GE_128-NEXT: ldp q5, q4, [x1, #32] ; VBITS_GE_128-NEXT: sdivr z0.s, p0/m, z0.s, z1.s @@ -664,8 +664,8 @@ define void @sdiv_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @sdiv_v8i64(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: sdiv_v8i64: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.d, vl2 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1] +; VBITS_GE_128-NEXT: ptrue p0.d, vl2 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0] ; VBITS_GE_128-NEXT: ldp q5, q4, [x1, #32] ; VBITS_GE_128-NEXT: sdivr z0.d, p0/m, z0.d, z1.d @@ -748,9 +748,9 @@ define void @sdiv_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 { define <8 x i8> @udiv_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { ; VBITS_GE_128-LABEL: udiv_v8i8: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll v1.8h, v1.8b, #0 ; VBITS_GE_128-NEXT: ushll v0.8h, v0.8b, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #0 ; VBITS_GE_128-NEXT: ushll v1.4s, v1.4h, #0 @@ -827,11 +827,11 @@ define <8 x i8> @udiv_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { define <16 x i8> @udiv_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 { ; VBITS_GE_128-LABEL: udiv_v16i8: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll2 v2.8h, v1.16b, #0 ; VBITS_GE_128-NEXT: ushll2 v3.8h, v0.16b, #0 ; VBITS_GE_128-NEXT: ushll v1.8h, v1.8b, #0 ; VBITS_GE_128-NEXT: ushll v0.8h, v0.8b, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll2 v4.4s, v2.8h, #0 ; VBITS_GE_128-NEXT: ushll2 v5.4s, v3.8h, #0 ; VBITS_GE_128-NEXT: ushll v2.4s, v2.4h, #0 @@ -980,14 +980,14 @@ define void @udiv_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-NEXT: udivr z3.s, p1/m, z3.s, z5.s ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: udiv z0.s, p1/m, z0.s, z1.s -; CHECK-NEXT: ptrue p1.h, vl64 ; CHECK-NEXT: uzp1 z1.h, z4.h, z4.h +; CHECK-NEXT: ptrue p1.h, vl64 ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z1.h, p1, z1.h, z2.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: splice z3.h, p1, z3.h, z0.h -; CHECK-NEXT: ptrue p1.b, vl128 ; CHECK-NEXT: uzp1 z0.b, z1.b, z1.b +; CHECK-NEXT: ptrue p1.b, vl128 ; CHECK-NEXT: uzp1 z1.b, z3.b, z3.b ; CHECK-NEXT: splice z0.b, p1, z0.b, z1.b ; CHECK-NEXT: st1b { z0.b }, p0, [x0] @@ -1004,18 +1004,18 @@ define void @udiv_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 { define <4 x i16> @udiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; VBITS_GE_128-LABEL: udiv_v4i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll v1.4s, v1.4h, #0 ; VBITS_GE_128-NEXT: ushll v0.4s, v0.4h, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: udiv z0.s, p0/m, z0.s, z1.s ; VBITS_GE_128-NEXT: xtn v0.4h, v0.4s ; VBITS_GE_128-NEXT: ret ; ; VBITS_GE_256-LABEL: udiv_v4i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: ushll v1.4s, v1.4h, #0 ; VBITS_GE_256-NEXT: ushll v0.4s, v0.4h, #0 +; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: udivr z1.s, p0/m, z1.s, z0.s ; VBITS_GE_256-NEXT: mov w8, v1.s[1] ; VBITS_GE_256-NEXT: mov v0.16b, v1.16b @@ -1029,9 +1029,9 @@ define <4 x i16> @udiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; ; VBITS_GE_512-LABEL: udiv_v4i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl4 ; VBITS_GE_512-NEXT: ushll v1.4s, v1.4h, #0 ; VBITS_GE_512-NEXT: ushll v0.4s, v0.4h, #0 +; VBITS_GE_512-NEXT: ptrue p0.s, vl4 ; VBITS_GE_512-NEXT: udivr z1.s, p0/m, z1.s, z0.s ; VBITS_GE_512-NEXT: mov w8, v1.s[1] ; VBITS_GE_512-NEXT: mov v0.16b, v1.16b @@ -1049,11 +1049,11 @@ define <4 x i16> @udiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { define <8 x i16> @udiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; VBITS_GE_128-LABEL: udiv_v8i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #0 ; VBITS_GE_128-NEXT: ushll v1.4s, v1.4h, #0 ; VBITS_GE_128-NEXT: ushll v0.4s, v0.4h, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_128-NEXT: udiv z0.s, p0/m, z0.s, z1.s ; VBITS_GE_128-NEXT: uzp1 v0.8h, v0.8h, v2.8h @@ -1061,9 +1061,9 @@ define <8 x i16> @udiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; ; VBITS_GE_256-LABEL: udiv_v8i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z1 ; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0 +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: uunpklo z1.s, z1.h ; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_256-NEXT: udiv z0.s, p0/m, z0.s, z1.s @@ -1073,9 +1073,9 @@ define <8 x i16> @udiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; ; VBITS_GE_512-LABEL: udiv_v8i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z1 ; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0 +; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: uunpklo z1.s, z1.h ; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_512-NEXT: udiv z0.s, p0/m, z0.s, z1.s @@ -1089,8 +1089,8 @@ define <8 x i16> @udiv_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { define void @udiv_v16i16(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: udiv_v16i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q4, q1, [x1] +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldr q0, [x0, #16] ; VBITS_GE_128-NEXT: ushll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #0 @@ -1253,8 +1253,8 @@ define void @udiv_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @udiv_v16i32(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: udiv_v16i32: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1] +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0] ; VBITS_GE_128-NEXT: ldp q5, q4, [x1, #32] ; VBITS_GE_128-NEXT: udivr z0.s, p0/m, z0.s, z1.s @@ -1375,8 +1375,8 @@ define void @udiv_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @udiv_v8i64(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: udiv_v8i64: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.d, vl2 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1] +; VBITS_GE_128-NEXT: ptrue p0.d, vl2 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0] ; VBITS_GE_128-NEXT: ldp q5, q4, [x1, #32] ; VBITS_GE_128-NEXT: udivr z0.d, p0/m, z0.d, z1.d diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll index 756e5f4cddf8..4feb86305f8f 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll @@ -58,8 +58,8 @@ define void @sext_v4i3_v4i64(<4 x i3> %a, ptr %out) vscale_range(2,0) #0 { define void @sext_v16i8_v16i16(<16 x i8> %a, ptr %out) vscale_range(2,0) #0 { ; CHECK-LABEL: sext_v16i8_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: sunpklo z0.h, z0.b ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret @@ -308,8 +308,8 @@ define void @sext_v32i8_v32i64(ptr %in, ptr %out) vscale_range(16,0) #0 { define void @sext_v8i16_v8i32(<8 x i16> %a, ptr %out) vscale_range(2,0) #0 { ; CHECK-LABEL: sext_v8i16_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: sunpklo z0.s, z0.h ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret @@ -472,8 +472,8 @@ define void @sext_v32i16_v32i64(ptr %in, ptr %out) vscale_range(16,0) #0 { define void @sext_v4i32_v4i64(<4 x i32> %a, ptr %out) vscale_range(2,0) #0 { ; CHECK-LABEL: sext_v4i32_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: sunpklo z0.d, z0.s ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret @@ -554,8 +554,8 @@ define void @sext_v32i32_v32i64(ptr %in, ptr %out) vscale_range(16,0) #0 { define void @zext_v16i8_v16i16(<16 x i8> %a, ptr %out) vscale_range(2,0) #0 { ; CHECK-LABEL: zext_v16i8_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: uunpklo z0.h, z0.b ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret @@ -804,8 +804,8 @@ define void @zext_v32i8_v32i64(ptr %in, ptr %out) vscale_range(16,0) #0 { define void @zext_v8i16_v8i32(<8 x i16> %a, ptr %out) vscale_range(2,0) #0 { ; CHECK-LABEL: zext_v8i16_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret @@ -968,8 +968,8 @@ define void @zext_v32i16_v32i64(ptr %in, ptr %out) vscale_range(16,0) #0 { define void @zext_v4i32_v4i64(<4 x i32> %a, ptr %out) vscale_range(2,0) #0 { ; CHECK-LABEL: zext_v4i32_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-rem.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-rem.ll index 38444d83c1d7..2d7894539917 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-rem.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-rem.ll @@ -15,9 +15,9 @@ target triple = "aarch64-unknown-linux-gnu" define <8 x i8> @srem_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { ; VBITS_GE_128-LABEL: srem_v8i8: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll v2.8h, v1.8b, #0 ; VBITS_GE_128-NEXT: sshll v3.8h, v0.8b, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll2 v4.4s, v2.8h, #0 ; VBITS_GE_128-NEXT: sshll2 v5.4s, v3.8h, #0 ; VBITS_GE_128-NEXT: sshll v2.4s, v2.4h, #0 @@ -97,9 +97,9 @@ define <8 x i8> @srem_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { define <16 x i8> @srem_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 { ; VBITS_GE_128-LABEL: srem_v16i8: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll2 v2.8h, v1.16b, #0 ; VBITS_GE_128-NEXT: sshll2 v3.8h, v0.16b, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll2 v4.4s, v2.8h, #0 ; VBITS_GE_128-NEXT: sshll2 v5.4s, v3.8h, #0 ; VBITS_GE_128-NEXT: sshll v2.4s, v2.4h, #0 @@ -277,8 +277,8 @@ define void @srem_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: sdivr z3.s, p1/m, z3.s, z5.s ; CHECK-NEXT: ptrue p1.h, vl64 -; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h ; CHECK-NEXT: splice z4.h, p1, z4.h, z2.h +; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h ; CHECK-NEXT: uzp1 z2.b, z4.b, z4.b ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z5.h, p1, z5.h, z3.h @@ -300,9 +300,9 @@ define void @srem_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 { define <4 x i16> @srem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; VBITS_GE_128-LABEL: srem_v4i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll v2.4s, v1.4h, #0 ; VBITS_GE_128-NEXT: sshll v3.4s, v0.4h, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_128-NEXT: xtn v2.4h, v2.4s ; VBITS_GE_128-NEXT: mls v0.4h, v2.4h, v1.4h @@ -310,9 +310,9 @@ define <4 x i16> @srem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; ; VBITS_GE_256-LABEL: srem_v4i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: sshll v2.4s, v1.4h, #0 ; VBITS_GE_256-NEXT: sshll v3.4s, v0.4h, #0 +; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_256-NEXT: mov w8, v2.s[1] ; VBITS_GE_256-NEXT: mov v3.16b, v2.16b @@ -326,9 +326,9 @@ define <4 x i16> @srem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; ; VBITS_GE_512-LABEL: srem_v4i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl4 ; VBITS_GE_512-NEXT: sshll v2.4s, v1.4h, #0 ; VBITS_GE_512-NEXT: sshll v3.4s, v0.4h, #0 +; VBITS_GE_512-NEXT: ptrue p0.s, vl4 ; VBITS_GE_512-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_512-NEXT: mov w8, v2.s[1] ; VBITS_GE_512-NEXT: mov v3.16b, v2.16b @@ -346,9 +346,9 @@ define <4 x i16> @srem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { define <8 x i16> @srem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; VBITS_GE_128-LABEL: srem_v8i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: sshll v4.4s, v0.4h, #0 ; VBITS_GE_128-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_128-NEXT: sshll v3.4s, v1.4h, #0 @@ -359,11 +359,11 @@ define <8 x i16> @srem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; ; VBITS_GE_256-LABEL: srem_v8i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z1 ; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0 ; VBITS_GE_256-NEXT: sunpklo z2.s, z1.h ; VBITS_GE_256-NEXT: sunpklo z3.s, z0.h +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h ; VBITS_GE_256-NEXT: mls v0.8h, v2.8h, v1.8h @@ -372,11 +372,11 @@ define <8 x i16> @srem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; ; VBITS_GE_512-LABEL: srem_v8i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z1 ; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0 ; VBITS_GE_512-NEXT: sunpklo z2.s, z1.h ; VBITS_GE_512-NEXT: sunpklo z3.s, z0.h +; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h ; VBITS_GE_512-NEXT: mls v0.8h, v2.8h, v1.8h @@ -389,8 +389,8 @@ define <8 x i16> @srem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { define void @srem_v16i16(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: srem_v16i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q4, q1, [x1] +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldr q0, [x0, #16] ; VBITS_GE_128-NEXT: sshll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: sshll2 v3.4s, v0.8h, #0 @@ -582,25 +582,25 @@ define void @srem_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @srem_v16i32(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: srem_v16i32: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1] +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0] ; VBITS_GE_128-NEXT: ldp q16, q5, [x0, #32] ; VBITS_GE_128-NEXT: ldp q17, q6, [x1, #32] ; VBITS_GE_128-NEXT: movprfx z4, z1 ; VBITS_GE_128-NEXT: sdiv z4.s, p0/m, z4.s, z0.s +; VBITS_GE_128-NEXT: movprfx z19, z2 +; VBITS_GE_128-NEXT: sdiv z19.s, p0/m, z19.s, z3.s ; VBITS_GE_128-NEXT: movprfx z7, z5 ; VBITS_GE_128-NEXT: sdiv z7.s, p0/m, z7.s, z6.s ; VBITS_GE_128-NEXT: movprfx z18, z16 ; VBITS_GE_128-NEXT: sdiv z18.s, p0/m, z18.s, z17.s -; VBITS_GE_128-NEXT: movprfx z19, z2 -; VBITS_GE_128-NEXT: sdiv z19.s, p0/m, z19.s, z3.s -; VBITS_GE_128-NEXT: mls v16.4s, v18.4s, v17.4s -; VBITS_GE_128-NEXT: mls v5.4s, v7.4s, v6.4s ; VBITS_GE_128-NEXT: mls v1.4s, v4.4s, v0.4s ; VBITS_GE_128-NEXT: mls v2.4s, v19.4s, v3.4s -; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32] +; VBITS_GE_128-NEXT: mls v16.4s, v18.4s, v17.4s +; VBITS_GE_128-NEXT: mls v5.4s, v7.4s, v6.4s ; VBITS_GE_128-NEXT: stp q1, q2, [x0] +; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32] ; VBITS_GE_128-NEXT: ret ; ; VBITS_GE_256-LABEL: srem_v16i32: @@ -730,26 +730,26 @@ define void @srem_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @srem_v8i64(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: srem_v8i64: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.d, vl2 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1] +; VBITS_GE_128-NEXT: ptrue p0.d, vl2 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0] ; VBITS_GE_128-NEXT: ldp q16, q5, [x0, #32] ; VBITS_GE_128-NEXT: ldp q17, q6, [x1, #32] ; VBITS_GE_128-NEXT: movprfx z4, z1 ; VBITS_GE_128-NEXT: sdiv z4.d, p0/m, z4.d, z0.d +; VBITS_GE_128-NEXT: movprfx z19, z2 +; VBITS_GE_128-NEXT: sdiv z19.d, p0/m, z19.d, z3.d ; VBITS_GE_128-NEXT: movprfx z7, z5 ; VBITS_GE_128-NEXT: sdiv z7.d, p0/m, z7.d, z6.d ; VBITS_GE_128-NEXT: movprfx z18, z16 ; VBITS_GE_128-NEXT: sdiv z18.d, p0/m, z18.d, z17.d -; VBITS_GE_128-NEXT: movprfx z19, z2 -; VBITS_GE_128-NEXT: sdiv z19.d, p0/m, z19.d, z3.d -; VBITS_GE_128-NEXT: mls z16.d, p0/m, z18.d, z17.d -; VBITS_GE_128-NEXT: mls z5.d, p0/m, z7.d, z6.d ; VBITS_GE_128-NEXT: msb z0.d, p0/m, z4.d, z1.d ; VBITS_GE_128-NEXT: movprfx z1, z2 ; VBITS_GE_128-NEXT: mls z1.d, p0/m, z19.d, z3.d -; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32] +; VBITS_GE_128-NEXT: mls z16.d, p0/m, z18.d, z17.d +; VBITS_GE_128-NEXT: mls z5.d, p0/m, z7.d, z6.d ; VBITS_GE_128-NEXT: stp q0, q1, [x0] +; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32] ; VBITS_GE_128-NEXT: ret ; ; VBITS_GE_256-LABEL: srem_v8i64: @@ -833,9 +833,9 @@ define void @srem_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 { define <8 x i8> @urem_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { ; VBITS_GE_128-LABEL: urem_v8i8: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll v2.8h, v1.8b, #0 ; VBITS_GE_128-NEXT: ushll v3.8h, v0.8b, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll2 v4.4s, v2.8h, #0 ; VBITS_GE_128-NEXT: ushll2 v5.4s, v3.8h, #0 ; VBITS_GE_128-NEXT: ushll v2.4s, v2.4h, #0 @@ -915,9 +915,9 @@ define <8 x i8> @urem_v8i8(<8 x i8> %op1, <8 x i8> %op2) #0 { define <16 x i8> @urem_v16i8(<16 x i8> %op1, <16 x i8> %op2) #0 { ; VBITS_GE_128-LABEL: urem_v16i8: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll2 v2.8h, v1.16b, #0 ; VBITS_GE_128-NEXT: ushll2 v3.8h, v0.16b, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll2 v4.4s, v2.8h, #0 ; VBITS_GE_128-NEXT: ushll2 v5.4s, v3.8h, #0 ; VBITS_GE_128-NEXT: ushll v2.4s, v2.4h, #0 @@ -1095,8 +1095,8 @@ define void @urem_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: udivr z3.s, p1/m, z3.s, z5.s ; CHECK-NEXT: ptrue p1.h, vl64 -; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h ; CHECK-NEXT: splice z4.h, p1, z4.h, z2.h +; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h ; CHECK-NEXT: uzp1 z2.b, z4.b, z4.b ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z5.h, p1, z5.h, z3.h @@ -1118,9 +1118,9 @@ define void @urem_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 { define <4 x i16> @urem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; VBITS_GE_128-LABEL: urem_v4i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll v2.4s, v1.4h, #0 ; VBITS_GE_128-NEXT: ushll v3.4s, v0.4h, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_128-NEXT: xtn v2.4h, v2.4s ; VBITS_GE_128-NEXT: mls v0.4h, v2.4h, v1.4h @@ -1128,9 +1128,9 @@ define <4 x i16> @urem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; ; VBITS_GE_256-LABEL: urem_v4i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: ushll v2.4s, v1.4h, #0 ; VBITS_GE_256-NEXT: ushll v3.4s, v0.4h, #0 +; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_256-NEXT: mov w8, v2.s[1] ; VBITS_GE_256-NEXT: mov v3.16b, v2.16b @@ -1144,9 +1144,9 @@ define <4 x i16> @urem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { ; ; VBITS_GE_512-LABEL: urem_v4i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl4 ; VBITS_GE_512-NEXT: ushll v2.4s, v1.4h, #0 ; VBITS_GE_512-NEXT: ushll v3.4s, v0.4h, #0 +; VBITS_GE_512-NEXT: ptrue p0.s, vl4 ; VBITS_GE_512-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_512-NEXT: mov w8, v2.s[1] ; VBITS_GE_512-NEXT: mov v3.16b, v2.16b @@ -1164,9 +1164,9 @@ define <4 x i16> @urem_v4i16(<4 x i16> %op1, <4 x i16> %op2) #0 { define <8 x i16> @urem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; VBITS_GE_128-LABEL: urem_v8i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #0 +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ushll v4.4s, v0.4h, #0 ; VBITS_GE_128-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_128-NEXT: ushll v3.4s, v1.4h, #0 @@ -1177,11 +1177,11 @@ define <8 x i16> @urem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; ; VBITS_GE_256-LABEL: urem_v8i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: // kill: def $q1 killed $q1 def $z1 ; VBITS_GE_256-NEXT: // kill: def $q0 killed $q0 def $z0 ; VBITS_GE_256-NEXT: uunpklo z2.s, z1.h ; VBITS_GE_256-NEXT: uunpklo z3.s, z0.h +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h ; VBITS_GE_256-NEXT: mls v0.8h, v2.8h, v1.8h @@ -1190,11 +1190,11 @@ define <8 x i16> @urem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { ; ; VBITS_GE_512-LABEL: urem_v8i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: // kill: def $q1 killed $q1 def $z1 ; VBITS_GE_512-NEXT: // kill: def $q0 killed $q0 def $z0 ; VBITS_GE_512-NEXT: uunpklo z2.s, z1.h ; VBITS_GE_512-NEXT: uunpklo z3.s, z0.h +; VBITS_GE_512-NEXT: ptrue p0.s, vl8 ; VBITS_GE_512-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; VBITS_GE_512-NEXT: uzp1 z2.h, z2.h, z2.h ; VBITS_GE_512-NEXT: mls v0.8h, v2.8h, v1.8h @@ -1207,8 +1207,8 @@ define <8 x i16> @urem_v8i16(<8 x i16> %op1, <8 x i16> %op2) #0 { define void @urem_v16i16(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: urem_v16i16: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q4, q1, [x1] +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldr q0, [x0, #16] ; VBITS_GE_128-NEXT: ushll2 v2.4s, v1.8h, #0 ; VBITS_GE_128-NEXT: ushll2 v3.4s, v0.8h, #0 @@ -1400,25 +1400,25 @@ define void @urem_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @urem_v16i32(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: urem_v16i32: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1] +; VBITS_GE_128-NEXT: ptrue p0.s, vl4 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0] ; VBITS_GE_128-NEXT: ldp q16, q5, [x0, #32] ; VBITS_GE_128-NEXT: ldp q17, q6, [x1, #32] ; VBITS_GE_128-NEXT: movprfx z4, z1 ; VBITS_GE_128-NEXT: udiv z4.s, p0/m, z4.s, z0.s +; VBITS_GE_128-NEXT: movprfx z19, z2 +; VBITS_GE_128-NEXT: udiv z19.s, p0/m, z19.s, z3.s ; VBITS_GE_128-NEXT: movprfx z7, z5 ; VBITS_GE_128-NEXT: udiv z7.s, p0/m, z7.s, z6.s ; VBITS_GE_128-NEXT: movprfx z18, z16 ; VBITS_GE_128-NEXT: udiv z18.s, p0/m, z18.s, z17.s -; VBITS_GE_128-NEXT: movprfx z19, z2 -; VBITS_GE_128-NEXT: udiv z19.s, p0/m, z19.s, z3.s -; VBITS_GE_128-NEXT: mls v16.4s, v18.4s, v17.4s -; VBITS_GE_128-NEXT: mls v5.4s, v7.4s, v6.4s ; VBITS_GE_128-NEXT: mls v1.4s, v4.4s, v0.4s ; VBITS_GE_128-NEXT: mls v2.4s, v19.4s, v3.4s -; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32] +; VBITS_GE_128-NEXT: mls v16.4s, v18.4s, v17.4s +; VBITS_GE_128-NEXT: mls v5.4s, v7.4s, v6.4s ; VBITS_GE_128-NEXT: stp q1, q2, [x0] +; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32] ; VBITS_GE_128-NEXT: ret ; ; VBITS_GE_256-LABEL: urem_v16i32: @@ -1548,26 +1548,26 @@ define void @urem_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @urem_v8i64(ptr %a, ptr %b) #0 { ; VBITS_GE_128-LABEL: urem_v8i64: ; VBITS_GE_128: // %bb.0: -; VBITS_GE_128-NEXT: ptrue p0.d, vl2 ; VBITS_GE_128-NEXT: ldp q0, q3, [x1] +; VBITS_GE_128-NEXT: ptrue p0.d, vl2 ; VBITS_GE_128-NEXT: ldp q1, q2, [x0] ; VBITS_GE_128-NEXT: ldp q16, q5, [x0, #32] ; VBITS_GE_128-NEXT: ldp q17, q6, [x1, #32] ; VBITS_GE_128-NEXT: movprfx z4, z1 ; VBITS_GE_128-NEXT: udiv z4.d, p0/m, z4.d, z0.d +; VBITS_GE_128-NEXT: movprfx z19, z2 +; VBITS_GE_128-NEXT: udiv z19.d, p0/m, z19.d, z3.d ; VBITS_GE_128-NEXT: movprfx z7, z5 ; VBITS_GE_128-NEXT: udiv z7.d, p0/m, z7.d, z6.d ; VBITS_GE_128-NEXT: movprfx z18, z16 ; VBITS_GE_128-NEXT: udiv z18.d, p0/m, z18.d, z17.d -; VBITS_GE_128-NEXT: movprfx z19, z2 -; VBITS_GE_128-NEXT: udiv z19.d, p0/m, z19.d, z3.d -; VBITS_GE_128-NEXT: mls z16.d, p0/m, z18.d, z17.d -; VBITS_GE_128-NEXT: mls z5.d, p0/m, z7.d, z6.d ; VBITS_GE_128-NEXT: msb z0.d, p0/m, z4.d, z1.d ; VBITS_GE_128-NEXT: movprfx z1, z2 ; VBITS_GE_128-NEXT: mls z1.d, p0/m, z19.d, z3.d -; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32] +; VBITS_GE_128-NEXT: mls z16.d, p0/m, z18.d, z17.d +; VBITS_GE_128-NEXT: mls z5.d, p0/m, z7.d, z6.d ; VBITS_GE_128-NEXT: stp q0, q1, [x0] +; VBITS_GE_128-NEXT: stp q16, q5, [x0, #32] ; VBITS_GE_128-NEXT: ret ; ; VBITS_GE_256-LABEL: urem_v8i64: diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-select.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-select.ll index 710dce4de6dd..37396ba7011b 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-select.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-select.ll @@ -34,14 +34,14 @@ define <16 x i8> @select_v16i8(<16 x i8> %op1, <16 x i8> %op2, i1 %mask) vscale_ define void @select_v32i8(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { ; CHECK-LABEL: select_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl32 ; CHECK-NEXT: mov z0.b, w2 -; CHECK-NEXT: ptrue p1.b -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.b, p1/z, z0.b, #0 -; CHECK-NEXT: sel z0.b, p1, z1.b, z2.b -; CHECK-NEXT: st1b { z0.b }, p0, [x0] +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: ptrue p1.b, vl32 +; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: ld1b { z0.b }, p1/z, [x0] +; CHECK-NEXT: ld1b { z1.b }, p1/z, [x1] +; CHECK-NEXT: sel z0.b, p0, z0.b, z1.b +; CHECK-NEXT: st1b { z0.b }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <32 x i8>, ptr %a %op2 = load volatile <32 x i8>, ptr %b @@ -53,31 +53,31 @@ define void @select_v32i8(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { define void @select_v64i8(ptr %a, ptr %b, i1 %mask) #0 { ; VBITS_GE_256-LABEL: select_v64i8: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.b, vl32 ; VBITS_GE_256-NEXT: mov z0.b, w2 +; VBITS_GE_256-NEXT: ptrue p0.b ; VBITS_GE_256-NEXT: mov w8, #32 // =0x20 -; VBITS_GE_256-NEXT: ptrue p1.b -; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0, x8] -; VBITS_GE_256-NEXT: ld1b { z2.b }, p0/z, [x0] -; VBITS_GE_256-NEXT: ld1b { z3.b }, p0/z, [x1, x8] -; VBITS_GE_256-NEXT: cmpne p1.b, p1/z, z0.b, #0 -; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x1] -; VBITS_GE_256-NEXT: sel z1.b, p1, z1.b, z3.b -; VBITS_GE_256-NEXT: mov z0.b, p1/m, z2.b -; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x0, x8] -; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0] +; VBITS_GE_256-NEXT: ptrue p1.b, vl32 +; VBITS_GE_256-NEXT: cmpne p0.b, p0/z, z0.b, #0 +; VBITS_GE_256-NEXT: ld1b { z0.b }, p1/z, [x0, x8] +; VBITS_GE_256-NEXT: ld1b { z1.b }, p1/z, [x0] +; VBITS_GE_256-NEXT: ld1b { z2.b }, p1/z, [x1, x8] +; VBITS_GE_256-NEXT: ld1b { z3.b }, p1/z, [x1] +; VBITS_GE_256-NEXT: sel z0.b, p0, z0.b, z2.b +; VBITS_GE_256-NEXT: sel z1.b, p0, z1.b, z3.b +; VBITS_GE_256-NEXT: st1b { z0.b }, p1, [x0, x8] +; VBITS_GE_256-NEXT: st1b { z1.b }, p1, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: select_v64i8: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.b, vl64 ; VBITS_GE_512-NEXT: mov z0.b, w2 -; VBITS_GE_512-NEXT: ptrue p1.b -; VBITS_GE_512-NEXT: ld1b { z1.b }, p0/z, [x0] -; VBITS_GE_512-NEXT: ld1b { z2.b }, p0/z, [x1] -; VBITS_GE_512-NEXT: cmpne p1.b, p1/z, z0.b, #0 -; VBITS_GE_512-NEXT: sel z0.b, p1, z1.b, z2.b -; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0] +; VBITS_GE_512-NEXT: ptrue p0.b +; VBITS_GE_512-NEXT: ptrue p1.b, vl64 +; VBITS_GE_512-NEXT: cmpne p0.b, p0/z, z0.b, #0 +; VBITS_GE_512-NEXT: ld1b { z0.b }, p1/z, [x0] +; VBITS_GE_512-NEXT: ld1b { z1.b }, p1/z, [x1] +; VBITS_GE_512-NEXT: sel z0.b, p0, z0.b, z1.b +; VBITS_GE_512-NEXT: st1b { z0.b }, p1, [x0] ; VBITS_GE_512-NEXT: ret %op1 = load volatile <64 x i8>, ptr %a %op2 = load volatile <64 x i8>, ptr %b @@ -89,14 +89,14 @@ define void @select_v64i8(ptr %a, ptr %b, i1 %mask) #0 { define void @select_v128i8(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { ; CHECK-LABEL: select_v128i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl128 ; CHECK-NEXT: mov z0.b, w2 -; CHECK-NEXT: ptrue p1.b -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.b, p1/z, z0.b, #0 -; CHECK-NEXT: sel z0.b, p1, z1.b, z2.b -; CHECK-NEXT: st1b { z0.b }, p0, [x0] +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: ptrue p1.b, vl128 +; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: ld1b { z0.b }, p1/z, [x0] +; CHECK-NEXT: ld1b { z1.b }, p1/z, [x1] +; CHECK-NEXT: sel z0.b, p0, z0.b, z1.b +; CHECK-NEXT: st1b { z0.b }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <128 x i8>, ptr %a %op2 = load volatile <128 x i8>, ptr %b @@ -108,14 +108,14 @@ define void @select_v128i8(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { define void @select_v256i8(ptr %a, ptr %b, i1 %mask) vscale_range(16,0) #0 { ; CHECK-LABEL: select_v256i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl256 ; CHECK-NEXT: mov z0.b, w2 -; CHECK-NEXT: ptrue p1.b -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0] -; CHECK-NEXT: ld1b { z2.b }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.b, p1/z, z0.b, #0 -; CHECK-NEXT: sel z0.b, p1, z1.b, z2.b -; CHECK-NEXT: st1b { z0.b }, p0, [x0] +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: ptrue p1.b, vl256 +; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: ld1b { z0.b }, p1/z, [x0] +; CHECK-NEXT: ld1b { z1.b }, p1/z, [x1] +; CHECK-NEXT: sel z0.b, p0, z0.b, z1.b +; CHECK-NEXT: st1b { z0.b }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <256 x i8>, ptr %a %op2 = load volatile <256 x i8>, ptr %b @@ -153,15 +153,15 @@ define <8 x i16> @select_v8i16(<8 x i16> %op1, <8 x i16> %op2, i1 %mask) vscale_ define void @select_v16i16(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { ; CHECK-LABEL: select_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: mov z0.h, w2 -; CHECK-NEXT: ptrue p1.h +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: ptrue p1.h, vl16 ; CHECK-NEXT: and z0.h, z0.h, #0x1 -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; CHECK-NEXT: sel z0.h, p1, z1.h, z2.h -; CHECK-NEXT: st1h { z0.h }, p0, [x0] +; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: ld1h { z1.h }, p1/z, [x1] +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h +; CHECK-NEXT: st1h { z0.h }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <16 x i16>, ptr %a %op2 = load volatile <16 x i16>, ptr %b @@ -173,33 +173,33 @@ define void @select_v16i16(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { define void @select_v32i16(ptr %a, ptr %b, i1 %mask) #0 { ; VBITS_GE_256-LABEL: select_v32i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: mov z0.h, w2 +; VBITS_GE_256-NEXT: ptrue p0.h ; VBITS_GE_256-NEXT: mov x8, #16 // =0x10 -; VBITS_GE_256-NEXT: ptrue p1.h +; VBITS_GE_256-NEXT: ptrue p1.h, vl16 ; VBITS_GE_256-NEXT: and z0.h, z0.h, #0x1 -; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0, x8, lsl #1] -; VBITS_GE_256-NEXT: ld1h { z2.h }, p0/z, [x0] -; VBITS_GE_256-NEXT: ld1h { z3.h }, p0/z, [x1, x8, lsl #1] -; VBITS_GE_256-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x1] -; VBITS_GE_256-NEXT: sel z1.h, p1, z1.h, z3.h -; VBITS_GE_256-NEXT: mov z0.h, p1/m, z2.h -; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x0, x8, lsl #1] -; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0] +; VBITS_GE_256-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; VBITS_GE_256-NEXT: ld1h { z0.h }, p1/z, [x0, x8, lsl #1] +; VBITS_GE_256-NEXT: ld1h { z1.h }, p1/z, [x0] +; VBITS_GE_256-NEXT: ld1h { z2.h }, p1/z, [x1, x8, lsl #1] +; VBITS_GE_256-NEXT: ld1h { z3.h }, p1/z, [x1] +; VBITS_GE_256-NEXT: sel z0.h, p0, z0.h, z2.h +; VBITS_GE_256-NEXT: sel z1.h, p0, z1.h, z3.h +; VBITS_GE_256-NEXT: st1h { z0.h }, p1, [x0, x8, lsl #1] +; VBITS_GE_256-NEXT: st1h { z1.h }, p1, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: select_v32i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.h, vl32 ; VBITS_GE_512-NEXT: mov z0.h, w2 -; VBITS_GE_512-NEXT: ptrue p1.h +; VBITS_GE_512-NEXT: ptrue p0.h +; VBITS_GE_512-NEXT: ptrue p1.h, vl32 ; VBITS_GE_512-NEXT: and z0.h, z0.h, #0x1 -; VBITS_GE_512-NEXT: ld1h { z1.h }, p0/z, [x0] -; VBITS_GE_512-NEXT: ld1h { z2.h }, p0/z, [x1] -; VBITS_GE_512-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; VBITS_GE_512-NEXT: sel z0.h, p1, z1.h, z2.h -; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0] +; VBITS_GE_512-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; VBITS_GE_512-NEXT: ld1h { z0.h }, p1/z, [x0] +; VBITS_GE_512-NEXT: ld1h { z1.h }, p1/z, [x1] +; VBITS_GE_512-NEXT: sel z0.h, p0, z0.h, z1.h +; VBITS_GE_512-NEXT: st1h { z0.h }, p1, [x0] ; VBITS_GE_512-NEXT: ret %op1 = load volatile <32 x i16>, ptr %a %op2 = load volatile <32 x i16>, ptr %b @@ -211,15 +211,15 @@ define void @select_v32i16(ptr %a, ptr %b, i1 %mask) #0 { define void @select_v64i16(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { ; CHECK-LABEL: select_v64i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: mov z0.h, w2 -; CHECK-NEXT: ptrue p1.h +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: ptrue p1.h, vl64 ; CHECK-NEXT: and z0.h, z0.h, #0x1 -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; CHECK-NEXT: sel z0.h, p1, z1.h, z2.h -; CHECK-NEXT: st1h { z0.h }, p0, [x0] +; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: ld1h { z1.h }, p1/z, [x1] +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h +; CHECK-NEXT: st1h { z0.h }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <64 x i16>, ptr %a %op2 = load volatile <64 x i16>, ptr %b @@ -231,15 +231,15 @@ define void @select_v64i16(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { define void @select_v128i16(ptr %a, ptr %b, i1 %mask) vscale_range(16,0) #0 { ; CHECK-LABEL: select_v128i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl128 ; CHECK-NEXT: mov z0.h, w2 -; CHECK-NEXT: ptrue p1.h +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: ptrue p1.h, vl128 ; CHECK-NEXT: and z0.h, z0.h, #0x1 -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0] -; CHECK-NEXT: ld1h { z2.h }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.h, p1/z, z0.h, #0 -; CHECK-NEXT: sel z0.h, p1, z1.h, z2.h -; CHECK-NEXT: st1h { z0.h }, p0, [x0] +; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: ld1h { z0.h }, p1/z, [x0] +; CHECK-NEXT: ld1h { z1.h }, p1/z, [x1] +; CHECK-NEXT: sel z0.h, p0, z0.h, z1.h +; CHECK-NEXT: st1h { z0.h }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <128 x i16>, ptr %a %op2 = load volatile <128 x i16>, ptr %b @@ -277,15 +277,15 @@ define <4 x i32> @select_v4i32(<4 x i32> %op1, <4 x i32> %op2, i1 %mask) vscale_ define void @select_v8i32(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { ; CHECK-LABEL: select_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: and w8, w2, #0x1 -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z0.s, w8 -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; CHECK-NEXT: sel z0.s, p1, z1.s, z2.s -; CHECK-NEXT: st1w { z0.s }, p0, [x0] +; CHECK-NEXT: ptrue p1.s, vl8 +; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0] +; CHECK-NEXT: ld1w { z1.s }, p1/z, [x1] +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: st1w { z0.s }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <8 x i32>, ptr %a %op2 = load volatile <8 x i32>, ptr %b @@ -297,33 +297,33 @@ define void @select_v8i32(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { define void @select_v16i32(ptr %a, ptr %b, i1 %mask) #0 { ; VBITS_GE_256-LABEL: select_v16i32: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: and w8, w2, #0x1 -; VBITS_GE_256-NEXT: ptrue p1.s +; VBITS_GE_256-NEXT: ptrue p0.s ; VBITS_GE_256-NEXT: mov z0.s, w8 +; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 -; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0, x8, lsl #2] -; VBITS_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0] -; VBITS_GE_256-NEXT: ld1w { z3.s }, p0/z, [x1, x8, lsl #2] -; VBITS_GE_256-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x1] -; VBITS_GE_256-NEXT: sel z1.s, p1, z1.s, z3.s -; VBITS_GE_256-NEXT: mov z0.s, p1/m, z2.s -; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0, x8, lsl #2] -; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0] +; VBITS_GE_256-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x0, x8, lsl #2] +; VBITS_GE_256-NEXT: ld1w { z1.s }, p1/z, [x0] +; VBITS_GE_256-NEXT: ld1w { z2.s }, p1/z, [x1, x8, lsl #2] +; VBITS_GE_256-NEXT: ld1w { z3.s }, p1/z, [x1] +; VBITS_GE_256-NEXT: sel z0.s, p0, z0.s, z2.s +; VBITS_GE_256-NEXT: sel z1.s, p0, z1.s, z3.s +; VBITS_GE_256-NEXT: st1w { z0.s }, p1, [x0, x8, lsl #2] +; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: select_v16i32: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl16 ; VBITS_GE_512-NEXT: and w8, w2, #0x1 -; VBITS_GE_512-NEXT: ptrue p1.s +; VBITS_GE_512-NEXT: ptrue p0.s ; VBITS_GE_512-NEXT: mov z0.s, w8 -; VBITS_GE_512-NEXT: ld1w { z1.s }, p0/z, [x0] -; VBITS_GE_512-NEXT: ld1w { z2.s }, p0/z, [x1] -; VBITS_GE_512-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; VBITS_GE_512-NEXT: sel z0.s, p1, z1.s, z2.s -; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0] +; VBITS_GE_512-NEXT: ptrue p1.s, vl16 +; VBITS_GE_512-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; VBITS_GE_512-NEXT: ld1w { z0.s }, p1/z, [x0] +; VBITS_GE_512-NEXT: ld1w { z1.s }, p1/z, [x1] +; VBITS_GE_512-NEXT: sel z0.s, p0, z0.s, z1.s +; VBITS_GE_512-NEXT: st1w { z0.s }, p1, [x0] ; VBITS_GE_512-NEXT: ret %op1 = load volatile <16 x i32>, ptr %a %op2 = load volatile <16 x i32>, ptr %b @@ -335,15 +335,15 @@ define void @select_v16i32(ptr %a, ptr %b, i1 %mask) #0 { define void @select_v32i32(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { ; CHECK-LABEL: select_v32i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: and w8, w2, #0x1 -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z0.s, w8 -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; CHECK-NEXT: sel z0.s, p1, z1.s, z2.s -; CHECK-NEXT: st1w { z0.s }, p0, [x0] +; CHECK-NEXT: ptrue p1.s, vl32 +; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0] +; CHECK-NEXT: ld1w { z1.s }, p1/z, [x1] +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: st1w { z0.s }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <32 x i32>, ptr %a %op2 = load volatile <32 x i32>, ptr %b @@ -355,15 +355,15 @@ define void @select_v32i32(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { define void @select_v64i32(ptr %a, ptr %b, i1 %mask) vscale_range(16,0) #0 { ; CHECK-LABEL: select_v64i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl64 ; CHECK-NEXT: and w8, w2, #0x1 -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z0.s, w8 -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.s, p1/z, z0.s, #0 -; CHECK-NEXT: sel z0.s, p1, z1.s, z2.s -; CHECK-NEXT: st1w { z0.s }, p0, [x0] +; CHECK-NEXT: ptrue p1.s, vl64 +; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: ld1w { z0.s }, p1/z, [x0] +; CHECK-NEXT: ld1w { z1.s }, p1/z, [x1] +; CHECK-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-NEXT: st1w { z0.s }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <64 x i32>, ptr %a %op2 = load volatile <64 x i32>, ptr %b @@ -401,16 +401,16 @@ define <2 x i64> @select_v2i64(<2 x i64> %op1, <2 x i64> %op2, i1 %mask) vscale_ define void @select_v4i64(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { ; CHECK-LABEL: select_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2 ; CHECK-NEXT: and x8, x2, #0x1 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, x8 -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; CHECK-NEXT: sel z0.d, p1, z1.d, z2.d -; CHECK-NEXT: st1d { z0.d }, p0, [x0] +; CHECK-NEXT: ptrue p1.d, vl4 +; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: st1d { z0.d }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <4 x i64>, ptr %a %op2 = load volatile <4 x i64>, ptr %b @@ -422,35 +422,35 @@ define void @select_v4i64(ptr %a, ptr %b, i1 %mask) vscale_range(2,0) #0 { define void @select_v8i64(ptr %a, ptr %b, i1 %mask) #0 { ; VBITS_GE_256-LABEL: select_v8i64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: // kill: def $w2 killed $w2 def $x2 ; VBITS_GE_256-NEXT: and x8, x2, #0x1 -; VBITS_GE_256-NEXT: ptrue p1.d +; VBITS_GE_256-NEXT: ptrue p0.d ; VBITS_GE_256-NEXT: mov z0.d, x8 +; VBITS_GE_256-NEXT: ptrue p1.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0, x8, lsl #3] -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x0] -; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1, x8, lsl #3] -; VBITS_GE_256-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1] -; VBITS_GE_256-NEXT: sel z1.d, p1, z1.d, z3.d -; VBITS_GE_256-NEXT: mov z0.d, p1/m, z2.d -; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0, x8, lsl #3] -; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0] +; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p1/z, [x0, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1d { z1.d }, p1/z, [x0] +; VBITS_GE_256-NEXT: ld1d { z2.d }, p1/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1d { z3.d }, p1/z, [x1] +; VBITS_GE_256-NEXT: sel z0.d, p0, z0.d, z2.d +; VBITS_GE_256-NEXT: sel z1.d, p0, z1.d, z3.d +; VBITS_GE_256-NEXT: st1d { z0.d }, p1, [x0, x8, lsl #3] +; VBITS_GE_256-NEXT: st1d { z1.d }, p1, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: select_v8i64: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: // kill: def $w2 killed $w2 def $x2 ; VBITS_GE_512-NEXT: and x8, x2, #0x1 -; VBITS_GE_512-NEXT: ptrue p1.d +; VBITS_GE_512-NEXT: ptrue p0.d ; VBITS_GE_512-NEXT: mov z0.d, x8 -; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x0] -; VBITS_GE_512-NEXT: ld1d { z2.d }, p0/z, [x1] -; VBITS_GE_512-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; VBITS_GE_512-NEXT: sel z0.d, p1, z1.d, z2.d -; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0] +; VBITS_GE_512-NEXT: ptrue p1.d, vl8 +; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; VBITS_GE_512-NEXT: ld1d { z0.d }, p1/z, [x0] +; VBITS_GE_512-NEXT: ld1d { z1.d }, p1/z, [x1] +; VBITS_GE_512-NEXT: sel z0.d, p0, z0.d, z1.d +; VBITS_GE_512-NEXT: st1d { z0.d }, p1, [x0] ; VBITS_GE_512-NEXT: ret %op1 = load volatile <8 x i64>, ptr %a %op2 = load volatile <8 x i64>, ptr %b @@ -462,16 +462,16 @@ define void @select_v8i64(ptr %a, ptr %b, i1 %mask) #0 { define void @select_v16i64(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { ; CHECK-LABEL: select_v16i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2 ; CHECK-NEXT: and x8, x2, #0x1 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, x8 -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; CHECK-NEXT: sel z0.d, p1, z1.d, z2.d -; CHECK-NEXT: st1d { z0.d }, p0, [x0] +; CHECK-NEXT: ptrue p1.d, vl16 +; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: st1d { z0.d }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <16 x i64>, ptr %a %op2 = load volatile <16 x i64>, ptr %b @@ -483,16 +483,16 @@ define void @select_v16i64(ptr %a, ptr %b, i1 %mask) vscale_range(8,0) #0 { define void @select_v32i64(ptr %a, ptr %b, i1 %mask) vscale_range(16,0) #0 { ; CHECK-LABEL: select_v32i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2 ; CHECK-NEXT: and x8, x2, #0x1 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, x8 -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p1.d, p1/z, z0.d, #0 -; CHECK-NEXT: sel z0.d, p1, z1.d, z2.d -; CHECK-NEXT: st1d { z0.d }, p0, [x0] +; CHECK-NEXT: ptrue p1.d, vl32 +; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p1/z, [x0] +; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] +; CHECK-NEXT: sel z0.d, p0, z0.d, z1.d +; CHECK-NEXT: st1d { z0.d }, p1, [x0] ; CHECK-NEXT: ret %op1 = load volatile <32 x i64>, ptr %a %op2 = load volatile <32 x i64>, ptr %b diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-to-fp.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-to-fp.ll index 50040eaa61e6..5bb012ae5750 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-to-fp.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-to-fp.ll @@ -131,8 +131,8 @@ define <4 x float> @ucvtf_v4i16_v4f32(<4 x i16> %op1) vscale_range(2,0) #0 { define void @ucvtf_v8i16_v8f32(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: ucvtf_v8i16_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: ucvtf z0.s, p0/m, z0.s ; CHECK-NEXT: st1w { z0.s }, p0, [x1] @@ -354,7 +354,6 @@ define void @ucvtf_v16i32_v16f16(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 -; VBITS_GE_256-NEXT: ptrue p1.h, vl16 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] ; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0] ; VBITS_GE_256-NEXT: ucvtf z0.h, p0/m, z0.s @@ -363,7 +362,8 @@ define void @ucvtf_v16i32_v16f16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: splice z1.h, p0, z1.h, z0.h -; VBITS_GE_256-NEXT: st1h { z1.h }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.h, vl16 +; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: ucvtf_v16i32_v16f16: @@ -535,8 +535,8 @@ define <2 x double> @ucvtf_v2i32_v2f64(<2 x i32> %op1) vscale_range(2,0) #0 { define void @ucvtf_v4i32_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: ucvtf_v4i32_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: ucvtf z0.d, p0/m, z0.d ; CHECK-NEXT: st1d { z0.d }, p0, [x1] @@ -759,7 +759,6 @@ define void @ucvtf_v8i64_v8f32(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] ; VBITS_GE_256-NEXT: ucvtf z0.s, p0/m, z0.d @@ -768,7 +767,8 @@ define void @ucvtf_v8i64_v8f32(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s -; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 +; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: ucvtf_v8i64_v8f32: @@ -1038,8 +1038,8 @@ define <4 x float> @scvtf_v4i16_v4f32(<4 x i16> %op1) vscale_range(2,0) #0 { define void @scvtf_v8i16_v8f32(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: scvtf_v8i16_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: sunpklo z0.s, z0.h ; CHECK-NEXT: scvtf z0.s, p0/m, z0.s ; CHECK-NEXT: st1w { z0.s }, p0, [x1] @@ -1273,7 +1273,6 @@ define void @scvtf_v16i32_v16f16(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 -; VBITS_GE_256-NEXT: ptrue p1.h, vl16 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] ; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0] ; VBITS_GE_256-NEXT: scvtf z0.h, p0/m, z0.s @@ -1282,7 +1281,8 @@ define void @scvtf_v16i32_v16f16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: splice z1.h, p0, z1.h, z0.h -; VBITS_GE_256-NEXT: st1h { z1.h }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.h, vl16 +; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: scvtf_v16i32_v16f16: @@ -1454,8 +1454,8 @@ define <2 x double> @scvtf_v2i32_v2f64(<2 x i32> %op1) vscale_range(2,0) #0 { define void @scvtf_v4i32_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: scvtf_v4i32_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: sunpklo z0.d, z0.s ; CHECK-NEXT: scvtf z0.d, p0/m, z0.d ; CHECK-NEXT: st1d { z0.d }, p0, [x1] @@ -1684,7 +1684,6 @@ define void @scvtf_v8i64_v8f32(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] ; VBITS_GE_256-NEXT: scvtf z0.s, p0/m, z0.d @@ -1693,7 +1692,8 @@ define void @scvtf_v8i64_v8f32(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s -; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 +; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: scvtf_v8i64_v8f32: diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-mask-opt.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-mask-opt.ll index e23151475014..f2ad98f8caec 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-mask-opt.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-mask-opt.ll @@ -246,16 +246,16 @@ define void @masked_gather_v8i32(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: ld1w { z0.d }, p0/z, [z0.d] ; VBITS_GE_256-NEXT: ld1w { z1.d }, p0/z, [z1.d] -; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s +; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s -; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x0] +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 +; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_gather_v8i32: diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-loads.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-loads.ll index 67a53d4e15f3..55d37d1bda5e 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-loads.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-loads.ll @@ -11,8 +11,8 @@ target triple = "aarch64-unknown-linux-gnu" define <16 x i8> @masked_load_v16i8(ptr %src, <16 x i1> %mask) { ; CHECK-LABEL: masked_load_v16i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: shl v0.16b, v0.16b, #7 +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: cmlt v0.16b, v0.16b, #0 ; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-stores.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-stores.ll index bdd6ce064701..1a19b77f53c6 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-stores.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-128bit-stores.ll @@ -11,12 +11,12 @@ target triple = "aarch64-unknown-linux-gnu" define void @masked_store_v16i8(ptr %dst, <16 x i1> %mask) { ; CHECK-LABEL: masked_store_v16i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: shl v0.16b, v0.16b, #7 -; CHECK-NEXT: movi v1.2d, #0000000000000000 +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: cmlt v0.16b, v0.16b, #0 ; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 -; CHECK-NEXT: st1b { z1.b }, p0, [x0] +; CHECK-NEXT: movi v0.2d, #0000000000000000 +; CHECK-NEXT: st1b { z0.b }, p0, [x0] ; CHECK-NEXT: ret call void @llvm.masked.store.v16i8(<16 x i8> zeroinitializer, ptr %dst, i32 8, <16 x i1> %mask) ret void @@ -27,11 +27,11 @@ define void @masked_store_v8f16(ptr %dst, <8 x i1> %mask) { ; CHECK: // %bb.0: ; CHECK-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-NEXT: ptrue p0.h, vl8 -; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: shl v0.8h, v0.8h, #15 ; CHECK-NEXT: cmlt v0.8h, v0.8h, #0 ; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 -; CHECK-NEXT: st1h { z1.h }, p0, [x0] +; CHECK-NEXT: movi v0.2d, #0000000000000000 +; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret call void @llvm.masked.store.v8f16(<8 x half> zeroinitializer, ptr %dst, i32 8, <8 x i1> %mask) ret void @@ -42,11 +42,11 @@ define void @masked_store_v4f32(ptr %dst, <4 x i1> %mask) { ; CHECK: // %bb.0: ; CHECK-NEXT: ushll v0.4s, v0.4h, #0 ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: shl v0.4s, v0.4s, #31 ; CHECK-NEXT: cmlt v0.4s, v0.4s, #0 ; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 -; CHECK-NEXT: st1w { z1.s }, p0, [x0] +; CHECK-NEXT: movi v0.2d, #0000000000000000 +; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret call void @llvm.masked.store.v4f32(<4 x float> zeroinitializer, ptr %dst, i32 8, <4 x i1> %mask) ret void @@ -57,11 +57,11 @@ define void @masked_store_v2f64(ptr %dst, <2 x i1> %mask) { ; CHECK: // %bb.0: ; CHECK-NEXT: ushll v0.2d, v0.2s, #0 ; CHECK-NEXT: ptrue p0.d, vl2 -; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: shl v0.2d, v0.2d, #63 ; CHECK-NEXT: cmlt v0.2d, v0.2d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: st1d { z1.d }, p0, [x0] +; CHECK-NEXT: movi v0.2d, #0000000000000000 +; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret call void @llvm.masked.store.v2f64(<2 x double> zeroinitializer, ptr %dst, i32 8, <2 x i1> %mask) ret void diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll index 92fce4584f6a..27e95489f8ad 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-gather.ll @@ -41,11 +41,11 @@ define void @masked_gather_v4i8(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-NEXT: cmeq v0.4h, v0.4h, #0 -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: sunpklo z0.s, z0.h ; CHECK-NEXT: sunpklo z0.d, z0.s ; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0 -; CHECK-NEXT: ld1b { z0.d }, p1/z, [z1.d] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] +; CHECK-NEXT: ld1b { z0.d }, p1/z, [z0.d] ; CHECK-NEXT: st1b { z0.d }, p0, [x0] ; CHECK-NEXT: ret %cval = load <4 x i8>, ptr %a @@ -65,7 +65,6 @@ define void @masked_gather_v8i8(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: cmeq v0.8b, v0.8b, #0 ; VBITS_GE_256-NEXT: zip2 v1.8b, v0.8b, v0.8b ; VBITS_GE_256-NEXT: zip1 v0.8b, v0.8b, v0.8b -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: shl v1.4h, v1.4h, #8 ; VBITS_GE_256-NEXT: shl v0.4h, v0.4h, #8 ; VBITS_GE_256-NEXT: sshr v1.4h, v1.4h, #8 @@ -75,10 +74,11 @@ define void @masked_gather_v8i8(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s ; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s ; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z1.d, #0 -; VBITS_GE_256-NEXT: ld1b { z1.d }, p1/z, [z2.d] -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1] -; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1b { z0.d }, p0/z, [z2.d] +; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1b { z1.d }, p1/z, [z1.d] +; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1] +; VBITS_GE_256-NEXT: ld1b { z0.d }, p1/z, [z0.d] ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s @@ -93,11 +93,11 @@ define void @masked_gather_v8i8(ptr %a, ptr %b) #0 { ; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: cmeq v0.8b, v0.8b, #0 ; VBITS_GE_512-NEXT: sunpklo z0.h, z0.b -; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h ; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s -; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; VBITS_GE_512-NEXT: ld1b { z0.d }, p0/z, [z1.d] +; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1] +; VBITS_GE_512-NEXT: ld1b { z0.d }, p1/z, [z0.d] ; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_512-NEXT: uzp1 z0.b, z0.b, z0.b @@ -118,11 +118,11 @@ define void @masked_gather_v16i8(ptr %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: cmeq v0.16b, v0.16b, #0 ; CHECK-NEXT: sunpklo z0.h, z0.b -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: sunpklo z0.s, z0.h ; CHECK-NEXT: sunpklo z0.d, z0.s -; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: ld1b { z0.d }, p0/z, [z1.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] +; CHECK-NEXT: ld1b { z0.d }, p1/z, [z0.d] ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b @@ -194,10 +194,10 @@ define void @masked_gather_v4i16(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: cmeq v0.4h, v0.4h, #0 ; CHECK-NEXT: sunpklo z0.s, z0.h -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: sunpklo z0.d, z0.s -; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: ld1h { z0.d }, p0/z, [z1.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] +; CHECK-NEXT: ld1h { z0.d }, p1/z, [z0.d] ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: str d0, [x0] @@ -219,15 +219,15 @@ define void @masked_gather_v8i16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: cmeq v0.8h, v0.8h, #0 ; VBITS_GE_256-NEXT: sunpklo z1.s, z0.h ; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v0.16b, #8 -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1] -; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h ; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s -; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s +; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h ; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z1.d, #0 -; VBITS_GE_256-NEXT: ld1h { z1.d }, p1/z, [z2.d] -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1h { z0.d }, p0/z, [z2.d] +; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s +; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1] +; VBITS_GE_256-NEXT: ld1h { z1.d }, p1/z, [z1.d] +; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1h { z0.d }, p1/z, [z0.d] ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s @@ -242,10 +242,10 @@ define void @masked_gather_v8i16(ptr %a, ptr %b) #0 { ; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: cmeq v0.8h, v0.8h, #0 ; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h -; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s -; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; VBITS_GE_512-NEXT: ld1h { z0.d }, p0/z, [z1.d] +; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1] +; VBITS_GE_512-NEXT: ld1h { z0.d }, p1/z, [z0.d] ; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_512-NEXT: str q0, [x0] @@ -332,9 +332,9 @@ define void @masked_gather_v4i32(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: cmeq v0.4s, v0.4s, #0 ; CHECK-NEXT: sunpklo z0.d, z0.s -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: ld1w { z0.d }, p0/z, [z1.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] +; CHECK-NEXT: ld1w { z0.d }, p1/z, [z0.d] ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: str q0, [x0] ; CHECK-NEXT: ret @@ -354,18 +354,18 @@ define void @masked_gather_v8i32(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: ptrue p2.d, vl4 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p2/z, [x1] -; VBITS_GE_256-NEXT: ld1d { z2.d }, p2/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: cmpeq p1.s, p0/z, z0.s, #0 -; VBITS_GE_256-NEXT: punpklo p3.h, p1.b ; VBITS_GE_256-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff +; VBITS_GE_256-NEXT: punpklo p1.h, p1.b +; VBITS_GE_256-NEXT: and p1.b, p1/z, p1.b, p2.b ; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16 -; VBITS_GE_256-NEXT: and p1.b, p3/z, p3.b, p2.b -; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s ; VBITS_GE_256-NEXT: ld1w { z1.d }, p1/z, [z1.d] +; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s ; VBITS_GE_256-NEXT: cmpne p1.d, p2/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1w { z0.d }, p1/z, [z2.d] -; VBITS_GE_256-NEXT: ptrue p1.s, vl4 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p2/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s +; VBITS_GE_256-NEXT: ld1w { z0.d }, p1/z, [z0.d] +; VBITS_GE_256-NEXT: ptrue p1.s, vl4 ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s ; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0] @@ -460,8 +460,8 @@ define void @masked_gather_v1i64(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @masked_gather_v2i64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: masked_gather_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: cmeq v0.2d, v0.2d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: ldr q0, [x1] @@ -481,9 +481,9 @@ define void @masked_gather_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #0 -; CHECK-NEXT: ld1d { z0.d }, p1/z, [z1.d] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] +; CHECK-NEXT: ld1d { z0.d }, p1/z, [z0.d] ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret %cval = load <4 x i64>, ptr %a @@ -500,13 +500,13 @@ define void @masked_gather_v8i64(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] -; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1, x8, lsl #3] -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1] -; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1d { z0.d }, p1/z, [z1.d] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] +; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z0.d, #0 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1d { z0.d }, p1/z, [z0.d] ; VBITS_GE_256-NEXT: cmpeq p1.d, p0/z, z1.d, #0 -; VBITS_GE_256-NEXT: ld1d { z1.d }, p1/z, [z2.d] +; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1] +; VBITS_GE_256-NEXT: ld1d { z1.d }, p1/z, [z1.d] ; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3] ; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0] ; VBITS_GE_256-NEXT: ret @@ -515,9 +515,9 @@ define void @masked_gather_v8i64(ptr %a, ptr %b) #0 { ; VBITS_GE_512: // %bb.0: ; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0] -; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0 -; VBITS_GE_512-NEXT: ld1d { z0.d }, p1/z, [z1.d] +; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1] +; VBITS_GE_512-NEXT: ld1d { z0.d }, p1/z, [z0.d] ; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0] ; VBITS_GE_512-NEXT: ret %cval = load <8 x i64>, ptr %a @@ -533,9 +533,9 @@ define void @masked_gather_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #0 -; CHECK-NEXT: ld1d { z0.d }, p1/z, [z1.d] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] +; CHECK-NEXT: ld1d { z0.d }, p1/z, [z0.d] ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret %cval = load <16 x i64>, ptr %a @@ -551,9 +551,9 @@ define void @masked_gather_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #0 -; CHECK-NEXT: ld1d { z0.d }, p1/z, [z1.d] +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] +; CHECK-NEXT: ld1d { z0.d }, p1/z, [z0.d] ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret %cval = load <32 x i64>, ptr %a @@ -603,10 +603,10 @@ define void @masked_gather_v4f16(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: fcmeq v0.4h, v0.4h, #0.0 ; CHECK-NEXT: sunpklo z0.s, z0.h -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] ; CHECK-NEXT: sunpklo z0.d, z0.s -; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: ld1h { z0.d }, p0/z, [z1.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] +; CHECK-NEXT: ld1h { z0.d }, p1/z, [z0.d] ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: str d0, [x0] @@ -626,17 +626,17 @@ define void @masked_gather_v8f16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: fcmeq v0.8h, v0.8h, #0.0 -; VBITS_GE_256-NEXT: sunpklo z2.s, z0.h +; VBITS_GE_256-NEXT: sunpklo z1.s, z0.h ; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v0.16b, #8 -; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1] +; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s ; VBITS_GE_256-NEXT: sunpklo z0.s, z0.h -; VBITS_GE_256-NEXT: sunpklo z2.d, z2.s +; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z1.d, #0 ; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s -; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z2.d, #0 -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: ld1h { z1.d }, p1/z, [z1.d] -; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1h { z0.d }, p0/z, [z2.d] +; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1h { z0.d }, p1/z, [z0.d] ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s @@ -651,10 +651,10 @@ define void @masked_gather_v8f16(ptr %a, ptr %b) #0 { ; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: fcmeq v0.8h, v0.8h, #0.0 ; VBITS_GE_512-NEXT: sunpklo z0.s, z0.h -; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: sunpklo z0.d, z0.s -; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; VBITS_GE_512-NEXT: ld1h { z0.d }, p0/z, [z1.d] +; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x1] +; VBITS_GE_512-NEXT: ld1h { z0.d }, p1/z, [z0.d] ; VBITS_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_512-NEXT: str q0, [x0] @@ -741,9 +741,9 @@ define void @masked_gather_v4f32(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: fcmeq v0.4s, v0.4s, #0.0 ; CHECK-NEXT: sunpklo z0.d, z0.s -; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: ld1w { z0.d }, p0/z, [z1.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1] +; CHECK-NEXT: ld1w { z0.d }, p1/z, [z0.d] ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: str q0, [x0] ; CHECK-NEXT: ret @@ -763,18 +763,18 @@ define void @masked_gather_v8f32(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: ptrue p2.d, vl4 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p2/z, [x1] -; VBITS_GE_256-NEXT: ld1d { z2.d }, p2/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: fcmeq p1.s, p0/z, z0.s, #0.0 -; VBITS_GE_256-NEXT: punpklo p3.h, p1.b ; VBITS_GE_256-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff +; VBITS_GE_256-NEXT: punpklo p1.h, p1.b +; VBITS_GE_256-NEXT: and p1.b, p1/z, p1.b, p2.b ; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16 -; VBITS_GE_256-NEXT: and p1.b, p3/z, p3.b, p2.b -; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s ; VBITS_GE_256-NEXT: ld1w { z1.d }, p1/z, [z1.d] +; VBITS_GE_256-NEXT: sunpklo z0.d, z0.s ; VBITS_GE_256-NEXT: cmpne p1.d, p2/z, z0.d, #0 -; VBITS_GE_256-NEXT: ld1w { z0.d }, p1/z, [z2.d] -; VBITS_GE_256-NEXT: ptrue p1.s, vl4 +; VBITS_GE_256-NEXT: ld1d { z0.d }, p2/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s +; VBITS_GE_256-NEXT: ld1w { z0.d }, p1/z, [z0.d] +; VBITS_GE_256-NEXT: ptrue p1.s, vl4 ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s ; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0] @@ -869,8 +869,8 @@ define void @masked_gather_v1f64(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @masked_gather_v2f64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: masked_gather_v2f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fcmeq v0.2d, v0.2d, #0.0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: ldr q0, [x1] @@ -1202,8 +1202,8 @@ define void @masked_gather_passthru(ptr %a, ptr %b, ptr %c) vscale_range(16,0) # ; CHECK-NEXT: ld1w { z1.s }, p0/z, [x2] ; CHECK-NEXT: fcmeq p1.s, p0/z, z0.s, #0.0 ; CHECK-NEXT: ld1d { z0.d }, p2/z, [x1] -; CHECK-NEXT: punpklo p3.h, p1.b -; CHECK-NEXT: ld1w { z0.d }, p3/z, [z0.d] +; CHECK-NEXT: punpklo p2.h, p1.b +; CHECK-NEXT: ld1w { z0.d }, p2/z, [z0.d] ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: sel z0.s, p1, z0.s, z1.s ; CHECK-NEXT: st1w { z0.s }, p0, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-loads.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-loads.ll index 467378e7da59..c22d9e71c51a 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-loads.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-loads.ll @@ -35,9 +35,9 @@ define <2 x half> @masked_load_v2f16(ptr %ap, ptr %bp) vscale_range(2,0) #0 { define <2 x float> @masked_load_v2f32(ptr %ap, ptr %bp) vscale_range(2,0) #0 { ; CHECK-LABEL: masked_load_v2f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: ldr d0, [x0] ; CHECK-NEXT: ldr d1, [x1] +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: fcmeq v0.2s, v0.2s, v1.2s ; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] @@ -53,9 +53,9 @@ define <2 x float> @masked_load_v2f32(ptr %ap, ptr %bp) vscale_range(2,0) #0 { define <4 x float> @masked_load_v4f32(ptr %ap, ptr %bp) vscale_range(1,0) #0 { ; CHECK-LABEL: masked_load_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fcmeq v0.4s, v0.4s, v1.4s ; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 ; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0] @@ -401,8 +401,8 @@ define void @masked_load_sext_v32i8i16(ptr %ap, ptr %bp, ptr %c) #0 { define void @masked_load_sext_v16i8i32(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-LABEL: masked_load_sext_v16i8i32: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.b, vl16 ; VBITS_GE_256-NEXT: ldr q0, [x1] +; VBITS_GE_256-NEXT: ptrue p0.b, vl16 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 ; VBITS_GE_256-NEXT: cmeq v0.16b, v0.16b, #0 ; VBITS_GE_256-NEXT: cmpne p0.b, p0/z, z0.b, #0 @@ -436,8 +436,8 @@ define void @masked_load_sext_v16i8i32(ptr %ap, ptr %bp, ptr %c) #0 { define void @masked_load_sext_v8i8i64(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-LABEL: masked_load_sext_v8i8i64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.b, vl8 ; VBITS_GE_256-NEXT: ldr d0, [x1] +; VBITS_GE_256-NEXT: ptrue p0.b, vl8 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: cmeq v0.8b, v0.8b, #0 ; VBITS_GE_256-NEXT: cmpne p0.b, p0/z, z0.b, #0 @@ -504,8 +504,8 @@ define void @masked_load_sext_v16i16i32(ptr %ap, ptr %bp, ptr %c) #0 { define void @masked_load_sext_v8i16i64(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-LABEL: masked_load_sext_v8i16i64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.h, vl8 ; VBITS_GE_256-NEXT: ldr q0, [x1] +; VBITS_GE_256-NEXT: ptrue p0.h, vl8 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: cmeq v0.8h, v0.8h, #0 ; VBITS_GE_256-NEXT: cmpne p0.h, p0/z, z0.h, #0 @@ -603,8 +603,8 @@ define void @masked_load_zext_v32i8i16(ptr %ap, ptr %bp, ptr %c) #0 { define void @masked_load_zext_v16i8i32(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-LABEL: masked_load_zext_v16i8i32: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.b, vl16 ; VBITS_GE_256-NEXT: ldr q0, [x1] +; VBITS_GE_256-NEXT: ptrue p0.b, vl16 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 ; VBITS_GE_256-NEXT: cmeq v0.16b, v0.16b, #0 ; VBITS_GE_256-NEXT: cmpne p0.b, p0/z, z0.b, #0 @@ -638,8 +638,8 @@ define void @masked_load_zext_v16i8i32(ptr %ap, ptr %bp, ptr %c) #0 { define void @masked_load_zext_v8i8i64(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-LABEL: masked_load_zext_v8i8i64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.b, vl8 ; VBITS_GE_256-NEXT: ldr d0, [x1] +; VBITS_GE_256-NEXT: ptrue p0.b, vl8 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: cmeq v0.8b, v0.8b, #0 ; VBITS_GE_256-NEXT: cmpne p0.b, p0/z, z0.b, #0 @@ -706,8 +706,8 @@ define void @masked_load_zext_v16i16i32(ptr %ap, ptr %bp, ptr %c) #0 { define void @masked_load_zext_v8i16i64(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-LABEL: masked_load_zext_v8i16i64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.h, vl8 ; VBITS_GE_256-NEXT: ldr q0, [x1] +; VBITS_GE_256-NEXT: ptrue p0.h, vl8 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: cmeq v0.8h, v0.8h, #0 ; VBITS_GE_256-NEXT: cmpne p0.h, p0/z, z0.h, #0 @@ -782,11 +782,11 @@ define void @masked_load_sext_v32i8i16_m16(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: ptrue p1.b, vl16 -; VBITS_GE_256-NEXT: ptrue p2.b, vl32 ; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b ; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b ; VBITS_GE_256-NEXT: splice z1.b, p1, z1.b, z0.b -; VBITS_GE_256-NEXT: cmpne p1.b, p2/z, z1.b, #0 +; VBITS_GE_256-NEXT: ptrue p1.b, vl32 +; VBITS_GE_256-NEXT: cmpne p1.b, p1/z, z1.b, #0 ; VBITS_GE_256-NEXT: ld1b { z0.b }, p1/z, [x0] ; VBITS_GE_256-NEXT: sunpklo z1.h, z0.b ; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16 @@ -1000,11 +1000,11 @@ define void @masked_load_sext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: mov z1.d, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: ptrue p1.s, vl4 -; VBITS_GE_256-NEXT: ptrue p2.s, vl8 ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s -; VBITS_GE_256-NEXT: cmpne p1.s, p2/z, z1.s, #0 +; VBITS_GE_256-NEXT: ptrue p1.s, vl8 +; VBITS_GE_256-NEXT: cmpne p1.s, p1/z, z1.s, #0 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x0] ; VBITS_GE_256-NEXT: sunpklo z1.d, z0.s ; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16 @@ -1041,11 +1041,11 @@ define void @masked_load_zext_v32i8i16_m16(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: ptrue p1.b, vl16 -; VBITS_GE_256-NEXT: ptrue p2.b, vl32 ; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b ; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b ; VBITS_GE_256-NEXT: splice z1.b, p1, z1.b, z0.b -; VBITS_GE_256-NEXT: cmpne p1.b, p2/z, z1.b, #0 +; VBITS_GE_256-NEXT: ptrue p1.b, vl32 +; VBITS_GE_256-NEXT: cmpne p1.b, p1/z, z1.b, #0 ; VBITS_GE_256-NEXT: ld1b { z0.b }, p1/z, [x0] ; VBITS_GE_256-NEXT: uunpklo z1.h, z0.b ; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16 @@ -1259,11 +1259,11 @@ define void @masked_load_zext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 { ; VBITS_GE_256-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: mov z1.d, p2/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: ptrue p1.s, vl4 -; VBITS_GE_256-NEXT: ptrue p2.s, vl8 ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: splice z1.s, p1, z1.s, z0.s -; VBITS_GE_256-NEXT: cmpne p1.s, p2/z, z1.s, #0 +; VBITS_GE_256-NEXT: ptrue p1.s, vl8 +; VBITS_GE_256-NEXT: cmpne p1.s, p1/z, z1.s, #0 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p1/z, [x0] ; VBITS_GE_256-NEXT: uunpklo z1.d, z0.s ; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16 diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll index e2d341c22efc..e3e06dcdf17f 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-scatter.ll @@ -39,12 +39,12 @@ define void @masked_scatter_v4i8(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-NEXT: cmeq v1.4h, v0.4h, #0 ; CHECK-NEXT: uunpklo z0.s, z0.h -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] ; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: sunpklo z1.d, z1.s -; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; CHECK-NEXT: st1b { z0.d }, p0, [z2.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] +; CHECK-NEXT: st1b { z0.d }, p1, [z1.d] ; CHECK-NEXT: ret %vals = load <4 x i8>, ptr %a %ptrs = load <4 x ptr>, ptr %b @@ -61,11 +61,11 @@ define void @masked_scatter_v8i8(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: cmeq v1.8b, v0.8b, #0 ; VBITS_GE_256-NEXT: zip1 v3.8b, v0.8b, v0.8b +; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: zip1 v2.8b, v1.8b, v0.8b ; VBITS_GE_256-NEXT: zip2 v1.8b, v1.8b, v0.8b ; VBITS_GE_256-NEXT: zip2 v0.8b, v0.8b, v0.8b ; VBITS_GE_256-NEXT: uunpklo z3.s, z3.h -; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: shl v2.4h, v2.4h, #8 ; VBITS_GE_256-NEXT: shl v1.4h, v1.4h, #8 ; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h @@ -78,10 +78,10 @@ define void @masked_scatter_v8i8(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: sunpklo z2.d, z2.s ; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s ; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z2.d, #0 -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; VBITS_GE_256-NEXT: st1b { z3.d }, p1, [z4.d] -; VBITS_GE_256-NEXT: st1b { z0.d }, p0, [z2.d] +; VBITS_GE_256-NEXT: st1b { z3.d }, p1, [z2.d] +; VBITS_GE_256-NEXT: st1b { z0.d }, p0, [z4.d] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_scatter_v8i8: @@ -92,12 +92,12 @@ define void @masked_scatter_v8i8(ptr %a, ptr %b) #0 { ; VBITS_GE_512-NEXT: uunpklo z0.h, z0.b ; VBITS_GE_512-NEXT: sunpklo z1.h, z1.b ; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h -; VBITS_GE_512-NEXT: ld1d { z2.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: sunpklo z1.s, z1.h ; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_512-NEXT: sunpklo z1.d, z1.s -; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; VBITS_GE_512-NEXT: st1b { z0.d }, p0, [z2.d] +; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] +; VBITS_GE_512-NEXT: st1b { z0.d }, p1, [z1.d] ; VBITS_GE_512-NEXT: ret %vals = load <8 x i8>, ptr %a %ptrs = load <8 x ptr>, ptr %b @@ -115,12 +115,12 @@ define void @masked_scatter_v16i8(ptr %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-NEXT: uunpklo z0.h, z0.b ; CHECK-NEXT: sunpklo z1.h, z1.b ; CHECK-NEXT: uunpklo z0.s, z0.h -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] ; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: sunpklo z1.d, z1.s -; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; CHECK-NEXT: st1b { z0.d }, p0, [z2.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] +; CHECK-NEXT: st1b { z0.d }, p1, [z1.d] ; CHECK-NEXT: ret %vals = load <16 x i8>, ptr %a %ptrs = load <16 x ptr>, ptr %b @@ -135,13 +135,13 @@ define void @masked_scatter_v32i8(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-NEXT: ptrue p0.b, vl32 ; CHECK-NEXT: ptrue p1.d, vl32 ; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0] -; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] +; CHECK-NEXT: uunpklo z1.h, z0.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0 -; CHECK-NEXT: uunpklo z0.h, z0.b +; CHECK-NEXT: uunpklo z0.s, z1.h ; CHECK-NEXT: punpklo p0.h, p0.b -; CHECK-NEXT: uunpklo z0.s, z0.h -; CHECK-NEXT: uunpklo z0.d, z0.s +; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] ; CHECK-NEXT: punpklo p0.h, p0.b +; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: st1b { z0.d }, p0, [z1.d] ; CHECK-NEXT: ret @@ -187,10 +187,10 @@ define void @masked_scatter_v4i16(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: uunpklo z0.d, z0.s -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] ; CHECK-NEXT: sunpklo z1.d, z1.s -; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; CHECK-NEXT: st1h { z0.d }, p0, [z2.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] +; CHECK-NEXT: st1h { z0.d }, p1, [z1.d] ; CHECK-NEXT: ret %vals = load <4 x i16>, ptr %a %ptrs = load <4 x ptr>, ptr %b @@ -208,20 +208,20 @@ define void @masked_scatter_v8i16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: cmeq v1.8h, v0.8h, #0 ; VBITS_GE_256-NEXT: uunpklo z3.s, z0.h ; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: sunpklo z2.s, z1.h ; VBITS_GE_256-NEXT: ext v1.16b, v1.16b, v1.16b, #8 ; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_256-NEXT: uunpklo z3.d, z3.s -; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h ; VBITS_GE_256-NEXT: sunpklo z2.d, z2.s ; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s ; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z2.d, #0 -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; VBITS_GE_256-NEXT: st1h { z3.d }, p1, [z4.d] +; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; VBITS_GE_256-NEXT: st1h { z0.d }, p0, [z2.d] +; VBITS_GE_256-NEXT: st1h { z3.d }, p1, [z2.d] +; VBITS_GE_256-NEXT: st1h { z0.d }, p0, [z4.d] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_scatter_v8i16: @@ -232,10 +232,10 @@ define void @masked_scatter_v8i16(ptr %a, ptr %b) #0 { ; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_512-NEXT: sunpklo z1.s, z1.h ; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s -; VBITS_GE_512-NEXT: ld1d { z2.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: sunpklo z1.d, z1.s -; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; VBITS_GE_512-NEXT: st1h { z0.d }, p0, [z2.d] +; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] +; VBITS_GE_512-NEXT: st1h { z0.d }, p1, [z1.d] ; VBITS_GE_512-NEXT: ret %vals = load <8 x i16>, ptr %a %ptrs = load <8 x ptr>, ptr %b @@ -253,9 +253,9 @@ define void @masked_scatter_v16i16(ptr %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 ; CHECK-NEXT: uunpklo z0.s, z0.h -; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: punpklo p0.h, p0.b +; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: st1h { z0.d }, p0, [z1.d] ; CHECK-NEXT: ret %vals = load <16 x i16>, ptr %a @@ -274,9 +274,9 @@ define void @masked_scatter_v32i16(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 ; CHECK-NEXT: uunpklo z0.s, z0.h -; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: punpklo p0.h, p0.b +; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: st1h { z0.d }, p0, [z1.d] ; CHECK-NEXT: ret %vals = load <32 x i16>, ptr %a @@ -317,9 +317,9 @@ define void @masked_scatter_v4i32(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: cmeq v1.4s, v0.4s, #0 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: sunpklo z1.d, z1.s -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; CHECK-NEXT: st1w { z0.d }, p0, [z2.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] +; CHECK-NEXT: st1w { z0.d }, p1, [z1.d] ; CHECK-NEXT: ret %vals = load <4 x i32>, ptr %a %ptrs = load <4 x ptr>, ptr %b @@ -333,21 +333,21 @@ define void @masked_scatter_v8i32(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 +; VBITS_GE_256-NEXT: ptrue p1.d, vl4 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0] -; VBITS_GE_256-NEXT: cmpeq p1.s, p0/z, z0.s, #0 -; VBITS_GE_256-NEXT: ptrue p0.d, vl4 +; VBITS_GE_256-NEXT: ld1d { z3.d }, p1/z, [x1] +; VBITS_GE_256-NEXT: ld1d { z4.d }, p1/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: cmpeq p0.s, p0/z, z0.s, #0 ; VBITS_GE_256-NEXT: uunpklo z2.d, z0.s ; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16 -; VBITS_GE_256-NEXT: mov z1.s, p1/z, #-1 // =0xffffffffffffffff -; VBITS_GE_256-NEXT: punpklo p2.h, p1.b ; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s +; VBITS_GE_256-NEXT: mov z1.s, p0/z, #-1 // =0xffffffffffffffff +; VBITS_GE_256-NEXT: punpklo p0.h, p0.b +; VBITS_GE_256-NEXT: and p0.b, p0/z, p0.b, p1.b ; VBITS_GE_256-NEXT: ext z1.b, z1.b, z1.b, #16 -; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1] -; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: st1w { z2.d }, p0, [z3.d] ; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s -; VBITS_GE_256-NEXT: and p1.b, p2/z, p2.b, p0.b -; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; VBITS_GE_256-NEXT: st1w { z2.d }, p1, [z3.d] +; VBITS_GE_256-NEXT: cmpne p0.d, p1/z, z1.d, #0 ; VBITS_GE_256-NEXT: st1w { z0.d }, p0, [z4.d] ; VBITS_GE_256-NEXT: ret ; @@ -434,8 +434,8 @@ define void @masked_scatter_v1i64(ptr %a, ptr %b) vscale_range(2,0) #0 { define void @masked_scatter_v2i64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: masked_scatter_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: cmeq v1.2d, v0.2d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 ; CHECK-NEXT: ldr q1, [x1] @@ -454,8 +454,8 @@ define void @masked_scatter_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] -; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: st1d { z0.d }, p0, [z1.d] +; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #0 +; CHECK-NEXT: st1d { z0.d }, p1, [z1.d] ; CHECK-NEXT: ret %vals = load <4 x i64>, ptr %a %ptrs = load <4 x ptr>, ptr %b @@ -484,8 +484,8 @@ define void @masked_scatter_v8i64(ptr %a, ptr %b) #0 { ; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0] ; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] -; VBITS_GE_512-NEXT: cmpeq p0.d, p0/z, z0.d, #0 -; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [z1.d] +; VBITS_GE_512-NEXT: cmpeq p1.d, p0/z, z0.d, #0 +; VBITS_GE_512-NEXT: st1d { z0.d }, p1, [z1.d] ; VBITS_GE_512-NEXT: ret %vals = load <8 x i64>, ptr %a %ptrs = load <8 x ptr>, ptr %b @@ -500,8 +500,8 @@ define void @masked_scatter_v16i64(ptr %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] -; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: st1d { z0.d }, p0, [z1.d] +; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #0 +; CHECK-NEXT: st1d { z0.d }, p1, [z1.d] ; CHECK-NEXT: ret %vals = load <16 x i64>, ptr %a %ptrs = load <16 x ptr>, ptr %b @@ -516,8 +516,8 @@ define void @masked_scatter_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] -; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: st1d { z0.d }, p0, [z1.d] +; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #0 +; CHECK-NEXT: st1d { z0.d }, p1, [z1.d] ; CHECK-NEXT: ret %vals = load <32 x i64>, ptr %a %ptrs = load <32 x ptr>, ptr %b @@ -539,15 +539,15 @@ define void @masked_scatter_v2f16(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: fcmeq v2.4h, v1.4h, #0.0 ; CHECK-NEXT: uunpklo z1.s, z1.h ; CHECK-NEXT: sshll v2.4s, v2.4h, #0 -; CHECK-NEXT: uunpklo z1.d, z1.s ; CHECK-NEXT: mov v0.h[0], v2.h[0] ; CHECK-NEXT: mov w8, v2.s[1] ; CHECK-NEXT: mov v0.h[1], w8 ; CHECK-NEXT: sunpklo z0.s, z0.h ; CHECK-NEXT: sunpklo z0.d, z0.s ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: ldr q0, [x1] -; CHECK-NEXT: st1h { z1.d }, p0, [z0.d] +; CHECK-NEXT: uunpklo z0.d, z1.s +; CHECK-NEXT: ldr q1, [x1] +; CHECK-NEXT: st1h { z0.d }, p0, [z1.d] ; CHECK-NEXT: ret %vals = load <2 x half>, ptr %a %ptrs = load <2 x ptr>, ptr %b @@ -565,10 +565,10 @@ define void @masked_scatter_v4f16(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: uunpklo z0.d, z0.s -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] ; CHECK-NEXT: sunpklo z1.d, z1.s -; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; CHECK-NEXT: st1h { z0.d }, p0, [z2.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] +; CHECK-NEXT: st1h { z0.d }, p1, [z1.d] ; CHECK-NEXT: ret %vals = load <4 x half>, ptr %a %ptrs = load <4 x ptr>, ptr %b @@ -586,20 +586,20 @@ define void @masked_scatter_v8f16(ptr %a, ptr %b) #0 { ; VBITS_GE_256-NEXT: fcmeq v1.8h, v0.8h, #0.0 ; VBITS_GE_256-NEXT: uunpklo z3.s, z0.h ; VBITS_GE_256-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: sunpklo z2.s, z1.h ; VBITS_GE_256-NEXT: ext v1.16b, v1.16b, v1.16b, #8 ; VBITS_GE_256-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_256-NEXT: uunpklo z3.d, z3.s -; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: sunpklo z1.s, z1.h ; VBITS_GE_256-NEXT: sunpklo z2.d, z2.s ; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s ; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s ; VBITS_GE_256-NEXT: cmpne p1.d, p0/z, z2.d, #0 -; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1, x8, lsl #3] -; VBITS_GE_256-NEXT: st1h { z3.d }, p1, [z4.d] +; VBITS_GE_256-NEXT: ld1d { z2.d }, p0/z, [x1] ; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; VBITS_GE_256-NEXT: st1h { z0.d }, p0, [z2.d] +; VBITS_GE_256-NEXT: st1h { z3.d }, p1, [z2.d] +; VBITS_GE_256-NEXT: st1h { z0.d }, p0, [z4.d] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_scatter_v8f16: @@ -610,10 +610,10 @@ define void @masked_scatter_v8f16(ptr %a, ptr %b) #0 { ; VBITS_GE_512-NEXT: uunpklo z0.s, z0.h ; VBITS_GE_512-NEXT: sunpklo z1.s, z1.h ; VBITS_GE_512-NEXT: uunpklo z0.d, z0.s -; VBITS_GE_512-NEXT: ld1d { z2.d }, p0/z, [x1] ; VBITS_GE_512-NEXT: sunpklo z1.d, z1.s -; VBITS_GE_512-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; VBITS_GE_512-NEXT: st1h { z0.d }, p0, [z2.d] +; VBITS_GE_512-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; VBITS_GE_512-NEXT: ld1d { z1.d }, p0/z, [x1] +; VBITS_GE_512-NEXT: st1h { z0.d }, p1, [z1.d] ; VBITS_GE_512-NEXT: ret %vals = load <8 x half>, ptr %a %ptrs = load <8 x ptr>, ptr %b @@ -631,9 +631,9 @@ define void @masked_scatter_v16f16(ptr %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] ; CHECK-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0 ; CHECK-NEXT: uunpklo z0.s, z0.h -; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: punpklo p0.h, p0.b +; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: st1h { z0.d }, p0, [z1.d] ; CHECK-NEXT: ret %vals = load <16 x half>, ptr %a @@ -652,9 +652,9 @@ define void @masked_scatter_v32f16(ptr %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-NEXT: ld1d { z1.d }, p1/z, [x1] ; CHECK-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0 ; CHECK-NEXT: uunpklo z0.s, z0.h -; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: punpklo p0.h, p0.b +; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: st1h { z0.d }, p0, [z1.d] ; CHECK-NEXT: ret %vals = load <32 x half>, ptr %a @@ -695,9 +695,9 @@ define void @masked_scatter_v4f32(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-NEXT: fcmeq v1.4s, v0.4s, #0.0 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: sunpklo z1.d, z1.s -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x1] -; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; CHECK-NEXT: st1w { z0.d }, p0, [z2.d] +; CHECK-NEXT: cmpne p1.d, p0/z, z1.d, #0 +; CHECK-NEXT: ld1d { z1.d }, p0/z, [x1] +; CHECK-NEXT: st1w { z0.d }, p1, [z1.d] ; CHECK-NEXT: ret %vals = load <4 x float>, ptr %a %ptrs = load <4 x ptr>, ptr %b @@ -711,21 +711,21 @@ define void @masked_scatter_v8f32(ptr %a, ptr %b) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 +; VBITS_GE_256-NEXT: ptrue p1.d, vl4 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0] -; VBITS_GE_256-NEXT: fcmeq p1.s, p0/z, z0.s, #0.0 -; VBITS_GE_256-NEXT: ptrue p0.d, vl4 +; VBITS_GE_256-NEXT: ld1d { z3.d }, p1/z, [x1] +; VBITS_GE_256-NEXT: ld1d { z4.d }, p1/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0 ; VBITS_GE_256-NEXT: uunpklo z2.d, z0.s ; VBITS_GE_256-NEXT: ext z0.b, z0.b, z0.b, #16 -; VBITS_GE_256-NEXT: mov z1.s, p1/z, #-1 // =0xffffffffffffffff -; VBITS_GE_256-NEXT: punpklo p2.h, p1.b ; VBITS_GE_256-NEXT: uunpklo z0.d, z0.s -; VBITS_GE_256-NEXT: ld1d { z3.d }, p0/z, [x1] -; VBITS_GE_256-NEXT: ld1d { z4.d }, p0/z, [x1, x8, lsl #3] +; VBITS_GE_256-NEXT: mov z1.s, p0/z, #-1 // =0xffffffffffffffff +; VBITS_GE_256-NEXT: punpklo p0.h, p0.b +; VBITS_GE_256-NEXT: and p0.b, p0/z, p0.b, p1.b ; VBITS_GE_256-NEXT: ext z1.b, z1.b, z1.b, #16 +; VBITS_GE_256-NEXT: st1w { z2.d }, p0, [z3.d] ; VBITS_GE_256-NEXT: sunpklo z1.d, z1.s -; VBITS_GE_256-NEXT: and p1.b, p2/z, p2.b, p0.b -; VBITS_GE_256-NEXT: cmpne p0.d, p0/z, z1.d, #0 -; VBITS_GE_256-NEXT: st1w { z2.d }, p1, [z3.d] +; VBITS_GE_256-NEXT: cmpne p0.d, p1/z, z1.d, #0 ; VBITS_GE_256-NEXT: st1w { z0.d }, p0, [z4.d] ; VBITS_GE_256-NEXT: ret ; @@ -812,8 +812,8 @@ define void @masked_scatter_v1f64(ptr %a, ptr %b) vscale_range(8,0) #0 { define void @masked_scatter_v2f64(ptr %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: masked_scatter_v2f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fcmeq v1.2d, v0.2d, #0.0 ; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 ; CHECK-NEXT: ldr q1, [x1] diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-stores.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-stores.ll index 68fb4cc6afb0..b0d4f79aea11 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-stores.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-stores.ll @@ -34,9 +34,9 @@ define void @masked_store_v2f16(ptr %ap, ptr %bp) vscale_range(2,0) #0 { define void @masked_store_v2f32(ptr %ap, ptr %bp) vscale_range(2,0) #0 { ; CHECK-LABEL: masked_store_v2f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: ldr d0, [x0] ; CHECK-NEXT: ldr d1, [x1] +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: fcmeq v1.2s, v0.2s, v1.2s ; CHECK-NEXT: cmpne p0.s, p0/z, z1.s, #0 ; CHECK-NEXT: st1w { z0.s }, p0, [x1] @@ -51,9 +51,9 @@ define void @masked_store_v2f32(ptr %ap, ptr %bp) vscale_range(2,0) #0 { define void @masked_store_v4f32(ptr %ap, ptr %bp) vscale_range(1,0) #0 { ; CHECK-LABEL: masked_store_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fcmeq v1.4s, v0.4s, v1.4s ; CHECK-NEXT: cmpne p0.s, p0/z, z1.s, #0 ; CHECK-NEXT: st1w { z0.s }, p0, [x1] @@ -161,11 +161,11 @@ define void @masked_store_trunc_v8i64i8(ptr %ap, ptr %bp, ptr %dest) #0 { ; VBITS_GE_256-NEXT: mov z3.d, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: uzp1 z2.s, z2.s, z2.s -; VBITS_GE_256-NEXT: uzp1 z3.s, z3.s, z3.s ; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s +; VBITS_GE_256-NEXT: uzp1 z3.s, z3.s, z3.s ; VBITS_GE_256-NEXT: splice z3.s, p0, z3.s, z2.s -; VBITS_GE_256-NEXT: cmpne p0.s, p1/z, z3.s, #0 -; VBITS_GE_256-NEXT: st1b { z1.s }, p0, [x2] +; VBITS_GE_256-NEXT: cmpne p1.s, p1/z, z3.s, #0 +; VBITS_GE_256-NEXT: st1b { z1.s }, p1, [x2] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_store_trunc_v8i64i8: @@ -197,14 +197,14 @@ define void @masked_store_trunc_v8i64i16(ptr %ap, ptr %bp, ptr %dest) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: cmpeq p0.d, p0/z, z1.d, z3.d ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s +; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_256-NEXT: mov z2.d, p1/z, #-1 // =0xffffffffffffffff +; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: mov z3.d, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: ptrue p0.s, vl4 -; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h -; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: uzp1 z2.s, z2.s, z2.s -; VBITS_GE_256-NEXT: uzp1 z3.s, z3.s, z3.s ; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0] +; VBITS_GE_256-NEXT: uzp1 z3.s, z3.s, z3.s ; VBITS_GE_256-NEXT: splice z3.s, p0, z3.s, z2.s ; VBITS_GE_256-NEXT: ptrue p0.h, vl8 ; VBITS_GE_256-NEXT: uzp1 z2.h, z3.h, z3.h @@ -246,11 +246,11 @@ define void @masked_store_trunc_v8i64i32(ptr %ap, ptr %bp, ptr %dest) #0 { ; VBITS_GE_256-NEXT: mov z3.d, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: uzp1 z2.s, z2.s, z2.s -; VBITS_GE_256-NEXT: uzp1 z3.s, z3.s, z3.s ; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s +; VBITS_GE_256-NEXT: uzp1 z3.s, z3.s, z3.s ; VBITS_GE_256-NEXT: splice z3.s, p0, z3.s, z2.s -; VBITS_GE_256-NEXT: cmpne p0.s, p1/z, z3.s, #0 -; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x2] +; VBITS_GE_256-NEXT: cmpne p1.s, p1/z, z3.s, #0 +; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x2] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_store_trunc_v8i64i32: @@ -282,14 +282,14 @@ define void @masked_store_trunc_v16i32i8(ptr %ap, ptr %bp, ptr %dest) #0 { ; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_256-NEXT: cmpeq p0.s, p0/z, z1.s, z3.s ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h +; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b ; VBITS_GE_256-NEXT: mov z2.s, p1/z, #-1 // =0xffffffffffffffff +; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b ; VBITS_GE_256-NEXT: mov z3.s, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: ptrue p0.b, vl16 -; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b -; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b ; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h -; VBITS_GE_256-NEXT: uzp1 z3.h, z3.h, z3.h ; VBITS_GE_256-NEXT: mov v1.d[1], v0.d[0] +; VBITS_GE_256-NEXT: uzp1 z3.h, z3.h, z3.h ; VBITS_GE_256-NEXT: uzp1 z2.b, z2.b, z2.b ; VBITS_GE_256-NEXT: uzp1 z3.b, z3.b, z3.b ; VBITS_GE_256-NEXT: mov v3.d[1], v2.d[0] @@ -327,17 +327,17 @@ define void @masked_store_trunc_v16i32i16(ptr %ap, ptr %bp, ptr %dest) #0 { ; VBITS_GE_256-NEXT: cmpeq p0.s, p0/z, z1.s, z3.s ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: mov z2.s, p1/z, #-1 // =0xffffffffffffffff -; VBITS_GE_256-NEXT: ptrue p1.h, vl16 +; VBITS_GE_256-NEXT: ptrue p1.h, vl8 ; VBITS_GE_256-NEXT: mov z3.s, p0/z, #-1 // =0xffffffffffffffff -; VBITS_GE_256-NEXT: ptrue p0.h, vl8 +; VBITS_GE_256-NEXT: ptrue p0.h, vl16 +; VBITS_GE_256-NEXT: splice z1.h, p1, z1.h, z0.h ; VBITS_GE_256-NEXT: uzp1 z2.h, z2.h, z2.h ; VBITS_GE_256-NEXT: uzp1 z3.h, z3.h, z3.h ; VBITS_GE_256-NEXT: uzp1 z2.b, z2.b, z2.b -; VBITS_GE_256-NEXT: splice z1.h, p0, z1.h, z0.h ; VBITS_GE_256-NEXT: uzp1 z3.b, z3.b, z3.b ; VBITS_GE_256-NEXT: mov v3.d[1], v2.d[0] ; VBITS_GE_256-NEXT: sunpklo z2.h, z3.b -; VBITS_GE_256-NEXT: cmpne p0.h, p1/z, z2.h, #0 +; VBITS_GE_256-NEXT: cmpne p0.h, p0/z, z2.h, #0 ; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x2] ; VBITS_GE_256-NEXT: ret ; @@ -375,11 +375,11 @@ define void @masked_store_trunc_v32i16i8(ptr %ap, ptr %bp, ptr %dest) #0 { ; VBITS_GE_256-NEXT: mov z3.h, p0/z, #-1 // =0xffffffffffffffff ; VBITS_GE_256-NEXT: ptrue p0.b, vl16 ; VBITS_GE_256-NEXT: uzp1 z2.b, z2.b, z2.b -; VBITS_GE_256-NEXT: uzp1 z3.b, z3.b, z3.b ; VBITS_GE_256-NEXT: splice z1.b, p0, z1.b, z0.b +; VBITS_GE_256-NEXT: uzp1 z3.b, z3.b, z3.b ; VBITS_GE_256-NEXT: splice z3.b, p0, z3.b, z2.b -; VBITS_GE_256-NEXT: cmpne p0.b, p1/z, z3.b, #0 -; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x2] +; VBITS_GE_256-NEXT: cmpne p1.b, p1/z, z3.b, #0 +; VBITS_GE_256-NEXT: st1b { z1.b }, p1, [x2] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: masked_store_trunc_v32i16i8: diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll index a5303c901b80..fb169491b0c9 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll @@ -64,7 +64,6 @@ define void @crash_when_lowering_extract_shuffle(ptr %dst, i1 %cond) vscale_rang ; CHECK-NEXT: uunpklo z0.h, z0.b ; CHECK-NEXT: mov x8, #16 // =0x10 ; CHECK-NEXT: mov x10, #8 // =0x8 -; CHECK-NEXT: ld1w { z4.s }, p0/z, [x0, x8, lsl #2] ; CHECK-NEXT: mov v2.b[7], w11 ; CHECK-NEXT: mov v1.b[7], w9 ; CHECK-NEXT: uunpklo z3.h, z3.b @@ -86,22 +85,23 @@ define void @crash_when_lowering_extract_shuffle(ptr %dst, i1 %cond) vscale_rang ; CHECK-NEXT: asr z2.s, z2.s, #31 ; CHECK-NEXT: asr z1.s, z1.s, #31 ; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0 -; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x9, lsl #2] +; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] ; CHECK-NEXT: cmpne p2.s, p0/z, z3.s, #0 -; CHECK-NEXT: ld1w { z3.s }, p0/z, [x0, x10, lsl #2] +; CHECK-NEXT: ld1w { z3.s }, p0/z, [x0, x9, lsl #2] ; CHECK-NEXT: and z2.s, z2.s, #0x1 ; CHECK-NEXT: and z1.s, z1.s, #0x1 -; CHECK-NEXT: mov z4.s, p1/m, #0 // =0x0 -; CHECK-NEXT: mov z0.s, p2/m, #0 // =0x0 +; CHECK-NEXT: mov z0.s, p1/m, #0 // =0x0 ; CHECK-NEXT: cmpne p3.s, p0/z, z2.s, #0 -; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0] -; CHECK-NEXT: cmpne p1.s, p0/z, z1.s, #0 -; CHECK-NEXT: st1w { z4.s }, p0, [x0, x8, lsl #2] -; CHECK-NEXT: st1w { z0.s }, p0, [x0, x9, lsl #2] -; CHECK-NEXT: mov z3.s, p3/m, #0 // =0x0 -; CHECK-NEXT: mov z2.s, p1/m, #0 // =0x0 -; CHECK-NEXT: st1w { z3.s }, p0, [x0, x10, lsl #2] -; CHECK-NEXT: st1w { z2.s }, p0, [x0] +; CHECK-NEXT: cmpne p4.s, p0/z, z1.s, #0 +; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, x10, lsl #2] +; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0] +; CHECK-NEXT: mov z3.s, p2/m, #0 // =0x0 +; CHECK-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2] +; CHECK-NEXT: mov z2.s, p3/m, #0 // =0x0 +; CHECK-NEXT: mov z1.s, p4/m, #0 // =0x0 +; CHECK-NEXT: st1w { z3.s }, p0, [x0, x9, lsl #2] +; CHECK-NEXT: st1w { z2.s }, p0, [x0, x10, lsl #2] +; CHECK-NEXT: st1w { z1.s }, p0, [x0] ; CHECK-NEXT: .LBB1_2: // %exit ; CHECK-NEXT: ret %broadcast.splat = shufflevector <32 x i1> zeroinitializer, <32 x i1> zeroinitializer, <32 x i32> zeroinitializer diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-splat-vector.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-splat-vector.ll index f97ca05f3bdd..b633057be139 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-splat-vector.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-splat-vector.ll @@ -34,8 +34,8 @@ define <16 x i8> @splat_v16i8(i8 %a) vscale_range(2,0) #0 { define void @splat_v32i8(i8 %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: splat_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl32 ; CHECK-NEXT: mov z0.b, w0 +; CHECK-NEXT: ptrue p0.b, vl32 ; CHECK-NEXT: st1b { z0.b }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <32 x i8> undef, i8 %a, i64 0 @@ -47,8 +47,8 @@ define void @splat_v32i8(i8 %a, ptr %b) vscale_range(2,0) #0 { define void @splat_v64i8(i8 %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: splat_v64i8: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.b, vl32 ; VBITS_GE_256-NEXT: mov z0.b, w0 +; VBITS_GE_256-NEXT: ptrue p0.b, vl32 ; VBITS_GE_256-NEXT: mov w8, #32 // =0x20 ; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x1, x8] ; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x1] @@ -56,8 +56,8 @@ define void @splat_v64i8(i8 %a, ptr %b) #0 { ; ; VBITS_GE_512-LABEL: splat_v64i8: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.b, vl64 ; VBITS_GE_512-NEXT: mov z0.b, w0 +; VBITS_GE_512-NEXT: ptrue p0.b, vl64 ; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x1] ; VBITS_GE_512-NEXT: ret %insert = insertelement <64 x i8> undef, i8 %a, i64 0 @@ -69,8 +69,8 @@ define void @splat_v64i8(i8 %a, ptr %b) #0 { define void @splat_v128i8(i8 %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: splat_v128i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl128 ; CHECK-NEXT: mov z0.b, w0 +; CHECK-NEXT: ptrue p0.b, vl128 ; CHECK-NEXT: st1b { z0.b }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <128 x i8> undef, i8 %a, i64 0 @@ -82,8 +82,8 @@ define void @splat_v128i8(i8 %a, ptr %b) vscale_range(8,0) #0 { define void @splat_v256i8(i8 %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: splat_v256i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl256 ; CHECK-NEXT: mov z0.b, w0 +; CHECK-NEXT: ptrue p0.b, vl256 ; CHECK-NEXT: st1b { z0.b }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <256 x i8> undef, i8 %a, i64 0 @@ -117,8 +117,8 @@ define <8 x i16> @splat_v8i16(i16 %a) vscale_range(2,0) #0 { define void @splat_v16i16(i16 %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: splat_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: mov z0.h, w0 +; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: st1h { z0.h }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <16 x i16> undef, i16 %a, i64 0 @@ -130,8 +130,8 @@ define void @splat_v16i16(i16 %a, ptr %b) vscale_range(2,0) #0 { define void @splat_v32i16(i16 %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: splat_v32i16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: mov z0.h, w0 +; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: mov x8, #16 // =0x10 ; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x1, x8, lsl #1] ; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x1] @@ -139,8 +139,8 @@ define void @splat_v32i16(i16 %a, ptr %b) #0 { ; ; VBITS_GE_512-LABEL: splat_v32i16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.h, vl32 ; VBITS_GE_512-NEXT: mov z0.h, w0 +; VBITS_GE_512-NEXT: ptrue p0.h, vl32 ; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x1] ; VBITS_GE_512-NEXT: ret %insert = insertelement <32 x i16> undef, i16 %a, i64 0 @@ -152,8 +152,8 @@ define void @splat_v32i16(i16 %a, ptr %b) #0 { define void @splat_v64i16(i16 %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: splat_v64i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: mov z0.h, w0 +; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: st1h { z0.h }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <64 x i16> undef, i16 %a, i64 0 @@ -165,8 +165,8 @@ define void @splat_v64i16(i16 %a, ptr %b) vscale_range(8,0) #0 { define void @splat_v128i16(i16 %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: splat_v128i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl128 ; CHECK-NEXT: mov z0.h, w0 +; CHECK-NEXT: ptrue p0.h, vl128 ; CHECK-NEXT: st1h { z0.h }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <128 x i16> undef, i16 %a, i64 0 @@ -200,8 +200,8 @@ define <4 x i32> @splat_v4i32(i32 %a) vscale_range(2,0) #0 { define void @splat_v8i32(i32 %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: splat_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: mov z0.s, w0 +; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: st1w { z0.s }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <8 x i32> undef, i32 %a, i64 0 @@ -213,8 +213,8 @@ define void @splat_v8i32(i32 %a, ptr %b) vscale_range(2,0) #0 { define void @splat_v16i32(i32 %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: splat_v16i32: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov z0.s, w0 +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 ; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x1, x8, lsl #2] ; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x1] @@ -222,8 +222,8 @@ define void @splat_v16i32(i32 %a, ptr %b) #0 { ; ; VBITS_GE_512-LABEL: splat_v16i32: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl16 ; VBITS_GE_512-NEXT: mov z0.s, w0 +; VBITS_GE_512-NEXT: ptrue p0.s, vl16 ; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x1] ; VBITS_GE_512-NEXT: ret %insert = insertelement <16 x i32> undef, i32 %a, i64 0 @@ -235,8 +235,8 @@ define void @splat_v16i32(i32 %a, ptr %b) #0 { define void @splat_v32i32(i32 %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: splat_v32i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: mov z0.s, w0 +; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: st1w { z0.s }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <32 x i32> undef, i32 %a, i64 0 @@ -248,8 +248,8 @@ define void @splat_v32i32(i32 %a, ptr %b) vscale_range(8,0) #0 { define void @splat_v64i32(i32 %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: splat_v64i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl64 ; CHECK-NEXT: mov z0.s, w0 +; CHECK-NEXT: ptrue p0.s, vl64 ; CHECK-NEXT: st1w { z0.s }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <64 x i32> undef, i32 %a, i64 0 @@ -283,8 +283,8 @@ define <2 x i64> @splat_v2i64(i64 %a) vscale_range(2,0) #0 { define void @splat_v4i64(i64 %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: splat_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: mov z0.d, x0 +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: st1d { z0.d }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <4 x i64> undef, i64 %a, i64 0 @@ -296,8 +296,8 @@ define void @splat_v4i64(i64 %a, ptr %b) vscale_range(2,0) #0 { define void @splat_v8i64(i64 %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: splat_v8i64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov z0.d, x0 +; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x1, x8, lsl #3] ; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x1] @@ -305,8 +305,8 @@ define void @splat_v8i64(i64 %a, ptr %b) #0 { ; ; VBITS_GE_512-LABEL: splat_v8i64: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: mov z0.d, x0 +; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x1] ; VBITS_GE_512-NEXT: ret %insert = insertelement <8 x i64> undef, i64 %a, i64 0 @@ -318,8 +318,8 @@ define void @splat_v8i64(i64 %a, ptr %b) #0 { define void @splat_v16i64(i64 %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: splat_v16i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: mov z0.d, x0 +; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: st1d { z0.d }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <16 x i64> undef, i64 %a, i64 0 @@ -331,8 +331,8 @@ define void @splat_v16i64(i64 %a, ptr %b) vscale_range(8,0) #0 { define void @splat_v32i64(i64 %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: splat_v32i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: mov z0.d, x0 +; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: st1d { z0.d }, p0, [x1] ; CHECK-NEXT: ret %insert = insertelement <32 x i64> undef, i64 %a, i64 0 @@ -372,8 +372,8 @@ define <8 x half> @splat_v8f16(half %a) vscale_range(2,0) #0 { define void @splat_v16f16(half %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: splat_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: // kill: def $h0 killed $h0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl16 ; CHECK-NEXT: mov z0.h, h0 ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret @@ -386,8 +386,8 @@ define void @splat_v16f16(half %a, ptr %b) vscale_range(2,0) #0 { define void @splat_v32f16(half %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: splat_v32f16: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: // kill: def $h0 killed $h0 def $z0 +; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: mov x8, #16 // =0x10 ; VBITS_GE_256-NEXT: mov z0.h, h0 ; VBITS_GE_256-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1] @@ -396,8 +396,8 @@ define void @splat_v32f16(half %a, ptr %b) #0 { ; ; VBITS_GE_512-LABEL: splat_v32f16: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.h, vl32 ; VBITS_GE_512-NEXT: // kill: def $h0 killed $h0 def $z0 +; VBITS_GE_512-NEXT: ptrue p0.h, vl32 ; VBITS_GE_512-NEXT: mov z0.h, h0 ; VBITS_GE_512-NEXT: st1h { z0.h }, p0, [x0] ; VBITS_GE_512-NEXT: ret @@ -410,8 +410,8 @@ define void @splat_v32f16(half %a, ptr %b) #0 { define void @splat_v64f16(half %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: splat_v64f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: // kill: def $h0 killed $h0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: mov z0.h, h0 ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret @@ -424,8 +424,8 @@ define void @splat_v64f16(half %a, ptr %b) vscale_range(8,0) #0 { define void @splat_v128f16(half %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: splat_v128f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl128 ; CHECK-NEXT: // kill: def $h0 killed $h0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl128 ; CHECK-NEXT: mov z0.h, h0 ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret @@ -462,8 +462,8 @@ define <4 x float> @splat_v4f32(float %a, <4 x float> %op2) vscale_range(2,0) #0 define void @splat_v8f32(float %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: splat_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: // kill: def $s0 killed $s0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl8 ; CHECK-NEXT: mov z0.s, s0 ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret @@ -476,8 +476,8 @@ define void @splat_v8f32(float %a, ptr %b) vscale_range(2,0) #0 { define void @splat_v16f32(float %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: splat_v16f32: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: // kill: def $s0 killed $s0 def $z0 +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 ; VBITS_GE_256-NEXT: mov z0.s, s0 ; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2] @@ -486,8 +486,8 @@ define void @splat_v16f32(float %a, ptr %b) #0 { ; ; VBITS_GE_512-LABEL: splat_v16f32: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.s, vl16 ; VBITS_GE_512-NEXT: // kill: def $s0 killed $s0 def $z0 +; VBITS_GE_512-NEXT: ptrue p0.s, vl16 ; VBITS_GE_512-NEXT: mov z0.s, s0 ; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0] ; VBITS_GE_512-NEXT: ret @@ -500,8 +500,8 @@ define void @splat_v16f32(float %a, ptr %b) #0 { define void @splat_v32f32(float %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: splat_v32f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: // kill: def $s0 killed $s0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: mov z0.s, s0 ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret @@ -514,8 +514,8 @@ define void @splat_v32f32(float %a, ptr %b) vscale_range(8,0) #0 { define void @splat_v64f32(float %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: splat_v64f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl64 ; CHECK-NEXT: // kill: def $s0 killed $s0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl64 ; CHECK-NEXT: mov z0.s, s0 ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret @@ -550,8 +550,8 @@ define <2 x double> @splat_v2f64(double %a, <2 x double> %op2) vscale_range(2,0) define void @splat_v4f64(double %a, ptr %b) vscale_range(2,0) #0 { ; CHECK-LABEL: splat_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl4 ; CHECK-NEXT: mov z0.d, d0 ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret @@ -564,8 +564,8 @@ define void @splat_v4f64(double %a, ptr %b) vscale_range(2,0) #0 { define void @splat_v8f64(double %a, ptr %b) #0 { ; VBITS_GE_256-LABEL: splat_v8f64: ; VBITS_GE_256: // %bb.0: -; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: // kill: def $d0 killed $d0 def $z0 +; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 ; VBITS_GE_256-NEXT: mov z0.d, d0 ; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3] @@ -574,8 +574,8 @@ define void @splat_v8f64(double %a, ptr %b) #0 { ; ; VBITS_GE_512-LABEL: splat_v8f64: ; VBITS_GE_512: // %bb.0: -; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: // kill: def $d0 killed $d0 def $z0 +; VBITS_GE_512-NEXT: ptrue p0.d, vl8 ; VBITS_GE_512-NEXT: mov z0.d, d0 ; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0] ; VBITS_GE_512-NEXT: ret @@ -588,8 +588,8 @@ define void @splat_v8f64(double %a, ptr %b) #0 { define void @splat_v16f64(double %a, ptr %b) vscale_range(8,0) #0 { ; CHECK-LABEL: splat_v16f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl16 ; CHECK-NEXT: mov z0.d, d0 ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret @@ -602,8 +602,8 @@ define void @splat_v16f64(double %a, ptr %b) vscale_range(8,0) #0 { define void @splat_v32f64(double %a, ptr %b) vscale_range(16,0) #0 { ; CHECK-LABEL: splat_v32f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl32 ; CHECK-NEXT: mov z0.d, d0 ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret @@ -620,8 +620,8 @@ define void @splat_v32f64(double %a, ptr %b) vscale_range(16,0) #0 { define void @splat_imm_v64i8(ptr %a) vscale_range(4,0) #0 { ; CHECK-LABEL: splat_imm_v64i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl64 ; CHECK-NEXT: mov z0.b, #1 // =0x1 +; CHECK-NEXT: ptrue p0.b, vl64 ; CHECK-NEXT: st1b { z0.b }, p0, [x0] ; CHECK-NEXT: ret %insert = insertelement <64 x i8> undef, i8 1, i64 0 @@ -633,8 +633,8 @@ define void @splat_imm_v64i8(ptr %a) vscale_range(4,0) #0 { define void @splat_imm_v32i16(ptr %a) vscale_range(4,0) #0 { ; CHECK-LABEL: splat_imm_v32i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl32 ; CHECK-NEXT: mov z0.h, #2 // =0x2 +; CHECK-NEXT: ptrue p0.h, vl32 ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret %insert = insertelement <32 x i16> undef, i16 2, i64 0 @@ -646,8 +646,8 @@ define void @splat_imm_v32i16(ptr %a) vscale_range(4,0) #0 { define void @splat_imm_v16i32(ptr %a) vscale_range(4,0) #0 { ; CHECK-LABEL: splat_imm_v16i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl16 ; CHECK-NEXT: mov z0.s, #3 // =0x3 +; CHECK-NEXT: ptrue p0.s, vl16 ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret %insert = insertelement <16 x i32> undef, i32 3, i64 0 @@ -659,8 +659,8 @@ define void @splat_imm_v16i32(ptr %a) vscale_range(4,0) #0 { define void @splat_imm_v8i64(ptr %a) vscale_range(4,0) #0 { ; CHECK-LABEL: splat_imm_v8i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl8 ; CHECK-NEXT: mov z0.d, #4 // =0x4 +; CHECK-NEXT: ptrue p0.d, vl8 ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret %insert = insertelement <8 x i64> undef, i64 4, i64 0 @@ -676,8 +676,8 @@ define void @splat_imm_v8i64(ptr %a) vscale_range(4,0) #0 { define void @splat_imm_v32f16(ptr %a) vscale_range(4,0) #0 { ; CHECK-LABEL: splat_imm_v32f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl32 ; CHECK-NEXT: fmov z0.h, #5.00000000 +; CHECK-NEXT: ptrue p0.h, vl32 ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret %insert = insertelement <32 x half> undef, half 5.0, i64 0 @@ -689,8 +689,8 @@ define void @splat_imm_v32f16(ptr %a) vscale_range(4,0) #0 { define void @splat_imm_v16f32(ptr %a) vscale_range(4,0) #0 { ; CHECK-LABEL: splat_imm_v16f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl16 ; CHECK-NEXT: fmov z0.s, #6.00000000 +; CHECK-NEXT: ptrue p0.s, vl16 ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret %insert = insertelement <16 x float> undef, float 6.0, i64 0 @@ -702,8 +702,8 @@ define void @splat_imm_v16f32(ptr %a) vscale_range(4,0) #0 { define void @splat_imm_v8f64(ptr %a) vscale_range(4,0) #0 { ; CHECK-LABEL: splat_imm_v8f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl8 ; CHECK-NEXT: fmov z0.d, #7.00000000 +; CHECK-NEXT: ptrue p0.d, vl8 ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret %insert = insertelement <8 x double> undef, double 7.0, i64 0 diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-trunc-stores.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-trunc-stores.ll index 2dc4bddb81a6..020d5cb53bf2 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-trunc-stores.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-trunc-stores.ll @@ -36,14 +36,14 @@ define void @store_trunc_v8i64i8(ptr %ap, ptr %dest) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] ; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s -; VBITS_GE_256-NEXT: st1b { z1.s }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 +; VBITS_GE_256-NEXT: st1b { z1.s }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: store_trunc_v8i64i8: @@ -117,14 +117,14 @@ define void @store_trunc_v8i64i32(ptr %ap, ptr %dest) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.d, vl4 ; VBITS_GE_256-NEXT: mov x8, #4 // =0x4 -; VBITS_GE_256-NEXT: ptrue p1.s, vl8 ; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] ; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] ; VBITS_GE_256-NEXT: ptrue p0.s, vl4 ; VBITS_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s ; VBITS_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s ; VBITS_GE_256-NEXT: splice z1.s, p0, z1.s, z0.s -; VBITS_GE_256-NEXT: st1w { z1.s }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.s, vl8 +; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: store_trunc_v8i64i32: @@ -172,14 +172,14 @@ define void @store_trunc_v16i32i16(ptr %ap, ptr %dest) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.s, vl8 ; VBITS_GE_256-NEXT: mov x8, #8 // =0x8 -; VBITS_GE_256-NEXT: ptrue p1.h, vl16 ; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] ; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0] ; VBITS_GE_256-NEXT: ptrue p0.h, vl8 ; VBITS_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h ; VBITS_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h ; VBITS_GE_256-NEXT: splice z1.h, p0, z1.h, z0.h -; VBITS_GE_256-NEXT: st1h { z1.h }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.h, vl16 +; VBITS_GE_256-NEXT: st1h { z1.h }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: store_trunc_v16i32i16: @@ -199,14 +199,14 @@ define void @store_trunc_v32i16i8(ptr %ap, ptr %dest) #0 { ; VBITS_GE_256: // %bb.0: ; VBITS_GE_256-NEXT: ptrue p0.h, vl16 ; VBITS_GE_256-NEXT: mov x8, #16 // =0x10 -; VBITS_GE_256-NEXT: ptrue p1.b, vl32 ; VBITS_GE_256-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1] ; VBITS_GE_256-NEXT: ld1h { z1.h }, p0/z, [x0] ; VBITS_GE_256-NEXT: ptrue p0.b, vl16 ; VBITS_GE_256-NEXT: uzp1 z0.b, z0.b, z0.b ; VBITS_GE_256-NEXT: uzp1 z1.b, z1.b, z1.b ; VBITS_GE_256-NEXT: splice z1.b, p0, z1.b, z0.b -; VBITS_GE_256-NEXT: st1b { z1.b }, p1, [x1] +; VBITS_GE_256-NEXT: ptrue p0.b, vl32 +; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x1] ; VBITS_GE_256-NEXT: ret ; ; VBITS_GE_512-LABEL: store_trunc_v32i16i8: diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll index 68c234a20d11..28094c7b68e7 100644 --- a/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-vector-shuffle-tbl.ll @@ -559,13 +559,13 @@ define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) { ; ; SVE2_128_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16: ; SVE2_128_NOMAX: // %bb.0: -; SVE2_128_NOMAX-NEXT: ptrue p0.h, vl8 ; SVE2_128_NOMAX-NEXT: cnth x8 ; SVE2_128_NOMAX-NEXT: adrp x9, .LCPI7_0 ; SVE2_128_NOMAX-NEXT: adrp x10, .LCPI7_1 ; SVE2_128_NOMAX-NEXT: mov z0.h, w8 ; SVE2_128_NOMAX-NEXT: ldr q1, [x9, :lo12:.LCPI7_0] ; SVE2_128_NOMAX-NEXT: ldr q2, [x10, :lo12:.LCPI7_1] +; SVE2_128_NOMAX-NEXT: ptrue p0.h, vl8 ; SVE2_128_NOMAX-NEXT: mad z0.h, p0/m, z1.h, z2.h ; SVE2_128_NOMAX-NEXT: ldr q1, [x0] ; SVE2_128_NOMAX-NEXT: ldr q2, [x1] @@ -575,13 +575,13 @@ define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) { ; ; SVE2_NOMIN_NOMAX-LABEL: shuffle_index_indices_from_both_ops_i16: ; SVE2_NOMIN_NOMAX: // %bb.0: -; SVE2_NOMIN_NOMAX-NEXT: ptrue p0.h, vl8 ; SVE2_NOMIN_NOMAX-NEXT: cnth x8 ; SVE2_NOMIN_NOMAX-NEXT: adrp x9, .LCPI7_0 ; SVE2_NOMIN_NOMAX-NEXT: adrp x10, .LCPI7_1 ; SVE2_NOMIN_NOMAX-NEXT: mov z0.h, w8 ; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x9, :lo12:.LCPI7_0] ; SVE2_NOMIN_NOMAX-NEXT: ldr q2, [x10, :lo12:.LCPI7_1] +; SVE2_NOMIN_NOMAX-NEXT: ptrue p0.h, vl8 ; SVE2_NOMIN_NOMAX-NEXT: mad z0.h, p0/m, z1.h, z2.h ; SVE2_NOMIN_NOMAX-NEXT: ldr q1, [x0] ; SVE2_NOMIN_NOMAX-NEXT: ldr q2, [x1] @@ -597,9 +597,9 @@ define <8 x i16> @shuffle_index_indices_from_both_ops_i16(ptr %a, ptr %b) { ; SVE2_MIN_256_NOMAX-NEXT: adrp x9, .LCPI7_1 ; SVE2_MIN_256_NOMAX-NEXT: add x9, x9, :lo12:.LCPI7_1 ; SVE2_MIN_256_NOMAX-NEXT: cnth x10 -; SVE2_MIN_256_NOMAX-NEXT: mov z2.h, w10 ; SVE2_MIN_256_NOMAX-NEXT: ld1h { z0.h }, p0/z, [x8] ; SVE2_MIN_256_NOMAX-NEXT: ld1h { z1.h }, p0/z, [x9] +; SVE2_MIN_256_NOMAX-NEXT: mov z2.h, w10 ; SVE2_MIN_256_NOMAX-NEXT: mad z0.h, p0/m, z2.h, z1.h ; SVE2_MIN_256_NOMAX-NEXT: ldr q1, [x0] ; SVE2_MIN_256_NOMAX-NEXT: ldr q2, [x1] diff --git a/llvm/test/CodeGen/AArch64/sve-fp-int-min-max.ll b/llvm/test/CodeGen/AArch64/sve-fp-int-min-max.ll index 5ff9f0f0df62..afe13851f0b9 100644 --- a/llvm/test/CodeGen/AArch64/sve-fp-int-min-max.ll +++ b/llvm/test/CodeGen/AArch64/sve-fp-int-min-max.ll @@ -7,11 +7,11 @@ define i64 @scalable_int_min_max(ptr %arg, ptr %arg1, %i37, < ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #3745 // =0xea1 ; CHECK-NEXT: movk w8, #16618, lsl #16 +; CHECK-NEXT: ld1w { z3.d }, p0/z, [x0] ; CHECK-NEXT: mov z4.s, w8 ; CHECK-NEXT: mov w8, #57344 // =0xe000 ; CHECK-NEXT: movk w8, #17535, lsl #16 ; CHECK-NEXT: mov z5.s, w8 -; CHECK-NEXT: ld1w { z3.d }, p0/z, [x0] ; CHECK-NEXT: fmul z4.s, p0/m, z4.s, z3.s ; CHECK-NEXT: fadd z4.s, p0/m, z4.s, z5.s ; CHECK-NEXT: mov z5.d, #1023 // =0x3ff diff --git a/llvm/test/CodeGen/AArch64/sve-fp-reciprocal.ll b/llvm/test/CodeGen/AArch64/sve-fp-reciprocal.ll index aefc8de43143..6420071b3dce 100644 --- a/llvm/test/CodeGen/AArch64/sve-fp-reciprocal.ll +++ b/llvm/test/CodeGen/AArch64/sve-fp-reciprocal.ll @@ -92,8 +92,8 @@ define @fsqrt_recip_8f16( %a) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: frsqrte z1.h, z0.h ; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: fmul z2.h, z1.h, z1.h ; CHECK-NEXT: fcmne p0.h, p0/z, z0.h, #0.0 +; CHECK-NEXT: fmul z2.h, z1.h, z1.h ; CHECK-NEXT: frsqrts z2.h, z0.h, z2.h ; CHECK-NEXT: fmul z1.h, z1.h, z2.h ; CHECK-NEXT: fmul z0.h, p0/m, z0.h, z1.h @@ -117,8 +117,8 @@ define @fsqrt_recip_4f32( %a) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: frsqrte z1.s, z0.s ; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: fmul z2.s, z1.s, z1.s ; CHECK-NEXT: fcmne p0.s, p0/z, z0.s, #0.0 +; CHECK-NEXT: fmul z2.s, z1.s, z1.s ; CHECK-NEXT: frsqrts z2.s, z0.s, z2.s ; CHECK-NEXT: fmul z1.s, z1.s, z2.s ; CHECK-NEXT: fmul z2.s, z1.s, z1.s @@ -145,8 +145,8 @@ define @fsqrt_recip_2f64( %a) #0 { ; CHECK: // %bb.0: ; CHECK-NEXT: frsqrte z1.d, z0.d ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: fmul z2.d, z1.d, z1.d ; CHECK-NEXT: fcmne p0.d, p0/z, z0.d, #0.0 +; CHECK-NEXT: fmul z2.d, z1.d, z1.d ; CHECK-NEXT: frsqrts z2.d, z0.d, z2.d ; CHECK-NEXT: fmul z1.d, z1.d, z2.d ; CHECK-NEXT: fmul z2.d, z1.d, z1.d diff --git a/llvm/test/CodeGen/AArch64/sve-fp-reduce-fadda.ll b/llvm/test/CodeGen/AArch64/sve-fp-reduce-fadda.ll index 460d8a8694bc..1a2ab8d4253a 100644 --- a/llvm/test/CodeGen/AArch64/sve-fp-reduce-fadda.ll +++ b/llvm/test/CodeGen/AArch64/sve-fp-reduce-fadda.ll @@ -51,10 +51,10 @@ define half @fadda_nxv6f16( %v, half %s) { ; CHECK-NEXT: addvl sp, sp, #-1 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #32768 // =0x8000 -; CHECK-NEXT: ptrue p1.d +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w8 +; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: st1h { z0.h }, p0, [sp] ; CHECK-NEXT: fmov s0, s1 ; CHECK-NEXT: st1h { z2.d }, p1, [sp, #3, mul vl] diff --git a/llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll b/llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll index 813f1601e809..584c29ebcfc0 100644 --- a/llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll +++ b/llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll @@ -14,8 +14,8 @@ declare @llvm.fptosi.sat.nxv4f32.nxv4i64( define @test_signed_v2f32_v2i32( %f) { ; CHECK-LABEL: test_signed_v2f32_v2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #-822083584 // =0xcf000000 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, #0xffffffff80000000 ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mov w8, #1325400063 // =0x4effffff @@ -38,8 +38,8 @@ define @test_signed_v2f32_v2i32( %f) { define @test_signed_v4f32_v4i32( %f) { ; CHECK-LABEL: test_signed_v4f32_v4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #-822083584 // =0xcf000000 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, #0x80000000 ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mov w8, #1325400063 // =0x4effffff @@ -67,29 +67,29 @@ define @test_signed_v8f32_v8i32( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #-822083584 // =0xcf000000 -; CHECK-NEXT: mov z3.s, #0x80000000 +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: mov z6.s, #0x7fffffff ; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: mov w8, #1325400063 // =0x4effffff -; CHECK-NEXT: mov z6.s, #0x7fffffff -; CHECK-NEXT: mov z4.s, w8 -; CHECK-NEXT: fcmge p1.s, p0/z, z0.s, z2.s -; CHECK-NEXT: fcmge p2.s, p0/z, z1.s, z2.s -; CHECK-NEXT: movprfx z2, z0 -; CHECK-NEXT: fcvtzs z2.s, p0/m, z0.s +; CHECK-NEXT: mov z3.s, w8 +; CHECK-NEXT: movprfx z4, z0 +; CHECK-NEXT: fcvtzs z4.s, p0/m, z0.s ; CHECK-NEXT: movprfx z5, z1 ; CHECK-NEXT: fcvtzs z5.s, p0/m, z1.s -; CHECK-NEXT: fcmgt p3.s, p0/z, z0.s, z4.s -; CHECK-NEXT: fcmgt p4.s, p0/z, z1.s, z4.s +; CHECK-NEXT: fcmge p1.s, p0/z, z0.s, z2.s +; CHECK-NEXT: fcmge p2.s, p0/z, z1.s, z2.s +; CHECK-NEXT: mov z2.s, #0x80000000 +; CHECK-NEXT: fcmgt p3.s, p0/z, z0.s, z3.s +; CHECK-NEXT: fcmgt p4.s, p0/z, z1.s, z3.s ; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: mov z2.s, p1/m, z3.s +; CHECK-NEXT: sel z3.s, p1, z2.s, z4.s ; CHECK-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s ; CHECK-NEXT: fcmuo p0.s, p0/z, z1.s, z1.s -; CHECK-NEXT: sel z3.s, p2, z3.s, z5.s -; CHECK-NEXT: sel z0.s, p3, z6.s, z2.s -; CHECK-NEXT: sel z1.s, p4, z6.s, z3.s +; CHECK-NEXT: sel z2.s, p2, z2.s, z5.s +; CHECK-NEXT: sel z0.s, p3, z6.s, z3.s +; CHECK-NEXT: sel z1.s, p4, z6.s, z2.s ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: mov z0.s, p1/m, #0 // =0x0 ; CHECK-NEXT: mov z1.s, p0/m, #0 // =0x0 @@ -103,8 +103,8 @@ define @test_signed_v8f32_v8i32( %f) { define @test_signed_v4f32_v4i16( %f) { ; CHECK-LABEL: test_signed_v4f32_v4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #-956301312 // =0xc7000000 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mov w8, #65024 // =0xfe00 ; CHECK-NEXT: movk w8, #18175, lsl #16 @@ -132,28 +132,28 @@ define @test_signed_v8f32_v8i16( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #-956301312 // =0xc7000000 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z5.s, #32767 // =0x7fff ; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: mov w8, #65024 // =0xfe00 ; CHECK-NEXT: movk w8, #18175, lsl #16 -; CHECK-NEXT: mov z3.s, w8 -; CHECK-NEXT: fcmge p1.s, p0/z, z1.s, z2.s -; CHECK-NEXT: fcmge p2.s, p0/z, z0.s, z2.s -; CHECK-NEXT: movprfx z2, z1 -; CHECK-NEXT: fcvtzs z2.s, p0/m, z1.s +; CHECK-NEXT: movprfx z3, z1 +; CHECK-NEXT: fcvtzs z3.s, p0/m, z1.s ; CHECK-NEXT: movprfx z4, z0 ; CHECK-NEXT: fcvtzs z4.s, p0/m, z0.s -; CHECK-NEXT: fcmgt p3.s, p0/z, z1.s, z3.s -; CHECK-NEXT: fcmgt p4.s, p0/z, z0.s, z3.s +; CHECK-NEXT: fcmge p1.s, p0/z, z1.s, z2.s +; CHECK-NEXT: fcmge p2.s, p0/z, z0.s, z2.s +; CHECK-NEXT: mov z2.s, w8 +; CHECK-NEXT: fcmgt p3.s, p0/z, z1.s, z2.s +; CHECK-NEXT: fcmgt p4.s, p0/z, z0.s, z2.s ; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: mov z2.s, p1/m, #-32768 // =0xffffffffffff8000 +; CHECK-NEXT: mov z3.s, p1/m, #-32768 // =0xffffffffffff8000 ; CHECK-NEXT: fcmuo p1.s, p0/z, z1.s, z1.s ; CHECK-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s ; CHECK-NEXT: mov z4.s, p2/m, #-32768 // =0xffffffffffff8000 -; CHECK-NEXT: sel z0.s, p3, z5.s, z2.s +; CHECK-NEXT: sel z0.s, p3, z5.s, z3.s ; CHECK-NEXT: sel z1.s, p4, z5.s, z4.s ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: mov z0.s, p1/m, #0 // =0x0 @@ -169,8 +169,8 @@ define @test_signed_v8f32_v8i16( %f) { define @test_signed_v2f32_v2i64( %f) { ; CHECK-LABEL: test_signed_v2f32_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #-553648128 // =0xdf000000 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, #0x8000000000000000 ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mov w8, #1593835519 // =0x5effffff @@ -198,31 +198,31 @@ define @test_signed_v4f32_v4i64( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #-553648128 // =0xdf000000 ; CHECK-NEXT: uunpklo z1.d, z0.s ; CHECK-NEXT: uunpkhi z0.d, z0.s ; CHECK-NEXT: mov z2.s, w8 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #1593835519 // =0x5effffff -; CHECK-NEXT: mov z3.d, #0x8000000000000000 -; CHECK-NEXT: mov z4.s, w8 +; CHECK-NEXT: mov z3.s, w8 ; CHECK-NEXT: mov z6.d, #0x7fffffffffffffff ; CHECK-NEXT: fcmge p1.s, p0/z, z1.s, z2.s ; CHECK-NEXT: fcmge p2.s, p0/z, z0.s, z2.s -; CHECK-NEXT: movprfx z2, z1 -; CHECK-NEXT: fcvtzs z2.d, p0/m, z1.s +; CHECK-NEXT: mov z2.d, #0x8000000000000000 +; CHECK-NEXT: movprfx z4, z1 +; CHECK-NEXT: fcvtzs z4.d, p0/m, z1.s ; CHECK-NEXT: movprfx z5, z0 ; CHECK-NEXT: fcvtzs z5.d, p0/m, z0.s -; CHECK-NEXT: fcmgt p3.s, p0/z, z1.s, z4.s -; CHECK-NEXT: fcmgt p4.s, p0/z, z0.s, z4.s +; CHECK-NEXT: fcmgt p3.s, p0/z, z1.s, z3.s +; CHECK-NEXT: fcmgt p4.s, p0/z, z0.s, z3.s ; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: mov z2.d, p1/m, z3.d +; CHECK-NEXT: sel z3.d, p1, z2.d, z4.d ; CHECK-NEXT: fcmuo p1.s, p0/z, z1.s, z1.s ; CHECK-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s -; CHECK-NEXT: sel z3.d, p2, z3.d, z5.d -; CHECK-NEXT: sel z0.d, p3, z6.d, z2.d -; CHECK-NEXT: sel z1.d, p4, z6.d, z3.d +; CHECK-NEXT: sel z2.d, p2, z2.d, z5.d +; CHECK-NEXT: sel z0.d, p3, z6.d, z3.d +; CHECK-NEXT: sel z1.d, p4, z6.d, z2.d ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: mov z0.d, p1/m, #0 // =0x0 ; CHECK-NEXT: mov z1.d, p0/m, #0 // =0x0 @@ -246,8 +246,8 @@ declare @llvm.fptosi.sat.nxv4f64.nxv4i64( @test_signed_v2f64_v2i32( %f) { ; CHECK-LABEL: test_signed_v2f64_v2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #-4476578029606273024 // =0xc1e0000000000000 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, #0xffffffff80000000 ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mov x8, #281474972516352 // =0xffffffc00000 @@ -276,30 +276,30 @@ define @test_signed_v4f64_v4i32( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #-4476578029606273024 // =0xc1e0000000000000 -; CHECK-NEXT: mov z3.d, #0xffffffff80000000 +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: mov z6.d, #0x7fffffff ; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: mov x8, #281474972516352 // =0xffffffc00000 -; CHECK-NEXT: mov z6.d, #0x7fffffff ; CHECK-NEXT: movk x8, #16863, lsl #48 -; CHECK-NEXT: mov z4.d, x8 -; CHECK-NEXT: fcmge p1.d, p0/z, z1.d, z2.d -; CHECK-NEXT: fcmge p2.d, p0/z, z0.d, z2.d -; CHECK-NEXT: movprfx z2, z1 -; CHECK-NEXT: fcvtzs z2.d, p0/m, z1.d +; CHECK-NEXT: movprfx z4, z1 +; CHECK-NEXT: fcvtzs z4.d, p0/m, z1.d ; CHECK-NEXT: movprfx z5, z0 ; CHECK-NEXT: fcvtzs z5.d, p0/m, z0.d -; CHECK-NEXT: fcmgt p3.d, p0/z, z1.d, z4.d -; CHECK-NEXT: fcmgt p4.d, p0/z, z0.d, z4.d +; CHECK-NEXT: mov z3.d, x8 +; CHECK-NEXT: fcmge p1.d, p0/z, z1.d, z2.d +; CHECK-NEXT: fcmge p2.d, p0/z, z0.d, z2.d +; CHECK-NEXT: mov z2.d, #0xffffffff80000000 +; CHECK-NEXT: fcmgt p3.d, p0/z, z1.d, z3.d +; CHECK-NEXT: fcmgt p4.d, p0/z, z0.d, z3.d ; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: mov z2.d, p1/m, z3.d +; CHECK-NEXT: sel z3.d, p1, z2.d, z4.d ; CHECK-NEXT: fcmuo p1.d, p0/z, z1.d, z1.d ; CHECK-NEXT: fcmuo p0.d, p0/z, z0.d, z0.d -; CHECK-NEXT: sel z3.d, p2, z3.d, z5.d -; CHECK-NEXT: sel z0.d, p3, z6.d, z2.d -; CHECK-NEXT: sel z1.d, p4, z6.d, z3.d +; CHECK-NEXT: sel z2.d, p2, z2.d, z5.d +; CHECK-NEXT: sel z0.d, p3, z6.d, z3.d +; CHECK-NEXT: sel z1.d, p4, z6.d, z2.d ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: mov z0.d, p1/m, #0 // =0x0 ; CHECK-NEXT: mov z1.d, p0/m, #0 // =0x0 @@ -322,48 +322,48 @@ define @test_signed_v8f64_v8i32( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #-4476578029606273024 // =0xc1e0000000000000 -; CHECK-NEXT: mov z26.d, #0x7fffffff +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: mov z5.d, #0xffffffff80000000 ; CHECK-NEXT: mov z4.d, x8 ; CHECK-NEXT: mov x8, #281474972516352 // =0xffffffc00000 +; CHECK-NEXT: mov z26.d, #0x7fffffff ; CHECK-NEXT: movk x8, #16863, lsl #48 -; CHECK-NEXT: mov z5.d, x8 -; CHECK-NEXT: fcmge p1.d, p0/z, z1.d, z4.d -; CHECK-NEXT: fcmge p2.d, p0/z, z0.d, z4.d -; CHECK-NEXT: fcmge p3.d, p0/z, z3.d, z4.d -; CHECK-NEXT: fcmge p4.d, p0/z, z2.d, z4.d -; CHECK-NEXT: mov z4.d, #0xffffffff80000000 -; CHECK-NEXT: movprfx z6, z1 -; CHECK-NEXT: fcvtzs z6.d, p0/m, z1.d ; CHECK-NEXT: movprfx z7, z0 ; CHECK-NEXT: fcvtzs z7.d, p0/m, z0.d ; CHECK-NEXT: movprfx z24, z3 ; CHECK-NEXT: fcvtzs z24.d, p0/m, z3.d +; CHECK-NEXT: mov z6.d, x8 ; CHECK-NEXT: movprfx z25, z2 ; CHECK-NEXT: fcvtzs z25.d, p0/m, z2.d -; CHECK-NEXT: fcmgt p5.d, p0/z, z1.d, z5.d -; CHECK-NEXT: fcmgt p6.d, p0/z, z0.d, z5.d -; CHECK-NEXT: fcmgt p7.d, p0/z, z3.d, z5.d +; CHECK-NEXT: fcmge p1.d, p0/z, z1.d, z4.d +; CHECK-NEXT: fcmge p2.d, p0/z, z0.d, z4.d +; CHECK-NEXT: fcmge p3.d, p0/z, z3.d, z4.d +; CHECK-NEXT: fcmge p4.d, p0/z, z2.d, z4.d +; CHECK-NEXT: movprfx z4, z1 +; CHECK-NEXT: fcvtzs z4.d, p0/m, z1.d +; CHECK-NEXT: fcmgt p5.d, p0/z, z1.d, z6.d +; CHECK-NEXT: fcmgt p6.d, p0/z, z0.d, z6.d +; CHECK-NEXT: fcmgt p7.d, p0/z, z3.d, z6.d ; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b ; CHECK-NEXT: not p3.b, p0/z, p3.b +; CHECK-NEXT: mov z4.d, p1/m, z5.d +; CHECK-NEXT: fcmgt p1.d, p0/z, z2.d, z6.d ; CHECK-NEXT: not p4.b, p0/z, p4.b -; CHECK-NEXT: mov z6.d, p1/m, z4.d -; CHECK-NEXT: fcmgt p1.d, p0/z, z2.d, z5.d -; CHECK-NEXT: sel z5.d, p2, z4.d, z7.d +; CHECK-NEXT: sel z6.d, p2, z5.d, z7.d ; CHECK-NEXT: fcmuo p2.d, p0/z, z1.d, z1.d -; CHECK-NEXT: sel z7.d, p3, z4.d, z24.d +; CHECK-NEXT: sel z7.d, p3, z5.d, z24.d ; CHECK-NEXT: fcmuo p3.d, p0/z, z0.d, z0.d -; CHECK-NEXT: sel z4.d, p4, z4.d, z25.d +; CHECK-NEXT: sel z5.d, p4, z5.d, z25.d ; CHECK-NEXT: fcmuo p4.d, p0/z, z3.d, z3.d ; CHECK-NEXT: fcmuo p0.d, p0/z, z2.d, z2.d -; CHECK-NEXT: sel z0.d, p5, z26.d, z6.d -; CHECK-NEXT: sel z1.d, p6, z26.d, z5.d +; CHECK-NEXT: sel z0.d, p5, z26.d, z4.d +; CHECK-NEXT: sel z1.d, p6, z26.d, z6.d ; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: sel z2.d, p7, z26.d, z7.d ; CHECK-NEXT: ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: sel z3.d, p1, z26.d, z4.d +; CHECK-NEXT: sel z3.d, p1, z26.d, z5.d ; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: mov z0.d, p2/m, #0 // =0x0 ; CHECK-NEXT: mov z1.d, p3/m, #0 // =0x0 @@ -387,28 +387,28 @@ define @test_signed_v4f64_v4i16( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #-4548635623644200960 // =0xc0e0000000000000 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z5.d, #32767 // =0x7fff ; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: mov x8, #281200098803712 // =0xffc000000000 ; CHECK-NEXT: movk x8, #16607, lsl #48 -; CHECK-NEXT: mov z3.d, x8 -; CHECK-NEXT: fcmge p1.d, p0/z, z1.d, z2.d -; CHECK-NEXT: fcmge p2.d, p0/z, z0.d, z2.d -; CHECK-NEXT: movprfx z2, z1 -; CHECK-NEXT: fcvtzs z2.d, p0/m, z1.d +; CHECK-NEXT: movprfx z3, z1 +; CHECK-NEXT: fcvtzs z3.d, p0/m, z1.d ; CHECK-NEXT: movprfx z4, z0 ; CHECK-NEXT: fcvtzs z4.d, p0/m, z0.d -; CHECK-NEXT: fcmgt p3.d, p0/z, z1.d, z3.d -; CHECK-NEXT: fcmgt p4.d, p0/z, z0.d, z3.d +; CHECK-NEXT: fcmge p1.d, p0/z, z1.d, z2.d +; CHECK-NEXT: fcmge p2.d, p0/z, z0.d, z2.d +; CHECK-NEXT: mov z2.d, x8 +; CHECK-NEXT: fcmgt p3.d, p0/z, z1.d, z2.d +; CHECK-NEXT: fcmgt p4.d, p0/z, z0.d, z2.d ; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: mov z2.d, p1/m, #-32768 // =0xffffffffffff8000 +; CHECK-NEXT: mov z3.d, p1/m, #-32768 // =0xffffffffffff8000 ; CHECK-NEXT: fcmuo p1.d, p0/z, z1.d, z1.d ; CHECK-NEXT: fcmuo p0.d, p0/z, z0.d, z0.d ; CHECK-NEXT: mov z4.d, p2/m, #-32768 // =0xffffffffffff8000 -; CHECK-NEXT: sel z0.d, p3, z5.d, z2.d +; CHECK-NEXT: sel z0.d, p3, z5.d, z3.d ; CHECK-NEXT: sel z1.d, p4, z5.d, z4.d ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: mov z0.d, p1/m, #0 // =0x0 @@ -432,34 +432,34 @@ define @test_signed_v8f64_v8i16( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #-4548635623644200960 // =0xc0e0000000000000 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z25.d, #32767 // =0x7fff ; CHECK-NEXT: mov z4.d, x8 ; CHECK-NEXT: mov x8, #281200098803712 // =0xffc000000000 ; CHECK-NEXT: movk x8, #16607, lsl #48 -; CHECK-NEXT: fcmge p1.d, p0/z, z3.d, z4.d -; CHECK-NEXT: fcmge p2.d, p0/z, z2.d, z4.d -; CHECK-NEXT: fcmge p3.d, p0/z, z1.d, z4.d -; CHECK-NEXT: fcmge p4.d, p0/z, z0.d, z4.d -; CHECK-NEXT: movprfx z5, z3 -; CHECK-NEXT: fcvtzs z5.d, p0/m, z3.d -; CHECK-NEXT: mov z4.d, x8 ; CHECK-NEXT: movprfx z6, z2 ; CHECK-NEXT: fcvtzs z6.d, p0/m, z2.d ; CHECK-NEXT: movprfx z7, z1 ; CHECK-NEXT: fcvtzs z7.d, p0/m, z1.d +; CHECK-NEXT: mov z5.d, x8 ; CHECK-NEXT: movprfx z24, z0 ; CHECK-NEXT: fcvtzs z24.d, p0/m, z0.d +; CHECK-NEXT: fcmge p1.d, p0/z, z3.d, z4.d +; CHECK-NEXT: fcmge p2.d, p0/z, z2.d, z4.d +; CHECK-NEXT: fcmge p3.d, p0/z, z1.d, z4.d +; CHECK-NEXT: fcmge p4.d, p0/z, z0.d, z4.d +; CHECK-NEXT: movprfx z4, z3 +; CHECK-NEXT: fcvtzs z4.d, p0/m, z3.d +; CHECK-NEXT: fcmgt p5.d, p0/z, z3.d, z5.d +; CHECK-NEXT: fcmgt p6.d, p0/z, z2.d, z5.d +; CHECK-NEXT: fcmgt p7.d, p0/z, z1.d, z5.d ; CHECK-NEXT: not p1.b, p0/z, p1.b -; CHECK-NEXT: fcmgt p5.d, p0/z, z3.d, z4.d -; CHECK-NEXT: fcmgt p6.d, p0/z, z2.d, z4.d ; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: fcmgt p7.d, p0/z, z1.d, z4.d ; CHECK-NEXT: not p3.b, p0/z, p3.b +; CHECK-NEXT: mov z4.d, p1/m, #-32768 // =0xffffffffffff8000 +; CHECK-NEXT: fcmgt p1.d, p0/z, z0.d, z5.d ; CHECK-NEXT: not p4.b, p0/z, p4.b -; CHECK-NEXT: mov z5.d, p1/m, #-32768 // =0xffffffffffff8000 -; CHECK-NEXT: fcmgt p1.d, p0/z, z0.d, z4.d ; CHECK-NEXT: mov z6.d, p2/m, #-32768 // =0xffffffffffff8000 ; CHECK-NEXT: fcmuo p2.d, p0/z, z3.d, z3.d ; CHECK-NEXT: mov z7.d, p3/m, #-32768 // =0xffffffffffff8000 @@ -467,7 +467,7 @@ define @test_signed_v8f64_v8i16( %f) { ; CHECK-NEXT: mov z24.d, p4/m, #-32768 // =0xffffffffffff8000 ; CHECK-NEXT: fcmuo p4.d, p0/z, z1.d, z1.d ; CHECK-NEXT: fcmuo p0.d, p0/z, z0.d, z0.d -; CHECK-NEXT: sel z2.d, p5, z25.d, z5.d +; CHECK-NEXT: sel z2.d, p5, z25.d, z4.d ; CHECK-NEXT: sel z0.d, p6, z25.d, z6.d ; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: sel z1.d, p7, z25.d, z7.d @@ -492,8 +492,8 @@ define @test_signed_v8f64_v8i16( %f) { define @test_signed_v2f64_v2i64( %f) { ; CHECK-LABEL: test_signed_v2f64_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #-4332462841530417152 // =0xc3e0000000000000 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, #0x8000000000000000 ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mov x8, #4890909195324358655 // =0x43dfffffffffffff @@ -521,29 +521,29 @@ define @test_signed_v4f64_v4i64( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #-4332462841530417152 // =0xc3e0000000000000 -; CHECK-NEXT: mov z3.d, #0x8000000000000000 +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: mov z6.d, #0x7fffffffffffffff ; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: mov x8, #4890909195324358655 // =0x43dfffffffffffff -; CHECK-NEXT: mov z6.d, #0x7fffffffffffffff -; CHECK-NEXT: mov z4.d, x8 -; CHECK-NEXT: fcmge p1.d, p0/z, z0.d, z2.d -; CHECK-NEXT: fcmge p2.d, p0/z, z1.d, z2.d -; CHECK-NEXT: movprfx z2, z0 -; CHECK-NEXT: fcvtzs z2.d, p0/m, z0.d +; CHECK-NEXT: mov z3.d, x8 +; CHECK-NEXT: movprfx z4, z0 +; CHECK-NEXT: fcvtzs z4.d, p0/m, z0.d ; CHECK-NEXT: movprfx z5, z1 ; CHECK-NEXT: fcvtzs z5.d, p0/m, z1.d -; CHECK-NEXT: fcmgt p3.d, p0/z, z0.d, z4.d -; CHECK-NEXT: fcmgt p4.d, p0/z, z1.d, z4.d +; CHECK-NEXT: fcmge p1.d, p0/z, z0.d, z2.d +; CHECK-NEXT: fcmge p2.d, p0/z, z1.d, z2.d +; CHECK-NEXT: mov z2.d, #0x8000000000000000 +; CHECK-NEXT: fcmgt p3.d, p0/z, z0.d, z3.d +; CHECK-NEXT: fcmgt p4.d, p0/z, z1.d, z3.d ; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: mov z2.d, p1/m, z3.d +; CHECK-NEXT: sel z3.d, p1, z2.d, z4.d ; CHECK-NEXT: fcmuo p1.d, p0/z, z0.d, z0.d ; CHECK-NEXT: fcmuo p0.d, p0/z, z1.d, z1.d -; CHECK-NEXT: sel z3.d, p2, z3.d, z5.d -; CHECK-NEXT: sel z0.d, p3, z6.d, z2.d -; CHECK-NEXT: sel z1.d, p4, z6.d, z3.d +; CHECK-NEXT: sel z2.d, p2, z2.d, z5.d +; CHECK-NEXT: sel z0.d, p3, z6.d, z3.d +; CHECK-NEXT: sel z1.d, p4, z6.d, z2.d ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: mov z0.d, p1/m, #0 // =0x0 ; CHECK-NEXT: mov z1.d, p0/m, #0 // =0x0 @@ -568,8 +568,8 @@ declare @llvm.fptosi.sat.nxv4f16.nxv4i64() define @test_signed_v2f16_v2i32( %f) { ; CHECK-LABEL: test_signed_v2f16_v2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #64511 // =0xfbff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, #0xffffffff80000000 ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: mov w8, #31743 // =0x7bff @@ -592,8 +592,8 @@ define @test_signed_v2f16_v2i32( %f) { define @test_signed_v4f16_v4i32( %f) { ; CHECK-LABEL: test_signed_v4f16_v4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #64511 // =0xfbff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, #0x80000000 ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: mov w8, #31743 // =0x7bff @@ -621,31 +621,31 @@ define @test_signed_v8f16_v8i32( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #64511 // =0xfbff ; CHECK-NEXT: uunpklo z1.s, z0.h ; CHECK-NEXT: uunpkhi z0.s, z0.h ; CHECK-NEXT: mov z2.h, w8 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #31743 // =0x7bff -; CHECK-NEXT: mov z3.s, #0x80000000 -; CHECK-NEXT: mov z4.h, w8 +; CHECK-NEXT: mov z3.h, w8 ; CHECK-NEXT: mov z6.s, #0x7fffffff ; CHECK-NEXT: fcmge p1.h, p0/z, z1.h, z2.h ; CHECK-NEXT: fcmge p2.h, p0/z, z0.h, z2.h -; CHECK-NEXT: movprfx z2, z1 -; CHECK-NEXT: fcvtzs z2.s, p0/m, z1.h +; CHECK-NEXT: mov z2.s, #0x80000000 +; CHECK-NEXT: movprfx z4, z1 +; CHECK-NEXT: fcvtzs z4.s, p0/m, z1.h ; CHECK-NEXT: movprfx z5, z0 ; CHECK-NEXT: fcvtzs z5.s, p0/m, z0.h -; CHECK-NEXT: fcmgt p3.h, p0/z, z1.h, z4.h -; CHECK-NEXT: fcmgt p4.h, p0/z, z0.h, z4.h +; CHECK-NEXT: fcmgt p3.h, p0/z, z1.h, z3.h +; CHECK-NEXT: fcmgt p4.h, p0/z, z0.h, z3.h ; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: mov z2.s, p1/m, z3.s +; CHECK-NEXT: sel z3.s, p1, z2.s, z4.s ; CHECK-NEXT: fcmuo p1.h, p0/z, z1.h, z1.h ; CHECK-NEXT: fcmuo p0.h, p0/z, z0.h, z0.h -; CHECK-NEXT: sel z3.s, p2, z3.s, z5.s -; CHECK-NEXT: sel z0.s, p3, z6.s, z2.s -; CHECK-NEXT: sel z1.s, p4, z6.s, z3.s +; CHECK-NEXT: sel z2.s, p2, z2.s, z5.s +; CHECK-NEXT: sel z0.s, p3, z6.s, z3.s +; CHECK-NEXT: sel z1.s, p4, z6.s, z2.s ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: mov z0.s, p1/m, #0 // =0x0 ; CHECK-NEXT: mov z1.s, p0/m, #0 // =0x0 @@ -659,8 +659,8 @@ define @test_signed_v8f16_v8i32( %f) { define @test_signed_v4f16_v4i16( %f) { ; CHECK-LABEL: test_signed_v4f16_v4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #63488 // =0xf800 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: mov w8, #30719 // =0x77ff ; CHECK-NEXT: mov z2.h, w8 @@ -682,8 +682,8 @@ define @test_signed_v4f16_v4i16( %f) { define @test_signed_v8f16_v8i16( %f) { ; CHECK-LABEL: test_signed_v8f16_v8i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #63488 // =0xf800 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: mov w8, #30719 // =0x77ff ; CHECK-NEXT: mov z2.h, w8 @@ -705,8 +705,8 @@ define @test_signed_v8f16_v8i16( %f) { define @test_signed_v2f16_v2i64( %f) { ; CHECK-LABEL: test_signed_v2f16_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #64511 // =0xfbff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, #0x8000000000000000 ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: mov w8, #31743 // =0x7bff @@ -734,31 +734,31 @@ define @test_signed_v4f16_v4i64( %f) { ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #64511 // =0xfbff ; CHECK-NEXT: uunpklo z1.d, z0.s ; CHECK-NEXT: uunpkhi z0.d, z0.s ; CHECK-NEXT: mov z2.h, w8 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #31743 // =0x7bff -; CHECK-NEXT: mov z3.d, #0x8000000000000000 -; CHECK-NEXT: mov z4.h, w8 +; CHECK-NEXT: mov z3.h, w8 ; CHECK-NEXT: mov z6.d, #0x7fffffffffffffff ; CHECK-NEXT: fcmge p1.h, p0/z, z1.h, z2.h ; CHECK-NEXT: fcmge p2.h, p0/z, z0.h, z2.h -; CHECK-NEXT: movprfx z2, z1 -; CHECK-NEXT: fcvtzs z2.d, p0/m, z1.h +; CHECK-NEXT: mov z2.d, #0x8000000000000000 +; CHECK-NEXT: movprfx z4, z1 +; CHECK-NEXT: fcvtzs z4.d, p0/m, z1.h ; CHECK-NEXT: movprfx z5, z0 ; CHECK-NEXT: fcvtzs z5.d, p0/m, z0.h -; CHECK-NEXT: fcmgt p3.h, p0/z, z1.h, z4.h -; CHECK-NEXT: fcmgt p4.h, p0/z, z0.h, z4.h +; CHECK-NEXT: fcmgt p3.h, p0/z, z1.h, z3.h +; CHECK-NEXT: fcmgt p4.h, p0/z, z0.h, z3.h ; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: mov z2.d, p1/m, z3.d +; CHECK-NEXT: sel z3.d, p1, z2.d, z4.d ; CHECK-NEXT: fcmuo p1.h, p0/z, z1.h, z1.h ; CHECK-NEXT: fcmuo p0.h, p0/z, z0.h, z0.h -; CHECK-NEXT: sel z3.d, p2, z3.d, z5.d -; CHECK-NEXT: sel z0.d, p3, z6.d, z2.d -; CHECK-NEXT: sel z1.d, p4, z6.d, z3.d +; CHECK-NEXT: sel z2.d, p2, z2.d, z5.d +; CHECK-NEXT: sel z0.d, p3, z6.d, z3.d +; CHECK-NEXT: sel z1.d, p4, z6.d, z2.d ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: mov z0.d, p1/m, #0 // =0x0 ; CHECK-NEXT: mov z1.d, p0/m, #0 // =0x0 diff --git a/llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll b/llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll index c56c0b37888d..ed352ffec339 100644 --- a/llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll +++ b/llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll @@ -82,8 +82,8 @@ define @test_signed_v4f32_v4i16( %f) { ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #65280 // =0xff00 ; CHECK-NEXT: movk w8, #18303, lsl #16 -; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: fcmge p1.s, p0/z, z0.s, #0.0 +; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: movprfx z2, z0 ; CHECK-NEXT: fcvtzu z2.s, p0/m, z0.s ; CHECK-NEXT: not p1.b, p0/z, p1.b @@ -102,9 +102,9 @@ define @test_signed_v8f32_v8i16( %f) { ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #65280 // =0xff00 ; CHECK-NEXT: movk w8, #18303, lsl #16 -; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: fcmge p1.s, p0/z, z1.s, #0.0 ; CHECK-NEXT: fcmge p2.s, p0/z, z0.s, #0.0 +; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: movprfx z3, z1 ; CHECK-NEXT: fcvtzu z3.s, p0/m, z1.s ; CHECK-NEXT: movprfx z4, z0 @@ -146,10 +146,10 @@ define @test_signed_v2f32_v2i64( %f) { define @test_signed_v4f32_v4i64( %f) { ; CHECK-LABEL: test_signed_v4f32_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z2.d, z0.s ; CHECK-NEXT: uunpkhi z3.d, z0.s ; CHECK-NEXT: mov w8, #1602224127 // =0x5f7fffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z4.s, w8 ; CHECK-NEXT: fcmge p1.s, p0/z, z2.s, #0.0 ; CHECK-NEXT: fcmge p2.s, p0/z, z3.s, #0.0 @@ -186,8 +186,8 @@ define @test_signed_v2f64_v2i32( %f) { ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #281474974613504 // =0xffffffe00000 ; CHECK-NEXT: movk x8, #16879, lsl #48 -; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: fcmge p1.d, p0/z, z0.d, #0.0 +; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: movprfx z2, z0 ; CHECK-NEXT: fcvtzu z2.d, p0/m, z0.d ; CHECK-NEXT: not p1.b, p0/z, p1.b @@ -206,9 +206,9 @@ define @test_signed_v4f64_v4i32( %f) { ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #281474974613504 // =0xffffffe00000 ; CHECK-NEXT: movk x8, #16879, lsl #48 -; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: fcmge p1.d, p0/z, z1.d, #0.0 ; CHECK-NEXT: fcmge p2.d, p0/z, z0.d, #0.0 +; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: movprfx z3, z1 ; CHECK-NEXT: fcvtzu z3.d, p0/m, z1.d ; CHECK-NEXT: movprfx z4, z0 @@ -241,28 +241,28 @@ define @test_signed_v8f64_v8i32( %f) { ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #281474974613504 // =0xffffffe00000 ; CHECK-NEXT: movk x8, #16879, lsl #48 -; CHECK-NEXT: mov z4.d, x8 ; CHECK-NEXT: fcmge p1.d, p0/z, z1.d, #0.0 ; CHECK-NEXT: fcmge p2.d, p0/z, z0.d, #0.0 ; CHECK-NEXT: fcmge p3.d, p0/z, z3.d, #0.0 ; CHECK-NEXT: fcmge p4.d, p0/z, z2.d, #0.0 ; CHECK-NEXT: movprfx z5, z1 ; CHECK-NEXT: fcvtzu z5.d, p0/m, z1.d +; CHECK-NEXT: mov z4.d, x8 ; CHECK-NEXT: movprfx z6, z0 ; CHECK-NEXT: fcvtzu z6.d, p0/m, z0.d ; CHECK-NEXT: movprfx z7, z3 ; CHECK-NEXT: fcvtzu z7.d, p0/m, z3.d ; CHECK-NEXT: movprfx z24, z2 ; CHECK-NEXT: fcvtzu z24.d, p0/m, z2.d +; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: fcmgt p5.d, p0/z, z1.d, z4.d ; CHECK-NEXT: fcmgt p6.d, p0/z, z0.d, z4.d -; CHECK-NEXT: mov z0.d, #0xffffffff -; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b +; CHECK-NEXT: mov z0.d, #0xffffffff ; CHECK-NEXT: not p3.b, p0/z, p3.b -; CHECK-NEXT: not p4.b, p0/z, p4.b ; CHECK-NEXT: mov z5.d, p1/m, #0 // =0x0 ; CHECK-NEXT: fcmgt p1.d, p0/z, z3.d, z4.d +; CHECK-NEXT: not p4.b, p0/z, p4.b ; CHECK-NEXT: fcmgt p0.d, p0/z, z2.d, z4.d ; CHECK-NEXT: mov z6.d, p2/m, #0 // =0x0 ; CHECK-NEXT: mov z7.d, p3/m, #0 // =0x0 @@ -289,9 +289,9 @@ define @test_signed_v4f64_v4i16( %f) { ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #281337537757184 // =0xffe000000000 ; CHECK-NEXT: movk x8, #16623, lsl #48 -; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: fcmge p1.d, p0/z, z1.d, #0.0 ; CHECK-NEXT: fcmge p2.d, p0/z, z0.d, #0.0 +; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: movprfx z3, z1 ; CHECK-NEXT: fcvtzu z3.d, p0/m, z1.d ; CHECK-NEXT: movprfx z4, z0 @@ -324,28 +324,28 @@ define @test_signed_v8f64_v8i16( %f) { ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x8, #281337537757184 // =0xffe000000000 ; CHECK-NEXT: movk x8, #16623, lsl #48 -; CHECK-NEXT: mov z4.d, x8 ; CHECK-NEXT: fcmge p1.d, p0/z, z3.d, #0.0 ; CHECK-NEXT: fcmge p2.d, p0/z, z2.d, #0.0 ; CHECK-NEXT: fcmge p3.d, p0/z, z1.d, #0.0 ; CHECK-NEXT: fcmge p4.d, p0/z, z0.d, #0.0 ; CHECK-NEXT: movprfx z5, z3 ; CHECK-NEXT: fcvtzu z5.d, p0/m, z3.d +; CHECK-NEXT: mov z4.d, x8 ; CHECK-NEXT: movprfx z6, z2 ; CHECK-NEXT: fcvtzu z6.d, p0/m, z2.d ; CHECK-NEXT: movprfx z7, z1 ; CHECK-NEXT: fcvtzu z7.d, p0/m, z1.d ; CHECK-NEXT: movprfx z24, z0 ; CHECK-NEXT: fcvtzu z24.d, p0/m, z0.d +; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: fcmgt p5.d, p0/z, z3.d, z4.d ; CHECK-NEXT: fcmgt p6.d, p0/z, z2.d, z4.d -; CHECK-NEXT: mov z2.d, #65535 // =0xffff -; CHECK-NEXT: not p1.b, p0/z, p1.b ; CHECK-NEXT: not p2.b, p0/z, p2.b +; CHECK-NEXT: mov z2.d, #65535 // =0xffff ; CHECK-NEXT: not p3.b, p0/z, p3.b -; CHECK-NEXT: not p4.b, p0/z, p4.b ; CHECK-NEXT: mov z5.d, p1/m, #0 // =0x0 ; CHECK-NEXT: fcmgt p1.d, p0/z, z1.d, z4.d +; CHECK-NEXT: not p4.b, p0/z, p4.b ; CHECK-NEXT: fcmgt p0.d, p0/z, z0.d, z4.d ; CHECK-NEXT: mov z6.d, p2/m, #0 // =0x0 ; CHECK-NEXT: mov z7.d, p3/m, #0 // =0x0 @@ -465,10 +465,10 @@ define @test_signed_v4f16_v4i32( %f) { define @test_signed_v8f16_v8i32( %f) { ; CHECK-LABEL: test_signed_v8f16_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpklo z2.s, z0.h ; CHECK-NEXT: uunpkhi z3.s, z0.h ; CHECK-NEXT: mov w8, #31743 // =0x7bff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z4.h, w8 ; CHECK-NEXT: fcmge p1.h, p0/z, z2.h, #0.0 ; CHECK-NEXT: fcmge p2.h, p0/z, z3.h, #0.0 @@ -549,10 +549,10 @@ define @test_signed_v2f16_v2i64( %f) { define @test_signed_v4f16_v4i64( %f) { ; CHECK-LABEL: test_signed_v4f16_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z2.d, z0.s ; CHECK-NEXT: uunpkhi z3.d, z0.s ; CHECK-NEXT: mov w8, #31743 // =0x7bff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z4.h, w8 ; CHECK-NEXT: fcmge p1.h, p0/z, z2.h, #0.0 ; CHECK-NEXT: fcmge p2.h, p0/z, z3.h, #0.0 diff --git a/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll b/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll index 5c4c9463528b..ad6371f78ec0 100644 --- a/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll +++ b/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll @@ -71,12 +71,12 @@ define @gather_i8_index_offset_8(ptr %base, i64 %offset, %pg, %data) #0 { ; CHECK-LABEL: scatter_f16_index_offset_var: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: index z1.d, #0, #1 ; CHECK-NEXT: mov z2.d, x1 -; CHECK-NEXT: punpklo p2.h, p0.b +; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: uunpklo z3.d, z0.s ; CHECK-NEXT: uunpkhi z0.d, z0.s +; CHECK-NEXT: punpklo p2.h, p0.b ; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: movprfx z4, z2 ; CHECK-NEXT: mla z4.d, p1/m, z1.d, z2.d @@ -101,16 +101,16 @@ define void @scatter_f16_index_offset_var(ptr %base, i64 %offset, i64 %scale, %pg, %data) #0 { ; CHECK-LABEL: scatter_i8_index_offset_maximum_plus_one: ; CHECK: // %bb.0: -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: mov w8, #33554432 // =0x2000000 ; CHECK-NEXT: uunpklo z2.d, z0.s -; CHECK-NEXT: index z1.d, #0, x8 ; CHECK-NEXT: rdvl x9, #1 -; CHECK-NEXT: add x8, x0, x1 +; CHECK-NEXT: index z1.d, #0, x8 +; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: lsr x9, x9, #4 +; CHECK-NEXT: add x8, x0, x1 ; CHECK-NEXT: mov w10, #67108864 // =0x4000000 -; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: uunpkhi z0.d, z0.s +; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: st1b { z2.d }, p1, [x8, z1.d] ; CHECK-NEXT: madd x8, x9, x10, x8 ; CHECK-NEXT: st1b { z0.d }, p0, [x8, z1.d] @@ -131,17 +131,17 @@ define void @scatter_i8_index_offset_maximum_plus_one(ptr %base, i64 %offset, %pg, %data) #0 { ; CHECK-LABEL: scatter_i8_index_offset_minimum_minus_one: ; CHECK: // %bb.0: -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: mov x8, #-33554433 // =0xfffffffffdffffff ; CHECK-NEXT: uunpklo z2.d, z0.s -; CHECK-NEXT: index z1.d, #0, x8 ; CHECK-NEXT: rdvl x9, #1 -; CHECK-NEXT: mov x10, #-2 // =0xfffffffffffffffe +; CHECK-NEXT: index z1.d, #0, x8 +; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: lsr x9, x9, #4 +; CHECK-NEXT: mov x10, #-2 // =0xfffffffffffffffe ; CHECK-NEXT: add x8, x0, x1 +; CHECK-NEXT: uunpkhi z0.d, z0.s ; CHECK-NEXT: movk x10, #64511, lsl #16 ; CHECK-NEXT: punpkhi p0.h, p0.b -; CHECK-NEXT: uunpkhi z0.d, z0.s ; CHECK-NEXT: st1b { z2.d }, p1, [x8, z1.d] ; CHECK-NEXT: madd x8, x9, x10, x8 ; CHECK-NEXT: st1b { z0.d }, p0, [x8, z1.d] @@ -162,16 +162,16 @@ define void @scatter_i8_index_offset_minimum_minus_one(ptr %base, i64 %offset, < define void @scatter_i8_index_stride_too_big(ptr %base, i64 %offset, %pg, %data) #0 { ; CHECK-LABEL: scatter_i8_index_stride_too_big: ; CHECK: // %bb.0: -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: mov x8, #4611686018427387904 // =0x4000000000000000 ; CHECK-NEXT: uunpklo z2.d, z0.s -; CHECK-NEXT: index z1.d, #0, x8 ; CHECK-NEXT: rdvl x9, #1 -; CHECK-NEXT: add x8, x0, x1 +; CHECK-NEXT: index z1.d, #0, x8 +; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: lsr x9, x9, #4 +; CHECK-NEXT: add x8, x0, x1 ; CHECK-NEXT: mov x10, #-9223372036854775808 // =0x8000000000000000 -; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: uunpkhi z0.d, z0.s +; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: st1b { z2.d }, p1, [x8, z1.d] ; CHECK-NEXT: madd x8, x9, x10, x8 ; CHECK-NEXT: st1b { z0.d }, p0, [x8, z1.d] diff --git a/llvm/test/CodeGen/AArch64/sve-hadd.ll b/llvm/test/CodeGen/AArch64/sve-hadd.ll index c73370d50287..f90aef8daa5d 100644 --- a/llvm/test/CodeGen/AArch64/sve-hadd.ll +++ b/llvm/test/CodeGen/AArch64/sve-hadd.ll @@ -129,9 +129,9 @@ define @haddu_v2i32( %s0, @haddu_v2i16( %s0, @haddu_v4i16( %s0, @haddu_v4i8( %s0, %s ; ; SVE2-LABEL: haddu_v4i8: ; SVE2: // %bb.0: // %entry -; SVE2-NEXT: ptrue p0.s ; SVE2-NEXT: and z0.s, z0.s, #0xff ; SVE2-NEXT: and z1.s, z1.s, #0xff +; SVE2-NEXT: ptrue p0.s ; SVE2-NEXT: uhadd z0.s, p0/m, z0.s, z1.s ; SVE2-NEXT: ret entry: @@ -557,9 +557,9 @@ define @haddu_v8i8( %s0, %s ; ; SVE2-LABEL: haddu_v8i8: ; SVE2: // %bb.0: // %entry -; SVE2-NEXT: ptrue p0.h ; SVE2-NEXT: and z0.h, z0.h, #0xff ; SVE2-NEXT: and z1.h, z1.h, #0xff +; SVE2-NEXT: ptrue p0.h ; SVE2-NEXT: uhadd z0.h, p0/m, z0.h, z1.h ; SVE2-NEXT: ret entry: @@ -787,9 +787,9 @@ define @rhaddu_v2i32( %s0, @rhaddu_v2i16( %s0, @rhaddu_v4i16( %s0, @rhaddu_v4i8( %s0, % ; ; SVE2-LABEL: rhaddu_v4i8: ; SVE2: // %bb.0: // %entry -; SVE2-NEXT: ptrue p0.s ; SVE2-NEXT: and z0.s, z0.s, #0xff ; SVE2-NEXT: and z1.s, z1.s, #0xff +; SVE2-NEXT: ptrue p0.s ; SVE2-NEXT: urhadd z0.s, p0/m, z0.s, z1.s ; SVE2-NEXT: ret entry: @@ -1241,9 +1241,9 @@ define @rhaddu_v8i8( %s0, % ; ; SVE2-LABEL: rhaddu_v8i8: ; SVE2: // %bb.0: // %entry -; SVE2-NEXT: ptrue p0.h ; SVE2-NEXT: and z0.h, z0.h, #0xff ; SVE2-NEXT: and z1.h, z1.h, #0xff +; SVE2-NEXT: ptrue p0.h ; SVE2-NEXT: urhadd z0.h, p0/m, z0.h, z1.h ; SVE2-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/sve-implicit-zero-filling.ll b/llvm/test/CodeGen/AArch64/sve-implicit-zero-filling.ll index e20399de70bf..73bbee094827 100644 --- a/llvm/test/CodeGen/AArch64/sve-implicit-zero-filling.ll +++ b/llvm/test/CodeGen/AArch64/sve-implicit-zero-filling.ll @@ -175,14 +175,14 @@ define @uminv_zero_fill( %pg, @zero_fill_non_zero_index( %pg, %a) #0 { ; CHECK-LABEL: zero_fill_non_zero_index: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z1.d, #0, #1 ; CHECK-NEXT: uminv d3, p0, z0.d ; CHECK-NEXT: mov z2.d, x8 +; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z0.d, #0 // =0x0 -; CHECK-NEXT: fmov x8, d3 ; CHECK-NEXT: cmpeq p0.d, p1/z, z1.d, z2.d +; CHECK-NEXT: fmov x8, d3 ; CHECK-NEXT: mov z0.d, p0/m, x8 ; CHECK-NEXT: ret %t1 = call i64 @llvm.aarch64.sve.uminv.nxv2i64( %pg, %a) @@ -210,11 +210,11 @@ define @zero_fill_type_mismatch( %pg, @zero_fill_no_zero_upper_lanes( %pg, %a) #0 { ; CHECK-LABEL: zero_fill_no_zero_upper_lanes: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d, vl1 ; CHECK-NEXT: umin z0.d, p0/m, z0.d, z0.d ; CHECK-NEXT: mov z1.d, #0 // =0x0 +; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: fmov x8, d0 -; CHECK-NEXT: mov z1.d, p1/m, x8 +; CHECK-NEXT: mov z1.d, p0/m, x8 ; CHECK-NEXT: mov z0.d, z1.d ; CHECK-NEXT: ret %t1 = call @llvm.aarch64.sve.umin.nxv2i64( %pg, %a, %a) diff --git a/llvm/test/CodeGen/AArch64/sve-insert-element.ll b/llvm/test/CodeGen/AArch64/sve-insert-element.ll index 2aa298f6d917..7344964f13bb 100644 --- a/llvm/test/CodeGen/AArch64/sve-insert-element.ll +++ b/llvm/test/CodeGen/AArch64/sve-insert-element.ll @@ -48,8 +48,8 @@ define @test_lane0_2xi64( %a) { define @test_lane0_2xf64( %a) { ; CHECK-LABEL: test_lane0_2xf64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: fmov d1, #1.00000000 +; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: mov z0.d, p0/m, z1.d ; CHECK-NEXT: ret %b = insertelement %a, double 1.0, i32 0 @@ -59,8 +59,8 @@ define @test_lane0_2xf64( %a) { define @test_lane0_4xf32( %a) { ; CHECK-LABEL: test_lane0_4xf32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl1 ; CHECK-NEXT: fmov s1, #1.00000000 +; CHECK-NEXT: ptrue p0.s, vl1 ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret %b = insertelement %a, float 1.0, i32 0 @@ -70,8 +70,8 @@ define @test_lane0_4xf32( %a) { define @test_lane0_8xf16( %a) { ; CHECK-LABEL: test_lane0_8xf16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl1 ; CHECK-NEXT: fmov h1, #1.00000000 +; CHECK-NEXT: ptrue p0.h, vl1 ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret %b = insertelement %a, half 1.0, i32 0 @@ -93,9 +93,9 @@ define @test_lane0_8xbf16( %a, bfloat define @test_lane4_2xi64( %a) { ; CHECK-LABEL: test_lane4_2xi64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #4 // =0x4 ; CHECK-NEXT: index z1.d, #0, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: mov w8, #30 // =0x1e ; CHECK-NEXT: cmpeq p0.d, p0/z, z1.d, z2.d @@ -109,9 +109,9 @@ define @test_lane4_2xi64( %a) { define @test_lane9_8xf16( %a) { ; CHECK-LABEL: test_lane9_8xf16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #9 // =0x9 ; CHECK-NEXT: index z1.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w8 ; CHECK-NEXT: cmpeq p0.h, p0/z, z1.h, z2.h ; CHECK-NEXT: fmov h1, #1.00000000 @@ -124,9 +124,9 @@ define @test_lane9_8xf16( %a) { define @test_lane9_8xbf16( %a, bfloat %x) { ; CHECK-LABEL: test_lane9_8xbf16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #9 // =0x9 ; CHECK-NEXT: index z2.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z3.h, w8 ; CHECK-NEXT: cmpeq p0.h, p0/z, z2.h, z3.h ; CHECK-NEXT: mov z0.h, p0/m, h1 @@ -138,9 +138,9 @@ define @test_lane9_8xbf16( %a, bfloat define @test_lane1_16xi8( %a) { ; CHECK-LABEL: test_lane1_16xi8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z1.b, #0, #1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w8 ; CHECK-NEXT: mov w8, #30 // =0x1e ; CHECK-NEXT: cmpeq p0.b, p0/z, z1.b, z2.b @@ -153,9 +153,9 @@ define @test_lane1_16xi8( %a) { define @test_lanex_16xi8( %a, i32 %x) { ; CHECK-LABEL: test_lanex_16xi8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: mov w8, w0 ; CHECK-NEXT: index z1.b, #0, #1 +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w8 ; CHECK-NEXT: mov w8, #30 // =0x1e ; CHECK-NEXT: cmpeq p0.b, p0/z, z1.b, z2.b @@ -179,9 +179,9 @@ define @extract_insert_4xi32( %a) { define @test_lane6_undef_8xi16(i16 %a) { ; CHECK-LABEL: test_lane6_undef_8xi16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #6 // =0x6 ; CHECK-NEXT: index z0.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, z1.h ; CHECK-NEXT: mov z0.h, p0/m, w0 @@ -202,8 +202,8 @@ define @test_lane0_undef_16xi8(i8 %a) { define @test_insert0_of_extract0_16xi8( %a, %b) { ; CHECK-LABEL: test_insert0_of_extract0_16xi8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl1 ; CHECK-NEXT: fmov w8, s1 +; CHECK-NEXT: ptrue p0.b, vl1 ; CHECK-NEXT: mov z0.b, p0/m, w8 ; CHECK-NEXT: ret %c = extractelement %b, i32 0 @@ -215,12 +215,12 @@ define @test_insert64_of_extract64_16xi8( % ; CHECK-LABEL: test_insert64_of_extract64_16xi8: ; CHECK: // %bb.0: ; CHECK-NEXT: mov w8, #64 // =0x40 -; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: whilels p0.b, xzr, x8 ; CHECK-NEXT: mov z2.b, w8 ; CHECK-NEXT: lastb w9, p0, z1.b ; CHECK-NEXT: index z1.b, #0, #1 -; CHECK-NEXT: cmpeq p0.b, p1/z, z1.b, z2.b +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: cmpeq p0.b, p0/z, z1.b, z2.b ; CHECK-NEXT: mov z0.b, p0/m, w9 ; CHECK-NEXT: ret %c = extractelement %b, i32 64 @@ -231,9 +231,9 @@ define @test_insert64_of_extract64_16xi8( % define @test_insert3_of_extract1_16xi8( %a, %b) { ; CHECK-LABEL: test_insert3_of_extract1_16xi8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z2.b, #0, #1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z3.b, w8 ; CHECK-NEXT: umov w8, v1.b[1] ; CHECK-NEXT: cmpeq p0.b, p0/z, z2.b, z3.b @@ -329,9 +329,9 @@ define @test_insert_into_undef_nxv2f64(double %a) { define @test_insert_with_index_nxv2f16(half %h, i64 %idx) { ; CHECK-LABEL: test_insert_with_index_nxv2f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: index z1.d, #0, #1 ; CHECK-NEXT: mov z2.d, x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cmpeq p0.d, p0/z, z1.d, z2.d ; CHECK-NEXT: mov z0.h, p0/m, h0 ; CHECK-NEXT: ret @@ -342,9 +342,9 @@ define @test_insert_with_index_nxv2f16(half %h, i64 %idx) { define @test_insert_with_index_nxv4f16(half %h, i64 %idx) { ; CHECK-LABEL: test_insert_with_index_nxv4f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: index z1.s, #0, #1 ; CHECK-NEXT: mov z2.s, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z1.s, z2.s ; CHECK-NEXT: mov z0.h, p0/m, h0 ; CHECK-NEXT: ret @@ -355,9 +355,9 @@ define @test_insert_with_index_nxv4f16(half %h, i64 %idx) { define @test_insert_with_index_nxv8f16(half %h, i64 %idx) { ; CHECK-LABEL: test_insert_with_index_nxv8f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: index z1.h, #0, #1 ; CHECK-NEXT: mov z2.h, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, h0 ; CHECK-NEXT: ret @@ -368,9 +368,9 @@ define @test_insert_with_index_nxv8f16(half %h, i64 %idx) { define @test_insert_with_index_nxv2bf16(bfloat %h, i64 %idx) { ; CHECK-LABEL: test_insert_with_index_nxv2bf16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: index z1.d, #0, #1 ; CHECK-NEXT: mov z2.d, x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cmpeq p0.d, p0/z, z1.d, z2.d ; CHECK-NEXT: mov z0.h, p0/m, h0 ; CHECK-NEXT: ret @@ -381,9 +381,9 @@ define @test_insert_with_index_nxv2bf16(bfloat %h, i64 %id define @test_insert_with_index_nxv4bf16(bfloat %h, i64 %idx) { ; CHECK-LABEL: test_insert_with_index_nxv4bf16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: index z1.s, #0, #1 ; CHECK-NEXT: mov z2.s, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z1.s, z2.s ; CHECK-NEXT: mov z0.h, p0/m, h0 ; CHECK-NEXT: ret @@ -394,9 +394,9 @@ define @test_insert_with_index_nxv4bf16(bfloat %h, i64 %id define @test_insert_with_index_nxv8bf16(bfloat %h, i64 %idx) { ; CHECK-LABEL: test_insert_with_index_nxv8bf16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: index z1.h, #0, #1 ; CHECK-NEXT: mov z2.h, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, h0 ; CHECK-NEXT: ret @@ -407,9 +407,9 @@ define @test_insert_with_index_nxv8bf16(bfloat %h, i64 %id define @test_insert_with_index_nxv2f32(float %f, i64 %idx) { ; CHECK-LABEL: test_insert_with_index_nxv2f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: index z1.d, #0, #1 ; CHECK-NEXT: mov z2.d, x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cmpeq p0.d, p0/z, z1.d, z2.d ; CHECK-NEXT: mov z0.s, p0/m, s0 ; CHECK-NEXT: ret @@ -420,9 +420,9 @@ define @test_insert_with_index_nxv2f32(float %f, i64 %idx) define @test_insert_with_index_nxv4f32(float %f, i64 %idx) { ; CHECK-LABEL: test_insert_with_index_nxv4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: index z1.s, #0, #1 ; CHECK-NEXT: mov z2.s, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, s0 ; CHECK-NEXT: ret @@ -433,9 +433,9 @@ define @test_insert_with_index_nxv4f32(float %f, i64 %idx) define @test_insert_with_index_nxv2f64(double %d, i64 %idx) { ; CHECK-LABEL: test_insert_with_index_nxv2f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: index z1.d, #0, #1 ; CHECK-NEXT: mov z2.d, x0 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cmpeq p0.d, p0/z, z1.d, z2.d ; CHECK-NEXT: mov z0.d, p0/m, d0 ; CHECK-NEXT: ret @@ -447,11 +447,11 @@ define @test_insert_with_index_nxv2f64(double %d, i64 %idx define @test_predicate_insert_2xi1_immediate ( %val, i1 %elt) { ; CHECK-LABEL: test_predicate_insert_2xi1_immediate: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d, vl1 ; CHECK-NEXT: mov z0.d, p0/z, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d, vl1 ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 +; CHECK-NEXT: mov z0.d, p0/m, x0 ; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov z0.d, p1/m, x0 ; CHECK-NEXT: and z0.d, z0.d, #0x1 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: ret @@ -462,9 +462,9 @@ define @test_predicate_insert_2xi1_immediate ( @test_predicate_insert_4xi1_immediate ( %val, i1 %elt) { ; CHECK-LABEL: test_predicate_insert_4xi1_immediate: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov w8, #2 // =0x2 ; CHECK-NEXT: index z0.s, #0, #1 +; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: cmpeq p2.s, p1/z, z0.s, z1.s ; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1 @@ -479,9 +479,9 @@ define @test_predicate_insert_4xi1_immediate ( @test_predicate_insert_8xi1_immediate ( %val, i32 %idx) { ; CHECK-LABEL: test_predicate_insert_8xi1_immediate: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.h -; CHECK-NEXT: mov w8, w0 ; CHECK-NEXT: index z0.h, #0, #1 +; CHECK-NEXT: mov w8, w0 +; CHECK-NEXT: ptrue p1.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: cmpeq p2.h, p1/z, z0.h, z1.h @@ -497,9 +497,9 @@ define @test_predicate_insert_8xi1_immediate ( @test_predicate_insert_16xi1_immediate ( %val) { ; CHECK-LABEL: test_predicate_insert_16xi1_immediate: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: mov w8, #4 // =0x4 ; CHECK-NEXT: index z0.b, #0, #1 +; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: mov z1.b, w8 ; CHECK-NEXT: mov w8, wzr ; CHECK-NEXT: cmpeq p2.b, p1/z, z0.b, z1.b @@ -516,9 +516,9 @@ define @test_predicate_insert_16xi1_immediate ( @test_predicate_insert_2xi1( %val, i1 %elt, i32 %idx) { ; CHECK-LABEL: test_predicate_insert_2xi1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d -; CHECK-NEXT: mov w8, w1 ; CHECK-NEXT: index z0.d, #0, #1 +; CHECK-NEXT: mov w8, w1 +; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: cmpeq p2.d, p1/z, z0.d, z1.d @@ -534,9 +534,9 @@ define @test_predicate_insert_2xi1( %val, i1 define @test_predicate_insert_4xi1( %val, i1 %elt, i32 %idx) { ; CHECK-LABEL: test_predicate_insert_4xi1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s -; CHECK-NEXT: mov w8, w1 ; CHECK-NEXT: index z0.s, #0, #1 +; CHECK-NEXT: mov w8, w1 +; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: cmpeq p2.s, p1/z, z0.s, z1.s ; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1 @@ -550,9 +550,9 @@ define @test_predicate_insert_4xi1( %val, i1 define @test_predicate_insert_8xi1( %val, i1 %elt, i32 %idx) { ; CHECK-LABEL: test_predicate_insert_8xi1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.h -; CHECK-NEXT: mov w8, w1 ; CHECK-NEXT: index z0.h, #0, #1 +; CHECK-NEXT: mov w8, w1 +; CHECK-NEXT: ptrue p1.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: cmpeq p2.h, p1/z, z0.h, z1.h ; CHECK-NEXT: mov z0.h, p0/z, #1 // =0x1 @@ -567,9 +567,9 @@ define @test_predicate_insert_8xi1( %val, i1 define @test_predicate_insert_16xi1( %val, i1 %elt, i32 %idx) { ; CHECK-LABEL: test_predicate_insert_16xi1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.b -; CHECK-NEXT: mov w8, w1 ; CHECK-NEXT: index z0.b, #0, #1 +; CHECK-NEXT: mov w8, w1 +; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: mov z1.b, w8 ; CHECK-NEXT: cmpeq p2.b, p1/z, z0.b, z1.b ; CHECK-NEXT: mov z0.b, p0/z, #1 // =0x1 @@ -589,24 +589,24 @@ define @test_predicate_insert_32xi1( %val, ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG -; CHECK-NEXT: ptrue p2.b ; CHECK-NEXT: rdvl x8, #2 ; CHECK-NEXT: mov z0.b, p1/z, #1 // =0x1 ; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1 ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: mov w9, w1 +; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: cmp x9, x8 ; CHECK-NEXT: csel x8, x9, x8, lo ; CHECK-NEXT: mov x9, sp -; CHECK-NEXT: st1b { z0.b }, p2, [sp, #1, mul vl] -; CHECK-NEXT: st1b { z1.b }, p2, [sp] +; CHECK-NEXT: st1b { z0.b }, p1, [sp, #1, mul vl] +; CHECK-NEXT: st1b { z1.b }, p1, [sp] ; CHECK-NEXT: strb w0, [x9, x8] -; CHECK-NEXT: ld1b { z0.b }, p2/z, [sp] -; CHECK-NEXT: ld1b { z1.b }, p2/z, [sp, #1, mul vl] +; CHECK-NEXT: ld1b { z0.b }, p1/z, [sp] +; CHECK-NEXT: ld1b { z1.b }, p1/z, [sp, #1, mul vl] ; CHECK-NEXT: and z0.b, z0.b, #0x1 ; CHECK-NEXT: and z1.b, z1.b, #0x1 -; CHECK-NEXT: cmpne p0.b, p2/z, z0.b, #0 -; CHECK-NEXT: cmpne p1.b, p2/z, z1.b, #0 +; CHECK-NEXT: cmpne p0.b, p1/z, z0.b, #0 +; CHECK-NEXT: cmpne p1.b, p1/z, z1.b, #0 ; CHECK-NEXT: addvl sp, sp, #2 ; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload diff --git a/llvm/test/CodeGen/AArch64/sve-insert-vector.ll b/llvm/test/CodeGen/AArch64/sve-insert-vector.ll index 4a5e272582d8..5efe9e2819d5 100644 --- a/llvm/test/CodeGen/AArch64/sve-insert-vector.ll +++ b/llvm/test/CodeGen/AArch64/sve-insert-vector.ll @@ -17,15 +17,15 @@ define @insert_v2i64_nxv2i64_idx2( %vec, <2 ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cntd x8 ; CHECK-NEXT: mov w9, #2 // =0x2 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: sub x8, x8, #2 ; CHECK-NEXT: cmp x8, #2 +; CHECK-NEXT: st1d { z0.d }, p0, [sp] ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #3 -; CHECK-NEXT: st1d { z0.d }, p0, [sp] ; CHECK-NEXT: str q1, [x9, x8] ; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp] ; CHECK-NEXT: addvl sp, sp, #1 @@ -51,15 +51,15 @@ define @insert_v4i32_nxv4i32_idx4( %vec, <4 ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cntw x8 ; CHECK-NEXT: mov w9, #4 // =0x4 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sub x8, x8, #4 ; CHECK-NEXT: cmp x8, #4 +; CHECK-NEXT: st1w { z0.s }, p0, [sp] ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #2 -; CHECK-NEXT: st1w { z0.s }, p0, [sp] ; CHECK-NEXT: str q1, [x9, x8] ; CHECK-NEXT: ld1w { z0.s }, p0/z, [sp] ; CHECK-NEXT: addvl sp, sp, #1 @@ -85,15 +85,15 @@ define @insert_v8i16_nxv8i16_idx8( %vec, <8 ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cnth x8 ; CHECK-NEXT: mov w9, #8 // =0x8 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: sub x8, x8, #8 ; CHECK-NEXT: cmp x8, #8 +; CHECK-NEXT: st1h { z0.h }, p0, [sp] ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: lsl x8, x8, #1 -; CHECK-NEXT: st1h { z0.h }, p0, [sp] ; CHECK-NEXT: str q1, [x9, x8] ; CHECK-NEXT: ld1h { z0.h }, p0/z, [sp] ; CHECK-NEXT: addvl sp, sp, #1 @@ -119,15 +119,15 @@ define @insert_v16i8_nxv16i8_idx16( %vec, < ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: rdvl x8, #1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov w9, #16 // =0x10 ; CHECK-NEXT: sub x8, x8, #16 -; CHECK-NEXT: mov x10, sp ; CHECK-NEXT: cmp x8, #16 -; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: st1b { z0.b }, p0, [sp] -; CHECK-NEXT: str q1, [x10, x8] +; CHECK-NEXT: csel x8, x8, x9, lo +; CHECK-NEXT: mov x9, sp +; CHECK-NEXT: str q1, [x9, x8] ; CHECK-NEXT: ld1b { z0.b }, p0/z, [sp] ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -239,8 +239,8 @@ define void @insert_v2i64_nxv16i64_lo2(ptr %psv, ptr %out) uwtable { ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldr q0, [x0] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: str q0, [sp, #16] ; CHECK-NEXT: ld1d { z0.d }, p0/z, [sp, #1, mul vl] ; CHECK-NEXT: ld1d { z1.d }, p0/z, [sp] diff --git a/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll b/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll index c0ddceb42e1d..52bd79e7a7e6 100644 --- a/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll +++ b/llvm/test/CodeGen/AArch64/sve-int-arith-imm.ll @@ -55,8 +55,8 @@ define @smax_i16_neg( %a) { define @smax_i16_out_of_range( %a) { ; CHECK-LABEL: smax_i16_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: dupm z1.b, #0x1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: smax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %elt = insertelement undef, i16 257, i32 0 @@ -93,8 +93,8 @@ define @smax_i32_neg( %a) { define @smax_i32_out_of_range( %a) { ; CHECK-LABEL: smax_i32_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, #-129 // =0xffffffffffffff7f +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %elt = insertelement undef, i32 -129, i32 0 @@ -131,8 +131,8 @@ define @smax_i64_neg( %a) { define @smax_i64_out_of_range( %a) { ; CHECK-LABEL: smax_i64_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, #65535 // =0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %elt = insertelement undef, i64 65535, i32 0 @@ -196,8 +196,8 @@ define @smin_i16_neg( %a) { define @smin_i16_out_of_range( %a) { ; CHECK-LABEL: smin_i16_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: dupm z1.b, #0x1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: smin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %elt = insertelement undef, i16 257, i32 0 @@ -234,8 +234,8 @@ define @smin_i32_neg( %a) { define @smin_i32_out_of_range( %a) { ; CHECK-LABEL: smin_i32_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, #-129 // =0xffffffffffffff7f +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: smin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %elt = insertelement undef, i32 -129, i32 0 @@ -272,8 +272,8 @@ define @smin_i64_neg( %a) { define @smin_i64_out_of_range( %a) { ; CHECK-LABEL: smin_i64_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, #65535 // =0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: smin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %elt = insertelement undef, i64 65535, i32 0 @@ -325,8 +325,8 @@ define @umax_i16_pos( %a) { define @umax_i16_out_of_range( %a) { ; CHECK-LABEL: umax_i16_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: dupm z1.b, #0x1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: umax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %elt = insertelement undef, i16 257, i32 0 @@ -351,8 +351,8 @@ define @umax_i32_pos( %a) { define @umax_i32_out_of_range( %a) { ; CHECK-LABEL: umax_i32_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #257 // =0x101 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: umax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -378,8 +378,8 @@ define @umax_i64_pos( %a) { define @umax_i64_out_of_range( %a) { ; CHECK-LABEL: umax_i64_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, #65535 // =0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: umax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %elt = insertelement undef, i64 65535, i32 0 @@ -431,8 +431,8 @@ define @umin_i16_pos( %a) { define @umin_i16_out_of_range( %a) { ; CHECK-LABEL: umin_i16_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: dupm z1.b, #0x1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: umin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %elt = insertelement undef, i16 257, i32 0 @@ -457,8 +457,8 @@ define @umin_i32_pos( %a) { define @umin_i32_out_of_range( %a) { ; CHECK-LABEL: umin_i32_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #257 // =0x101 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: umin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -484,8 +484,8 @@ define @umin_i64_pos( %a) { define @umin_i64_out_of_range( %a) { ; CHECK-LABEL: umin_i64_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, #65535 // =0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: umin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %elt = insertelement undef, i64 65535, i32 0 @@ -589,8 +589,8 @@ define @mul_i64_pos( %a) { define @mul_i16_range( %a) { ; CHECK-LABEL: mul_i16_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, #255 // =0xff +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mul z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %elt = insertelement undef, i16 255, i32 0 @@ -602,8 +602,8 @@ define @mul_i16_range( %a) { define @mul_i32_range( %a) { ; CHECK-LABEL: mul_i32_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, #255 // =0xff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %elt = insertelement undef, i32 255, i32 0 @@ -615,8 +615,8 @@ define @mul_i32_range( %a) { define @mul_i64_range( %a) { ; CHECK-LABEL: mul_i64_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, #255 // =0xff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %elt = insertelement undef, i64 255, i32 0 @@ -766,8 +766,8 @@ define @lsr_i64( %a){ define @sdiv_const( %a) #0 { ; CHECK-LABEL: sdiv_const: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, #3 // =0x3 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret entry: @@ -778,8 +778,8 @@ entry: define @udiv_const( %a) #0 { ; CHECK-LABEL: udiv_const: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, #3 // =0x3 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/sve-int-arith.ll b/llvm/test/CodeGen/AArch64/sve-int-arith.ll index fc2672f8c80a..5f92dee3b530 100644 --- a/llvm/test/CodeGen/AArch64/sve-int-arith.ll +++ b/llvm/test/CodeGen/AArch64/sve-int-arith.ll @@ -532,8 +532,8 @@ define @mls_i64( %a, %b, define @muladd_i64_positiveAddend( %a, %b) ; CHECK-LABEL: muladd_i64_positiveAddend: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, #0xffffffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mad z0.d, p0/m, z1.d, z2.d ; CHECK-NEXT: ret { @@ -545,8 +545,8 @@ define @mls_i64( %a, %b, define @muladd_i64_negativeAddend( %a, %b) ; CHECK-LABEL: muladd_i64_negativeAddend: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, #0xffffffff00000001 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mad z0.d, p0/m, z1.d, z2.d ; CHECK-NEXT: ret { @@ -559,8 +559,8 @@ define @muladd_i64_negativeAddend( %a, @muladd_i32_positiveAddend( %a, %b) ; CHECK-LABEL: muladd_i32_positiveAddend: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, #0x10000 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mad z0.s, p0/m, z1.s, z2.s ; CHECK-NEXT: ret { @@ -572,8 +572,8 @@ define @muladd_i32_positiveAddend( %a, @muladd_i32_negativeAddend( %a, %b) ; CHECK-LABEL: muladd_i32_negativeAddend: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, #0xffff0000 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mad z0.s, p0/m, z1.s, z2.s ; CHECK-NEXT: ret { @@ -585,8 +585,8 @@ define @muladd_i32_negativeAddend( %a, @muladd_i16_positiveAddend( %a, %b) ; CHECK-LABEL: muladd_i16_positiveAddend: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, #255 // =0xff +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mad z0.h, p0/m, z1.h, z2.h ; CHECK-NEXT: ret { @@ -598,8 +598,8 @@ define @muladd_i16_positiveAddend( %a, @muladd_i16_negativeAddend( %a, %b) ; CHECK-LABEL: muladd_i16_negativeAddend: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, #-255 // =0xffffffffffffff01 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mad z0.h, p0/m, z1.h, z2.h ; CHECK-NEXT: ret { @@ -611,8 +611,8 @@ define @muladd_i16_negativeAddend( %a, @muladd_i8_positiveAddend( %a, %b) ; CHECK-LABEL: muladd_i8_positiveAddend: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, #15 // =0xf +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mad z0.b, p0/m, z1.b, z2.b ; CHECK-NEXT: ret { @@ -624,8 +624,8 @@ define @muladd_i8_positiveAddend( %a, @muladd_i8_negativeAddend( %a, %b) ; CHECK-LABEL: muladd_i8_negativeAddend: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, #-15 // =0xfffffffffffffff1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mad z0.b, p0/m, z1.b, z2.b ; CHECK-NEXT: ret { @@ -748,8 +748,8 @@ define @mulsub_i8_negativeAddend( %a, @multiple_fused_ops( %a, %b) ; CHECK-LABEL: multiple_fused_ops: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #200 // =0xc8 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w8 ; CHECK-NEXT: mla z2.h, p0/m, z0.h, z1.h ; CHECK-NEXT: mul z0.h, p0/m, z0.h, z2.h @@ -770,8 +770,8 @@ define void @mad_in_loop(ptr %dst, ptr %src1, ptr %src2, i32 %n) { ; CHECK-NEXT: b.lt .LBB70_3 ; CHECK-NEXT: // %bb.1: // %for.body.preheader ; CHECK-NEXT: mov w9, w3 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z0.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: whilelo p1.s, xzr, x9 ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: cntw x10 diff --git a/llvm/test/CodeGen/AArch64/sve-int-reduce.ll b/llvm/test/CodeGen/AArch64/sve-int-reduce.ll index d04da6245177..8c1b5225b7f2 100644 --- a/llvm/test/CodeGen/AArch64/sve-int-reduce.ll +++ b/llvm/test/CodeGen/AArch64/sve-int-reduce.ll @@ -378,29 +378,29 @@ declare i8 @llvm.vector.reduce.smin.nxv10i8() define i8 @smin_nxv10i8( %a) { ; CHECK-LABEL: smin_nxv10i8: ; CHECK: // %bb.0: -; CHECK-NEXT: uunpkhi z2.h, z0.b -; CHECK-NEXT: mov z1.d, #127 // =0x7f +; CHECK-NEXT: uunpkhi z1.h, z0.b +; CHECK-NEXT: mov z3.d, #127 // =0x7f ; CHECK-NEXT: uunpklo z0.h, z0.b ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: uunpklo z3.s, z2.h -; CHECK-NEXT: uunpkhi z2.s, z2.h -; CHECK-NEXT: uunpklo z3.d, z3.s -; CHECK-NEXT: uzp1 z3.s, z3.s, z1.s -; CHECK-NEXT: uzp1 z2.h, z3.h, z2.h -; CHECK-NEXT: uzp1 z2.b, z0.b, z2.b -; CHECK-NEXT: uunpkhi z2.h, z2.b -; CHECK-NEXT: uunpkhi z3.s, z2.h -; CHECK-NEXT: uunpklo z2.s, z2.h -; CHECK-NEXT: uunpkhi z3.d, z3.s -; CHECK-NEXT: uzp1 z3.s, z1.s, z3.s -; CHECK-NEXT: uzp1 z2.h, z2.h, z3.h -; CHECK-NEXT: uzp1 z2.b, z0.b, z2.b -; CHECK-NEXT: uunpkhi z2.h, z2.b -; CHECK-NEXT: uunpkhi z3.s, z2.h -; CHECK-NEXT: uunpklo z2.s, z2.h -; CHECK-NEXT: uunpklo z3.d, z3.s -; CHECK-NEXT: uzp1 z1.s, z3.s, z1.s +; CHECK-NEXT: uunpklo z2.s, z1.h +; CHECK-NEXT: uunpkhi z1.s, z1.h +; CHECK-NEXT: uunpklo z2.d, z2.s +; CHECK-NEXT: uzp1 z2.s, z2.s, z3.s ; CHECK-NEXT: uzp1 z1.h, z2.h, z1.h +; CHECK-NEXT: uzp1 z1.b, z0.b, z1.b +; CHECK-NEXT: uunpkhi z1.h, z1.b +; CHECK-NEXT: uunpkhi z2.s, z1.h +; CHECK-NEXT: uunpklo z1.s, z1.h +; CHECK-NEXT: uunpkhi z2.d, z2.s +; CHECK-NEXT: uzp1 z2.s, z3.s, z2.s +; CHECK-NEXT: uzp1 z1.h, z1.h, z2.h +; CHECK-NEXT: uzp1 z1.b, z0.b, z1.b +; CHECK-NEXT: uunpkhi z1.h, z1.b +; CHECK-NEXT: uunpkhi z2.s, z1.h +; CHECK-NEXT: uunpklo z1.s, z1.h +; CHECK-NEXT: uunpklo z2.d, z2.s +; CHECK-NEXT: uzp1 z2.s, z2.s, z3.s +; CHECK-NEXT: uzp1 z1.h, z1.h, z2.h ; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b ; CHECK-NEXT: sminv b0, p0, z0.b ; CHECK-NEXT: fmov w0, s0 @@ -414,12 +414,12 @@ declare i8 @llvm.vector.reduce.add.nxv12i8() define i8 @uaddv_nxv12i8( %a) { ; CHECK-LABEL: uaddv_nxv12i8: ; CHECK: // %bb.0: -; CHECK-NEXT: uunpkhi z2.h, z0.b -; CHECK-NEXT: mov z1.s, #0 // =0x0 +; CHECK-NEXT: uunpkhi z1.h, z0.b +; CHECK-NEXT: mov z2.s, #0 // =0x0 ; CHECK-NEXT: uunpklo z0.h, z0.b ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: uunpklo z2.s, z2.h -; CHECK-NEXT: uzp1 z1.h, z2.h, z1.h +; CHECK-NEXT: uunpklo z1.s, z1.h +; CHECK-NEXT: uzp1 z1.h, z1.h, z2.h ; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b ; CHECK-NEXT: uaddv d0, p0, z0.b ; CHECK-NEXT: fmov x0, d0 @@ -434,15 +434,15 @@ declare i8 @llvm.vector.reduce.umax.nxv14i8() define i8 @umax_nxv14i8( %a) { ; CHECK-LABEL: umax_nxv14i8: ; CHECK: // %bb.0: -; CHECK-NEXT: uunpkhi z2.h, z0.b -; CHECK-NEXT: mov z1.d, #0 // =0x0 +; CHECK-NEXT: uunpkhi z1.h, z0.b +; CHECK-NEXT: mov z3.d, #0 // =0x0 ; CHECK-NEXT: uunpklo z0.h, z0.b ; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: uunpkhi z3.s, z2.h -; CHECK-NEXT: uunpklo z2.s, z2.h -; CHECK-NEXT: uunpklo z3.d, z3.s -; CHECK-NEXT: uzp1 z1.s, z3.s, z1.s -; CHECK-NEXT: uzp1 z1.h, z2.h, z1.h +; CHECK-NEXT: uunpkhi z2.s, z1.h +; CHECK-NEXT: uunpklo z1.s, z1.h +; CHECK-NEXT: uunpklo z2.d, z2.s +; CHECK-NEXT: uzp1 z2.s, z2.s, z3.s +; CHECK-NEXT: uzp1 z1.h, z1.h, z2.h ; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b ; CHECK-NEXT: umaxv b0, p0, z0.b ; CHECK-NEXT: fmov w0, s0 diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-index.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-index.ll index 2464eacd185d..bc94c087ef5f 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-index.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-index.ll @@ -239,10 +239,10 @@ define @index_rr_i32_combine(i32 %a, i32 %b) { define @index_rr_i32_not_combine(i32 %a, i32 %b) { ; CHECK-LABEL: index_rr_i32_not_combine: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: index z0.s, #0, #1 ; CHECK-NEXT: mov z1.s, w0 ; CHECK-NEXT: mov z2.s, w1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mla z1.s, p0/m, z0.s, z2.s ; CHECK-NEXT: add z0.s, z1.s, z0.s ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll index 3e453a6b7817..5648e8244e6e 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll @@ -247,8 +247,8 @@ define @sub_i32_ptrue_all_h( %a) #0 { define @sub_i32_ptrue_all_d( %a) #0 { ; CHECK-LABEL: sub_i32_ptrue_all_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: sub z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg.d = tail call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -402,8 +402,8 @@ define @subr_i32_ptrue_all_h( %a) #0 { define @subr_i32_ptrue_all_d( %a) #0 { ; CHECK-LABEL: subr_i32_ptrue_all_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: subr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg.d = tail call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -449,8 +449,8 @@ define @smax_i16( %a) { define @smax_i16_out_of_range( %a) { ; CHECK-LABEL: smax_i16_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #129 // =0x81 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: smax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret @@ -480,8 +480,8 @@ define @smax_i32( %a) { define @smax_i32_out_of_range( %a) { ; CHECK-LABEL: smax_i32_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, #-129 // =0xffffffffffffff7f +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg = call @llvm.aarch64.sve.ptrue.nxv4i1(i32 31) @@ -510,8 +510,8 @@ define @smax_i64( %a) { define @smax_i64_out_of_range( %a) { ; CHECK-LABEL: smax_i64_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, #65535 // =0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %pg = call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -559,8 +559,8 @@ define @smax_i32_ptrue_all_h( %a) #0 { define @smax_i32_ptrue_all_d( %a) #0 { ; CHECK-LABEL: smax_i32_ptrue_all_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg.d = tail call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -606,8 +606,8 @@ define @smin_i16( %a) { define @smin_i16_out_of_range( %a) { ; CHECK-LABEL: smin_i16_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, #-129 // =0xffffffffffffff7f +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: smin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %pg = call @llvm.aarch64.sve.ptrue.nxv8i1(i32 31) @@ -636,8 +636,8 @@ define @smin_i32( %a) { define @smin_i32_out_of_range( %a) { ; CHECK-LABEL: smin_i32_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #257 // =0x101 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: smin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -668,8 +668,8 @@ define @smin_i64( %a) { define @smin_i64_out_of_range( %a) { ; CHECK-LABEL: smin_i64_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, #-256 // =0xffffffffffffff00 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: smin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %pg = call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -717,8 +717,8 @@ define @smin_i32_ptrue_all_h( %a) #0 { define @smin_i32_ptrue_all_d( %a) #0 { ; CHECK-LABEL: smin_i32_ptrue_all_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: smin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg.d = tail call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -764,8 +764,8 @@ define @umax_i16( %a) { define @umax_i16_out_of_range( %a) { ; CHECK-LABEL: umax_i16_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: dupm z1.b, #0x1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: umax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %pg = call @llvm.aarch64.sve.ptrue.nxv8i1(i32 31) @@ -794,8 +794,8 @@ define @umax_i32( %a) { define @umax_i32_out_of_range( %a) { ; CHECK-LABEL: umax_i32_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #257 // =0x101 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: umax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -825,8 +825,8 @@ define @umax_i64( %a) { define @umax_i64_out_of_range( %a) { ; CHECK-LABEL: umax_i64_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, #65535 // =0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: umax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %pg = call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -874,8 +874,8 @@ define @umax_i32_ptrue_all_h( %a) #0 { define @umax_i32_ptrue_all_d( %a) #0 { ; CHECK-LABEL: umax_i32_ptrue_all_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: umax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg.d = tail call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -921,8 +921,8 @@ define @umin_i16( %a) { define @umin_i16_out_of_range( %a) { ; CHECK-LABEL: umin_i16_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: dupm z1.b, #0x1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: umin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %pg = call @llvm.aarch64.sve.ptrue.nxv8i1(i32 31) @@ -951,8 +951,8 @@ define @umin_i32( %a) { define @umin_i32_out_of_range( %a) { ; CHECK-LABEL: umin_i32_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #257 // =0x101 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: umin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret @@ -982,8 +982,8 @@ define @umin_i64( %a) { define @umin_i64_out_of_range( %a) { ; CHECK-LABEL: umin_i64_out_of_range: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, #65535 // =0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: umin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %pg = call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -1031,8 +1031,8 @@ define @umin_i32_ptrue_all_h( %a) #0 { define @umin_i32_ptrue_all_d( %a) #0 { ; CHECK-LABEL: umin_i32_ptrue_all_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: umin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg.d = tail call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) @@ -2120,8 +2120,8 @@ define @mul_i32_ptrue_all_h( %a) #0 { define @mul_i32_ptrue_all_d( %a) #0 { ; CHECK-LABEL: mul_i32_ptrue_all_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg.d = tail call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-logical-imm.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-logical-imm.ll index 7cdedeee2cad..5db7ee75c2a8 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-logical-imm.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-logical-imm.ll @@ -261,8 +261,8 @@ define @orr_i32_ptrue_all_h( %a) { define @orr_i32_ptrue_all_d( %a) { ; CHECK-LABEL: orr_i32_ptrue_all_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.s, #65535 // =0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: orr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg.d = tail call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) diff --git a/llvm/test/CodeGen/AArch64/sve-ld-post-inc.ll b/llvm/test/CodeGen/AArch64/sve-ld-post-inc.ll index da5dc5c5b34d..619134dc4a69 100644 --- a/llvm/test/CodeGen/AArch64/sve-ld-post-inc.ll +++ b/llvm/test/CodeGen/AArch64/sve-ld-post-inc.ll @@ -38,18 +38,18 @@ define @test_post_ld1_dup(ptr %a, ptr %ptr, i64 %inc) { define void @test_post_ld1_int_fixed(ptr %data, i64 %idx, ptr %addr, ptr %res_ptr) #1 { ; CHECK-LABEL: test_post_ld1_int_fixed: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #2 // =0x2 ; CHECK-NEXT: index z0.d, #0, #1 -; CHECK-NEXT: ptrue p1.d, vl1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: ldr x8, [x0] -; CHECK-NEXT: ldr x9, [x0, x1, lsl #3] +; CHECK-NEXT: ptrue p2.d, vl1 ; CHECK-NEXT: ld1d { z2.d }, p0/z, [x2] -; CHECK-NEXT: cmpeq p2.d, p0/z, z0.d, z1.d +; CHECK-NEXT: ldr x9, [x0, x1, lsl #3] +; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d ; CHECK-NEXT: mov z0.d, z2.d -; CHECK-NEXT: mov z2.d, p2/m, x9 -; CHECK-NEXT: mov z0.d, p1/m, x8 +; CHECK-NEXT: mov z0.d, p2/m, x8 +; CHECK-NEXT: mov z2.d, p1/m, x9 ; CHECK-NEXT: add z0.d, z0.d, z2.d ; CHECK-NEXT: st1d { z0.d }, p0, [x3] ; CHECK-NEXT: ret @@ -67,18 +67,18 @@ define void @test_post_ld1_int_fixed(ptr %data, i64 %idx, ptr %addr, ptr %res_pt define void @test_post_ld1_double_fixed(ptr %data, i64 %idx, ptr %addr, ptr %res_ptr) #1 { ; CHECK-LABEL: test_post_ld1_double_fixed: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #2 // =0x2 ; CHECK-NEXT: index z0.d, #0, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 -; CHECK-NEXT: ptrue p1.d, vl1 -; CHECK-NEXT: ld1d { z2.d }, p0/z, [x2] -; CHECK-NEXT: cmpeq p2.d, p0/z, z0.d, z1.d -; CHECK-NEXT: ldr d0, [x0] -; CHECK-NEXT: ldr d1, [x0, x1, lsl #3] -; CHECK-NEXT: sel z0.d, p1, z0.d, z2.d -; CHECK-NEXT: mov z2.d, p2/m, d1 -; CHECK-NEXT: fadd z0.d, z0.d, z2.d +; CHECK-NEXT: ptrue p2.d, vl1 +; CHECK-NEXT: ldr d2, [x0, x1, lsl #3] +; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d +; CHECK-NEXT: ld1d { z0.d }, p0/z, [x2] +; CHECK-NEXT: ldr d1, [x0] +; CHECK-NEXT: sel z1.d, p2, z1.d, z0.d +; CHECK-NEXT: mov z0.d, p1/m, d2 +; CHECK-NEXT: fadd z0.d, z1.d, z0.d ; CHECK-NEXT: st1d { z0.d }, p0, [x3] ; CHECK-NEXT: ret %A = load <4 x double>, ptr %addr diff --git a/llvm/test/CodeGen/AArch64/sve-ld1r.ll b/llvm/test/CodeGen/AArch64/sve-ld1r.ll index e42e2272a2d4..fbe82e8591fd 100644 --- a/llvm/test/CodeGen/AArch64/sve-ld1r.ll +++ b/llvm/test/CodeGen/AArch64/sve-ld1r.ll @@ -20,8 +20,8 @@ define @ld1r_stack() { ; CHECK: // %bb.0: ; CHECK-NEXT: sub sp, sp, #16 ; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: adrp x8, :got:g8 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: ldr x8, [x8, :got_lo12:g8] ; CHECK-NEXT: ldrb w8, [x8] ; CHECK-NEXT: strb w8, [sp, #12] @@ -1433,10 +1433,10 @@ define ptr @avoid_preindex_load(ptr %src, ptr %out) { define ptr @avoid_preindex_load_dup(ptr %src, %pg, ptr %out) { ; CHECK-LABEL: avoid_preindex_load_dup: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: ld1rsb { z0.d }, p0/z, [x0, #1] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: add x0, x0, #1 -; CHECK-NEXT: st1d { z0.d }, p1, [x1] +; CHECK-NEXT: st1d { z0.d }, p0, [x1] ; CHECK-NEXT: ret %ptr = getelementptr inbounds i8, ptr %src, i64 1 %tmp = load i8, ptr %ptr, align 4 @@ -1450,10 +1450,10 @@ define ptr @avoid_preindex_load_dup(ptr %src, %pg, ptr %out) { define ptr @avoid_preindex_load_dup_passthru_zero(ptr %src, %pg, ptr %out) { ; CHECK-LABEL: avoid_preindex_load_dup_passthru_zero: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: ld1rsb { z0.d }, p0/z, [x0, #1] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: add x0, x0, #1 -; CHECK-NEXT: st1d { z0.d }, p1, [x1] +; CHECK-NEXT: st1d { z0.d }, p0, [x1] ; CHECK-NEXT: ret %ptr = getelementptr inbounds i8, ptr %src, i64 1 %tmp = load i8, ptr %ptr, align 4 @@ -1467,10 +1467,10 @@ define ptr @avoid_preindex_load_dup_passthru_zero(ptr %src, %p define ptr @preindex_load_dup_passthru( %passthru, ptr %src, %pg, ptr %out) { ; CHECK-LABEL: preindex_load_dup_passthru: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: ldrsb x8, [x0, #1]! ; CHECK-NEXT: mov z0.d, p0/m, x8 -; CHECK-NEXT: st1d { z0.d }, p1, [x1] +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: st1d { z0.d }, p0, [x1] ; CHECK-NEXT: ret %ptr = getelementptr inbounds i8, ptr %src, i64 1 %tmp = load i8, ptr %ptr, align 4 @@ -1485,8 +1485,8 @@ define ptr @preindex_load_dup_passthru( %passthru, ptr %src, < define ptr @preidx8sext64_instead_of_ld1r(ptr %src, ptr %out, ptr %dst) { ; CHECK-LABEL: preidx8sext64_instead_of_ld1r: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldrsb x8, [x0, #1]! +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, x8 ; CHECK-NEXT: st1d { z0.d }, p0, [x1] ; CHECK-NEXT: str x8, [x2] diff --git a/llvm/test/CodeGen/AArch64/sve-lsr-scaled-index-addressing-mode.ll b/llvm/test/CodeGen/AArch64/sve-lsr-scaled-index-addressing-mode.ll index 06ec13280815..5d53c00c5272 100644 --- a/llvm/test/CodeGen/AArch64/sve-lsr-scaled-index-addressing-mode.ll +++ b/llvm/test/CodeGen/AArch64/sve-lsr-scaled-index-addressing-mode.ll @@ -38,8 +38,8 @@ define void @ld_st_nxv8i16(ptr %in, ptr %out) { ; ; ASM-LABEL: ld_st_nxv8i16: ; ASM: // %bb.0: // %entry -; ASM-NEXT: ptrue p0.h ; ASM-NEXT: mov z0.h, #3 // =0x3 +; ASM-NEXT: ptrue p0.h ; ASM-NEXT: mov x8, xzr ; ASM-NEXT: cnth x9 ; ASM-NEXT: .LBB0_1: // %loop @@ -111,8 +111,8 @@ define void @masked_ld_st_nxv8i16(ptr %in, ptr %out, i64 %n) { ; ; ASM-LABEL: masked_ld_st_nxv8i16: ; ASM: // %bb.0: // %entry -; ASM-NEXT: ptrue p0.h ; ASM-NEXT: mov z0.h, #3 // =0x3 +; ASM-NEXT: ptrue p0.h ; ASM-NEXT: mov x8, xzr ; ASM-NEXT: cnth x9 ; ASM-NEXT: .LBB1_1: // %loop diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll index e40d65efb158..dfdfc456ccdb 100644 --- a/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll +++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll @@ -126,9 +126,9 @@ define @masked_gather_nxv8f16( %ptrs, @masked_gather_nxv8bf16(ptr %base, %indices, %mask) #0 { ; CHECK-LABEL: masked_gather_nxv8bf16: ; CHECK: // %bb.0: -; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: sunpkhi z1.s, z0.h ; CHECK-NEXT: sunpklo z0.s, z0.h +; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: ld1h { z1.s }, p1/z, [x0, z1.s, sxtw #1] ; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0, z0.s, sxtw #1] @@ -175,14 +175,14 @@ define @masked_gather_nxv8f32(ptr %base, define @masked_gather_nxv16i8(ptr %base, %indices, %mask) #0 { ; CHECK-LABEL: masked_gather_nxv16i8: ; CHECK: // %bb.0: -; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: sunpkhi z1.h, z0.b +; CHECK-NEXT: punpkhi p1.h, p0.b ; CHECK-NEXT: sunpklo z0.h, z0.b ; CHECK-NEXT: punpklo p0.h, p0.b -; CHECK-NEXT: sunpkhi z2.s, z1.h -; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: punpkhi p2.h, p1.b ; CHECK-NEXT: punpklo p1.h, p1.b +; CHECK-NEXT: sunpkhi z2.s, z1.h +; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: ld1b { z2.s }, p2/z, [x0, z2.s, sxtw] ; CHECK-NEXT: ld1b { z1.s }, p1/z, [x0, z1.s, sxtw] ; CHECK-NEXT: punpkhi p1.h, p0.b diff --git a/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll b/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll index 40d889f1b501..d397424cb162 100644 --- a/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll +++ b/llvm/test/CodeGen/AArch64/sve-masked-ldst-sext.ll @@ -235,9 +235,9 @@ define @masked_sload_x2_8i8_8i64(ptr %a, ptr %b, @masked_zload_nxv8i16(ptr %a, %mask) define @masked_zload_2i16_2f64(ptr noalias %in, %mask) { ; CHECK-LABEL: masked_zload_2i16_2f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0] -; CHECK-NEXT: ucvtf z0.d, p1/m, z0.d +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: ucvtf z0.d, p0/m, z0.d ; CHECK-NEXT: ret %wide.load = call @llvm.masked.load.nxv2i16(ptr %in, i32 2, %mask, undef) %zext = zext %wide.load to @@ -230,9 +230,9 @@ define @masked_zload_x2_8i8_8i64(ptr %a, ptr %b, %data, ptr %base, %offsets, %mask) #0 { ; CHECK-LABEL: masked_scatter_nxv16i8: ; CHECK: // %bb.0: -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: sunpklo z2.h, z1.b ; CHECK-NEXT: uunpklo z4.h, z0.b -; CHECK-NEXT: punpkhi p0.h, p0.b +; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: sunpkhi z1.h, z1.b ; CHECK-NEXT: uunpkhi z0.h, z0.b +; CHECK-NEXT: punpkhi p0.h, p0.b +; CHECK-NEXT: punpklo p2.h, p1.b +; CHECK-NEXT: punpkhi p1.h, p1.b ; CHECK-NEXT: sunpklo z3.s, z2.h ; CHECK-NEXT: uunpklo z5.s, z4.h ; CHECK-NEXT: sunpkhi z2.s, z2.h -; CHECK-NEXT: punpklo p2.h, p1.b -; CHECK-NEXT: punpkhi p1.h, p1.b ; CHECK-NEXT: st1b { z5.s }, p2, [x0, z3.s, sxtw] ; CHECK-NEXT: uunpkhi z3.s, z4.h ; CHECK-NEXT: st1b { z3.s }, p1, [x0, z2.s, sxtw] -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: sunpklo z2.s, z1.h -; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: uunpklo z3.s, z0.h ; CHECK-NEXT: sunpkhi z1.s, z1.h ; CHECK-NEXT: uunpkhi z0.s, z0.h +; CHECK-NEXT: punpklo p1.h, p0.b +; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: st1b { z3.s }, p1, [x0, z2.s, sxtw] ; CHECK-NEXT: st1b { z0.s }, p0, [x0, z1.s, sxtw] ; CHECK-NEXT: ret @@ -40,12 +40,12 @@ define void @masked_scatter_nxv16i8( %data, ptr %base, %data, ptr %base, %offsets, %mask) #0 { ; CHECK-LABEL: masked_scatter_nxv8i16: ; CHECK: // %bb.0: -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: sunpklo z2.s, z1.h ; CHECK-NEXT: uunpklo z3.s, z0.h -; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: sunpkhi z1.s, z1.h ; CHECK-NEXT: uunpkhi z0.s, z0.h +; CHECK-NEXT: punpklo p1.h, p0.b +; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: st1h { z3.s }, p1, [x0, z2.s, sxtw #1] ; CHECK-NEXT: st1h { z0.s }, p0, [x0, z1.s, sxtw #1] ; CHECK-NEXT: ret @@ -57,12 +57,12 @@ define void @masked_scatter_nxv8i16( %data, ptr %base, %data, ptr %base, %offsets, %mask) #0 { ; CHECK-LABEL: masked_scatter_nxv8bf16: ; CHECK: // %bb.0: -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: sunpklo z2.s, z1.h ; CHECK-NEXT: uunpklo z3.s, z0.h -; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: sunpkhi z1.s, z1.h ; CHECK-NEXT: uunpkhi z0.s, z0.h +; CHECK-NEXT: punpklo p1.h, p0.b +; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: st1h { z3.s }, p1, [x0, z2.s, sxtw #1] ; CHECK-NEXT: st1h { z0.s }, p0, [x0, z1.s, sxtw #1] ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll b/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll index e866474942cd..94e525d22b82 100644 --- a/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll +++ b/llvm/test/CodeGen/AArch64/sve-masked-scatter.ll @@ -76,8 +76,8 @@ define void @masked_scatter_nxv2f64( %data, %pg) { ; CHECK-LABEL: masked_scatter_splat_constant_pointer: ; CHECK: // %bb.0: // %vector.body -; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: mov z0.d, #0 // =0x0 +; CHECK-NEXT: punpklo p1.h, p0.b ; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: st1w { z0.d }, p1, [z0.d] ; CHECK-NEXT: st1w { z0.d }, p0, [z0.d] diff --git a/llvm/test/CodeGen/AArch64/sve-pr62151.ll b/llvm/test/CodeGen/AArch64/sve-pr62151.ll index 7cec20fda429..5ed34f14a0b1 100644 --- a/llvm/test/CodeGen/AArch64/sve-pr62151.ll +++ b/llvm/test/CodeGen/AArch64/sve-pr62151.ll @@ -5,8 +5,8 @@ define i32 @build_interpolation(<2 x i32> %0, <2 x i32> %1, <2 x i32> %2) { ; CHECK-LABEL: build_interpolation: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: mul v0.2s, v1.2s, v0.2s +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: // kill: def $d2 killed $d2 def $z2 ; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z2.s ; CHECK-NEXT: mla v0.2s, v1.2s, v0.s[1] diff --git a/llvm/test/CodeGen/AArch64/sve-pred-arith.ll b/llvm/test/CodeGen/AArch64/sve-pred-arith.ll index 4d46ac5ecbaa..6e08606db953 100644 --- a/llvm/test/CodeGen/AArch64/sve-pred-arith.ll +++ b/llvm/test/CodeGen/AArch64/sve-pred-arith.ll @@ -54,24 +54,24 @@ define aarch64_sve_vector_pcs @add_nxv64i1( ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: addvl sp, sp, #-1 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG -; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: ptrue p4.b ; CHECK-NEXT: str p8, [sp, #3, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p7, [sp, #4, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p6, [sp, #5, mul vl] // 2-byte Folded Spill +; CHECK-NEXT: ptrue p6.b ; CHECK-NEXT: str p5, [sp, #6, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: ldr p5, [x0] -; CHECK-NEXT: ldr p6, [x1] +; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill +; CHECK-NEXT: ldr p4, [x0] +; CHECK-NEXT: ldr p5, [x1] ; CHECK-NEXT: ldr p7, [x2] ; CHECK-NEXT: ldr p8, [x3] -; CHECK-NEXT: eor p0.b, p4/z, p0.b, p5.b -; CHECK-NEXT: eor p1.b, p4/z, p1.b, p6.b -; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: eor p2.b, p4/z, p2.b, p7.b +; CHECK-NEXT: eor p0.b, p6/z, p0.b, p4.b +; CHECK-NEXT: eor p1.b, p6/z, p1.b, p5.b +; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: eor p2.b, p6/z, p2.b, p7.b ; CHECK-NEXT: ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: eor p3.b, p4/z, p3.b, p8.b +; CHECK-NEXT: eor p3.b, p6/z, p3.b, p8.b ; CHECK-NEXT: ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: .cfi_def_cfa wsp, 16 @@ -138,24 +138,24 @@ define aarch64_sve_vector_pcs @sub_nxv64i1( ; CHECK-NEXT: .cfi_offset w29, -16 ; CHECK-NEXT: addvl sp, sp, #-1 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG -; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: ptrue p4.b ; CHECK-NEXT: str p8, [sp, #3, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p7, [sp, #4, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p6, [sp, #5, mul vl] // 2-byte Folded Spill +; CHECK-NEXT: ptrue p6.b ; CHECK-NEXT: str p5, [sp, #6, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: ldr p5, [x0] -; CHECK-NEXT: ldr p6, [x1] +; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill +; CHECK-NEXT: ldr p4, [x0] +; CHECK-NEXT: ldr p5, [x1] ; CHECK-NEXT: ldr p7, [x2] ; CHECK-NEXT: ldr p8, [x3] -; CHECK-NEXT: eor p0.b, p4/z, p0.b, p5.b -; CHECK-NEXT: eor p1.b, p4/z, p1.b, p6.b -; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: eor p2.b, p4/z, p2.b, p7.b +; CHECK-NEXT: eor p0.b, p6/z, p0.b, p4.b +; CHECK-NEXT: eor p1.b, p6/z, p1.b, p5.b +; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: eor p2.b, p6/z, p2.b, p7.b ; CHECK-NEXT: ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: eor p3.b, p4/z, p3.b, p8.b +; CHECK-NEXT: eor p3.b, p6/z, p3.b, p8.b ; CHECK-NEXT: ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: .cfi_def_cfa wsp, 16 diff --git a/llvm/test/CodeGen/AArch64/sve-pred-selectop.ll b/llvm/test/CodeGen/AArch64/sve-pred-selectop.ll index 600e9c4805ff..8438e9d88f5d 100644 --- a/llvm/test/CodeGen/AArch64/sve-pred-selectop.ll +++ b/llvm/test/CodeGen/AArch64/sve-pred-selectop.ll @@ -322,10 +322,10 @@ entry: define @ornot_v4i32( %z, %x, %y) { ; CHECK-LABEL: ornot_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z3.s, #-1 // =0xffffffffffffffff -; CHECK-NEXT: eor z2.d, z2.d, z3.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: eor z2.d, z2.d, z3.d ; CHECK-NEXT: orr z1.d, z1.d, z2.d ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret @@ -340,10 +340,10 @@ entry: define @ornot_v8i16( %z, %x, %y) { ; CHECK-LABEL: ornot_v8i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z3.h, #-1 // =0xffffffffffffffff -; CHECK-NEXT: eor z2.d, z2.d, z3.d +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: eor z2.d, z2.d, z3.d ; CHECK-NEXT: orr z1.d, z1.d, z2.d ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret @@ -358,10 +358,10 @@ entry: define @ornot_v16i8( %z, %x, %y) { ; CHECK-LABEL: ornot_v16i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z3.b, #-1 // =0xffffffffffffffff -; CHECK-NEXT: eor z2.d, z2.d, z3.d +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: eor z2.d, z2.d, z3.d ; CHECK-NEXT: orr z1.d, z1.d, z2.d ; CHECK-NEXT: mov z0.b, p0/m, z1.b ; CHECK-NEXT: ret @@ -904,8 +904,8 @@ define @addqr_v4i32( %z, ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w0 -; CHECK-NEXT: add z1.s, z1.s, z2.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: add z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -922,8 +922,8 @@ define @addqr_v8i16( %z, ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 -; CHECK-NEXT: add z1.h, z1.h, z2.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: add z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -940,8 +940,8 @@ define @addqr_v16i8( %z, ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w0 -; CHECK-NEXT: add z1.b, z1.b, z2.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: add z1.b, z1.b, z2.b ; CHECK-NEXT: mov z0.b, p0/m, z1.b ; CHECK-NEXT: ret entry: @@ -958,8 +958,8 @@ define @subqr_v4i32( %z, ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w0 -; CHECK-NEXT: sub z1.s, z1.s, z2.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: sub z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -976,8 +976,8 @@ define @subqr_v8i16( %z, ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 -; CHECK-NEXT: sub z1.h, z1.h, z2.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: sub z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -994,8 +994,8 @@ define @subqr_v16i8( %z, ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w0 -; CHECK-NEXT: sub z1.b, z1.b, z2.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: sub z1.b, z1.b, z2.b ; CHECK-NEXT: mov z0.b, p0/m, z1.b ; CHECK-NEXT: ret entry: @@ -1010,10 +1010,10 @@ entry: define @mulqr_v4i32( %z, %x, i32 %y) { ; CHECK-LABEL: mulqr_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w0 -; CHECK-NEXT: mul z1.s, z1.s, z2.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: mul z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -1028,10 +1028,10 @@ entry: define @mulqr_v8i16( %z, %x, i16 %y) { ; CHECK-LABEL: mulqr_v8i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 -; CHECK-NEXT: mul z1.h, z1.h, z2.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: mul z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -1046,10 +1046,10 @@ entry: define @mulqr_v16i8( %z, %x, i8 %y) { ; CHECK-LABEL: mulqr_v16i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w0 -; CHECK-NEXT: mul z1.b, z1.b, z2.b +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: mul z1.b, z1.b, z2.b ; CHECK-NEXT: mov z0.b, p0/m, z1.b ; CHECK-NEXT: ret entry: @@ -1064,11 +1064,11 @@ entry: define @faddqr_v4f32( %z, %x, float %y) { ; CHECK-LABEL: faddqr_v4f32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: // kill: def $s2 killed $s2 def $z2 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, s2 -; CHECK-NEXT: fadd z1.s, z1.s, z2.s ; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0 +; CHECK-NEXT: fadd z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -1083,11 +1083,11 @@ entry: define @faddqr_v8f16( %z, %x, half %y) { ; CHECK-LABEL: faddqr_v8f16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: // kill: def $h2 killed $h2 def $z2 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, h2 -; CHECK-NEXT: fadd z1.h, z1.h, z2.h ; CHECK-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0 +; CHECK-NEXT: fadd z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -1102,11 +1102,11 @@ entry: define @fsubqr_v4f32( %z, %x, float %y) { ; CHECK-LABEL: fsubqr_v4f32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: // kill: def $s2 killed $s2 def $z2 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, s2 -; CHECK-NEXT: fsub z1.s, z1.s, z2.s ; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0 +; CHECK-NEXT: fsub z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -1121,11 +1121,11 @@ entry: define @fsubqr_v8f16( %z, %x, half %y) { ; CHECK-LABEL: fsubqr_v8f16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: // kill: def $h2 killed $h2 def $z2 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, h2 -; CHECK-NEXT: fsub z1.h, z1.h, z2.h ; CHECK-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0 +; CHECK-NEXT: fsub z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -1140,11 +1140,11 @@ entry: define @fmulqr_v4f32( %z, %x, float %y) { ; CHECK-LABEL: fmulqr_v4f32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: // kill: def $s2 killed $s2 def $z2 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, s2 -; CHECK-NEXT: fmul z1.s, z1.s, z2.s ; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, #0.0 +; CHECK-NEXT: fmul z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -1159,11 +1159,11 @@ entry: define @fmulqr_v8f16( %z, %x, half %y) { ; CHECK-LABEL: fmulqr_v8f16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: // kill: def $h2 killed $h2 def $z2 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, h2 -; CHECK-NEXT: fmul z1.h, z1.h, z2.h ; CHECK-NEXT: fcmeq p0.h, p0/z, z0.h, #0.0 +; CHECK-NEXT: fmul z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -1178,10 +1178,10 @@ entry: define @sadd_satqr_v4i32( %z, %x, i32 %y) { ; CHECK-LABEL: sadd_satqr_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w0 -; CHECK-NEXT: sqadd z1.s, z1.s, z2.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: sqadd z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -1196,10 +1196,10 @@ entry: define @sadd_satqr_v8i16( %z, %x, i16 %y) { ; CHECK-LABEL: sadd_satqr_v8i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 -; CHECK-NEXT: sqadd z1.h, z1.h, z2.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: sqadd z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -1214,10 +1214,10 @@ entry: define @sadd_satqr_v16i8( %z, %x, i8 %y) { ; CHECK-LABEL: sadd_satqr_v16i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w0 -; CHECK-NEXT: sqadd z1.b, z1.b, z2.b +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: sqadd z1.b, z1.b, z2.b ; CHECK-NEXT: mov z0.b, p0/m, z1.b ; CHECK-NEXT: ret entry: @@ -1232,10 +1232,10 @@ entry: define @uadd_satqr_v4i32( %z, %x, i32 %y) { ; CHECK-LABEL: uadd_satqr_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w0 -; CHECK-NEXT: uqadd z1.s, z1.s, z2.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: uqadd z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -1250,10 +1250,10 @@ entry: define @uadd_satqr_v8i16( %z, %x, i16 %y) { ; CHECK-LABEL: uadd_satqr_v8i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 -; CHECK-NEXT: uqadd z1.h, z1.h, z2.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: uqadd z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -1268,10 +1268,10 @@ entry: define @uadd_satqr_v16i8( %z, %x, i8 %y) { ; CHECK-LABEL: uadd_satqr_v16i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w0 -; CHECK-NEXT: uqadd z1.b, z1.b, z2.b +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: uqadd z1.b, z1.b, z2.b ; CHECK-NEXT: mov z0.b, p0/m, z1.b ; CHECK-NEXT: ret entry: @@ -1286,10 +1286,10 @@ entry: define @ssub_satqr_v4i32( %z, %x, i32 %y) { ; CHECK-LABEL: ssub_satqr_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w0 -; CHECK-NEXT: sqsub z1.s, z1.s, z2.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: sqsub z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -1304,10 +1304,10 @@ entry: define @ssub_satqr_v8i16( %z, %x, i16 %y) { ; CHECK-LABEL: ssub_satqr_v8i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 -; CHECK-NEXT: sqsub z1.h, z1.h, z2.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: sqsub z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -1322,10 +1322,10 @@ entry: define @ssub_satqr_v16i8( %z, %x, i8 %y) { ; CHECK-LABEL: ssub_satqr_v16i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w0 -; CHECK-NEXT: sqsub z1.b, z1.b, z2.b +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: sqsub z1.b, z1.b, z2.b ; CHECK-NEXT: mov z0.b, p0/m, z1.b ; CHECK-NEXT: ret entry: @@ -1340,10 +1340,10 @@ entry: define @usub_satqr_v4i32( %z, %x, i32 %y) { ; CHECK-LABEL: usub_satqr_v4i32: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w0 -; CHECK-NEXT: uqsub z1.s, z1.s, z2.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, #0 +; CHECK-NEXT: uqsub z1.s, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, z1.s ; CHECK-NEXT: ret entry: @@ -1358,10 +1358,10 @@ entry: define @usub_satqr_v8i16( %z, %x, i16 %y) { ; CHECK-LABEL: usub_satqr_v8i16: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 -; CHECK-NEXT: uqsub z1.h, z1.h, z2.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: uqsub z1.h, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, z1.h ; CHECK-NEXT: ret entry: @@ -1376,10 +1376,10 @@ entry: define @usub_satqr_v16i8( %z, %x, i8 %y) { ; CHECK-LABEL: usub_satqr_v16i8: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w0 -; CHECK-NEXT: uqsub z1.b, z1.b, z2.b +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z0.b, #0 +; CHECK-NEXT: uqsub z1.b, z1.b, z2.b ; CHECK-NEXT: mov z0.b, p0/m, z1.b ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/sve-pred-selectop2.ll b/llvm/test/CodeGen/AArch64/sve-pred-selectop2.ll index 14bc1b45e79e..2541910e080e 100644 --- a/llvm/test/CodeGen/AArch64/sve-pred-selectop2.ll +++ b/llvm/test/CodeGen/AArch64/sve-pred-selectop2.ll @@ -202,9 +202,9 @@ entry: define @sdiv_nxv8i16_x( %x, %y, %n) { ; CHECK-LABEL: sdiv_nxv8i16_x: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpkhi z3.s, z1.h ; CHECK-NEXT: sunpkhi z4.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: sdivr z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: sunpklo z4.s, z0.h @@ -288,9 +288,9 @@ entry: define @udiv_nxv8i16_x( %x, %y, %n) { ; CHECK-LABEL: udiv_nxv8i16_x: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpkhi z3.s, z1.h ; CHECK-NEXT: uunpkhi z4.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpklo z1.s, z1.h ; CHECK-NEXT: udivr z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: uunpklo z4.s, z0.h @@ -376,17 +376,17 @@ entry: define @srem_nxv8i16_x( %x, %y, %n) { ; CHECK-LABEL: srem_nxv8i16_x: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpkhi z3.s, z1.h ; CHECK-NEXT: sunpkhi z4.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpklo z5.s, z0.h ; CHECK-NEXT: sdivr z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: sunpklo z4.s, z1.h ; CHECK-NEXT: sdivr z4.s, p0/m, z4.s, z5.s ; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpgt p0.h, p0/z, z2.h, #0 -; CHECK-NEXT: uzp1 z3.h, z4.h, z3.h -; CHECK-NEXT: mls z0.h, p0/m, z3.h, z1.h +; CHECK-NEXT: uzp1 z2.h, z4.h, z3.h +; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h ; CHECK-NEXT: ret entry: %c = icmp sgt %n, zeroinitializer @@ -419,8 +419,8 @@ define @srem_nxv16i8_x( %x, %n, zeroinitializer @@ -464,17 +464,17 @@ entry: define @urem_nxv8i16_x( %x, %y, %n) { ; CHECK-LABEL: urem_nxv8i16_x: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpkhi z3.s, z1.h ; CHECK-NEXT: uunpkhi z4.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpklo z5.s, z0.h ; CHECK-NEXT: udivr z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: uunpklo z4.s, z1.h ; CHECK-NEXT: udivr z4.s, p0/m, z4.s, z5.s ; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpgt p0.h, p0/z, z2.h, #0 -; CHECK-NEXT: uzp1 z3.h, z4.h, z3.h -; CHECK-NEXT: mls z0.h, p0/m, z3.h, z1.h +; CHECK-NEXT: uzp1 z2.h, z4.h, z3.h +; CHECK-NEXT: mls z0.h, p0/m, z2.h, z1.h ; CHECK-NEXT: ret entry: %c = icmp sgt %n, zeroinitializer @@ -507,8 +507,8 @@ define @urem_nxv16i8_x( %x, %n, zeroinitializer @@ -1140,8 +1140,8 @@ define @fdiv_nxv8f16_x( %x, @sdiv_nxv8i16_y( %x, %y, %n) { ; CHECK-LABEL: sdiv_nxv8i16_y: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpkhi z3.s, z1.h ; CHECK-NEXT: sunpkhi z4.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpklo z0.s, z0.h ; CHECK-NEXT: sdivr z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: sunpklo z4.s, z1.h @@ -1740,9 +1740,9 @@ entry: define @udiv_nxv8i16_y( %x, %y, %n) { ; CHECK-LABEL: udiv_nxv8i16_y: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpkhi z3.s, z1.h ; CHECK-NEXT: uunpkhi z4.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: udivr z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: uunpklo z4.s, z1.h @@ -1830,17 +1830,17 @@ entry: define @srem_nxv8i16_y( %x, %y, %n) { ; CHECK-LABEL: srem_nxv8i16_y: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpkhi z3.s, z1.h ; CHECK-NEXT: sunpkhi z4.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpklo z5.s, z0.h ; CHECK-NEXT: sdivr z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: sunpklo z4.s, z1.h ; CHECK-NEXT: sdivr z4.s, p0/m, z4.s, z5.s ; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpgt p0.h, p0/z, z2.h, #0 -; CHECK-NEXT: uzp1 z3.h, z4.h, z3.h -; CHECK-NEXT: msb z1.h, p0/m, z3.h, z0.h +; CHECK-NEXT: uzp1 z2.h, z4.h, z3.h +; CHECK-NEXT: msb z1.h, p0/m, z2.h, z0.h ; CHECK-NEXT: mov z0.d, z1.d ; CHECK-NEXT: ret entry: @@ -1874,8 +1874,8 @@ define @srem_nxv16i8_y( %x, @urem_nxv8i16_y( %x, %y, %n) { ; CHECK-LABEL: urem_nxv8i16_y: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpkhi z3.s, z1.h ; CHECK-NEXT: uunpkhi z4.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpklo z5.s, z0.h ; CHECK-NEXT: udivr z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: uunpklo z4.s, z1.h ; CHECK-NEXT: udivr z4.s, p0/m, z4.s, z5.s ; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpgt p0.h, p0/z, z2.h, #0 -; CHECK-NEXT: uzp1 z3.h, z4.h, z3.h -; CHECK-NEXT: msb z1.h, p0/m, z3.h, z0.h +; CHECK-NEXT: uzp1 z2.h, z4.h, z3.h +; CHECK-NEXT: msb z1.h, p0/m, z2.h, z0.h ; CHECK-NEXT: mov z0.d, z1.d ; CHECK-NEXT: ret entry: @@ -1966,8 +1966,8 @@ define @urem_nxv16i8_y( %x, @fdiv_nxv8f16_y( %x, @fmai_nxv4f32_y( %x, %n, zeroinitializer @@ -2871,8 +2871,8 @@ define @fmai_nxv8f16_y( %x, %n, zeroinitializer @@ -2887,8 +2887,8 @@ define @fmai_nxv2f64_y( %x, %n, zeroinitializer @@ -2903,8 +2903,8 @@ define @fma_nxv4f32_y( %x, %n, zeroinitializer @@ -2920,8 +2920,8 @@ define @fma_nxv8f16_y( %x, %n, zeroinitializer @@ -2937,8 +2937,8 @@ define @fma_nxv2f64_y( %x, %n, zeroinitializer diff --git a/llvm/test/CodeGen/AArch64/sve-pred-selectop3.ll b/llvm/test/CodeGen/AArch64/sve-pred-selectop3.ll index 0f09f7dac298..bafd5abcc7b2 100644 --- a/llvm/test/CodeGen/AArch64/sve-pred-selectop3.ll +++ b/llvm/test/CodeGen/AArch64/sve-pred-selectop3.ll @@ -792,8 +792,8 @@ define @fdiv_nxv8f16_x( %x, @fdiv_nxv8f16_y( %x, @fmai_nxv4f32_y( %x, %n, zeroinitializer @@ -1750,8 +1750,8 @@ define @fmai_nxv8f16_y( %x, %n, zeroinitializer @@ -1766,8 +1766,8 @@ define @fmai_nxv2f64_y( %x, %n, zeroinitializer @@ -1782,8 +1782,8 @@ define @fma_nxv4f32_y( %x, %n, zeroinitializer @@ -1799,8 +1799,8 @@ define @fma_nxv8f16_y( %x, %n, zeroinitializer @@ -1816,8 +1816,8 @@ define @fma_nxv2f64_y( %x, %n, zeroinitializer diff --git a/llvm/test/CodeGen/AArch64/sve-ptest-removal-cmple.ll b/llvm/test/CodeGen/AArch64/sve-ptest-removal-cmple.ll index fbadbf7226fd..8bd38d7bc44d 100644 --- a/llvm/test/CodeGen/AArch64/sve-ptest-removal-cmple.ll +++ b/llvm/test/CodeGen/AArch64/sve-ptest-removal-cmple.ll @@ -320,8 +320,8 @@ define i1 @cmp32_ptest_any_xx( %pg, %a, %pg, %a, %b) { ; CHECK-LABEL: cmp8_ptest_first_ax: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: cmpge p0.b, p0/z, z0.b, z1.b +; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: ptest p1, p0.b ; CHECK-NEXT: cset w0, mi ; CHECK-NEXT: ret @@ -338,8 +338,8 @@ define i1 @cmp8_ptest_first_ax( %pg, %a, %pg, %a, %b) { ; CHECK-LABEL: cmp8_ptest_last_ax: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: cmpge p0.b, p0/z, z0.b, z1.b +; CHECK-NEXT: ptrue p1.b ; CHECK-NEXT: ptest p1, p0.b ; CHECK-NEXT: cset w0, lo ; CHECK-NEXT: ret @@ -371,8 +371,8 @@ define i1 @cmp8_ptest_any_ax( %pg, %a, %pg, %a, %b) { ; CHECK-LABEL: cmp32_ptest_first_ax: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: cmpge p0.s, p0/z, z0.s, z1.s +; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: ptest p1, p0.b ; CHECK-NEXT: cset w0, mi ; CHECK-NEXT: ret @@ -390,8 +390,8 @@ define i1 @cmp32_ptest_first_ax( %pg, %a, < define i1 @cmp32_ptest_last_ax( %pg, %a, %b) { ; CHECK-LABEL: cmp32_ptest_last_ax: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: cmpge p0.s, p0/z, z0.s, z1.s +; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: ptest p1, p0.b ; CHECK-NEXT: cset w0, lo ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-redundant-store.ll b/llvm/test/CodeGen/AArch64/sve-redundant-store.ll index 6873404724f1..508fe5d5a58a 100644 --- a/llvm/test/CodeGen/AArch64/sve-redundant-store.ll +++ b/llvm/test/CodeGen/AArch64/sve-redundant-store.ll @@ -35,8 +35,8 @@ entry: define void @keep_scalable_store(ptr writeonly %ptr, ptr %a, %b) { ; CHECK-LABEL: keep_scalable_store: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldp q2, q1, [x1] +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: stp q2, q1, [x0] ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll b/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll index a1c2ec9c7e1d..76190eba870d 100644 --- a/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll +++ b/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll @@ -22,15 +22,15 @@ define i8 @split_extract_32i8_idx( %a, i32 %idx) { ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: rdvl x8, #2 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov w9, w0 ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x9, x8 -; CHECK-NEXT: csel x8, x9, x8, lo -; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: st1b { z1.b }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1b { z0.b }, p0, [sp] +; CHECK-NEXT: csel x8, x9, x8, lo +; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: ldrb w0, [x9, x8] ; CHECK-NEXT: addvl sp, sp, #2 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -46,15 +46,15 @@ define i16 @split_extract_16i16_idx( %a, i32 %idx) { ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: rdvl x8, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w9, w0 ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x9, x8 -; CHECK-NEXT: csel x8, x9, x8, lo -; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: st1h { z1.h }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1h { z0.h }, p0, [sp] +; CHECK-NEXT: csel x8, x9, x8, lo +; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: ldrh w0, [x9, x8, lsl #1] ; CHECK-NEXT: addvl sp, sp, #2 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -70,15 +70,15 @@ define i32 @split_extract_8i32_idx( %a, i32 %idx) { ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cnth x8 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w9, w0 ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x9, x8 -; CHECK-NEXT: csel x8, x9, x8, lo -; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: st1w { z1.s }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1w { z0.s }, p0, [sp] +; CHECK-NEXT: csel x8, x9, x8, lo +; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: ldr w0, [x9, x8, lsl #2] ; CHECK-NEXT: addvl sp, sp, #2 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -94,15 +94,15 @@ define i64 @split_extract_8i64_idx( %a, i32 %idx) { ; CHECK-NEXT: addvl sp, sp, #-4 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cnth x8 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w9, w0 ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x9, x8 -; CHECK-NEXT: csel x8, x9, x8, lo -; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: st1d { z3.d }, p0, [sp, #3, mul vl] ; CHECK-NEXT: st1d { z2.d }, p0, [sp, #2, mul vl] +; CHECK-NEXT: csel x8, x9, x8, lo +; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: st1d { z1.d }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1d { z0.d }, p0, [sp] ; CHECK-NEXT: ldr x0, [x9, x8, lsl #3] @@ -140,15 +140,15 @@ define i16 @split_extract_16i16( %a) { ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: rdvl x8, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w9, #128 // =0x80 ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x8, #128 -; CHECK-NEXT: csel x8, x8, x9, lo -; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: st1h { z1.h }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1h { z0.h }, p0, [sp] +; CHECK-NEXT: csel x8, x8, x9, lo +; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: ldrh w0, [x9, x8, lsl #1] ; CHECK-NEXT: addvl sp, sp, #2 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -164,16 +164,16 @@ define i32 @split_extract_16i32( %a) { ; CHECK-NEXT: addvl sp, sp, #-4 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: rdvl x8, #1 ; CHECK-NEXT: mov w9, #34464 // =0x86a0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: movk w9, #1, lsl #16 ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x8, x9 -; CHECK-NEXT: csel x8, x8, x9, lo -; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: st1w { z3.s }, p0, [sp, #3, mul vl] ; CHECK-NEXT: st1w { z2.s }, p0, [sp, #2, mul vl] +; CHECK-NEXT: csel x8, x8, x9, lo +; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: st1w { z1.s }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1w { z0.s }, p0, [sp] ; CHECK-NEXT: ldr w0, [x9, x8, lsl #2] @@ -191,15 +191,15 @@ define i64 @split_extract_4i64( %a) { ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cntw x8 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w9, #10 // =0xa ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x8, #10 -; CHECK-NEXT: csel x8, x8, x9, lo -; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: st1d { z1.d }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: csel x8, x8, x9, lo +; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: ldr x0, [x9, x8, lsl #3] ; CHECK-NEXT: addvl sp, sp, #2 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload diff --git a/llvm/test/CodeGen/AArch64/sve-split-fcvt.ll b/llvm/test/CodeGen/AArch64/sve-split-fcvt.ll index 3997409172d0..bc015116917d 100644 --- a/llvm/test/CodeGen/AArch64/sve-split-fcvt.ll +++ b/llvm/test/CodeGen/AArch64/sve-split-fcvt.ll @@ -6,9 +6,9 @@ define @fcvts_nxv8f16( %a) { ; CHECK-LABEL: fcvts_nxv8f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpklo z1.s, z0.h ; CHECK-NEXT: uunpkhi z2.s, z0.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: movprfx z0, z1 ; CHECK-NEXT: fcvt z0.s, p0/m, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -21,9 +21,9 @@ define @fcvts_nxv8f16( %a) { define @fcvtd_nxv4f16( %a) { ; CHECK-LABEL: fcvtd_nxv4f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z1.d, z0.s ; CHECK-NEXT: uunpkhi z2.d, z0.s +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movprfx z0, z1 ; CHECK-NEXT: fcvt z0.d, p0/m, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -37,8 +37,8 @@ define @fcvtd_nxv8f16( %a) { ; CHECK-LABEL: fcvtd_nxv8f16: ; CHECK: // %bb.0: ; CHECK-NEXT: uunpklo z1.s, z0.h -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpkhi z0.s, z0.h +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z2.d, z1.s ; CHECK-NEXT: uunpkhi z1.d, z1.s ; CHECK-NEXT: uunpklo z3.d, z0.s @@ -58,9 +58,9 @@ define @fcvtd_nxv8f16( %a) { define @fcvtd_nxv4f32( %a) { ; CHECK-LABEL: fcvtd_nxv4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z1.d, z0.s ; CHECK-NEXT: uunpkhi z2.d, z0.s +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movprfx z0, z1 ; CHECK-NEXT: fcvt z0.d, p0/m, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -73,11 +73,11 @@ define @fcvtd_nxv4f32( %a) { define @fcvtd_nxv8f32( %a) { ; CHECK-LABEL: fcvtd_nxv8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z2.d, z0.s ; CHECK-NEXT: uunpkhi z3.d, z0.s ; CHECK-NEXT: uunpklo z4.d, z1.s ; CHECK-NEXT: uunpkhi z5.d, z1.s +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movprfx z0, z2 ; CHECK-NEXT: fcvt z0.d, p0/m, z2.s ; CHECK-NEXT: movprfx z1, z3 @@ -195,9 +195,9 @@ define @fcvtzs_h_nxv8f64( %a) { define @fcvtzs_d_nxv4f32( %a) { ; CHECK-LABEL: fcvtzs_d_nxv4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z1.d, z0.s ; CHECK-NEXT: uunpkhi z2.d, z0.s +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movprfx z0, z1 ; CHECK-NEXT: fcvtzs z0.d, p0/m, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -210,11 +210,11 @@ define @fcvtzs_d_nxv4f32( %a) { define @fcvtzs_s_nxv16f16( %a) { ; CHECK-LABEL: fcvtzs_s_nxv16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uunpklo z2.s, z0.h ; CHECK-NEXT: uunpkhi z3.s, z0.h ; CHECK-NEXT: uunpklo z4.s, z1.h ; CHECK-NEXT: uunpkhi z5.s, z1.h +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: movprfx z0, z2 ; CHECK-NEXT: fcvtzs z0.s, p0/m, z2.h ; CHECK-NEXT: movprfx z1, z3 @@ -247,9 +247,9 @@ define @fcvtzu_s_nxv4f64( %a) { define @fcvtzu_d_nxv4f32( %a) { ; CHECK-LABEL: fcvtzu_d_nxv4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z1.d, z0.s ; CHECK-NEXT: uunpkhi z2.d, z0.s +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movprfx z0, z1 ; CHECK-NEXT: fcvtzu z0.d, p0/m, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -295,8 +295,8 @@ define @scvtf_s_nxv16i8( %a) { ; CHECK-LABEL: scvtf_s_nxv16i8: ; CHECK: // %bb.0: ; CHECK-NEXT: sunpklo z1.h, z0.b -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpkhi z0.h, z0.b +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpklo z2.s, z1.h ; CHECK-NEXT: sunpkhi z1.s, z1.h ; CHECK-NEXT: sunpklo z3.s, z0.h @@ -316,9 +316,9 @@ define @scvtf_s_nxv16i8( %a) { define @scvtf_d_nxv4i32( %a) { ; CHECK-LABEL: scvtf_d_nxv4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: sunpklo z1.d, z0.s ; CHECK-NEXT: sunpkhi z2.d, z0.s +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movprfx z0, z1 ; CHECK-NEXT: scvtf z0.d, p0/m, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -333,8 +333,8 @@ define @scvtf_d_nxv4i1( %a) { ; CHECK: // %bb.0: ; CHECK-NEXT: punpklo p2.h, p0.b ; CHECK-NEXT: punpkhi p0.h, p0.b -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z0.d, p2/z, #-1 // =0xffffffffffffffff +; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z1.d, p0/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: scvtf z0.d, p1/m, z0.d ; CHECK-NEXT: scvtf z1.d, p1/m, z1.d @@ -378,9 +378,9 @@ define @ucvtf_h_nxv8i64( %a) { define @ucvtf_d_nxv4i32( %a) { ; CHECK-LABEL: ucvtf_d_nxv4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uunpklo z1.d, z0.s ; CHECK-NEXT: uunpkhi z2.d, z0.s +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: movprfx z0, z1 ; CHECK-NEXT: ucvtf z0.d, p0/m, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -395,8 +395,8 @@ define @ucvtf_d_nxv4i1( %a) { ; CHECK: // %bb.0: ; CHECK-NEXT: punpklo p2.h, p0.b ; CHECK-NEXT: punpkhi p0.h, p0.b -; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z0.d, p2/z, #1 // =0x1 +; CHECK-NEXT: ptrue p1.d ; CHECK-NEXT: mov z1.d, p0/z, #1 // =0x1 ; CHECK-NEXT: ucvtf z0.d, p1/m, z0.d ; CHECK-NEXT: ucvtf z1.d, p1/m, z1.d diff --git a/llvm/test/CodeGen/AArch64/sve-split-fp-reduce.ll b/llvm/test/CodeGen/AArch64/sve-split-fp-reduce.ll index 7f642882eddb..696b6c34ef04 100644 --- a/llvm/test/CodeGen/AArch64/sve-split-fp-reduce.ll +++ b/llvm/test/CodeGen/AArch64/sve-split-fp-reduce.ll @@ -23,8 +23,8 @@ define double @fadda_nxv8f64(double %init, %a) { define float @faddv_nxv8f32(float %init, %a) { ; CHECK-LABEL: faddv_nxv8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: fadd z1.s, z1.s, z2.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: faddv s1, p0, z1.s ; CHECK-NEXT: fadd s0, s0, s1 ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll b/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll index 5441659fa5cb..75366384cb75 100644 --- a/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll +++ b/llvm/test/CodeGen/AArch64/sve-split-insert-elt.ll @@ -6,9 +6,9 @@ define @promote_insert_8i8( %a, i8 %elt, i64 %idx) { ; CHECK-LABEL: promote_insert_8i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: index z1.h, #0, #1 ; CHECK-NEXT: mov z2.h, w1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z1.h, z2.h ; CHECK-NEXT: mov z0.h, p0/m, w0 ; CHECK-NEXT: ret @@ -23,13 +23,13 @@ define @split_insert_32i8_idx( %a, i8 %elt, ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: rdvl x8, #2 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x1, x8 -; CHECK-NEXT: csel x8, x1, x8, lo ; CHECK-NEXT: st1b { z1.b }, p0, [sp, #1, mul vl] +; CHECK-NEXT: csel x8, x1, x8, lo ; CHECK-NEXT: st1b { z0.b }, p0, [sp] ; CHECK-NEXT: strb w0, [x9, x8] ; CHECK-NEXT: ld1b { z0.b }, p0/z, [sp] @@ -48,13 +48,13 @@ define @split_insert_8f32_idx( %a, floa ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cnth x8 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x0, x8 -; CHECK-NEXT: csel x8, x0, x8, lo ; CHECK-NEXT: st1w { z1.s }, p0, [sp, #1, mul vl] +; CHECK-NEXT: csel x8, x0, x8, lo ; CHECK-NEXT: st1w { z0.s }, p0, [sp] ; CHECK-NEXT: str s2, [x9, x8, lsl #2] ; CHECK-NEXT: ld1w { z0.s }, p0/z, [sp] @@ -73,13 +73,13 @@ define @split_insert_8i64_idx( %a, i64 %elt ; CHECK-NEXT: addvl sp, sp, #-4 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cnth x8 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov x9, sp ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x1, x8 -; CHECK-NEXT: csel x8, x1, x8, lo ; CHECK-NEXT: st1d { z3.d }, p0, [sp, #3, mul vl] +; CHECK-NEXT: csel x8, x1, x8, lo ; CHECK-NEXT: st1d { z2.d }, p0, [sp, #2, mul vl] ; CHECK-NEXT: st1d { z1.d }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1d { z0.d }, p0, [sp] @@ -100,9 +100,9 @@ define @split_insert_8i64_idx( %a, i64 %elt define @promote_insert_4i16( %a, i16 %elt) { ; CHECK-LABEL: promote_insert_4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #5 // =0x5 ; CHECK-NEXT: index z1.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: cmpeq p0.s, p0/z, z1.s, z2.s ; CHECK-NEXT: mov z0.s, p0/m, w0 @@ -117,9 +117,9 @@ define @promote_insert_4i16( %a, i16 %elt) define @split_insert_32i8( %a, i8 %elt) { ; CHECK-LABEL: split_insert_32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z2.b, #0, #1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z3.b, w8 ; CHECK-NEXT: cmpeq p0.b, p0/z, z2.b, z3.b ; CHECK-NEXT: mov z0.b, p0/m, w0 @@ -135,14 +135,14 @@ define @split_insert_32i16( %a, i16 %elt) ; CHECK-NEXT: addvl sp, sp, #-4 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: rdvl x8, #2 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w9, #128 // =0x80 ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x8, #128 +; CHECK-NEXT: st1h { z3.h }, p0, [sp, #3, mul vl] ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp -; CHECK-NEXT: st1h { z3.h }, p0, [sp, #3, mul vl] ; CHECK-NEXT: st1h { z2.h }, p0, [sp, #2, mul vl] ; CHECK-NEXT: st1h { z1.h }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1h { z0.h }, p0, [sp] @@ -165,15 +165,15 @@ define @split_insert_8i32( %a, i32 %elt) { ; CHECK-NEXT: addvl sp, sp, #-2 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cnth x8 ; CHECK-NEXT: mov w9, #16960 // =0x4240 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: movk w9, #15, lsl #16 ; CHECK-NEXT: sub x8, x8, #1 ; CHECK-NEXT: cmp x8, x9 +; CHECK-NEXT: st1w { z1.s }, p0, [sp, #1, mul vl] ; CHECK-NEXT: csel x8, x8, x9, lo ; CHECK-NEXT: mov x9, sp -; CHECK-NEXT: st1w { z1.s }, p0, [sp, #1, mul vl] ; CHECK-NEXT: st1w { z0.s }, p0, [sp] ; CHECK-NEXT: str w0, [x9, x8, lsl #2] ; CHECK-NEXT: ld1w { z0.s }, p0/z, [sp] diff --git a/llvm/test/CodeGen/AArch64/sve-split-int-reduce.ll b/llvm/test/CodeGen/AArch64/sve-split-int-reduce.ll index 42f3a163d14c..dd7b15ef5ee6 100644 --- a/llvm/test/CodeGen/AArch64/sve-split-int-reduce.ll +++ b/llvm/test/CodeGen/AArch64/sve-split-int-reduce.ll @@ -17,8 +17,8 @@ define i8 @andv_nxv8i8( %a) { define i32 @andv_nxv8i32( %a) { ; CHECK-LABEL: andv_nxv8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z0.d, z0.d, z1.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: andv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 ; CHECK-NEXT: ret @@ -71,8 +71,8 @@ define i16 @xorv_nxv2i16( %a) { define i32 @xorv_nxv8i32( %a) { ; CHECK-LABEL: xorv_nxv8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: eor z0.d, z0.d, z1.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: eorv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 ; CHECK-NEXT: ret @@ -97,8 +97,8 @@ define i16 @uaddv_nxv4i16( %a) { define i16 @uaddv_nxv16i16( %a) { ; CHECK-LABEL: uaddv_nxv16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: add z0.h, z0.h, z1.h +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: uaddv d0, p0, z0.h ; CHECK-NEXT: fmov x0, d0 ; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 @@ -127,8 +127,8 @@ define i32 @uaddv_nxv16i32( %a) { define i32 @umin_nxv2i32( %a) { ; CHECK-LABEL: umin_nxv2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z0.d, z0.d, #0xffffffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uminv d0, p0, z0.d ; CHECK-NEXT: fmov x0, d0 ; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 diff --git a/llvm/test/CodeGen/AArch64/sve-split-load.ll b/llvm/test/CodeGen/AArch64/sve-split-load.ll index af03059cf0d8..754f0339702d 100644 --- a/llvm/test/CodeGen/AArch64/sve-split-load.ll +++ b/llvm/test/CodeGen/AArch64/sve-split-load.ll @@ -93,8 +93,8 @@ define @masked_load_split_32i16(ptr %a, % ; CHECK-NEXT: punpklo p2.h, p0.b ; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: punpklo p3.h, p1.b -; CHECK-NEXT: punpkhi p1.h, p1.b ; CHECK-NEXT: ld1h { z0.h }, p2/z, [x0] +; CHECK-NEXT: punpkhi p1.h, p1.b ; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0, #1, mul vl] ; CHECK-NEXT: ld1h { z2.h }, p3/z, [x0, #2, mul vl] ; CHECK-NEXT: ld1h { z3.h }, p1/z, [x0, #3, mul vl] @@ -123,8 +123,8 @@ define @masked_load_split_8i64(ptr %a, %pg) ; CHECK-NEXT: punpklo p2.h, p1.b ; CHECK-NEXT: punpkhi p1.h, p1.b ; CHECK-NEXT: punpklo p3.h, p0.b -; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: ld1d { z0.d }, p2/z, [x0] +; CHECK-NEXT: punpkhi p0.h, p0.b ; CHECK-NEXT: ld1d { z1.d }, p1/z, [x0, #1, mul vl] ; CHECK-NEXT: ld1d { z2.d }, p3/z, [x0, #2, mul vl] ; CHECK-NEXT: ld1d { z3.d }, p0/z, [x0, #3, mul vl] diff --git a/llvm/test/CodeGen/AArch64/sve-split-store.ll b/llvm/test/CodeGen/AArch64/sve-split-store.ll index 90ec783ea4db..affa9a18ac18 100644 --- a/llvm/test/CodeGen/AArch64/sve-split-store.ll +++ b/llvm/test/CodeGen/AArch64/sve-split-store.ll @@ -81,8 +81,8 @@ define void @masked_store_split_32i16( %data, ptr %a, %data, ptr %a, %data, ptr %a, i32 1, %pg) diff --git a/llvm/test/CodeGen/AArch64/sve-srem-combine-loop.ll b/llvm/test/CodeGen/AArch64/sve-srem-combine-loop.ll index 9c3d4b1e5a81..f556d60d23b8 100644 --- a/llvm/test/CodeGen/AArch64/sve-srem-combine-loop.ll +++ b/llvm/test/CodeGen/AArch64/sve-srem-combine-loop.ll @@ -6,8 +6,8 @@ target triple = "aarch64-unknown-linux-gnu" define @srem_combine_loop( %a) #0 { ; CHECK-LABEL: srem_combine_loop: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.d, z0.d +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, #2 // =0x2 ; CHECK-NEXT: asrd z1.s, p0/m, z1.s, #1 ; CHECK-NEXT: mls z0.s, p0/m, z1.s, z2.s diff --git a/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll b/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll index 728041d3f916..3273e6b384f6 100644 --- a/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll +++ b/llvm/test/CodeGen/AArch64/sve-st1-addressing-mode-reg-imm.ll @@ -105,8 +105,8 @@ define void @st1d_inbound( %data, ptr %a) { define void @store_nxv2f32(ptr %out) { ; CHECK-LABEL: store_nxv2f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: fmov z0.s, #1.00000000 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: st1w { z0.d }, p0, [x0] ; CHECK-NEXT: ret %ins = insertelement undef, float 1.0, i32 0 @@ -118,8 +118,8 @@ define void @store_nxv2f32(ptr %out) { define void @store_nxv4f16(ptr %out) { ; CHECK-LABEL: store_nxv4f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: fmov z0.h, #1.00000000 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: st1h { z0.s }, p0, [x0] ; CHECK-NEXT: ret %ins = insertelement undef, half 1.0, i32 0 @@ -133,9 +133,9 @@ define void @store_nxv4f16(ptr %out) { define void @store_nxv6f32(ptr %out) { ; CHECK-LABEL: store_nxv6f32: ; CHECK: // %bb.0: +; CHECK-NEXT: fmov z0.s, #1.00000000 ; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ptrue p1.s -; CHECK-NEXT: fmov z0.s, #1.00000000 ; CHECK-NEXT: st1w { z0.d }, p0, [x0, #2, mul vl] ; CHECK-NEXT: st1w { z0.s }, p1, [x0] ; CHECK-NEXT: ret @@ -148,9 +148,9 @@ define void @store_nxv6f32(ptr %out) { define void @store_nxv12f16(ptr %out) { ; CHECK-LABEL: store_nxv12f16: ; CHECK: // %bb.0: +; CHECK-NEXT: fmov z0.h, #1.00000000 ; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ptrue p1.h -; CHECK-NEXT: fmov z0.h, #1.00000000 ; CHECK-NEXT: st1h { z0.s }, p0, [x0, #2, mul vl] ; CHECK-NEXT: st1h { z0.h }, p1, [x0] ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-stepvector.ll b/llvm/test/CodeGen/AArch64/sve-stepvector.ll index 6f5a31248de7..4c5f27d3e709 100644 --- a/llvm/test/CodeGen/AArch64/sve-stepvector.ll +++ b/llvm/test/CodeGen/AArch64/sve-stepvector.ll @@ -208,9 +208,9 @@ entry: define @multiple_use_stepvector_nxv4i32_1(i32 %data) { ; CHECK-LABEL: multiple_use_stepvector_nxv4i32_1: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: index z0.s, w0, #1 ; CHECK-NEXT: mov z1.s, w0 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mul z1.s, p0/m, z1.s, z0.s ; CHECK-NEXT: sub z0.s, z1.s, z0.s ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bit-counting.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bit-counting.ll index be5c318e675d..d547f99a0230 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bit-counting.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bit-counting.ll @@ -11,8 +11,8 @@ target triple = "aarch64-unknown-linux-gnu" define <4 x i8> @ctlz_v4i8(<4 x i8> %op) { ; CHECK-LABEL: ctlz_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: and z0.h, z0.h, #0xff ; CHECK-NEXT: clz z0.h, p0/m, z0.h ; CHECK-NEXT: sub z0.h, z0.h, #8 // =0x8 @@ -49,8 +49,8 @@ define <16 x i8> @ctlz_v16i8(<16 x i8> %op) { define void @ctlz_v32i8(ptr %a) { ; CHECK-LABEL: ctlz_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: clz z0.b, p0/m, z0.b ; CHECK-NEXT: clz z1.b, p0/m, z1.b ; CHECK-NEXT: stp q0, q1, [x0] @@ -64,8 +64,8 @@ define void @ctlz_v32i8(ptr %a) { define <2 x i16> @ctlz_v2i16(<2 x i16> %op) { ; CHECK-LABEL: ctlz_v2i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: and z0.s, z0.s, #0xffff ; CHECK-NEXT: clz z0.s, p0/m, z0.s ; CHECK-NEXT: sub z0.s, z0.s, #16 // =0x10 @@ -102,8 +102,8 @@ define <8 x i16> @ctlz_v8i16(<8 x i16> %op) { define void @ctlz_v16i16(ptr %a) { ; CHECK-LABEL: ctlz_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: clz z0.h, p0/m, z0.h ; CHECK-NEXT: clz z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -141,8 +141,8 @@ define <4 x i32> @ctlz_v4i32(<4 x i32> %op) { define void @ctlz_v8i32(ptr %a) { ; CHECK-LABEL: ctlz_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: clz z0.s, p0/m, z0.s ; CHECK-NEXT: clz z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -180,8 +180,8 @@ define <2 x i64> @ctlz_v2i64(<2 x i64> %op) { define void @ctlz_v4i64(ptr %a) { ; CHECK-LABEL: ctlz_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: clz z0.d, p0/m, z0.d ; CHECK-NEXT: clz z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -199,8 +199,8 @@ define void @ctlz_v4i64(ptr %a) { define <4 x i8> @ctpop_v4i8(<4 x i8> %op) { ; CHECK-LABEL: ctpop_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: and z0.h, z0.h, #0xff ; CHECK-NEXT: cnt z0.h, p0/m, z0.h ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -236,8 +236,8 @@ define <16 x i8> @ctpop_v16i8(<16 x i8> %op) { define void @ctpop_v32i8(ptr %a) { ; CHECK-LABEL: ctpop_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: cnt z0.b, p0/m, z0.b ; CHECK-NEXT: cnt z1.b, p0/m, z1.b ; CHECK-NEXT: stp q0, q1, [x0] @@ -251,8 +251,8 @@ define void @ctpop_v32i8(ptr %a) { define <2 x i16> @ctpop_v2i16(<2 x i16> %op) { ; CHECK-LABEL: ctpop_v2i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: and z0.s, z0.s, #0xffff ; CHECK-NEXT: cnt z0.s, p0/m, z0.s ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -288,8 +288,8 @@ define <8 x i16> @ctpop_v8i16(<8 x i16> %op) { define void @ctpop_v16i16(ptr %a) { ; CHECK-LABEL: ctpop_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: cnt z0.h, p0/m, z0.h ; CHECK-NEXT: cnt z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -327,8 +327,8 @@ define <4 x i32> @ctpop_v4i32(<4 x i32> %op) { define void @ctpop_v8i32(ptr %a) { ; CHECK-LABEL: ctpop_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: cnt z0.s, p0/m, z0.s ; CHECK-NEXT: cnt z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -366,8 +366,8 @@ define <2 x i64> @ctpop_v2i64(<2 x i64> %op) { define void @ctpop_v4i64(ptr %a) { ; CHECK-LABEL: ctpop_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: cnt z0.d, p0/m, z0.d ; CHECK-NEXT: cnt z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -385,8 +385,8 @@ define void @ctpop_v4i64(ptr %a) { define <4 x i8> @cttz_v4i8(<4 x i8> %op) { ; CHECK-LABEL: cttz_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: orr z0.h, z0.h, #0x100 ; CHECK-NEXT: rbit z0.h, p0/m, z0.h ; CHECK-NEXT: clz z0.h, p0/m, z0.h @@ -425,8 +425,8 @@ define <16 x i8> @cttz_v16i8(<16 x i8> %op) { define void @cttz_v32i8(ptr %a) { ; CHECK-LABEL: cttz_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: rbit z0.b, p0/m, z0.b ; CHECK-NEXT: rbit z1.b, p0/m, z1.b ; CHECK-NEXT: clz z0.b, p0/m, z0.b @@ -442,8 +442,8 @@ define void @cttz_v32i8(ptr %a) { define <2 x i16> @cttz_v2i16(<2 x i16> %op) { ; CHECK-LABEL: cttz_v2i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: orr z0.s, z0.s, #0x10000 ; CHECK-NEXT: rbit z0.s, p0/m, z0.s ; CHECK-NEXT: clz z0.s, p0/m, z0.s @@ -482,8 +482,8 @@ define <8 x i16> @cttz_v8i16(<8 x i16> %op) { define void @cttz_v16i16(ptr %a) { ; CHECK-LABEL: cttz_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: rbit z0.h, p0/m, z0.h ; CHECK-NEXT: rbit z1.h, p0/m, z1.h ; CHECK-NEXT: clz z0.h, p0/m, z0.h @@ -525,8 +525,8 @@ define <4 x i32> @cttz_v4i32(<4 x i32> %op) { define void @cttz_v8i32(ptr %a) { ; CHECK-LABEL: cttz_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: rbit z0.s, p0/m, z0.s ; CHECK-NEXT: rbit z1.s, p0/m, z1.s ; CHECK-NEXT: clz z0.s, p0/m, z0.s @@ -568,8 +568,8 @@ define <2 x i64> @cttz_v2i64(<2 x i64> %op) { define void @cttz_v4i64(ptr %a) { ; CHECK-LABEL: cttz_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: rbit z0.d, p0/m, z0.d ; CHECK-NEXT: rbit z1.d, p0/m, z1.d ; CHECK-NEXT: clz z0.d, p0/m, z0.d diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ext-loads.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ext-loads.ll index 251a7c3b18a9..0aefba2d4c6a 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ext-loads.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ext-loads.ll @@ -169,15 +169,15 @@ define <16 x i64> @load_zext_v16i16i64(ptr %ap) { ; CHECK-NEXT: ld1h { z1.d }, p0/z, [x0, x8, lsl #1] ; CHECK-NEXT: mov x8, #8 // =0x8 ; CHECK-NEXT: ld1h { z2.d }, p0/z, [x0, x9, lsl #1] -; CHECK-NEXT: mov x9, #10 // =0xa ; CHECK-NEXT: ld1h { z3.d }, p0/z, [x0, x10, lsl #1] -; CHECK-NEXT: mov x10, #12 // =0xc +; CHECK-NEXT: mov x9, #10 // =0xa ; CHECK-NEXT: ld1h { z4.d }, p0/z, [x0, x8, lsl #1] -; CHECK-NEXT: mov x8, #14 // =0xe +; CHECK-NEXT: mov x8, #12 // =0xc +; CHECK-NEXT: mov x10, #14 // =0xe ; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0] ; CHECK-NEXT: ld1h { z5.d }, p0/z, [x0, x9, lsl #1] -; CHECK-NEXT: ld1h { z6.d }, p0/z, [x0, x10, lsl #1] -; CHECK-NEXT: ld1h { z7.d }, p0/z, [x0, x8, lsl #1] +; CHECK-NEXT: ld1h { z6.d }, p0/z, [x0, x8, lsl #1] +; CHECK-NEXT: ld1h { z7.d }, p0/z, [x0, x10, lsl #1] ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 ; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z1 ; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll index 2ace0bca274a..0d6675def8b5 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fcopysign.ll @@ -241,8 +241,8 @@ define void @test_copysign_v2f32_v2f64(ptr %ap, ptr %bp) { ; SVE-NEXT: ptrue p0.d ; SVE-NEXT: ldr q0, [x1] ; SVE-NEXT: ldr d1, [x0] -; SVE-NEXT: and z1.s, z1.s, #0x7fffffff ; SVE-NEXT: fcvt z0.s, p0/m, z0.d +; SVE-NEXT: and z1.s, z1.s, #0x7fffffff ; SVE-NEXT: uzp1 z0.s, z0.s, z0.s ; SVE-NEXT: and z0.s, z0.s, #0x80000000 ; SVE-NEXT: orr z0.d, z1.d, z0.d @@ -274,8 +274,8 @@ define void @test_copysign_v2f32_v2f64(ptr %ap, ptr %bp) { define void @test_copysign_v4f32_v4f64(ptr %ap, ptr %bp) { ; SVE-LABEL: test_copysign_v4f32_v4f64: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.d ; SVE-NEXT: ldp q0, q1, [x1] +; SVE-NEXT: ptrue p0.d ; SVE-NEXT: fcvt z1.s, p0/m, z1.d ; SVE-NEXT: fcvt z0.s, p0/m, z0.d ; SVE-NEXT: ptrue p0.s, vl2 @@ -291,8 +291,8 @@ define void @test_copysign_v4f32_v4f64(ptr %ap, ptr %bp) { ; ; SVE2-LABEL: test_copysign_v4f32_v4f64: ; SVE2: // %bb.0: -; SVE2-NEXT: ptrue p0.d ; SVE2-NEXT: ldp q0, q1, [x1] +; SVE2-NEXT: ptrue p0.d ; SVE2-NEXT: ldr q2, [x0] ; SVE2-NEXT: fcvt z1.s, p0/m, z1.d ; SVE2-NEXT: fcvt z0.s, p0/m, z0.d @@ -319,8 +319,8 @@ define void @test_copysign_v2f64_v2f32(ptr %ap, ptr %bp) { ; SVE: // %bb.0: ; SVE-NEXT: ptrue p0.d, vl2 ; SVE-NEXT: ldr q0, [x0] -; SVE-NEXT: and z0.d, z0.d, #0x7fffffffffffffff ; SVE-NEXT: ld1w { z1.d }, p0/z, [x1] +; SVE-NEXT: and z0.d, z0.d, #0x7fffffffffffffff ; SVE-NEXT: fcvt z1.d, p0/m, z1.s ; SVE-NEXT: and z1.d, z1.d, #0x8000000000000000 ; SVE-NEXT: orr z0.d, z0.d, z1.d @@ -354,10 +354,10 @@ define void @test_copysign_v4f64_v4f32(ptr %ap, ptr %bp) { ; SVE-NEXT: ptrue p0.d, vl2 ; SVE-NEXT: mov x8, #2 // =0x2 ; SVE-NEXT: ldp q2, q3, [x0] -; SVE-NEXT: and z2.d, z2.d, #0x7fffffffffffffff -; SVE-NEXT: and z3.d, z3.d, #0x7fffffffffffffff ; SVE-NEXT: ld1w { z0.d }, p0/z, [x1] ; SVE-NEXT: ld1w { z1.d }, p0/z, [x1, x8, lsl #2] +; SVE-NEXT: and z2.d, z2.d, #0x7fffffffffffffff +; SVE-NEXT: and z3.d, z3.d, #0x7fffffffffffffff ; SVE-NEXT: fcvt z0.d, p0/m, z0.s ; SVE-NEXT: fcvt z1.d, p0/m, z1.s ; SVE-NEXT: and z0.d, z0.d, #0x8000000000000000 @@ -397,8 +397,8 @@ define void @test_copysign_v4f16_v4f32(ptr %ap, ptr %bp) { ; SVE-NEXT: ptrue p0.s ; SVE-NEXT: ldr q0, [x1] ; SVE-NEXT: ldr d1, [x0] -; SVE-NEXT: and z1.h, z1.h, #0x7fff ; SVE-NEXT: fcvt z0.h, p0/m, z0.s +; SVE-NEXT: and z1.h, z1.h, #0x7fff ; SVE-NEXT: uzp1 z0.h, z0.h, z0.h ; SVE-NEXT: and z0.h, z0.h, #0x8000 ; SVE-NEXT: orr z0.d, z1.d, z0.d @@ -429,13 +429,13 @@ define void @test_copysign_v4f16_v4f64(ptr %ap, ptr %bp) { ; SVE: // %bb.0: ; SVE-NEXT: ldp q0, q1, [x1] ; SVE-NEXT: ptrue p0.s, vl2 -; SVE-NEXT: ptrue p1.s ; SVE-NEXT: fcvtxn v1.2s, v1.2d ; SVE-NEXT: fcvtxn v0.2s, v0.2d ; SVE-NEXT: splice z0.s, p0, z0.s, z1.s +; SVE-NEXT: ptrue p0.s ; SVE-NEXT: ldr d1, [x0] ; SVE-NEXT: and z1.h, z1.h, #0x7fff -; SVE-NEXT: fcvt z0.h, p1/m, z0.s +; SVE-NEXT: fcvt z0.h, p0/m, z0.s ; SVE-NEXT: uzp1 z0.h, z0.h, z0.h ; SVE-NEXT: and z0.h, z0.h, #0x8000 ; SVE-NEXT: orr z0.d, z1.d, z0.d @@ -446,13 +446,13 @@ define void @test_copysign_v4f16_v4f64(ptr %ap, ptr %bp) { ; SVE2: // %bb.0: ; SVE2-NEXT: ldp q0, q1, [x1] ; SVE2-NEXT: ptrue p0.s, vl2 -; SVE2-NEXT: ptrue p1.s ; SVE2-NEXT: ldr d2, [x0] ; SVE2-NEXT: fcvtxn v1.2s, v1.2d ; SVE2-NEXT: fcvtxn v0.2s, v0.2d ; SVE2-NEXT: splice z0.s, p0, z0.s, z1.s +; SVE2-NEXT: ptrue p0.s ; SVE2-NEXT: mov z1.h, #32767 // =0x7fff -; SVE2-NEXT: fcvt z0.h, p1/m, z0.s +; SVE2-NEXT: fcvt z0.h, p0/m, z0.s ; SVE2-NEXT: uzp1 z0.h, z0.h, z0.h ; SVE2-NEXT: bsl z2.d, z2.d, z0.d, z1.d ; SVE2-NEXT: str d2, [x0] @@ -470,8 +470,8 @@ define void @test_copysign_v4f16_v4f64(ptr %ap, ptr %bp) { define void @test_copysign_v8f16_v8f32(ptr %ap, ptr %bp) { ; SVE-LABEL: test_copysign_v8f16_v8f32: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s ; SVE-NEXT: ldp q0, q1, [x1] +; SVE-NEXT: ptrue p0.s ; SVE-NEXT: fcvt z1.h, p0/m, z1.s ; SVE-NEXT: fcvt z0.h, p0/m, z0.s ; SVE-NEXT: ptrue p0.h, vl4 @@ -487,8 +487,8 @@ define void @test_copysign_v8f16_v8f32(ptr %ap, ptr %bp) { ; ; SVE2-LABEL: test_copysign_v8f16_v8f32: ; SVE2: // %bb.0: -; SVE2-NEXT: ptrue p0.s ; SVE2-NEXT: ldp q0, q1, [x1] +; SVE2-NEXT: ptrue p0.s ; SVE2-NEXT: ldr q2, [x0] ; SVE2-NEXT: fcvt z1.h, p0/m, z1.s ; SVE2-NEXT: fcvt z0.h, p0/m, z0.s diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-arith.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-arith.ll index c436dea8ff1b..c2d6ed4e9ccf 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-arith.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-arith.ll @@ -50,8 +50,8 @@ define <8 x half> @fadd_v8f16(<8 x half> %op1, <8 x half> %op2) { define void @fadd_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fadd_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fadd z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -94,8 +94,8 @@ define <4 x float> @fadd_v4f32(<4 x float> %op1, <4 x float> %op2) { define void @fadd_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: fadd_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fadd z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -125,8 +125,8 @@ define <2 x double> @fadd_v2f64(<2 x double> %op1, <2 x double> %op2) { define void @fadd_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: fadd_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fadd z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -186,8 +186,8 @@ define <8 x half> @fdiv_v8f16(<8 x half> %op1, <8 x half> %op2) { define void @fdiv_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fdiv_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fdivr z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -230,8 +230,8 @@ define <4 x float> @fdiv_v4f32(<4 x float> %op1, <4 x float> %op2) { define void @fdiv_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: fdiv_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fdivr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -261,8 +261,8 @@ define <2 x double> @fdiv_v2f64(<2 x double> %op1, <2 x double> %op2) { define void @fdiv_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: fdiv_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fdivr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -325,8 +325,8 @@ define <8 x half> @fma_v8f16(<8 x half> %op1, <8 x half> %op2, <8 x half> %op3) define void @fma_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fma_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q4, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q5, [x2] ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fmad z0.h, p0/m, z2.h, z1.h @@ -373,8 +373,8 @@ define <4 x float> @fma_v4f32(<4 x float> %op1, <4 x float> %op2, <4 x float> %o define void @fma_v8f32(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fma_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q4, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q5, [x2] ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fmad z0.s, p0/m, z2.s, z1.s @@ -407,8 +407,8 @@ define <2 x double> @fma_v2f64(<2 x double> %op1, <2 x double> %op2, <2 x double define void @fma_v4f64(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fma_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q4, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q5, [x2] ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fmad z0.d, p0/m, z2.d, z1.d @@ -470,8 +470,8 @@ define <8 x half> @fmul_v8f16(<8 x half> %op1, <8 x half> %op2) { define void @fmul_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fmul_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmul z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -514,8 +514,8 @@ define <4 x float> @fmul_v4f32(<4 x float> %op1, <4 x float> %op2) { define void @fmul_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: fmul_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -545,8 +545,8 @@ define <2 x double> @fmul_v2f64(<2 x double> %op1, <2 x double> %op2) { define void @fmul_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: fmul_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -603,8 +603,8 @@ define <8 x half> @fneg_v8f16(<8 x half> %op) { define void @fneg_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fneg_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fneg z0.h, p0/m, z0.h ; CHECK-NEXT: fneg z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -642,8 +642,8 @@ define <4 x float> @fneg_v4f32(<4 x float> %op) { define void @fneg_v8f32(ptr %a) { ; CHECK-LABEL: fneg_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fneg z0.s, p0/m, z0.s ; CHECK-NEXT: fneg z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -669,8 +669,8 @@ define <2 x double> @fneg_v2f64(<2 x double> %op) { define void @fneg_v4f64(ptr %a) { ; CHECK-LABEL: fneg_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fneg z0.d, p0/m, z0.d ; CHECK-NEXT: fneg z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -724,8 +724,8 @@ define <8 x half> @fsqrt_v8f16(<8 x half> %op) { define void @fsqrt_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fsqrt_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fsqrt z0.h, p0/m, z0.h ; CHECK-NEXT: fsqrt z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -763,8 +763,8 @@ define <4 x float> @fsqrt_v4f32(<4 x float> %op) { define void @fsqrt_v8f32(ptr %a) { ; CHECK-LABEL: fsqrt_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fsqrt z0.s, p0/m, z0.s ; CHECK-NEXT: fsqrt z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -790,8 +790,8 @@ define <2 x double> @fsqrt_v2f64(<2 x double> %op) { define void @fsqrt_v4f64(ptr %a) { ; CHECK-LABEL: fsqrt_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fsqrt z0.d, p0/m, z0.d ; CHECK-NEXT: fsqrt z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -848,8 +848,8 @@ define <8 x half> @fsub_v8f16(<8 x half> %op1, <8 x half> %op2) { define void @fsub_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fsub_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fsubr z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -892,8 +892,8 @@ define <4 x float> @fsub_v4f32(<4 x float> %op1, <4 x float> %op2) { define void @fsub_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: fsub_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fsubr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -923,8 +923,8 @@ define <2 x double> @fsub_v2f64(<2 x double> %op1, <2 x double> %op2) { define void @fsub_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: fsub_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fsubr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -981,8 +981,8 @@ define <8 x half> @fabs_v8f16(<8 x half> %op) { define void @fabs_v16f16(ptr %a) { ; CHECK-LABEL: fabs_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fabs z0.h, p0/m, z0.h ; CHECK-NEXT: fabs z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -1020,8 +1020,8 @@ define <4 x float> @fabs_v4f32(<4 x float> %op) { define void @fabs_v8f32(ptr %a) { ; CHECK-LABEL: fabs_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fabs z0.s, p0/m, z0.s ; CHECK-NEXT: fabs z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -1047,8 +1047,8 @@ define <2 x double> @fabs_v2f64(<2 x double> %op) { define void @fabs_v4f64(ptr %a) { ; CHECK-LABEL: fabs_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fabs z0.d, p0/m, z0.d ; CHECK-NEXT: fabs z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-compares.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-compares.ll index aad078f035f7..e92694d1fc80 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-compares.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-compares.ll @@ -57,8 +57,8 @@ define <8 x i16> @fcmp_oeq_v8f16(<8 x half> %op1, <8 x half> %op2) { define void @fcmp_oeq_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_oeq_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmeq p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmeq p0.h, p0/z, z2.h, z3.h @@ -107,8 +107,8 @@ define <4 x i32> @fcmp_oeq_v4f32(<4 x float> %op1, <4 x float> %op2) { define void @fcmp_oeq_v8f32(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_oeq_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmeq p1.s, p0/z, z1.s, z0.s ; CHECK-NEXT: fcmeq p0.s, p0/z, z2.s, z3.s @@ -157,8 +157,8 @@ define <2 x i64> @fcmp_oeq_v2f64(<2 x double> %op1, <2 x double> %op2) { define void @fcmp_oeq_v4f64(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_oeq_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmeq p1.d, p0/z, z1.d, z0.d ; CHECK-NEXT: fcmeq p0.d, p0/z, z2.d, z3.d @@ -181,8 +181,8 @@ define void @fcmp_oeq_v4f64(ptr %a, ptr %b, ptr %c) { define void @fcmp_ueq_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_ueq_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmuo p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmeq p2.h, p0/z, z1.h, z0.h @@ -209,8 +209,8 @@ define void @fcmp_ueq_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_one_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_one_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmgt p1.h, p0/z, z0.h, z1.h ; CHECK-NEXT: fcmgt p2.h, p0/z, z1.h, z0.h @@ -237,8 +237,8 @@ define void @fcmp_one_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_une_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_une_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmne p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmne p0.h, p0/z, z2.h, z3.h @@ -261,8 +261,8 @@ define void @fcmp_une_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_ogt_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_ogt_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmgt p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmgt p0.h, p0/z, z2.h, z3.h @@ -285,8 +285,8 @@ define void @fcmp_ogt_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_ugt_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_ugt_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmge p1.h, p0/z, z0.h, z1.h ; CHECK-NEXT: fcmge p0.h, p0/z, z3.h, z2.h @@ -312,8 +312,8 @@ define void @fcmp_ugt_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_olt_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_olt_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmgt p1.h, p0/z, z0.h, z1.h ; CHECK-NEXT: fcmgt p0.h, p0/z, z3.h, z2.h @@ -336,8 +336,8 @@ define void @fcmp_olt_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_ult_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_ult_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmge p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmge p0.h, p0/z, z2.h, z3.h @@ -363,8 +363,8 @@ define void @fcmp_ult_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_oge_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_oge_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmge p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmge p0.h, p0/z, z2.h, z3.h @@ -387,8 +387,8 @@ define void @fcmp_oge_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_uge_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_uge_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmgt p1.h, p0/z, z0.h, z1.h ; CHECK-NEXT: fcmgt p0.h, p0/z, z3.h, z2.h @@ -414,8 +414,8 @@ define void @fcmp_uge_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_ole_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_ole_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmge p1.h, p0/z, z0.h, z1.h ; CHECK-NEXT: fcmge p0.h, p0/z, z3.h, z2.h @@ -438,8 +438,8 @@ define void @fcmp_ole_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_ule_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_ule_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmgt p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmgt p0.h, p0/z, z2.h, z3.h @@ -465,8 +465,8 @@ define void @fcmp_ule_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_uno_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_uno_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmuo p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmuo p0.h, p0/z, z2.h, z3.h @@ -489,8 +489,8 @@ define void @fcmp_uno_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_ord_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_ord_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmuo p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmuo p0.h, p0/z, z2.h, z3.h @@ -516,8 +516,8 @@ define void @fcmp_ord_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_eq_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_eq_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmeq p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmeq p0.h, p0/z, z2.h, z3.h @@ -540,8 +540,8 @@ define void @fcmp_eq_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_ne_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_ne_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmne p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmne p0.h, p0/z, z2.h, z3.h @@ -564,8 +564,8 @@ define void @fcmp_ne_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_gt_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_gt_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmgt p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmgt p0.h, p0/z, z2.h, z3.h @@ -588,8 +588,8 @@ define void @fcmp_gt_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_lt_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_lt_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmgt p1.h, p0/z, z0.h, z1.h ; CHECK-NEXT: fcmgt p0.h, p0/z, z3.h, z2.h @@ -612,8 +612,8 @@ define void @fcmp_lt_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_ge_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_ge_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmge p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: fcmge p0.h, p0/z, z2.h, z3.h @@ -636,8 +636,8 @@ define void @fcmp_ge_v16f16(ptr %a, ptr %b, ptr %c) { define void @fcmp_le_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fcmp_le_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fcmge p1.h, p0/z, z0.h, z1.h ; CHECK-NEXT: fcmge p0.h, p0/z, z3.h, z2.h diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-convert.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-convert.ll index 18f9a4d371d0..9bdde14e8d83 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-convert.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-convert.ll @@ -8,9 +8,9 @@ target triple = "aarch64-unknown-linux-gnu" define void @fp_convert_combine_crash(ptr %a, ptr %b) { ; CHECK-LABEL: fp_convert_combine_crash: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmov z0.s, #8.00000000 +; CHECK-NEXT: ldp q1, q2, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fmul z1.s, p0/m, z1.s, z0.s ; CHECK-NEXT: fmul z0.s, p0/m, z0.s, z2.s ; CHECK-NEXT: fcvtzs z1.s, p0/m, z1.s diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll index 28e02da53af4..244a40510173 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll @@ -11,8 +11,8 @@ target triple = "aarch64-unknown-linux-gnu" define void @fcvt_v2f16_to_v2f32(<2 x half> %a, ptr %b) { ; CHECK-LABEL: fcvt_v2f16_to_v2f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: fcvt z0.s, p0/m, z0.h ; CHECK-NEXT: str d0, [x0] @@ -25,8 +25,8 @@ define void @fcvt_v2f16_to_v2f32(<2 x half> %a, ptr %b) { define void @fcvt_v4f16_to_v4f32(<4 x half> %a, ptr %b) { ; CHECK-LABEL: fcvt_v4f16_to_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: fcvt z0.s, p0/m, z0.h ; CHECK-NEXT: str q0, [x0] @@ -371,8 +371,8 @@ define void @fcvt_v4f32_v4f16(ptr %a, ptr %b) { define void @fcvt_v8f32_v8f16(ptr %a, ptr %b) { ; CHECK-LABEL: fcvt_v8f32_v8f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: mov x8, #4 // =0x4 ; CHECK-NEXT: fcvt z0.h, p0/m, z0.s ; CHECK-NEXT: fcvt z1.h, p0/m, z1.s @@ -420,8 +420,8 @@ define void @fcvt_v2f64_v2f16(ptr %a, ptr %b) { define void @fcvt_v4f64_v4f16(ptr %a, ptr %b) { ; CHECK-LABEL: fcvt_v4f64_v4f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: mov x8, #2 // =0x2 ; CHECK-NEXT: fcvt z0.h, p0/m, z0.d ; CHECK-NEXT: fcvt z1.h, p0/m, z1.d @@ -467,8 +467,8 @@ define void @fcvt_v2f64_v2f32(<2 x double> %op1, ptr %b) { define void @fcvt_v4f64_v4f32(ptr %a, ptr %b) { ; CHECK-LABEL: fcvt_v4f64_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: mov x8, #2 // =0x2 ; CHECK-NEXT: fcvt z0.s, p0/m, z0.d ; CHECK-NEXT: fcvt z1.s, p0/m, z1.d diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-fma.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-fma.ll index b5df97f767c1..478be9ab76dd 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-fma.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-fma.ll @@ -40,8 +40,8 @@ define <8 x half> @fma_v8f16(<8 x half> %op1, <8 x half> %op2, <8 x half> %op3) define void @fma_v16f16(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fma_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q4, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q5, [x2] ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fmad z0.h, p0/m, z2.h, z1.h @@ -91,8 +91,8 @@ define <4 x float> @fma_v4f32(<4 x float> %op1, <4 x float> %op2, <4 x float> %o define void @fma_v8f32(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fma_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q4, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q5, [x2] ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fmad z0.s, p0/m, z2.s, z1.s @@ -140,8 +140,8 @@ define <2 x double> @fma_v2f64(<2 x double> %op1, <2 x double> %op2, <2 x double define void @fma_v4f64(ptr %a, ptr %b, ptr %c) { ; CHECK-LABEL: fma_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q4, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q5, [x2] ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fmad z0.d, p0/m, z2.d, z1.d diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-minmax.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-minmax.ll index 07a67e265029..4dc034adf459 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-minmax.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-minmax.ll @@ -37,8 +37,8 @@ define <8 x half> @fmaxnm_v8f16(<8 x half> %op1, <8 x half> %op2) { define void @fmaxnm_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fmaxnm_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmaxnm z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -81,8 +81,8 @@ define <4 x float> @fmaxnm_v4f32(<4 x float> %op1, <4 x float> %op2) { define void @fmaxnm_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: fmaxnm_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmaxnm z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -123,8 +123,8 @@ define <2 x double> @fmaxnm_v2f64(<2 x double> %op1, <2 x double> %op2) { define void @fmaxnm_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: fmaxnm_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmaxnm z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -171,8 +171,8 @@ define <8 x half> @fminnm_v8f16(<8 x half> %op1, <8 x half> %op2) { define void @fminnm_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fminnm_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fminnm z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -215,8 +215,8 @@ define <4 x float> @fminnm_v4f32(<4 x float> %op1, <4 x float> %op2) { define void @fminnm_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: fminnm_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fminnm z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -257,8 +257,8 @@ define <2 x double> @fminnm_v2f64(<2 x double> %op1, <2 x double> %op2) { define void @fminnm_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: fminnm_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fminnm z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -305,8 +305,8 @@ define <8 x half> @fmax_v8f16(<8 x half> %op1, <8 x half> %op2) { define void @fmax_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fmax_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -349,8 +349,8 @@ define <4 x float> @fmax_v4f32(<4 x float> %op1, <4 x float> %op2) { define void @fmax_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: fmax_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -391,8 +391,8 @@ define <2 x double> @fmax_v2f64(<2 x double> %op1, <2 x double> %op2) { define void @fmax_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: fmax_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -439,8 +439,8 @@ define <8 x half> @fmin_v8f16(<8 x half> %op1, <8 x half> %op2) { define void @fmin_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fmin_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -483,8 +483,8 @@ define <4 x float> @fmin_v4f32(<4 x float> %op1, <4 x float> %op2) { define void @fmin_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: fmin_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -525,8 +525,8 @@ define <2 x double> @fmin_v2f64(<2 x double> %op1, <2 x double> %op2) { define void @fmin_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: fmin_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fmin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-reduce.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-reduce.ll index d2d771c48c20..bd10a0e091c0 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-reduce.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-reduce.ll @@ -211,8 +211,8 @@ define half @faddv_v8f16(half %start, <8 x half> %a) { define half @faddv_v16f16(half %start, ptr %a) { ; CHECK-LABEL: faddv_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q2, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fadd z1.h, p0/m, z1.h, z2.h ; CHECK-NEXT: faddv h1, p0, z1.h ; CHECK-NEXT: fadd h0, h0, h1 @@ -249,8 +249,8 @@ define float @faddv_v4f32(float %start, <4 x float> %a) { define float @faddv_v8f32(float %start, ptr %a) { ; CHECK-LABEL: faddv_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q2, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fadd z1.s, p0/m, z1.s, z2.s ; CHECK-NEXT: faddv s1, p0, z1.s ; CHECK-NEXT: fadd s0, s0, s1 @@ -285,8 +285,8 @@ define double @faddv_v2f64(double %start, <2 x double> %a) { define double @faddv_v4f64(double %start, ptr %a) { ; CHECK-LABEL: faddv_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q2, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fadd z1.d, p0/m, z1.d, z2.d ; CHECK-NEXT: faddv d1, p0, z1.d ; CHECK-NEXT: fadd d0, d0, d1 @@ -327,8 +327,8 @@ define half @fmaxv_v8f16(<8 x half> %a) { define half @fmaxv_v16f16(ptr %a) { ; CHECK-LABEL: fmaxv_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fmaxnm z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: fmaxnmv h0, p0, z0.h ; CHECK-NEXT: // kill: def $h0 killed $h0 killed $z0 @@ -365,8 +365,8 @@ define float @fmaxv_v4f32(<4 x float> %a) { define float @fmaxv_v8f32(ptr %a) { ; CHECK-LABEL: fmaxv_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fmaxnm z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: fmaxnmv s0, p0, z0.s ; CHECK-NEXT: // kill: def $s0 killed $s0 killed $z0 @@ -401,8 +401,8 @@ define double @fmaxv_v2f64(<2 x double> %a) { define double @fmaxv_v4f64(ptr %a) { ; CHECK-LABEL: fmaxv_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fmaxnm z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: fmaxnmv d0, p0, z0.d ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -443,8 +443,8 @@ define half @fminv_v8f16(<8 x half> %a) { define half @fminv_v16f16(ptr %a) { ; CHECK-LABEL: fminv_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fminnm z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: fminnmv h0, p0, z0.h ; CHECK-NEXT: // kill: def $h0 killed $h0 killed $z0 @@ -481,8 +481,8 @@ define float @fminv_v4f32(<4 x float> %a) { define float @fminv_v8f32(ptr %a) { ; CHECK-LABEL: fminv_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fminnm z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: fminnmv s0, p0, z0.s ; CHECK-NEXT: // kill: def $s0 killed $s0 killed $z0 @@ -517,8 +517,8 @@ define double @fminv_v2f64(<2 x double> %a) { define double @fminv_v4f64(ptr %a) { ; CHECK-LABEL: fminv_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fminnm z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: fminnmv d0, p0, z0.d ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -559,8 +559,8 @@ define half @fmaximumv_v8f16(<8 x half> %a) { define half @fmaximumv_v16f16(ptr %a) { ; CHECK-LABEL: fmaximumv_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fmax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: fmaxv h0, p0, z0.h ; CHECK-NEXT: // kill: def $h0 killed $h0 killed $z0 @@ -597,8 +597,8 @@ define float @fmaximumv_v4f32(<4 x float> %a) { define float @fmaximumv_v8f32(ptr %a) { ; CHECK-LABEL: fmaximumv_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fmax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: fmaxv s0, p0, z0.s ; CHECK-NEXT: // kill: def $s0 killed $s0 killed $z0 @@ -633,8 +633,8 @@ define double @fmaximumv_v2f64(<2 x double> %a) { define double @fmaximumv_v4f64(ptr %a) { ; CHECK-LABEL: fmaximumv_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fmax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: fmaxv d0, p0, z0.d ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -675,8 +675,8 @@ define half @fminimumv_v8f16(<8 x half> %a) { define half @fminimumv_v16f16(ptr %a) { ; CHECK-LABEL: fminimumv_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fmin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: fminv h0, p0, z0.h ; CHECK-NEXT: // kill: def $h0 killed $h0 killed $z0 @@ -713,8 +713,8 @@ define float @fminimumv_v4f32(<4 x float> %a) { define float @fminimumv_v8f32(ptr %a) { ; CHECK-LABEL: fminimumv_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fmin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: fminv s0, p0, z0.s ; CHECK-NEXT: // kill: def $s0 killed $s0 killed $z0 @@ -749,8 +749,8 @@ define double @fminimumv_v2f64(<2 x double> %a) { define double @fminimumv_v4f64(ptr %a) { ; CHECK-LABEL: fminimumv_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fmin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: fminv d0, p0, z0.d ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-rounding.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-rounding.ll index 580b43531070..24832d807c64 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-rounding.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-rounding.ll @@ -47,8 +47,8 @@ define <8 x half> @frintp_v8f16(<8 x half> %op) { define void @frintp_v16f16(ptr %a) { ; CHECK-LABEL: frintp_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: frintp z0.h, p0/m, z0.h ; CHECK-NEXT: frintp z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -86,8 +86,8 @@ define <4 x float> @frintp_v4f32(<4 x float> %op) { define void @frintp_v8f32(ptr %a) { ; CHECK-LABEL: frintp_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: frintp z0.s, p0/m, z0.s ; CHECK-NEXT: frintp z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -123,8 +123,8 @@ define <2 x double> @frintp_v2f64(<2 x double> %op) { define void @frintp_v4f64(ptr %a) { ; CHECK-LABEL: frintp_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: frintp z0.d, p0/m, z0.d ; CHECK-NEXT: frintp z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -178,8 +178,8 @@ define <8 x half> @frintm_v8f16(<8 x half> %op) { define void @frintm_v16f16(ptr %a) { ; CHECK-LABEL: frintm_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: frintm z0.h, p0/m, z0.h ; CHECK-NEXT: frintm z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -217,8 +217,8 @@ define <4 x float> @frintm_v4f32(<4 x float> %op) { define void @frintm_v8f32(ptr %a) { ; CHECK-LABEL: frintm_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: frintm z0.s, p0/m, z0.s ; CHECK-NEXT: frintm z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -254,8 +254,8 @@ define <2 x double> @frintm_v2f64(<2 x double> %op) { define void @frintm_v4f64(ptr %a) { ; CHECK-LABEL: frintm_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: frintm z0.d, p0/m, z0.d ; CHECK-NEXT: frintm z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -309,8 +309,8 @@ define <8 x half> @frinti_v8f16(<8 x half> %op) { define void @frinti_v16f16(ptr %a) { ; CHECK-LABEL: frinti_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: frinti z0.h, p0/m, z0.h ; CHECK-NEXT: frinti z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -348,8 +348,8 @@ define <4 x float> @frinti_v4f32(<4 x float> %op) { define void @frinti_v8f32(ptr %a) { ; CHECK-LABEL: frinti_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: frinti z0.s, p0/m, z0.s ; CHECK-NEXT: frinti z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -385,8 +385,8 @@ define <2 x double> @frinti_v2f64(<2 x double> %op) { define void @frinti_v4f64(ptr %a) { ; CHECK-LABEL: frinti_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: frinti z0.d, p0/m, z0.d ; CHECK-NEXT: frinti z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -440,8 +440,8 @@ define <8 x half> @frintx_v8f16(<8 x half> %op) { define void @frintx_v16f16(ptr %a) { ; CHECK-LABEL: frintx_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: frintx z0.h, p0/m, z0.h ; CHECK-NEXT: frintx z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -479,8 +479,8 @@ define <4 x float> @frintx_v4f32(<4 x float> %op) { define void @frintx_v8f32(ptr %a) { ; CHECK-LABEL: frintx_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: frintx z0.s, p0/m, z0.s ; CHECK-NEXT: frintx z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -516,8 +516,8 @@ define <2 x double> @frintx_v2f64(<2 x double> %op) { define void @frintx_v4f64(ptr %a) { ; CHECK-LABEL: frintx_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: frintx z0.d, p0/m, z0.d ; CHECK-NEXT: frintx z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -571,8 +571,8 @@ define <8 x half> @frinta_v8f16(<8 x half> %op) { define void @frinta_v16f16(ptr %a) { ; CHECK-LABEL: frinta_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: frinta z0.h, p0/m, z0.h ; CHECK-NEXT: frinta z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -610,8 +610,8 @@ define <4 x float> @frinta_v4f32(<4 x float> %op) { define void @frinta_v8f32(ptr %a) { ; CHECK-LABEL: frinta_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: frinta z0.s, p0/m, z0.s ; CHECK-NEXT: frinta z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -647,8 +647,8 @@ define <2 x double> @frinta_v2f64(<2 x double> %op) { define void @frinta_v4f64(ptr %a) { ; CHECK-LABEL: frinta_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: frinta z0.d, p0/m, z0.d ; CHECK-NEXT: frinta z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -702,8 +702,8 @@ define <8 x half> @frintn_v8f16(<8 x half> %op) { define void @frintn_v16f16(ptr %a) { ; CHECK-LABEL: frintn_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: frintn z0.h, p0/m, z0.h ; CHECK-NEXT: frintn z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -741,8 +741,8 @@ define <4 x float> @frintn_v4f32(<4 x float> %op) { define void @frintn_v8f32(ptr %a) { ; CHECK-LABEL: frintn_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: frintn z0.s, p0/m, z0.s ; CHECK-NEXT: frintn z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -778,8 +778,8 @@ define <2 x double> @frintn_v2f64(<2 x double> %op) { define void @frintn_v4f64(ptr %a) { ; CHECK-LABEL: frintn_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: frintn z0.d, p0/m, z0.d ; CHECK-NEXT: frintn z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -833,8 +833,8 @@ define <8 x half> @frintz_v8f16(<8 x half> %op) { define void @frintz_v16f16(ptr %a) { ; CHECK-LABEL: frintz_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: frintz z0.h, p0/m, z0.h ; CHECK-NEXT: frintz z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -872,8 +872,8 @@ define <4 x float> @frintz_v4f32(<4 x float> %op) { define void @frintz_v8f32(ptr %a) { ; CHECK-LABEL: frintz_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: frintz z0.s, p0/m, z0.s ; CHECK-NEXT: frintz z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -909,8 +909,8 @@ define <2 x double> @frintz_v2f64(<2 x double> %op) { define void @frintz_v4f64(ptr %a) { ; CHECK-LABEL: frintz_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: frintz z0.d, p0/m, z0.d ; CHECK-NEXT: frintz z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-select.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-select.ll index 73fd7e146534..132225546fc4 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-select.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-select.ll @@ -7,8 +7,8 @@ target triple = "aarch64-unknown-linux-gnu" define <2 x half> @select_v2f16(<2 x half> %op1, <2 x half> %op2, i1 %mask) { ; CHECK-LABEL: select_v2f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: and z2.h, z2.h, #0x1 @@ -23,8 +23,8 @@ define <2 x half> @select_v2f16(<2 x half> %op1, <2 x half> %op2, i1 %mask) { define <4 x half> @select_v4f16(<4 x half> %op1, <4 x half> %op2, i1 %mask) { ; CHECK-LABEL: select_v4f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: and z2.h, z2.h, #0x1 @@ -39,8 +39,8 @@ define <4 x half> @select_v4f16(<4 x half> %op1, <4 x half> %op2, i1 %mask) { define <8 x half> @select_v8f16(<8 x half> %op1, <8 x half> %op2, i1 %mask) { ; CHECK-LABEL: select_v8f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 ; CHECK-NEXT: and z2.h, z2.h, #0x1 @@ -55,8 +55,8 @@ define <8 x half> @select_v8f16(<8 x half> %op1, <8 x half> %op2, i1 %mask) { define void @select_v16f16(ptr %a, ptr %b, i1 %mask) { ; CHECK-LABEL: select_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z0.h, w2 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and z0.h, z0.h, #0x1 ; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 ; CHECK-NEXT: ldr q0, [x0] @@ -77,8 +77,8 @@ define void @select_v16f16(ptr %a, ptr %b, i1 %mask) { define <2 x float> @select_v2f32(<2 x float> %op1, <2 x float> %op2, i1 %mask) { ; CHECK-LABEL: select_v2f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w0, #0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: mov z2.s, w8 @@ -93,8 +93,8 @@ define <2 x float> @select_v2f32(<2 x float> %op1, <2 x float> %op2, i1 %mask) { define <4 x float> @select_v4f32(<4 x float> %op1, <4 x float> %op2, i1 %mask) { ; CHECK-LABEL: select_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w0, #0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 ; CHECK-NEXT: mov z2.s, w8 @@ -109,8 +109,8 @@ define <4 x float> @select_v4f32(<4 x float> %op1, <4 x float> %op2, i1 %mask) { define void @select_v8f32(ptr %a, ptr %b, i1 %mask) { ; CHECK-LABEL: select_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w2, #0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z0.s, w8 ; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 ; CHECK-NEXT: ldr q0, [x0] @@ -150,9 +150,9 @@ define <1 x double> @select_v1f64(<1 x double> %op1, <1 x double> %op2, i1 %mask define <2 x double> @select_v2f64(<2 x double> %op1, <2 x double> %op2, i1 %mask) { ; CHECK-LABEL: select_v2f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: and x8, x0, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 ; CHECK-NEXT: mov z2.d, x8 @@ -167,9 +167,9 @@ define <2 x double> @select_v2f64(<2 x double> %op1, <2 x double> %op2, i1 %mask define void @select_v4f64(ptr %a, ptr %b, i1 %mask) { ; CHECK-LABEL: select_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2 ; CHECK-NEXT: and x8, x2, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, x8 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: ldr q0, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll index d6adf9cf0ad6..58eae212d799 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll @@ -36,8 +36,8 @@ define void @fcvtzu_v8f16_v8i16(ptr %a, ptr %b) { define void @fcvtzu_v16f16_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzu_v16f16_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fcvtzu z0.h, p0/m, z0.h ; CHECK-NEXT: fcvtzu z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x1] @@ -55,8 +55,8 @@ define void @fcvtzu_v16f16_v16i16(ptr %a, ptr %b) { define <2 x i32> @fcvtzu_v2f16_v2i32(<2 x half> %op1) { ; CHECK-LABEL: fcvtzu_v2f16_v2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: fcvtzu z0.s, p0/m, z0.h ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -68,8 +68,8 @@ define <2 x i32> @fcvtzu_v2f16_v2i32(<2 x half> %op1) { define <4 x i32> @fcvtzu_v4f16_v4i32(<4 x half> %op1) { ; CHECK-LABEL: fcvtzu_v4f16_v4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: fcvtzu z0.s, p0/m, z0.h ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 @@ -302,8 +302,8 @@ define <4 x i16> @fcvtzu_v4f32_v4i16(<4 x float> %op1) { define <8 x i16> @fcvtzu_v8f32_v8i16(ptr %a) { ; CHECK-LABEL: fcvtzu_v8f32_v8i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fcvtzu z1.s, p0/m, z1.s ; CHECK-NEXT: fcvtzu z0.s, p0/m, z0.s ; CHECK-NEXT: ptrue p0.h, vl4 @@ -320,8 +320,8 @@ define <8 x i16> @fcvtzu_v8f32_v8i16(ptr %a) { define void @fcvtzu_v16f32_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzu_v16f32_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fcvtzu z1.s, p0/m, z1.s ; CHECK-NEXT: fcvtzu z0.s, p0/m, z0.s @@ -373,8 +373,8 @@ define <4 x i32> @fcvtzu_v4f32_v4i32(<4 x float> %op1) { define void @fcvtzu_v8f32_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzu_v8f32_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fcvtzu z0.s, p0/m, z0.s ; CHECK-NEXT: fcvtzu z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x1] @@ -392,8 +392,8 @@ define void @fcvtzu_v8f32_v8i32(ptr %a, ptr %b) { define <1 x i64> @fcvtzu_v1f32_v1i64(<1 x float> %op1) { ; CHECK-LABEL: fcvtzu_v1f32_v1i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: fcvtzu z0.d, p0/m, z0.s ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -405,8 +405,8 @@ define <1 x i64> @fcvtzu_v1f32_v1i64(<1 x float> %op1) { define <2 x i64> @fcvtzu_v2f32_v2i64(<2 x float> %op1) { ; CHECK-LABEL: fcvtzu_v2f32_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: fcvtzu z0.d, p0/m, z0.s ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 @@ -491,8 +491,8 @@ define <4 x i16> @fcvtzu_v4f64_v4i16(ptr %a) { ; CHECK: // %bb.0: ; CHECK-NEXT: sub sp, sp, #16 ; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d ; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s @@ -520,8 +520,8 @@ define <8 x i16> @fcvtzu_v8f64_v8i16(ptr %a) { ; CHECK: // %bb.0: ; CHECK-NEXT: sub sp, sp, #16 ; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0, #32] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q3, q2, [x0] ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d ; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d @@ -563,24 +563,22 @@ define void @fcvtzu_v16f64_v16i16(ptr %a, ptr %b) { ; CHECK: // %bb.0: ; CHECK-NEXT: sub sp, sp, #32 ; CHECK-NEXT: .cfi_def_cfa_offset 32 -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q3, q2, [x0] -; CHECK-NEXT: ldr q6, [x0, #112] -; CHECK-NEXT: ldp q4, q5, [x0, #80] -; CHECK-NEXT: ldr q7, [x0, #64] +; CHECK-NEXT: ldp q4, q5, [x0, #96] ; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d -; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.d ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d +; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.d +; CHECK-NEXT: ldp q6, q7, [x0, #64] ; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.d -; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.d ; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.d ; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.d ; CHECK-NEXT: uzp1 z1.s, z1.s, z1.s -; CHECK-NEXT: uzp1 z2.s, z2.s, z2.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.d +; CHECK-NEXT: uzp1 z2.s, z2.s, z2.s ; CHECK-NEXT: uzp1 z3.s, z3.s, z3.s -; CHECK-NEXT: uzp1 z6.s, z6.s, z6.s ; CHECK-NEXT: uzp1 z5.s, z5.s, z5.s ; CHECK-NEXT: fmov w8, s1 ; CHECK-NEXT: mov z16.s, z1.s[1] @@ -606,25 +604,26 @@ define void @fcvtzu_v16f64_v16i16(ptr %a, ptr %b) { ; CHECK-NEXT: mov z3.s, z5.s[1] ; CHECK-NEXT: strh w8, [sp, #6] ; CHECK-NEXT: fmov w8, s2 -; CHECK-NEXT: mov z2.s, z6.s[1] +; CHECK-NEXT: uzp1 z2.s, z6.s, z6.s ; CHECK-NEXT: strh w8, [sp, #2] -; CHECK-NEXT: fmov w8, s6 -; CHECK-NEXT: strh w8, [sp, #28] ; CHECK-NEXT: fmov w8, s5 -; CHECK-NEXT: strh w8, [sp, #24] +; CHECK-NEXT: strh w8, [sp, #28] ; CHECK-NEXT: fmov w8, s1 ; CHECK-NEXT: mov z1.s, z1.s[1] -; CHECK-NEXT: strh w8, [sp, #20] +; CHECK-NEXT: strh w8, [sp, #24] ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: mov z0.s, z0.s[1] -; CHECK-NEXT: strh w8, [sp, #16] +; CHECK-NEXT: strh w8, [sp, #20] ; CHECK-NEXT: fmov w8, s2 -; CHECK-NEXT: strh w8, [sp, #30] +; CHECK-NEXT: mov z2.s, z2.s[1] +; CHECK-NEXT: strh w8, [sp, #16] ; CHECK-NEXT: fmov w8, s3 -; CHECK-NEXT: strh w8, [sp, #26] +; CHECK-NEXT: strh w8, [sp, #30] ; CHECK-NEXT: fmov w8, s1 -; CHECK-NEXT: strh w8, [sp, #22] +; CHECK-NEXT: strh w8, [sp, #26] ; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: strh w8, [sp, #22] +; CHECK-NEXT: fmov w8, s2 ; CHECK-NEXT: strh w8, [sp, #18] ; CHECK-NEXT: ldp q1, q0, [sp] ; CHECK-NEXT: stp q1, q0, [x1] @@ -669,8 +668,8 @@ define <2 x i32> @fcvtzu_v2f64_v2i32(<2 x double> %op1) { define <4 x i32> @fcvtzu_v4f64_v4i32(ptr %a) { ; CHECK-LABEL: fcvtzu_v4f64_v4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fcvtzu z1.d, p0/m, z1.d ; CHECK-NEXT: fcvtzu z0.d, p0/m, z0.d ; CHECK-NEXT: ptrue p0.s, vl2 @@ -687,8 +686,8 @@ define <4 x i32> @fcvtzu_v4f64_v4i32(ptr %a) { define void @fcvtzu_v8f64_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzu_v8f64_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fcvtzu z1.d, p0/m, z1.d ; CHECK-NEXT: fcvtzu z0.d, p0/m, z0.d @@ -740,8 +739,8 @@ define <2 x i64> @fcvtzu_v2f64_v2i64(<2 x double> %op1) { define void @fcvtzu_v4f64_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzu_v4f64_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fcvtzu z0.d, p0/m, z0.d ; CHECK-NEXT: fcvtzu z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x1] @@ -785,8 +784,8 @@ define void @fcvtzs_v8f16_v8i16(ptr %a, ptr %b) { define void @fcvtzs_v16f16_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzs_v16f16_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: fcvtzs z0.h, p0/m, z0.h ; CHECK-NEXT: fcvtzs z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x1] @@ -804,8 +803,8 @@ define void @fcvtzs_v16f16_v16i16(ptr %a, ptr %b) { define <2 x i32> @fcvtzs_v2f16_v2i32(<2 x half> %op1) { ; CHECK-LABEL: fcvtzs_v2f16_v2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.h ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -817,8 +816,8 @@ define <2 x i32> @fcvtzs_v2f16_v2i32(<2 x half> %op1) { define <4 x i32> @fcvtzs_v4f16_v4i32(<4 x half> %op1) { ; CHECK-LABEL: fcvtzs_v4f16_v4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.h ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 @@ -1052,8 +1051,8 @@ define <4 x i16> @fcvtzs_v4f32_v4i16(<4 x float> %op1) { define <8 x i16> @fcvtzs_v8f32_v8i16(ptr %a) { ; CHECK-LABEL: fcvtzs_v8f32_v8i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fcvtzs z1.s, p0/m, z1.s ; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.s ; CHECK-NEXT: ptrue p0.h, vl4 @@ -1070,8 +1069,8 @@ define <8 x i16> @fcvtzs_v8f32_v8i16(ptr %a) { define void @fcvtzs_v16f32_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzs_v16f32_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fcvtzs z1.s, p0/m, z1.s ; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.s @@ -1123,8 +1122,8 @@ define <4 x i32> @fcvtzs_v4f32_v4i32(<4 x float> %op1) { define void @fcvtzs_v8f32_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzs_v8f32_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.s ; CHECK-NEXT: fcvtzs z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x1] @@ -1142,8 +1141,8 @@ define void @fcvtzs_v8f32_v8i32(ptr %a, ptr %b) { define <1 x i64> @fcvtzs_v1f32_v1i64(<1 x float> %op1) { ; CHECK-LABEL: fcvtzs_v1f32_v1i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.s ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -1155,8 +1154,8 @@ define <1 x i64> @fcvtzs_v1f32_v1i64(<1 x float> %op1) { define <2 x i64> @fcvtzs_v2f32_v2i64(<2 x float> %op1) { ; CHECK-LABEL: fcvtzs_v2f32_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.s ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 @@ -1243,8 +1242,8 @@ define <4 x i16> @fcvtzs_v4f64_v4i16(ptr %a) { ; CHECK: // %bb.0: ; CHECK-NEXT: sub sp, sp, #16 ; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d ; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s @@ -1272,8 +1271,8 @@ define <8 x i16> @fcvtzs_v8f64_v8i16(ptr %a) { ; CHECK: // %bb.0: ; CHECK-NEXT: sub sp, sp, #16 ; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0, #32] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q3, q2, [x0] ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d ; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d @@ -1315,24 +1314,22 @@ define void @fcvtzs_v16f64_v16i16(ptr %a, ptr %b) { ; CHECK: // %bb.0: ; CHECK-NEXT: sub sp, sp, #32 ; CHECK-NEXT: .cfi_def_cfa_offset 32 -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q3, q2, [x0] -; CHECK-NEXT: ldr q6, [x0, #112] -; CHECK-NEXT: ldp q4, q5, [x0, #80] -; CHECK-NEXT: ldr q7, [x0, #64] +; CHECK-NEXT: ldp q4, q5, [x0, #96] ; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d -; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.d ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d +; CHECK-NEXT: fcvtzs z2.d, p0/m, z2.d +; CHECK-NEXT: ldp q6, q7, [x0, #64] ; CHECK-NEXT: fcvtzs z3.d, p0/m, z3.d -; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.d ; CHECK-NEXT: fcvtzs z5.d, p0/m, z5.d ; CHECK-NEXT: fcvtzs z4.d, p0/m, z4.d ; CHECK-NEXT: uzp1 z1.s, z1.s, z1.s -; CHECK-NEXT: uzp1 z2.s, z2.s, z2.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-NEXT: fcvtzs z6.d, p0/m, z6.d +; CHECK-NEXT: uzp1 z2.s, z2.s, z2.s ; CHECK-NEXT: uzp1 z3.s, z3.s, z3.s -; CHECK-NEXT: uzp1 z6.s, z6.s, z6.s ; CHECK-NEXT: uzp1 z5.s, z5.s, z5.s ; CHECK-NEXT: fmov w8, s1 ; CHECK-NEXT: mov z16.s, z1.s[1] @@ -1358,25 +1355,26 @@ define void @fcvtzs_v16f64_v16i16(ptr %a, ptr %b) { ; CHECK-NEXT: mov z3.s, z5.s[1] ; CHECK-NEXT: strh w8, [sp, #6] ; CHECK-NEXT: fmov w8, s2 -; CHECK-NEXT: mov z2.s, z6.s[1] +; CHECK-NEXT: uzp1 z2.s, z6.s, z6.s ; CHECK-NEXT: strh w8, [sp, #2] -; CHECK-NEXT: fmov w8, s6 -; CHECK-NEXT: strh w8, [sp, #28] ; CHECK-NEXT: fmov w8, s5 -; CHECK-NEXT: strh w8, [sp, #24] +; CHECK-NEXT: strh w8, [sp, #28] ; CHECK-NEXT: fmov w8, s1 ; CHECK-NEXT: mov z1.s, z1.s[1] -; CHECK-NEXT: strh w8, [sp, #20] +; CHECK-NEXT: strh w8, [sp, #24] ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: mov z0.s, z0.s[1] -; CHECK-NEXT: strh w8, [sp, #16] +; CHECK-NEXT: strh w8, [sp, #20] ; CHECK-NEXT: fmov w8, s2 -; CHECK-NEXT: strh w8, [sp, #30] +; CHECK-NEXT: mov z2.s, z2.s[1] +; CHECK-NEXT: strh w8, [sp, #16] ; CHECK-NEXT: fmov w8, s3 -; CHECK-NEXT: strh w8, [sp, #26] +; CHECK-NEXT: strh w8, [sp, #30] ; CHECK-NEXT: fmov w8, s1 -; CHECK-NEXT: strh w8, [sp, #22] +; CHECK-NEXT: strh w8, [sp, #26] ; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: strh w8, [sp, #22] +; CHECK-NEXT: fmov w8, s2 ; CHECK-NEXT: strh w8, [sp, #18] ; CHECK-NEXT: ldp q1, q0, [sp] ; CHECK-NEXT: stp q1, q0, [x1] @@ -1421,8 +1419,8 @@ define <2 x i32> @fcvtzs_v2f64_v2i32(<2 x double> %op1) { define <4 x i32> @fcvtzs_v4f64_v4i32(ptr %a) { ; CHECK-LABEL: fcvtzs_v4f64_v4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d ; CHECK-NEXT: ptrue p0.s, vl2 @@ -1439,8 +1437,8 @@ define <4 x i32> @fcvtzs_v4f64_v4i32(ptr %a) { define void @fcvtzs_v8f64_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzs_v8f64_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d @@ -1492,8 +1490,8 @@ define <2 x i64> @fcvtzs_v2f64_v2i64(<2 x double> %op1) { define void @fcvtzs_v4f64_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: fcvtzs_v4f64_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: fcvtzs z0.d, p0/m, z0.d ; CHECK-NEXT: fcvtzs z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x1] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-vselect.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-vselect.ll index ee8704284def..4c5a6fe2fd23 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-vselect.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-vselect.ll @@ -71,8 +71,8 @@ define <8 x half> @select_v8f16(<8 x half> %op1, <8 x half> %op2, <8 x i1> %mask define void @select_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: select_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q2, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q3, [x1] ; CHECK-NEXT: fcmeq p1.h, p0/z, z0.h, z1.h ; CHECK-NEXT: fcmeq p0.h, p0/z, z2.h, z3.h @@ -128,8 +128,8 @@ define <4 x float> @select_v4f32(<4 x float> %op1, <4 x float> %op2, <4 x i1> %m define void @select_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: select_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q2, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q3, [x1] ; CHECK-NEXT: fcmeq p1.s, p0/z, z0.s, z1.s ; CHECK-NEXT: fcmeq p0.s, p0/z, z2.s, z3.s @@ -186,8 +186,8 @@ define <2 x double> @select_v2f64(<2 x double> %op1, <2 x double> %op2, <2 x i1> define void @select_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: select_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q2, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q3, [x1] ; CHECK-NEXT: fcmeq p1.d, p0/z, z0.d, z1.d ; CHECK-NEXT: fcmeq p0.d, p0/z, z2.d, z3.d diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-insert-vector-elt.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-insert-vector-elt.ll index 0b3e7695e6a0..4aa965777c74 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-insert-vector-elt.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-insert-vector-elt.ll @@ -11,9 +11,9 @@ target triple = "aarch64-unknown-linux-gnu" define <4 x i8> @insertelement_v4i8(<4 x i8> %op1) { ; CHECK-LABEL: insertelement_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z1.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w8 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -28,9 +28,9 @@ define <4 x i8> @insertelement_v4i8(<4 x i8> %op1) { define <8 x i8> @insertelement_v8i8(<8 x i8> %op1) { ; CHECK-LABEL: insertelement_v8i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov w8, #7 // =0x7 ; CHECK-NEXT: index z1.b, #0, #1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w8 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -45,9 +45,9 @@ define <8 x i8> @insertelement_v8i8(<8 x i8> %op1) { define <16 x i8> @insertelement_v16i8(<16 x i8> %op1) { ; CHECK-LABEL: insertelement_v16i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov w8, #15 // =0xf ; CHECK-NEXT: index z1.b, #0, #1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w8 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -62,9 +62,9 @@ define <16 x i8> @insertelement_v16i8(<16 x i8> %op1) { define <32 x i8> @insertelement_v32i8(<32 x i8> %op1) { ; CHECK-LABEL: insertelement_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov w8, #15 // =0xf ; CHECK-NEXT: index z2.b, #0, #1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z3.b, w8 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -80,9 +80,9 @@ define <32 x i8> @insertelement_v32i8(<32 x i8> %op1) { define <2 x i16> @insertelement_v2i16(<2 x i16> %op1) { ; CHECK-LABEL: insertelement_v2i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z1.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -97,9 +97,9 @@ define <2 x i16> @insertelement_v2i16(<2 x i16> %op1) { define <4 x i16> @insertelement_v4i16(<4 x i16> %op1) { ; CHECK-LABEL: insertelement_v4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z1.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w8 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -114,9 +114,9 @@ define <4 x i16> @insertelement_v4i16(<4 x i16> %op1) { define <8 x i16> @insertelement_v8i16(<8 x i16> %op1) { ; CHECK-LABEL: insertelement_v8i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #7 // =0x7 ; CHECK-NEXT: index z1.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w8 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -131,9 +131,9 @@ define <8 x i16> @insertelement_v8i16(<8 x i16> %op1) { define <16 x i16> @insertelement_v16i16(<16 x i16> %op1) { ; CHECK-LABEL: insertelement_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #7 // =0x7 ; CHECK-NEXT: index z2.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z3.h, w8 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -149,9 +149,9 @@ define <16 x i16> @insertelement_v16i16(<16 x i16> %op1) { define <2 x i32> @insertelement_v2i32(<2 x i32> %op1) { ; CHECK-LABEL: insertelement_v2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z1.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -166,9 +166,9 @@ define <2 x i32> @insertelement_v2i32(<2 x i32> %op1) { define <4 x i32> @insertelement_v4i32(<4 x i32> %op1) { ; CHECK-LABEL: insertelement_v4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z1.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -183,9 +183,9 @@ define <4 x i32> @insertelement_v4i32(<4 x i32> %op1) { define <8 x i32> @insertelement_v8i32(ptr %a) { ; CHECK-LABEL: insertelement_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z0.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: mov w8, #5 // =0x5 ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s @@ -212,9 +212,9 @@ define <1 x i64> @insertelement_v1i64(<1 x i64> %op1) { define <2 x i64> @insertelement_v2i64(<2 x i64> %op1) { ; CHECK-LABEL: insertelement_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z1.d, #0, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: mov w8, #5 // =0x5 @@ -229,9 +229,9 @@ define <2 x i64> @insertelement_v2i64(<2 x i64> %op1) { define <4 x i64> @insertelement_v4i64(ptr %a) { ; CHECK-LABEL: insertelement_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z0.d, #0, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: mov w8, #5 // =0x5 ; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d @@ -264,9 +264,9 @@ define <2 x half> @insertelement_v2f16(<2 x half> %op1) { define <4 x half> @insertelement_v4f16(<4 x half> %op1) { ; CHECK-LABEL: insertelement_v4f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z1.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w8 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: cmpeq p0.h, p0/z, z1.h, z2.h @@ -281,9 +281,9 @@ define <4 x half> @insertelement_v4f16(<4 x half> %op1) { define <8 x half> @insertelement_v8f16(<8 x half> %op1) { ; CHECK-LABEL: insertelement_v8f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #7 // =0x7 ; CHECK-NEXT: index z1.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w8 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: cmpeq p0.h, p0/z, z1.h, z2.h @@ -298,9 +298,9 @@ define <8 x half> @insertelement_v8f16(<8 x half> %op1) { define <16 x half> @insertelement_v16f16(ptr %a) { ; CHECK-LABEL: insertelement_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov w8, #7 // =0x7 ; CHECK-NEXT: index z0.h, #0, #1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z1.h, w8 ; CHECK-NEXT: fmov h2, #5.00000000 ; CHECK-NEXT: cmpeq p0.h, p0/z, z0.h, z1.h @@ -317,9 +317,9 @@ define <16 x half> @insertelement_v16f16(ptr %a) { define <2 x float> @insertelement_v2f32(<2 x float> %op1) { ; CHECK-LABEL: insertelement_v2f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z1.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: cmpeq p0.s, p0/z, z1.s, z2.s @@ -334,9 +334,9 @@ define <2 x float> @insertelement_v2f32(<2 x float> %op1) { define <4 x float> @insertelement_v4f32(<4 x float> %op1) { ; CHECK-LABEL: insertelement_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z1.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z2.s, w8 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: cmpeq p0.s, p0/z, z1.s, z2.s @@ -351,9 +351,9 @@ define <4 x float> @insertelement_v4f32(<4 x float> %op1) { define <8 x float> @insertelement_v8f32(ptr %a) { ; CHECK-LABEL: insertelement_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov w8, #3 // =0x3 ; CHECK-NEXT: index z0.s, #0, #1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: fmov s2, #5.00000000 ; CHECK-NEXT: cmpeq p0.s, p0/z, z0.s, z1.s @@ -379,9 +379,9 @@ define <1 x double> @insertelement_v1f64(<1 x double> %op1) { define <2 x double> @insertelement_v2f64(<2 x double> %op1) { ; CHECK-LABEL: insertelement_v2f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z1.d, #0, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z2.d, x8 ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: cmpeq p0.d, p0/z, z1.d, z2.d @@ -396,9 +396,9 @@ define <2 x double> @insertelement_v2f64(<2 x double> %op1) { define <4 x double> @insertelement_v4f64(ptr %a) { ; CHECK-LABEL: insertelement_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov w8, #1 // =0x1 ; CHECK-NEXT: index z0.d, #0, #1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.d, x8 ; CHECK-NEXT: fmov d2, #5.00000000 ; CHECK-NEXT: cmpeq p0.d, p0/z, z0.d, z1.d diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll index e3c4b6f1cb53..8baa87c6d686 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll @@ -262,8 +262,8 @@ define <16 x i8> @mul_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @mul_v32i8(ptr %a, ptr %b) { ; SVE-LABEL: mul_v32i8: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.b, vl16 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.b, vl16 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: mul z0.b, p0/m, z0.b, z1.b ; SVE-NEXT: movprfx z1, z2 @@ -352,8 +352,8 @@ define <8 x i16> @mul_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @mul_v16i16(ptr %a, ptr %b) { ; SVE-LABEL: mul_v16i16: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.h, vl8 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.h, vl8 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: mul z0.h, p0/m, z0.h, z1.h ; SVE-NEXT: movprfx z1, z2 @@ -421,8 +421,8 @@ define <4 x i32> @mul_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @mul_v8i32(ptr %a, ptr %b) { ; SVE-LABEL: mul_v8i32: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s, vl4 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.s, vl4 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: mul z0.s, p0/m, z0.s, z1.s ; SVE-NEXT: movprfx z1, z2 @@ -490,8 +490,8 @@ define <2 x i64> @mul_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @mul_v4i64(ptr %a, ptr %b) { ; SVE-LABEL: mul_v4i64: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.d, vl2 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.d, vl2 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: mul z0.d, p0/m, z0.d, z1.d ; SVE-NEXT: movprfx z1, z2 @@ -746,8 +746,8 @@ define <16 x i8> @abs_v16i8(<16 x i8> %op1) { define void @abs_v32i8(ptr %a) { ; CHECK-LABEL: abs_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: abs z0.b, p0/m, z0.b ; CHECK-NEXT: abs z1.b, p0/m, z1.b ; CHECK-NEXT: stp q0, q1, [x0] @@ -798,8 +798,8 @@ define <8 x i16> @abs_v8i16(<8 x i16> %op1) { define void @abs_v16i16(ptr %a) { ; CHECK-LABEL: abs_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: abs z0.h, p0/m, z0.h ; CHECK-NEXT: abs z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -837,8 +837,8 @@ define <4 x i32> @abs_v4i32(<4 x i32> %op1) { define void @abs_v8i32(ptr %a) { ; CHECK-LABEL: abs_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: abs z0.s, p0/m, z0.s ; CHECK-NEXT: abs z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -876,8 +876,8 @@ define <2 x i64> @abs_v2i64(<2 x i64> %op1) { define void @abs_v4i64(ptr %a) { ; CHECK-LABEL: abs_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: abs z0.d, p0/m, z0.d ; CHECK-NEXT: abs z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-compares.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-compares.ll index 6200e44218a9..73c1eac99dd3 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-compares.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-compares.ll @@ -41,8 +41,8 @@ define <16 x i8> @icmp_eq_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @icmp_eq_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: icmp_eq_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: cmpeq p1.b, p0/z, z1.b, z0.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z2.b, z3.b @@ -91,8 +91,8 @@ define <8 x i16> @icmp_eq_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @icmp_eq_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: icmp_eq_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: cmpeq p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z2.h, z3.h @@ -141,8 +141,8 @@ define <4 x i32> @icmp_eq_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @icmp_eq_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: icmp_eq_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: cmpeq p1.s, p0/z, z1.s, z0.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z2.s, z3.s @@ -191,8 +191,8 @@ define <2 x i64> @icmp_eq_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @icmp_eq_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: icmp_eq_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: cmpeq p1.d, p0/z, z1.d, z0.d ; CHECK-NEXT: cmpeq p0.d, p0/z, z2.d, z3.d @@ -215,8 +215,8 @@ define void @icmp_eq_v4i64(ptr %a, ptr %b) { define void @icmp_ne_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: icmp_ne_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: cmpne p1.b, p0/z, z1.b, z0.b ; CHECK-NEXT: cmpne p0.b, p0/z, z2.b, z3.b @@ -261,8 +261,8 @@ define void @icmp_sge_v8i16(ptr %a, ptr %b) { define void @icmp_sgt_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: icmp_sgt_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: cmpgt p1.h, p0/z, z1.h, z0.h ; CHECK-NEXT: cmpgt p0.h, p0/z, z2.h, z3.h @@ -307,8 +307,8 @@ define void @icmp_sle_v4i32(ptr %a, ptr %b) { define void @icmp_slt_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: icmp_slt_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: cmpgt p1.s, p0/z, z0.s, z1.s ; CHECK-NEXT: cmpgt p0.s, p0/z, z3.s, z2.s diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll index fcf4f21c6ea8..5158dda37a8b 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll @@ -86,8 +86,8 @@ define <16 x i8> @sdiv_v16i8(<16 x i8> %op1, <16 x i8> %op2) { ; CHECK-NEXT: sdivr z3.s, p0/m, z3.s, z5.s ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s -; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z1.h, z4.h, z4.h +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z1.h, p0, z1.h, z2.h ; CHECK-NEXT: uzp1 z1.b, z1.b, z1.b @@ -163,18 +163,18 @@ define void @sdiv_v32i8(ptr %a, ptr %b) { ; CHECK-NEXT: uzp1 z5.h, z5.h, z5.h ; CHECK-NEXT: sdiv z6.s, p0/m, z6.s, z7.s ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: uzp1 z7.h, z16.h, z16.h ; CHECK-NEXT: splice z4.h, p0, z4.h, z5.h ; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h ; CHECK-NEXT: splice z2.h, p0, z2.h, z3.h +; CHECK-NEXT: uzp1 z7.h, z16.h, z16.h ; CHECK-NEXT: uzp1 z1.b, z4.b, z4.b ; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b ; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b ; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h ; CHECK-NEXT: splice z7.h, p0, z7.h, z6.h ; CHECK-NEXT: ptrue p0.b, vl8 -; CHECK-NEXT: uzp1 z3.b, z7.b, z7.b ; CHECK-NEXT: splice z2.b, p0, z2.b, z0.b +; CHECK-NEXT: uzp1 z3.b, z7.b, z7.b ; CHECK-NEXT: splice z3.b, p0, z3.b, z1.b ; CHECK-NEXT: stp q3, q2, [x0] ; CHECK-NEXT: ret @@ -203,9 +203,9 @@ define <2 x i16> @sdiv_v2i16(<2 x i16> %op1, <2 x i16> %op2) { define <4 x i16> @sdiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) { ; CHECK-LABEL: sdiv_v4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: sunpklo z1.s, z1.h ; CHECK-NEXT: sunpklo z0.s, z0.h ; CHECK-NEXT: sdiv z0.s, p0/m, z0.s, z1.s @@ -272,8 +272,8 @@ define void @sdiv_v16i16(ptr %a, ptr %b) { ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: sdiv z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: uzp1 z1.h, z5.h, z5.h ; CHECK-NEXT: splice z0.h, p0, z0.h, z2.h +; CHECK-NEXT: uzp1 z1.h, z5.h, z5.h ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z3.h, p0, z3.h, z1.h ; CHECK-NEXT: stp q3, q0, [x0] @@ -314,8 +314,8 @@ define <4 x i32> @sdiv_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @sdiv_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: sdiv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: sdivr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -358,8 +358,8 @@ define <2 x i64> @sdiv_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @sdiv_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: sdiv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: sdivr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -453,8 +453,8 @@ define <16 x i8> @udiv_v16i8(<16 x i8> %op1, <16 x i8> %op2) { ; CHECK-NEXT: udivr z3.s, p0/m, z3.s, z5.s ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s -; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z1.h, z4.h, z4.h +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z1.h, p0, z1.h, z2.h ; CHECK-NEXT: uzp1 z1.b, z1.b, z1.b @@ -530,18 +530,18 @@ define void @udiv_v32i8(ptr %a, ptr %b) { ; CHECK-NEXT: uzp1 z5.h, z5.h, z5.h ; CHECK-NEXT: udiv z6.s, p0/m, z6.s, z7.s ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: uzp1 z7.h, z16.h, z16.h ; CHECK-NEXT: splice z4.h, p0, z4.h, z5.h ; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h ; CHECK-NEXT: splice z2.h, p0, z2.h, z3.h +; CHECK-NEXT: uzp1 z7.h, z16.h, z16.h ; CHECK-NEXT: uzp1 z1.b, z4.b, z4.b ; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b ; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b ; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h ; CHECK-NEXT: splice z7.h, p0, z7.h, z6.h ; CHECK-NEXT: ptrue p0.b, vl8 -; CHECK-NEXT: uzp1 z3.b, z7.b, z7.b ; CHECK-NEXT: splice z2.b, p0, z2.b, z0.b +; CHECK-NEXT: uzp1 z3.b, z7.b, z7.b ; CHECK-NEXT: splice z3.b, p0, z3.b, z1.b ; CHECK-NEXT: stp q3, q2, [x0] ; CHECK-NEXT: ret @@ -555,9 +555,9 @@ define void @udiv_v32i8(ptr %a, ptr %b) { define <2 x i16> @udiv_v2i16(<2 x i16> %op1, <2 x i16> %op2) { ; CHECK-LABEL: udiv_v2i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: and z1.s, z1.s, #0xffff ; CHECK-NEXT: and z0.s, z0.s, #0xffff ; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s @@ -570,9 +570,9 @@ define <2 x i16> @udiv_v2i16(<2 x i16> %op1, <2 x i16> %op2) { define <4 x i16> @udiv_v4i16(<4 x i16> %op1, <4 x i16> %op2) { ; CHECK-LABEL: udiv_v4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: uunpklo z1.s, z1.h ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: udiv z0.s, p0/m, z0.s, z1.s @@ -639,8 +639,8 @@ define void @udiv_v16i16(ptr %a, ptr %b) { ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: udiv z3.s, p0/m, z3.s, z4.s ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: uzp1 z1.h, z5.h, z5.h ; CHECK-NEXT: splice z0.h, p0, z0.h, z2.h +; CHECK-NEXT: uzp1 z1.h, z5.h, z5.h ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z3.h, p0, z3.h, z1.h ; CHECK-NEXT: stp q3, q0, [x0] @@ -681,8 +681,8 @@ define <4 x i32> @udiv_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @udiv_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: udiv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: udivr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -725,8 +725,8 @@ define <2 x i64> @udiv_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @udiv_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: udiv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: udivr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -743,10 +743,10 @@ define void @udiv_v4i64(ptr %a, ptr %b) { define void @udiv_constantsplat_v8i32(ptr %a) { ; SVE-LABEL: udiv_constantsplat_v8i32: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s, vl4 ; SVE-NEXT: mov w8, #8969 // =0x2309 -; SVE-NEXT: movk w8, #22765, lsl #16 ; SVE-NEXT: ldp q1, q2, [x0] +; SVE-NEXT: movk w8, #22765, lsl #16 +; SVE-NEXT: ptrue p0.s, vl4 ; SVE-NEXT: mov z0.s, w8 ; SVE-NEXT: movprfx z3, z1 ; SVE-NEXT: umulh z3.s, p0/m, z3.s, z0.s diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-immediates.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-immediates.ll index 0785c67ce6f4..f028b3eeca25 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-immediates.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-immediates.ll @@ -221,8 +221,8 @@ define void @ashr_v4i64(ptr %a) { define void @icmp_eq_v32i8(ptr %a) { ; CHECK-LABEL: icmp_eq_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: cmpeq p1.b, p0/z, z0.b, #7 ; CHECK-NEXT: cmpeq p0.b, p0/z, z1.b, #7 ; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff @@ -241,8 +241,8 @@ define void @icmp_eq_v32i8(ptr %a) { define void @icmp_sge_v16i16(ptr %a) { ; CHECK-LABEL: icmp_sge_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: cmpge p1.h, p0/z, z0.h, #15 ; CHECK-NEXT: cmpge p0.h, p0/z, z1.h, #15 ; CHECK-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff @@ -261,8 +261,8 @@ define void @icmp_sge_v16i16(ptr %a) { define void @icmp_sgt_v8i32(ptr %a) { ; CHECK-LABEL: icmp_sgt_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: cmpgt p1.s, p0/z, z0.s, #-8 ; CHECK-NEXT: cmpgt p0.s, p0/z, z1.s, #-8 ; CHECK-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff @@ -281,8 +281,8 @@ define void @icmp_sgt_v8i32(ptr %a) { define void @icmp_ult_v4i64(ptr %a) { ; CHECK-LABEL: icmp_ult_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: cmplo p1.d, p0/z, z0.d, #63 ; CHECK-NEXT: cmplo p0.d, p0/z, z1.d, #63 ; CHECK-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-minmax.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-minmax.ll index d7600c6e6192..50cf9b73d9a7 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-minmax.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-minmax.ll @@ -37,8 +37,8 @@ define <16 x i8> @smax_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @smax_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: smax_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: smax z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: movprfx z1, z2 @@ -81,8 +81,8 @@ define <8 x i16> @smax_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @smax_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: smax_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: smax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -125,8 +125,8 @@ define <4 x i32> @smax_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @smax_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: smax_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -171,8 +171,8 @@ define <2 x i64> @smax_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @smax_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: smax_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -219,8 +219,8 @@ define <16 x i8> @smin_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @smin_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: smin_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: smin z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: movprfx z1, z2 @@ -263,8 +263,8 @@ define <8 x i16> @smin_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @smin_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: smin_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: smin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -307,8 +307,8 @@ define <4 x i32> @smin_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @smin_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: smin_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: smin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -353,8 +353,8 @@ define <2 x i64> @smin_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @smin_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: smin_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: smin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -401,8 +401,8 @@ define <16 x i8> @umax_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @umax_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: umax_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: umax z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: movprfx z1, z2 @@ -445,8 +445,8 @@ define <8 x i16> @umax_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @umax_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: umax_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: umax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -489,8 +489,8 @@ define <4 x i32> @umax_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @umax_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: umax_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: umax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -535,8 +535,8 @@ define <2 x i64> @umax_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @umax_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: umax_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: umax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -583,8 +583,8 @@ define <16 x i8> @umin_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @umin_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: umin_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: umin z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: movprfx z1, z2 @@ -627,8 +627,8 @@ define <8 x i16> @umin_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @umin_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: umin_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: umin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -671,8 +671,8 @@ define <4 x i32> @umin_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @umin_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: umin_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: umin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -717,8 +717,8 @@ define <2 x i64> @umin_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @umin_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: umin_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: umin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll index c48cb315a7aa..cb7fa53eac51 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll @@ -101,8 +101,8 @@ define <16 x i8> @smulh_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @smulh_v32i8(ptr %a, ptr %b) { ; SVE-LABEL: smulh_v32i8: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.b, vl16 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.b, vl16 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: smulh z0.b, p0/m, z0.b, z1.b ; SVE-NEXT: movprfx z1, z2 @@ -214,8 +214,8 @@ define <8 x i16> @smulh_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @smulh_v16i16(ptr %a, ptr %b) { ; SVE-LABEL: smulh_v16i16: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.h, vl8 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.h, vl8 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: smulh z0.h, p0/m, z0.h, z1.h ; SVE-NEXT: movprfx z1, z2 @@ -295,8 +295,8 @@ define <4 x i32> @smulh_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @smulh_v8i32(ptr %a, ptr %b) { ; SVE-LABEL: smulh_v8i32: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s, vl4 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.s, vl4 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: smulh z0.s, p0/m, z0.s, z1.s ; SVE-NEXT: movprfx z1, z2 @@ -378,8 +378,8 @@ define <2 x i64> @smulh_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @smulh_v4i64(ptr %a, ptr %b) { ; SVE-LABEL: smulh_v4i64: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.d, vl2 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.d, vl2 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: smulh z0.d, p0/m, z0.d, z1.d ; SVE-NEXT: movprfx z1, z2 @@ -413,9 +413,9 @@ define void @smulh_v4i64(ptr %a, ptr %b) { define <4 x i8> @umulh_v4i8(<4 x i8> %op1, <4 x i8> %op2) { ; SVE-LABEL: umulh_v4i8: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.h, vl4 ; SVE-NEXT: // kill: def $d1 killed $d1 def $z1 ; SVE-NEXT: // kill: def $d0 killed $d0 def $z0 +; SVE-NEXT: ptrue p0.h, vl4 ; SVE-NEXT: and z0.h, z0.h, #0xff ; SVE-NEXT: and z1.h, z1.h, #0xff ; SVE-NEXT: mul z0.h, p0/m, z0.h, z1.h @@ -494,8 +494,8 @@ define <16 x i8> @umulh_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @umulh_v32i8(ptr %a, ptr %b) { ; SVE-LABEL: umulh_v32i8: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.b, vl16 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.b, vl16 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: umulh z0.b, p0/m, z0.b, z1.b ; SVE-NEXT: movprfx z1, z2 @@ -525,9 +525,9 @@ define void @umulh_v32i8(ptr %a, ptr %b) { define <2 x i16> @umulh_v2i16(<2 x i16> %op1, <2 x i16> %op2) { ; SVE-LABEL: umulh_v2i16: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s, vl2 ; SVE-NEXT: // kill: def $d1 killed $d1 def $z1 ; SVE-NEXT: // kill: def $d0 killed $d0 def $z0 +; SVE-NEXT: ptrue p0.s, vl2 ; SVE-NEXT: and z0.s, z0.s, #0xffff ; SVE-NEXT: and z1.s, z1.s, #0xffff ; SVE-NEXT: mul z0.s, p0/m, z0.s, z1.s @@ -606,8 +606,8 @@ define <8 x i16> @umulh_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @umulh_v16i16(ptr %a, ptr %b) { ; SVE-LABEL: umulh_v16i16: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.h, vl8 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.h, vl8 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: umulh z0.h, p0/m, z0.h, z1.h ; SVE-NEXT: movprfx z1, z2 @@ -687,8 +687,8 @@ define <4 x i32> @umulh_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @umulh_v8i32(ptr %a, ptr %b) { ; SVE-LABEL: umulh_v8i32: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s, vl4 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.s, vl4 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: umulh z0.s, p0/m, z0.s, z1.s ; SVE-NEXT: movprfx z1, z2 @@ -770,8 +770,8 @@ define <2 x i64> @umulh_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @umulh_v4i64(ptr %a, ptr %b) { ; SVE-LABEL: umulh_v4i64: ; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.d, vl2 ; SVE-NEXT: ldp q0, q3, [x1] +; SVE-NEXT: ptrue p0.d, vl2 ; SVE-NEXT: ldp q1, q2, [x0] ; SVE-NEXT: umulh z0.d, p0/m, z0.d, z1.d ; SVE-NEXT: movprfx z1, z2 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-reduce.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-reduce.ll index c51630ecd752..751f43768a51 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-reduce.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-reduce.ll @@ -37,8 +37,8 @@ define i8 @uaddv_v16i8(<16 x i8> %a) { define i8 @uaddv_v32i8(ptr %a) { ; CHECK-LABEL: uaddv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: add z0.b, z1.b, z0.b ; CHECK-NEXT: uaddv d0, p0, z0.b ; CHECK-NEXT: fmov x0, d0 @@ -78,8 +78,8 @@ define i16 @uaddv_v8i16(<8 x i16> %a) { define i16 @uaddv_v16i16(ptr %a) { ; CHECK-LABEL: uaddv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: add z0.h, z1.h, z0.h ; CHECK-NEXT: uaddv d0, p0, z0.h ; CHECK-NEXT: fmov x0, d0 @@ -119,8 +119,8 @@ define i32 @uaddv_v4i32(<4 x i32> %a) { define i32 @uaddv_v8i32(ptr %a) { ; CHECK-LABEL: uaddv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: add z0.s, z1.s, z0.s ; CHECK-NEXT: uaddv d0, p0, z0.s ; CHECK-NEXT: fmov x0, d0 @@ -146,8 +146,8 @@ define i64 @uaddv_v2i64(<2 x i64> %a) { define i64 @uaddv_v4i64(ptr %a) { ; CHECK-LABEL: uaddv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: add z0.d, z1.d, z0.d ; CHECK-NEXT: uaddv d0, p0, z0.d ; CHECK-NEXT: fmov x0, d0 @@ -188,8 +188,8 @@ define i8 @smaxv_v16i8(<16 x i8> %a) { define i8 @smaxv_v32i8(ptr %a) { ; CHECK-LABEL: smaxv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: smax z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: smaxv b0, p0, z0.b ; CHECK-NEXT: fmov w0, s0 @@ -226,8 +226,8 @@ define i16 @smaxv_v8i16(<8 x i16> %a) { define i16 @smaxv_v16i16(ptr %a) { ; CHECK-LABEL: smaxv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: smax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: smaxv h0, p0, z0.h ; CHECK-NEXT: fmov w0, s0 @@ -264,8 +264,8 @@ define i32 @smaxv_v4i32(<4 x i32> %a) { define i32 @smaxv_v8i32(ptr %a) { ; CHECK-LABEL: smaxv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: smaxv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 @@ -291,8 +291,8 @@ define i64 @smaxv_v2i64(<2 x i64> %a) { define i64 @smaxv_v4i64(ptr %a) { ; CHECK-LABEL: smaxv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: smaxv d0, p0, z0.d ; CHECK-NEXT: fmov x0, d0 @@ -333,8 +333,8 @@ define i8 @sminv_v16i8(<16 x i8> %a) { define i8 @sminv_v32i8(ptr %a) { ; CHECK-LABEL: sminv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: smin z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: sminv b0, p0, z0.b ; CHECK-NEXT: fmov w0, s0 @@ -371,8 +371,8 @@ define i16 @sminv_v8i16(<8 x i16> %a) { define i16 @sminv_v16i16(ptr %a) { ; CHECK-LABEL: sminv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: smin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: sminv h0, p0, z0.h ; CHECK-NEXT: fmov w0, s0 @@ -409,8 +409,8 @@ define i32 @sminv_v4i32(<4 x i32> %a) { define i32 @sminv_v8i32(ptr %a) { ; CHECK-LABEL: sminv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: smin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: sminv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 @@ -436,8 +436,8 @@ define i64 @sminv_v2i64(<2 x i64> %a) { define i64 @sminv_v4i64(ptr %a) { ; CHECK-LABEL: sminv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: smin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: sminv d0, p0, z0.d ; CHECK-NEXT: fmov x0, d0 @@ -478,8 +478,8 @@ define i8 @umaxv_v16i8(<16 x i8> %a) { define i8 @umaxv_v32i8(ptr %a) { ; CHECK-LABEL: umaxv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: umax z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: umaxv b0, p0, z0.b ; CHECK-NEXT: fmov w0, s0 @@ -516,8 +516,8 @@ define i16 @umaxv_v8i16(<8 x i16> %a) { define i16 @umaxv_v16i16(ptr %a) { ; CHECK-LABEL: umaxv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: umax z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: umaxv h0, p0, z0.h ; CHECK-NEXT: fmov w0, s0 @@ -554,8 +554,8 @@ define i32 @umaxv_v4i32(<4 x i32> %a) { define i32 @umaxv_v8i32(ptr %a) { ; CHECK-LABEL: umaxv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: umax z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: umaxv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 @@ -581,8 +581,8 @@ define i64 @umaxv_v2i64(<2 x i64> %a) { define i64 @umaxv_v4i64(ptr %a) { ; CHECK-LABEL: umaxv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: umax z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: umaxv d0, p0, z0.d ; CHECK-NEXT: fmov x0, d0 @@ -623,8 +623,8 @@ define i8 @uminv_v16i8(<16 x i8> %a) { define i8 @uminv_v32i8(ptr %a) { ; CHECK-LABEL: uminv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: umin z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: uminv b0, p0, z0.b ; CHECK-NEXT: fmov w0, s0 @@ -661,8 +661,8 @@ define i16 @uminv_v8i16(<8 x i16> %a) { define i16 @uminv_v16i16(ptr %a) { ; CHECK-LABEL: uminv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: umin z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: uminv h0, p0, z0.h ; CHECK-NEXT: fmov w0, s0 @@ -699,8 +699,8 @@ define i32 @uminv_v4i32(<4 x i32> %a) { define i32 @uminv_v8i32(ptr %a) { ; CHECK-LABEL: uminv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: umin z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: uminv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 @@ -726,8 +726,8 @@ define i64 @uminv_v2i64(<2 x i64> %a) { define i64 @uminv_v4i64(ptr %a) { ; CHECK-LABEL: uminv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: umin z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: uminv d0, p0, z0.d ; CHECK-NEXT: fmov x0, d0 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll index 4a1209b942f4..d373a9063f85 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll @@ -65,7 +65,6 @@ define <16 x i8> @srem_v16i8(<16 x i8> %op1, <16 x i8> %op2) { ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ptrue p1.b, vl16 ; CHECK-NEXT: ext z2.b, z2.b, z1.b, #8 ; CHECK-NEXT: ext z3.b, z3.b, z0.b, #8 ; CHECK-NEXT: sunpklo z2.h, z2.b @@ -91,15 +90,16 @@ define <16 x i8> @srem_v16i8(<16 x i8> %op1, <16 x i8> %op2) { ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: sdivr z3.s, p0/m, z3.s, z5.s ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h ; CHECK-NEXT: splice z4.h, p0, z4.h, z2.h +; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h ; CHECK-NEXT: uzp1 z2.b, z4.b, z4.b ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z5.h, p0, z5.h, z3.h ; CHECK-NEXT: ptrue p0.b, vl8 ; CHECK-NEXT: uzp1 z3.b, z5.b, z5.b ; CHECK-NEXT: splice z3.b, p0, z3.b, z2.b -; CHECK-NEXT: mls z0.b, p1/m, z3.b, z1.b +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: mls z0.b, p0/m, z3.b, z1.b ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 ; CHECK-NEXT: ret %res = srem <16 x i8> %op1, %op2 @@ -112,7 +112,6 @@ define void @srem_v32i8(ptr %a, ptr %b) { ; CHECK-NEXT: ldr q0, [x0, #16] ; CHECK-NEXT: ldr q1, [x1, #16] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ptrue p1.b, vl16 ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: sunpklo z7.h, z1.b @@ -171,22 +170,23 @@ define void @srem_v32i8(ptr %a, ptr %b) { ; CHECK-NEXT: uzp1 z17.h, z17.h, z17.h ; CHECK-NEXT: sdivr z18.s, p0/m, z18.s, z20.s ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: uzp1 z19.h, z21.h, z21.h ; CHECK-NEXT: splice z16.h, p0, z16.h, z17.h ; CHECK-NEXT: splice z2.h, p0, z2.h, z5.h ; CHECK-NEXT: splice z6.h, p0, z6.h, z7.h +; CHECK-NEXT: uzp1 z19.h, z21.h, z21.h ; CHECK-NEXT: uzp1 z5.b, z16.b, z16.b ; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b ; CHECK-NEXT: uzp1 z6.b, z6.b, z6.b ; CHECK-NEXT: uzp1 z18.h, z18.h, z18.h ; CHECK-NEXT: splice z19.h, p0, z19.h, z18.h ; CHECK-NEXT: ptrue p0.b, vl8 -; CHECK-NEXT: uzp1 z7.b, z19.b, z19.b ; CHECK-NEXT: splice z6.b, p0, z6.b, z2.b +; CHECK-NEXT: uzp1 z7.b, z19.b, z19.b ; CHECK-NEXT: splice z7.b, p0, z7.b, z5.b -; CHECK-NEXT: mls z0.b, p1/m, z6.b, z1.b +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: mls z0.b, p0/m, z6.b, z1.b ; CHECK-NEXT: movprfx z2, z3 -; CHECK-NEXT: mls z2.b, p1/m, z7.b, z4.b +; CHECK-NEXT: mls z2.b, p0/m, z7.b, z4.b ; CHECK-NEXT: stp q2, q0, [x0] ; CHECK-NEXT: ret %op1 = load <32 x i8>, ptr %a @@ -199,11 +199,11 @@ define void @srem_v32i8(ptr %a, ptr %b) { define <4 x i16> @srem_v4i16(<4 x i16> %op1, <4 x i16> %op2) { ; CHECK-LABEL: srem_v4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: sunpklo z2.s, z1.h ; CHECK-NEXT: sunpklo z3.s, z0.h +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: sdivr z2.s, p0/m, z2.s, z3.s ; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h @@ -223,7 +223,6 @@ define <8 x i16> @srem_v8i16(<8 x i16> %op1, <8 x i16> %op2) { ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: sunpklo z4.s, z0.h -; CHECK-NEXT: ptrue p1.h, vl8 ; CHECK-NEXT: ext z2.b, z2.b, z1.b, #8 ; CHECK-NEXT: ext z3.b, z3.b, z0.b, #8 ; CHECK-NEXT: sunpklo z2.s, z2.h @@ -235,7 +234,8 @@ define <8 x i16> @srem_v8i16(<8 x i16> %op1, <8 x i16> %op2) { ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z3.h, p0, z3.h, z2.h -; CHECK-NEXT: mls z0.h, p1/m, z3.h, z1.h +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: mls z0.h, p0/m, z3.h, z1.h ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 ; CHECK-NEXT: ret %res = srem <8 x i16> %op1, %op2 @@ -248,7 +248,6 @@ define void @srem_v16i16(ptr %a, ptr %b) { ; CHECK-NEXT: ldp q4, q1, [x1] ; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldr q0, [x0, #16] -; CHECK-NEXT: ptrue p1.h, vl8 ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: mov z5.d, z4.d @@ -277,9 +276,10 @@ define void @srem_v16i16(ptr %a, ptr %b) { ; CHECK-NEXT: splice z6.h, p0, z6.h, z5.h ; CHECK-NEXT: uzp1 z7.h, z7.h, z7.h ; CHECK-NEXT: splice z7.h, p0, z7.h, z2.h +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: movprfx z2, z3 -; CHECK-NEXT: mls z2.h, p1/m, z6.h, z4.h -; CHECK-NEXT: mls z0.h, p1/m, z7.h, z1.h +; CHECK-NEXT: mls z2.h, p0/m, z6.h, z4.h +; CHECK-NEXT: mls z0.h, p0/m, z7.h, z1.h ; CHECK-NEXT: stp q2, q0, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i16>, ptr %a @@ -322,8 +322,8 @@ define <4 x i32> @srem_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @srem_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: srem_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: movprfx z4, z1 ; CHECK-NEXT: sdiv z4.s, p0/m, z4.s, z0.s @@ -374,8 +374,8 @@ define <2 x i64> @srem_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @srem_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: srem_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: movprfx z4, z1 ; CHECK-NEXT: sdiv z4.d, p0/m, z4.d, z0.d @@ -454,7 +454,6 @@ define <16 x i8> @urem_v16i8(<16 x i8> %op1, <16 x i8> %op2) { ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ptrue p1.b, vl16 ; CHECK-NEXT: ext z2.b, z2.b, z1.b, #8 ; CHECK-NEXT: ext z3.b, z3.b, z0.b, #8 ; CHECK-NEXT: uunpklo z2.h, z2.b @@ -480,15 +479,16 @@ define <16 x i8> @urem_v16i8(<16 x i8> %op1, <16 x i8> %op2) { ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: udivr z3.s, p0/m, z3.s, z5.s ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h ; CHECK-NEXT: splice z4.h, p0, z4.h, z2.h +; CHECK-NEXT: uzp1 z5.h, z6.h, z6.h ; CHECK-NEXT: uzp1 z2.b, z4.b, z4.b ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z5.h, p0, z5.h, z3.h ; CHECK-NEXT: ptrue p0.b, vl8 ; CHECK-NEXT: uzp1 z3.b, z5.b, z5.b ; CHECK-NEXT: splice z3.b, p0, z3.b, z2.b -; CHECK-NEXT: mls z0.b, p1/m, z3.b, z1.b +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: mls z0.b, p0/m, z3.b, z1.b ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 ; CHECK-NEXT: ret %res = urem <16 x i8> %op1, %op2 @@ -501,7 +501,6 @@ define void @urem_v32i8(ptr %a, ptr %b) { ; CHECK-NEXT: ldr q0, [x0, #16] ; CHECK-NEXT: ldr q1, [x1, #16] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ptrue p1.b, vl16 ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: uunpklo z7.h, z1.b @@ -560,22 +559,23 @@ define void @urem_v32i8(ptr %a, ptr %b) { ; CHECK-NEXT: uzp1 z17.h, z17.h, z17.h ; CHECK-NEXT: udivr z18.s, p0/m, z18.s, z20.s ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: uzp1 z19.h, z21.h, z21.h ; CHECK-NEXT: splice z16.h, p0, z16.h, z17.h ; CHECK-NEXT: splice z2.h, p0, z2.h, z5.h ; CHECK-NEXT: splice z6.h, p0, z6.h, z7.h +; CHECK-NEXT: uzp1 z19.h, z21.h, z21.h ; CHECK-NEXT: uzp1 z5.b, z16.b, z16.b ; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b ; CHECK-NEXT: uzp1 z6.b, z6.b, z6.b ; CHECK-NEXT: uzp1 z18.h, z18.h, z18.h ; CHECK-NEXT: splice z19.h, p0, z19.h, z18.h ; CHECK-NEXT: ptrue p0.b, vl8 -; CHECK-NEXT: uzp1 z7.b, z19.b, z19.b ; CHECK-NEXT: splice z6.b, p0, z6.b, z2.b +; CHECK-NEXT: uzp1 z7.b, z19.b, z19.b ; CHECK-NEXT: splice z7.b, p0, z7.b, z5.b -; CHECK-NEXT: mls z0.b, p1/m, z6.b, z1.b +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: mls z0.b, p0/m, z6.b, z1.b ; CHECK-NEXT: movprfx z2, z3 -; CHECK-NEXT: mls z2.b, p1/m, z7.b, z4.b +; CHECK-NEXT: mls z2.b, p0/m, z7.b, z4.b ; CHECK-NEXT: stp q2, q0, [x0] ; CHECK-NEXT: ret %op1 = load <32 x i8>, ptr %a @@ -588,11 +588,11 @@ define void @urem_v32i8(ptr %a, ptr %b) { define <4 x i16> @urem_v4i16(<4 x i16> %op1, <4 x i16> %op2) { ; CHECK-LABEL: urem_v4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: uunpklo z2.s, z1.h ; CHECK-NEXT: uunpklo z3.s, z0.h +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: udivr z2.s, p0/m, z2.s, z3.s ; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h @@ -612,7 +612,6 @@ define <8 x i16> @urem_v8i16(<8 x i16> %op1, <8 x i16> %op2) { ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: uunpklo z4.s, z0.h -; CHECK-NEXT: ptrue p1.h, vl8 ; CHECK-NEXT: ext z2.b, z2.b, z1.b, #8 ; CHECK-NEXT: ext z3.b, z3.b, z0.b, #8 ; CHECK-NEXT: uunpklo z2.s, z2.h @@ -624,7 +623,8 @@ define <8 x i16> @urem_v8i16(<8 x i16> %op1, <8 x i16> %op2) { ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: splice z3.h, p0, z3.h, z2.h -; CHECK-NEXT: mls z0.h, p1/m, z3.h, z1.h +; CHECK-NEXT: ptrue p0.h, vl8 +; CHECK-NEXT: mls z0.h, p0/m, z3.h, z1.h ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 ; CHECK-NEXT: ret %res = urem <8 x i16> %op1, %op2 @@ -637,7 +637,6 @@ define void @urem_v16i16(ptr %a, ptr %b) { ; CHECK-NEXT: ldp q4, q1, [x1] ; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldr q0, [x0, #16] -; CHECK-NEXT: ptrue p1.h, vl8 ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: mov z3.d, z0.d ; CHECK-NEXT: mov z5.d, z4.d @@ -666,9 +665,10 @@ define void @urem_v16i16(ptr %a, ptr %b) { ; CHECK-NEXT: splice z6.h, p0, z6.h, z5.h ; CHECK-NEXT: uzp1 z7.h, z7.h, z7.h ; CHECK-NEXT: splice z7.h, p0, z7.h, z2.h +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: movprfx z2, z3 -; CHECK-NEXT: mls z2.h, p1/m, z6.h, z4.h -; CHECK-NEXT: mls z0.h, p1/m, z7.h, z1.h +; CHECK-NEXT: mls z2.h, p0/m, z6.h, z4.h +; CHECK-NEXT: mls z0.h, p0/m, z7.h, z1.h ; CHECK-NEXT: stp q2, q0, [x0] ; CHECK-NEXT: ret %op1 = load <16 x i16>, ptr %a @@ -711,8 +711,8 @@ define <4 x i32> @urem_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @urem_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: urem_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: movprfx z4, z1 ; CHECK-NEXT: udiv z4.s, p0/m, z4.s, z0.s @@ -763,8 +763,8 @@ define <2 x i64> @urem_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @urem_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: urem_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: movprfx z4, z1 ; CHECK-NEXT: udiv z4.d, p0/m, z4.d, z0.d diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-select.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-select.ll index 3b58e35bd844..906112f7ac39 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-select.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-select.ll @@ -7,8 +7,8 @@ target triple = "aarch64-unknown-linux-gnu" define <4 x i8> @select_v4i8(<4 x i8> %op1, <4 x i8> %op2, i1 %mask) { ; CHECK-LABEL: select_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: and z2.h, z2.h, #0x1 @@ -23,8 +23,8 @@ define <4 x i8> @select_v4i8(<4 x i8> %op1, <4 x i8> %op2, i1 %mask) { define <8 x i8> @select_v8i8(<8 x i8> %op1, <8 x i8> %op2, i1 %mask) { ; CHECK-LABEL: select_v8i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w0 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: cmpne p0.b, p0/z, z2.b, #0 @@ -38,8 +38,8 @@ define <8 x i8> @select_v8i8(<8 x i8> %op1, <8 x i8> %op2, i1 %mask) { define <16 x i8> @select_v16i8(<16 x i8> %op1, <16 x i8> %op2, i1 %mask) { ; CHECK-LABEL: select_v16i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z2.b, w0 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 ; CHECK-NEXT: cmpne p0.b, p0/z, z2.b, #0 @@ -53,8 +53,8 @@ define <16 x i8> @select_v16i8(<16 x i8> %op1, <16 x i8> %op2, i1 %mask) { define void @select_v32i8(ptr %a, ptr %b, i1 %mask) { ; CHECK-LABEL: select_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: mov z0.b, w2 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x0, #16] @@ -74,8 +74,8 @@ define void @select_v32i8(ptr %a, ptr %b, i1 %mask) { define <2 x i16> @select_v2i16(<2 x i16> %op1, <2 x i16> %op2, i1 %mask) { ; CHECK-LABEL: select_v2i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w0, #0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: mov z2.s, w8 @@ -90,8 +90,8 @@ define <2 x i16> @select_v2i16(<2 x i16> %op1, <2 x i16> %op2, i1 %mask) { define <4 x i16> @select_v4i16(<4 x i16> %op1, <4 x i16> %op2, i1 %mask) { ; CHECK-LABEL: select_v4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: and z2.h, z2.h, #0x1 @@ -106,8 +106,8 @@ define <4 x i16> @select_v4i16(<4 x i16> %op1, <4 x i16> %op2, i1 %mask) { define <8 x i16> @select_v8i16(<8 x i16> %op1, <8 x i16> %op2, i1 %mask) { ; CHECK-LABEL: select_v8i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z2.h, w0 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 ; CHECK-NEXT: and z2.h, z2.h, #0x1 @@ -122,8 +122,8 @@ define <8 x i16> @select_v8i16(<8 x i16> %op1, <8 x i16> %op2, i1 %mask) { define void @select_v16i16(ptr %a, ptr %b, i1 %mask) { ; CHECK-LABEL: select_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mov z0.h, w2 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and z0.h, z0.h, #0x1 ; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 ; CHECK-NEXT: ldr q0, [x0] @@ -144,8 +144,8 @@ define void @select_v16i16(ptr %a, ptr %b, i1 %mask) { define <2 x i32> @select_v2i32(<2 x i32> %op1, <2 x i32> %op2, i1 %mask) { ; CHECK-LABEL: select_v2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w0, #0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: mov z2.s, w8 @@ -160,8 +160,8 @@ define <2 x i32> @select_v2i32(<2 x i32> %op1, <2 x i32> %op2, i1 %mask) { define <4 x i32> @select_v4i32(<4 x i32> %op1, <4 x i32> %op2, i1 %mask) { ; CHECK-LABEL: select_v4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w0, #0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 ; CHECK-NEXT: mov z2.s, w8 @@ -176,8 +176,8 @@ define <4 x i32> @select_v4i32(<4 x i32> %op1, <4 x i32> %op2, i1 %mask) { define void @select_v8i32(ptr %a, ptr %b, i1 %mask) { ; CHECK-LABEL: select_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and w8, w2, #0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mov z0.s, w8 ; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 ; CHECK-NEXT: ldr q0, [x0] @@ -198,9 +198,9 @@ define void @select_v8i32(ptr %a, ptr %b, i1 %mask) { define <1 x i64> @select_v1i64(<1 x i64> %op1, <1 x i64> %op2, i1 %mask) { ; CHECK-LABEL: select_v1i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: and x8, x0, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: mov z2.d, x8 @@ -215,9 +215,9 @@ define <1 x i64> @select_v1i64(<1 x i64> %op1, <1 x i64> %op2, i1 %mask) { define <2 x i64> @select_v2i64(<2 x i64> %op1, <2 x i64> %op2, i1 %mask) { ; CHECK-LABEL: select_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: and x8, x0, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1 ; CHECK-NEXT: mov z2.d, x8 @@ -232,9 +232,9 @@ define <2 x i64> @select_v2i64(<2 x i64> %op1, <2 x i64> %op2, i1 %mask) { define void @select_v4i64(ptr %a, ptr %b, i1 %mask) { ; CHECK-LABEL: select_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w2 killed $w2 def $x2 ; CHECK-NEXT: and x8, x2, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z0.d, x8 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: ldr q0, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-shifts.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-shifts.ll index c7fa0e8ad5e4..9ed52e321d9a 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-shifts.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-shifts.ll @@ -52,8 +52,8 @@ define <16 x i8> @ashr_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @ashr_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: ashr_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: asrr z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: movprfx z1, z2 @@ -111,8 +111,8 @@ define <8 x i16> @ashr_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @ashr_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: ashr_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: asrr z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -155,8 +155,8 @@ define <4 x i32> @ashr_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @ashr_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: ashr_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: asrr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -199,8 +199,8 @@ define <2 x i64> @ashr_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @ashr_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: ashr_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: asrr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -221,9 +221,9 @@ define void @ashr_v4i64(ptr %a, ptr %b) { define <4 x i8> @lshr_v4i8(<4 x i8> %op1, <4 x i8> %op2) { ; CHECK-LABEL: lshr_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: and z1.h, z1.h, #0xff ; CHECK-NEXT: and z0.h, z0.h, #0xff ; CHECK-NEXT: lsr z0.h, p0/m, z0.h, z1.h @@ -262,8 +262,8 @@ define <16 x i8> @lshr_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @lshr_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: lshr_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: lsrr z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: movprfx z1, z2 @@ -280,9 +280,9 @@ define void @lshr_v32i8(ptr %a, ptr %b) { define <2 x i16> @lshr_v2i16(<2 x i16> %op1, <2 x i16> %op2) { ; CHECK-LABEL: lshr_v2i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: and z1.s, z1.s, #0xffff ; CHECK-NEXT: and z0.s, z0.s, #0xffff ; CHECK-NEXT: lsr z0.s, p0/m, z0.s, z1.s @@ -321,8 +321,8 @@ define <8 x i16> @lshr_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @lshr_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: lshr_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: lsrr z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -365,8 +365,8 @@ define <4 x i32> @lshr_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @lshr_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: lshr_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: lsrr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -409,8 +409,8 @@ define <2 x i64> @lshr_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @lshr_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: lshr_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: lsrr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 @@ -431,8 +431,8 @@ define void @lshr_v4i64(ptr %a, ptr %b) { define <2 x i8> @shl_v2i8(<2 x i8> %op1, <2 x i8> %op2) { ; CHECK-LABEL: shl_v2i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: and z1.s, z1.s, #0xff ; CHECK-NEXT: lsl z0.s, p0/m, z0.s, z1.s @@ -445,8 +445,8 @@ define <2 x i8> @shl_v2i8(<2 x i8> %op1, <2 x i8> %op2) { define <4 x i8> @shl_v4i8(<4 x i8> %op1, <4 x i8> %op2) { ; CHECK-LABEL: shl_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: and z1.h, z1.h, #0xff ; CHECK-NEXT: lsl z0.h, p0/m, z0.h, z1.h @@ -485,8 +485,8 @@ define <16 x i8> @shl_v16i8(<16 x i8> %op1, <16 x i8> %op2) { define void @shl_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: shl_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: lslr z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: movprfx z1, z2 @@ -529,8 +529,8 @@ define <8 x i16> @shl_v8i16(<8 x i16> %op1, <8 x i16> %op2) { define void @shl_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: shl_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: lslr z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -573,8 +573,8 @@ define <4 x i32> @shl_v4i32(<4 x i32> %op1, <4 x i32> %op2) { define void @shl_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: shl_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: lslr z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -617,8 +617,8 @@ define <2 x i64> @shl_v2i64(<2 x i64> %op1, <2 x i64> %op2) { define void @shl_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: shl_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: lslr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll index 5c5cf68135bf..b285659258f3 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll @@ -36,8 +36,8 @@ define void @ucvtf_v8i16_v8f16(ptr %a, ptr %b) { define void @ucvtf_v16i16_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: ucvtf_v16i16_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ucvtf z0.h, p0/m, z0.h ; CHECK-NEXT: ucvtf z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x1] @@ -55,8 +55,8 @@ define void @ucvtf_v16i16_v16f16(ptr %a, ptr %b) { define <2 x float> @ucvtf_v2i16_v2f32(<2 x i16> %op1) { ; CHECK-LABEL: ucvtf_v2i16_v2f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: and z0.s, z0.s, #0xffff ; CHECK-NEXT: ucvtf z0.s, p0/m, z0.s ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 @@ -68,8 +68,8 @@ define <2 x float> @ucvtf_v2i16_v2f32(<2 x i16> %op1) { define <4 x float> @ucvtf_v4i16_v4f32(<4 x i16> %op1) { ; CHECK-LABEL: ucvtf_v4i16_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: uunpklo z0.s, z0.h ; CHECK-NEXT: ucvtf z0.s, p0/m, z0.s ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 @@ -277,8 +277,8 @@ define <4 x half> @ucvtf_v4i32_v4f16(<4 x i32> %op1) { define <8 x half> @ucvtf_v8i32_v8f16(ptr %a) { ; CHECK-LABEL: ucvtf_v8i32_v8f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ucvtf z1.h, p0/m, z1.s ; CHECK-NEXT: ucvtf z0.h, p0/m, z0.s ; CHECK-NEXT: ptrue p0.h, vl4 @@ -295,8 +295,8 @@ define <8 x half> @ucvtf_v8i32_v8f16(ptr %a) { define void @ucvtf_v16i32_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: ucvtf_v16i32_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: ucvtf z1.h, p0/m, z1.s ; CHECK-NEXT: ucvtf z0.h, p0/m, z0.s @@ -348,8 +348,8 @@ define <4 x float> @ucvtf_v4i32_v4f32(<4 x i32> %op1) { define void @ucvtf_v8i32_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: ucvtf_v8i32_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ucvtf z0.s, p0/m, z0.s ; CHECK-NEXT: ucvtf z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x1] @@ -367,8 +367,8 @@ define void @ucvtf_v8i32_v8f32(ptr %a, ptr %b) { define <2 x double> @ucvtf_v2i32_v2f64(<2 x i32> %op1) { ; CHECK-LABEL: ucvtf_v2i32_v2f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: uunpklo z0.d, z0.s ; CHECK-NEXT: ucvtf z0.d, p0/m, z0.d ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 @@ -446,16 +446,16 @@ define <2 x half> @ucvtf_v2i64_v2f16(<2 x i64> %op1) { define <4 x half> @ucvtf_v4i64_v4f16(ptr %a) { ; CHECK-LABEL: ucvtf_v4i64_v4f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ucvtf z1.s, p0/m, z1.d ; CHECK-NEXT: ucvtf z0.s, p0/m, z0.d ; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: uzp1 z1.s, z1.s, z1.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s -; CHECK-NEXT: fcvt z0.h, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: fcvt z0.h, p0/m, z0.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 ; CHECK-NEXT: ret @@ -467,10 +467,9 @@ define <4 x half> @ucvtf_v4i64_v4f16(ptr %a) { define <8 x half> @ucvtf_v8i64_v8f16(ptr %a) { ; CHECK-LABEL: ucvtf_v8i64_v8f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0, #32] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q2, q3, [x0] -; CHECK-NEXT: ptrue p1.s ; CHECK-NEXT: ucvtf z0.s, p0/m, z0.d ; CHECK-NEXT: ucvtf z1.s, p0/m, z1.d ; CHECK-NEXT: ucvtf z3.s, p0/m, z3.d @@ -482,11 +481,12 @@ define <8 x half> @ucvtf_v8i64_v8f16(ptr %a) { ; CHECK-NEXT: uzp1 z2.s, z2.s, z2.s ; CHECK-NEXT: splice z1.s, p0, z1.s, z0.s ; CHECK-NEXT: splice z2.s, p0, z2.s, z3.s -; CHECK-NEXT: ptrue p0.h, vl4 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: movprfx z0, z1 -; CHECK-NEXT: fcvt z0.h, p1/m, z1.s +; CHECK-NEXT: fcvt z0.h, p0/m, z1.s ; CHECK-NEXT: movprfx z1, z2 -; CHECK-NEXT: fcvt z1.h, p1/m, z2.s +; CHECK-NEXT: fcvt z1.h, p0/m, z2.s +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z2.h, z0.h, z0.h ; CHECK-NEXT: uzp1 z0.h, z1.h, z1.h ; CHECK-NEXT: splice z0.h, p0, z0.h, z2.h @@ -517,8 +517,8 @@ define <2 x float> @ucvtf_v2i64_v2f32(<2 x i64> %op1) { define <4 x float> @ucvtf_v4i64_v4f32(ptr %a) { ; CHECK-LABEL: ucvtf_v4i64_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ucvtf z1.s, p0/m, z1.d ; CHECK-NEXT: ucvtf z0.s, p0/m, z0.d ; CHECK-NEXT: ptrue p0.s, vl2 @@ -535,8 +535,8 @@ define <4 x float> @ucvtf_v4i64_v4f32(ptr %a) { define void @ucvtf_v8i64_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: ucvtf_v8i64_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: ucvtf z1.s, p0/m, z1.d ; CHECK-NEXT: ucvtf z0.s, p0/m, z0.d @@ -576,8 +576,8 @@ define <2 x double> @ucvtf_v2i64_v2f64(<2 x i64> %op1) { define void @ucvtf_v4i64_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: ucvtf_v4i64_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ucvtf z0.d, p0/m, z0.d ; CHECK-NEXT: ucvtf z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x1] @@ -621,8 +621,8 @@ define void @scvtf_v8i16_v8f16(ptr %a, ptr %b) { define void @scvtf_v16i16_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: scvtf_v16i16_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: scvtf z0.h, p0/m, z0.h ; CHECK-NEXT: scvtf z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x1] @@ -652,8 +652,8 @@ define <2 x float> @scvtf_v2i16_v2f32(<2 x i16> %op1) { define <4 x float> @scvtf_v4i16_v4f32(<4 x i16> %op1) { ; CHECK-LABEL: scvtf_v4i16_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: sunpklo z0.s, z0.h ; CHECK-NEXT: scvtf z0.s, p0/m, z0.s ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 @@ -850,8 +850,8 @@ define <4 x half> @scvtf_v4i32_v4f16(<4 x i32> %op1) { define <8 x half> @scvtf_v8i32_v8f16(ptr %a) { ; CHECK-LABEL: scvtf_v8i32_v8f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: scvtf z1.h, p0/m, z1.s ; CHECK-NEXT: scvtf z0.h, p0/m, z0.s ; CHECK-NEXT: ptrue p0.h, vl4 @@ -896,8 +896,8 @@ define <4 x float> @scvtf_v4i32_v4f32(<4 x i32> %op1) { define void @scvtf_v8i32_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: scvtf_v8i32_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: scvtf z0.s, p0/m, z0.s ; CHECK-NEXT: scvtf z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x1] @@ -915,8 +915,8 @@ define void @scvtf_v8i32_v8f32(ptr %a, ptr %b) { define <2 x double> @scvtf_v2i32_v2f64(<2 x i32> %op1) { ; CHECK-LABEL: scvtf_v2i32_v2f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: sunpklo z0.d, z0.s ; CHECK-NEXT: scvtf z0.d, p0/m, z0.d ; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 @@ -1038,16 +1038,16 @@ define <2 x half> @scvtf_v2i64_v2f16(<2 x i64> %op1) { define <4 x half> @scvtf_v4i64_v4f16(ptr %a) { ; CHECK-LABEL: scvtf_v4i64_v4f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] -; CHECK-NEXT: ptrue p1.s +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: scvtf z1.s, p0/m, z1.d ; CHECK-NEXT: scvtf z0.s, p0/m, z0.d ; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: uzp1 z1.s, z1.s, z1.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s -; CHECK-NEXT: fcvt z0.h, p1/m, z0.s +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: fcvt z0.h, p0/m, z0.s ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 ; CHECK-NEXT: ret @@ -1076,8 +1076,8 @@ define <2 x float> @scvtf_v2i64_v2f32(<2 x i64> %op1) { define <4 x float> @scvtf_v4i64_v4f32(ptr %a) { ; CHECK-LABEL: scvtf_v4i64_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: scvtf z1.s, p0/m, z1.d ; CHECK-NEXT: scvtf z0.s, p0/m, z0.d ; CHECK-NEXT: ptrue p0.s, vl2 @@ -1110,8 +1110,8 @@ define <2 x double> @scvtf_v2i64_v2f64(<2 x i64> %op1) { define void @scvtf_v4i64_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: scvtf_v4i64_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: scvtf z0.d, p0/m, z0.d ; CHECK-NEXT: scvtf z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x1] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-vselect.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-vselect.ll index 1809cfcf3db6..81bbaa92d4b4 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-vselect.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-vselect.ll @@ -61,8 +61,8 @@ define <16 x i8> @select_v16i8(<16 x i8> %op1, <16 x i8> %op2, <16 x i1> %mask) define void @select_v32i8(ptr %a, ptr %b) { ; CHECK-LABEL: select_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q2, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q3, [x1] ; CHECK-NEXT: cmpeq p1.b, p0/z, z0.b, z1.b ; CHECK-NEXT: cmpeq p0.b, p0/z, z2.b, z3.b @@ -136,8 +136,8 @@ define <8 x i16> @select_v8i16(<8 x i16> %op1, <8 x i16> %op2, <8 x i1> %mask) { define void @select_v16i16(ptr %a, ptr %b) { ; CHECK-LABEL: select_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q2, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q3, [x1] ; CHECK-NEXT: cmpeq p1.h, p0/z, z0.h, z1.h ; CHECK-NEXT: cmpeq p0.h, p0/z, z2.h, z3.h @@ -193,8 +193,8 @@ define <4 x i32> @select_v4i32(<4 x i32> %op1, <4 x i32> %op2, <4 x i1> %mask) { define void @select_v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: select_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q2, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q3, [x1] ; CHECK-NEXT: cmpeq p1.s, p0/z, z0.s, z1.s ; CHECK-NEXT: cmpeq p0.s, p0/z, z2.s, z3.s @@ -213,9 +213,9 @@ define void @select_v8i32(ptr %a, ptr %b) { define <1 x i64> @select_v1i64(<1 x i64> %op1, <1 x i64> %op2, <1 x i1> %mask) { ; CHECK-LABEL: select_v1i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 ; CHECK-NEXT: and x8, x0, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0 ; CHECK-NEXT: // kill: def $d1 killed $d1 def $z1 ; CHECK-NEXT: mov z2.d, x8 @@ -249,8 +249,8 @@ define <2 x i64> @select_v2i64(<2 x i64> %op1, <2 x i64> %op2, <2 x i1> %mask) { define void @select_v4i64(ptr %a, ptr %b) { ; CHECK-LABEL: select_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q2, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q3, [x1] ; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d ; CHECK-NEXT: cmpeq p0.d, p0/z, z2.d, z3.d diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-log-reduce.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-log-reduce.ll index bb1bd8fe72b2..c4aeb4465c53 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-log-reduce.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-log-reduce.ll @@ -48,8 +48,8 @@ define i8 @andv_v16i8(<16 x i8> %a) { define i8 @andv_v32i8(ptr %a) { ; CHECK-LABEL: andv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: and z0.d, z1.d, z0.d ; CHECK-NEXT: andv b0, p0, z0.b ; CHECK-NEXT: fmov w0, s0 @@ -98,8 +98,8 @@ define i16 @andv_v8i16(<8 x i16> %a) { define i16 @andv_v16i16(ptr %a) { ; CHECK-LABEL: andv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: and z0.d, z1.d, z0.d ; CHECK-NEXT: andv h0, p0, z0.h ; CHECK-NEXT: fmov w0, s0 @@ -136,8 +136,8 @@ define i32 @andv_v4i32(<4 x i32> %a) { define i32 @andv_v8i32(ptr %a) { ; CHECK-LABEL: andv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: and z0.d, z1.d, z0.d ; CHECK-NEXT: andv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 @@ -162,8 +162,8 @@ define i64 @andv_v2i64(<2 x i64> %a) { define i64 @andv_v4i64(ptr %a) { ; CHECK-LABEL: andv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: and z0.d, z1.d, z0.d ; CHECK-NEXT: andv d0, p0, z0.d ; CHECK-NEXT: fmov x0, d0 @@ -216,8 +216,8 @@ define i8 @eorv_v16i8(<16 x i8> %a) { define i8 @eorv_v32i8(ptr %a) { ; CHECK-LABEL: eorv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: eor z0.d, z1.d, z0.d ; CHECK-NEXT: eorv b0, p0, z0.b ; CHECK-NEXT: fmov w0, s0 @@ -266,8 +266,8 @@ define i16 @eorv_v8i16(<8 x i16> %a) { define i16 @eorv_v16i16(ptr %a) { ; CHECK-LABEL: eorv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: eor z0.d, z1.d, z0.d ; CHECK-NEXT: eorv h0, p0, z0.h ; CHECK-NEXT: fmov w0, s0 @@ -304,8 +304,8 @@ define i32 @eorv_v4i32(<4 x i32> %a) { define i32 @eorv_v8i32(ptr %a) { ; CHECK-LABEL: eorv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: eor z0.d, z1.d, z0.d ; CHECK-NEXT: eorv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 @@ -330,8 +330,8 @@ define i64 @eorv_v2i64(<2 x i64> %a) { define i64 @eorv_v4i64(ptr %a) { ; CHECK-LABEL: eorv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: eor z0.d, z1.d, z0.d ; CHECK-NEXT: eorv d0, p0, z0.d ; CHECK-NEXT: fmov x0, d0 @@ -384,8 +384,8 @@ define i8 @orv_v16i8(<16 x i8> %a) { define i8 @orv_v32i8(ptr %a) { ; CHECK-LABEL: orv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: orr z0.d, z1.d, z0.d ; CHECK-NEXT: orv b0, p0, z0.b ; CHECK-NEXT: fmov w0, s0 @@ -434,8 +434,8 @@ define i16 @orv_v8i16(<8 x i16> %a) { define i16 @orv_v16i16(ptr %a) { ; CHECK-LABEL: orv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: orr z0.d, z1.d, z0.d ; CHECK-NEXT: orv h0, p0, z0.h ; CHECK-NEXT: fmov w0, s0 @@ -472,8 +472,8 @@ define i32 @orv_v4i32(<4 x i32> %a) { define i32 @orv_v8i32(ptr %a) { ; CHECK-LABEL: orv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: orr z0.d, z1.d, z0.d ; CHECK-NEXT: orv s0, p0, z0.s ; CHECK-NEXT: fmov w0, s0 @@ -498,8 +498,8 @@ define i64 @orv_v2i64(<2 x i64> %a) { define i64 @orv_v4i64(ptr %a) { ; CHECK-LABEL: orv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q0, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: orr z0.d, z1.d, z0.d ; CHECK-NEXT: orv d0, p0, z0.d ; CHECK-NEXT: fmov x0, d0 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll index e81270674474..f2b3f9b12ea7 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll @@ -123,8 +123,8 @@ define void @masked_store_v32i8(ptr %dst, <32 x i1> %mask) { ; CHECK-NEXT: asr z0.b, z0.b, #7 ; CHECK-NEXT: asr z1.b, z1.b, #7 ; CHECK-NEXT: cmpne p1.b, p0/z, z0.b, #0 -; CHECK-NEXT: mov z0.b, #0 // =0x0 ; CHECK-NEXT: cmpne p0.b, p0/z, z1.b, #0 +; CHECK-NEXT: mov z0.b, #0 // =0x0 ; CHECK-NEXT: st1b { z0.b }, p1, [x0, x8] ; CHECK-NEXT: st1b { z0.b }, p0, [x0] ; CHECK-NEXT: add sp, sp, #32 @@ -204,8 +204,8 @@ define void @masked_store_v16f16(ptr %dst, <16 x i1> %mask) { ; CHECK-NEXT: asr z0.h, z0.h, #15 ; CHECK-NEXT: asr z1.h, z1.h, #15 ; CHECK-NEXT: cmpne p1.h, p0/z, z1.h, #0 -; CHECK-NEXT: mov z1.h, #0 // =0x0 ; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; CHECK-NEXT: mov z1.h, #0 // =0x0 ; CHECK-NEXT: st1h { z1.h }, p1, [x0, x8, lsl #1] ; CHECK-NEXT: st1h { z1.h }, p0, [x0] ; CHECK-NEXT: ret @@ -242,7 +242,7 @@ define void @masked_store_v8f32(ptr %dst, <8 x i1> %mask) { ; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: fmov w8, s1 ; CHECK-NEXT: fmov w9, s2 -; CHECK-NEXT: mov z2.s, #0 // =0x0 +; CHECK-NEXT: mov z2.b, z0.b[3] ; CHECK-NEXT: strh w8, [sp, #14] ; CHECK-NEXT: fmov w8, s3 ; CHECK-NEXT: mov z3.b, z0.b[2] @@ -258,9 +258,9 @@ define void @masked_store_v8f32(ptr %dst, <8 x i1> %mask) { ; CHECK-NEXT: lsl z1.s, z1.s, #31 ; CHECK-NEXT: asr z1.s, z1.s, #31 ; CHECK-NEXT: cmpne p1.s, p0/z, z1.s, #0 -; CHECK-NEXT: mov z1.b, z0.b[3] -; CHECK-NEXT: st1w { z2.s }, p1, [x0, x8, lsl #2] -; CHECK-NEXT: fmov w8, s1 +; CHECK-NEXT: mov z1.s, #0 // =0x0 +; CHECK-NEXT: st1w { z1.s }, p1, [x0, x8, lsl #2] +; CHECK-NEXT: fmov w8, s2 ; CHECK-NEXT: strh w9, [sp] ; CHECK-NEXT: strh w8, [sp, #6] ; CHECK-NEXT: fmov w8, s3 @@ -272,7 +272,7 @@ define void @masked_store_v8f32(ptr %dst, <8 x i1> %mask) { ; CHECK-NEXT: lsl z0.s, z0.s, #31 ; CHECK-NEXT: asr z0.s, z0.s, #31 ; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 -; CHECK-NEXT: st1w { z2.s }, p0, [x0] +; CHECK-NEXT: st1w { z1.s }, p0, [x0] ; CHECK-NEXT: add sp, sp, #16 ; CHECK-NEXT: ret call void @llvm.masked.store.v8f32(<8 x float> zeroinitializer, ptr %dst, i32 8, <8 x i1> %mask) @@ -310,8 +310,8 @@ define void @masked_store_v4f64(ptr %dst, <4 x i1> %mask) { ; CHECK-NEXT: asr z1.d, z1.d, #63 ; CHECK-NEXT: asr z0.d, z0.d, #63 ; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0 -; CHECK-NEXT: mov z0.d, #0 // =0x0 ; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 +; CHECK-NEXT: mov z0.d, #0 // =0x0 ; CHECK-NEXT: st1d { z0.d }, p1, [x0, x8, lsl #3] ; CHECK-NEXT: st1d { z0.d }, p0, [x0] ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-optimize-ptrue.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-optimize-ptrue.ll index f0b0b3269e98..6fcb95f28333 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-optimize-ptrue.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-optimize-ptrue.ll @@ -170,8 +170,8 @@ define void @abs_v4i32(ptr %a) { define void @abs_v8i32(ptr %a) { ; CHECK-LABEL: abs_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: abs z0.s, p0/m, z0.s ; CHECK-NEXT: abs z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -199,8 +199,8 @@ define void @abs_v2i64(ptr %a) { define void @abs_v4i64(ptr %a) { ; CHECK-LABEL: abs_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: abs z0.d, p0/m, z0.d ; CHECK-NEXT: abs z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -263,8 +263,8 @@ define void @fadd_v8f16(ptr %a, ptr %b) { define void @fadd_v16f16(ptr %a, ptr %b) { ; CHECK-LABEL: fadd_v16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fadd z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: movprfx z1, z2 @@ -313,8 +313,8 @@ define void @fadd_v4f32(ptr %a, ptr %b) { define void @fadd_v8f32(ptr %a, ptr %b) { ; CHECK-LABEL: fadd_v8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fadd z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: movprfx z1, z2 @@ -347,8 +347,8 @@ define void @fadd_v2f64(ptr %a, ptr %b) { define void @fadd_v4f64(ptr %a, ptr %b) { ; CHECK-LABEL: fadd_v4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q3, [x1] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q1, q2, [x0] ; CHECK-NEXT: fadd z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: movprfx z1, z2 diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll index d1bff4fa21a1..00413302798c 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll @@ -9,8 +9,8 @@ target triple = "aarch64-unknown-linux-gnu" define void @test_revbv16i16(ptr %a) { ; CHECK-LABEL: test_revbv16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: revb z0.h, p0/m, z0.h ; CHECK-NEXT: revb z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -25,8 +25,8 @@ define void @test_revbv16i16(ptr %a) { define void @test_revbv8i32(ptr %a) { ; CHECK-LABEL: test_revbv8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: revb z0.s, p0/m, z0.s ; CHECK-NEXT: revb z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -41,8 +41,8 @@ define void @test_revbv8i32(ptr %a) { define void @test_revbv4i64(ptr %a) { ; CHECK-LABEL: test_revbv4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: revb z0.d, p0/m, z0.d ; CHECK-NEXT: revb z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -57,8 +57,8 @@ define void @test_revbv4i64(ptr %a) { define void @test_revhv8i32(ptr %a) { ; CHECK-LABEL: test_revhv8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: revh z0.s, p0/m, z0.s ; CHECK-NEXT: revh z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -73,8 +73,8 @@ define void @test_revhv8i32(ptr %a) { define void @test_revhv8f32(ptr %a) { ; CHECK-LABEL: test_revhv8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: revh z0.s, p0/m, z0.s ; CHECK-NEXT: revh z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -89,8 +89,8 @@ define void @test_revhv8f32(ptr %a) { define void @test_revhv4i64(ptr %a) { ; CHECK-LABEL: test_revhv4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: revh z0.d, p0/m, z0.d ; CHECK-NEXT: revh z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -105,8 +105,8 @@ define void @test_revhv4i64(ptr %a) { define void @test_revwv4i64(ptr %a) { ; CHECK-LABEL: test_revwv4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: revw z0.d, p0/m, z0.d ; CHECK-NEXT: revw z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -121,8 +121,8 @@ define void @test_revwv4i64(ptr %a) { define void @test_revwv4f64(ptr %a) { ; CHECK-LABEL: test_revwv4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: revw z0.d, p0/m, z0.d ; CHECK-NEXT: revw z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -150,8 +150,8 @@ define <16 x i8> @test_revv16i8(ptr %a) { define void @test_revwv8i32v8i32(ptr %a, ptr %b) { ; CHECK-LABEL: test_revwv8i32v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldp q0, q1, [x1] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: revw z0.d, p0/m, z0.d ; CHECK-NEXT: revw z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -166,8 +166,8 @@ define void @test_revwv8i32v8i32(ptr %a, ptr %b) { define void @test_revhv32i16(ptr %a) { ; CHECK-LABEL: test_revhv32i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: revh z0.d, p0/m, z0.d ; CHECK-NEXT: revh z1.d, p0/m, z1.d @@ -202,8 +202,8 @@ define void @test_rev_elts_fail(ptr %a) { define void @test_revdv4i64_sve2p1(ptr %a) #1 { ; CHECK-LABEL: test_revdv4i64_sve2p1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: revd z0.q, p0/m, z0.q ; CHECK-NEXT: revd z1.q, p0/m, z1.q ; CHECK-NEXT: stp q0, q1, [x0] @@ -217,8 +217,8 @@ define void @test_revdv4i64_sve2p1(ptr %a) #1 { define void @test_revdv4f64_sve2p1(ptr %a) #1 { ; CHECK-LABEL: test_revdv4f64_sve2p1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: revd z0.q, p0/m, z0.q ; CHECK-NEXT: revd z1.q, p0/m, z1.q ; CHECK-NEXT: stp q0, q1, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll index d7bfb6b2680e..cb73030306b0 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-zip-uzp-trn.ll @@ -82,11 +82,11 @@ define void @zip_v32i16(ptr %a, ptr %b) { ; CHECK-NEXT: .cfi_def_cfa_offset 64 ; CHECK-NEXT: ldp q1, q3, [x1] ; CHECK-NEXT: ldp q0, q4, [x0] -; CHECK-NEXT: ldp q2, q6, [x0, #32] +; CHECK-NEXT: ldp q2, q5, [x0, #32] ; CHECK-NEXT: mov z16.h, z3.h[7] ; CHECK-NEXT: mov z18.h, z3.h[6] ; CHECK-NEXT: mov z17.h, z4.h[7] -; CHECK-NEXT: ldp q5, q7, [x1, #32] +; CHECK-NEXT: ldp q6, q7, [x1, #32] ; CHECK-NEXT: mov z19.h, z4.h[6] ; CHECK-NEXT: fmov w8, s16 ; CHECK-NEXT: mov z16.h, z3.h[5] @@ -98,13 +98,13 @@ define void @zip_v32i16(ptr %a, ptr %b) { ; CHECK-NEXT: mov z18.h, z3.h[4] ; CHECK-NEXT: strh w9, [sp, #28] ; CHECK-NEXT: fmov w9, s19 -; CHECK-NEXT: mov z19.h, z6.h[7] +; CHECK-NEXT: mov z19.h, z5.h[7] ; CHECK-NEXT: zip1 z3.h, z4.h, z3.h ; CHECK-NEXT: strh w8, [sp, #26] ; CHECK-NEXT: fmov w8, s16 ; CHECK-NEXT: mov z16.h, z4.h[4] ; CHECK-NEXT: strh w9, [sp, #24] -; CHECK-NEXT: zip1 z4.h, z6.h, z7.h +; CHECK-NEXT: zip1 z4.h, z5.h, z7.h ; CHECK-NEXT: strh w8, [sp, #22] ; CHECK-NEXT: fmov w8, s17 ; CHECK-NEXT: mov z17.h, z1.h[7] @@ -131,7 +131,7 @@ define void @zip_v32i16(ptr %a, ptr %b) { ; CHECK-NEXT: fmov w8, s18 ; CHECK-NEXT: mov z18.h, z0.h[4] ; CHECK-NEXT: zip1 z0.h, z0.h, z1.h -; CHECK-NEXT: zip1 z1.h, z2.h, z5.h +; CHECK-NEXT: zip1 z1.h, z2.h, z6.h ; CHECK-NEXT: strh w8, [sp, #54] ; CHECK-NEXT: fmov w8, s16 ; CHECK-NEXT: ldr q16, [sp, #16] @@ -143,41 +143,41 @@ define void @zip_v32i16(ptr %a, ptr %b) { ; CHECK-NEXT: mov z18.h, z7.h[7] ; CHECK-NEXT: strh w8, [sp, #48] ; CHECK-NEXT: fmov w8, s18 -; CHECK-NEXT: mov z18.h, z6.h[6] +; CHECK-NEXT: mov z18.h, z5.h[6] ; CHECK-NEXT: ldr q17, [sp, #48] ; CHECK-NEXT: strh w8, [sp, #46] ; CHECK-NEXT: fmov w8, s19 ; CHECK-NEXT: mov z19.h, z7.h[5] ; CHECK-NEXT: strh w8, [sp, #44] ; CHECK-NEXT: fmov w8, s20 -; CHECK-NEXT: mov z20.h, z6.h[5] +; CHECK-NEXT: mov z20.h, z5.h[5] ; CHECK-NEXT: strh w8, [sp, #42] ; CHECK-NEXT: fmov w8, s18 ; CHECK-NEXT: mov z18.h, z7.h[4] ; CHECK-NEXT: strh w8, [sp, #40] ; CHECK-NEXT: fmov w8, s19 -; CHECK-NEXT: mov z19.h, z6.h[4] +; CHECK-NEXT: mov z19.h, z5.h[4] ; CHECK-NEXT: strh w8, [sp, #38] ; CHECK-NEXT: fmov w8, s20 -; CHECK-NEXT: mov z20.h, z5.h[7] +; CHECK-NEXT: mov z20.h, z6.h[7] ; CHECK-NEXT: strh w8, [sp, #36] ; CHECK-NEXT: fmov w8, s18 ; CHECK-NEXT: mov z18.h, z2.h[7] ; CHECK-NEXT: strh w8, [sp, #34] ; CHECK-NEXT: fmov w8, s19 -; CHECK-NEXT: mov z19.h, z5.h[6] +; CHECK-NEXT: mov z19.h, z6.h[6] ; CHECK-NEXT: strh w8, [sp, #32] ; CHECK-NEXT: fmov w8, s20 ; CHECK-NEXT: mov z20.h, z2.h[6] ; CHECK-NEXT: strh w8, [sp, #14] ; CHECK-NEXT: fmov w8, s18 -; CHECK-NEXT: mov z18.h, z5.h[5] +; CHECK-NEXT: mov z18.h, z6.h[5] ; CHECK-NEXT: strh w8, [sp, #12] ; CHECK-NEXT: fmov w8, s19 ; CHECK-NEXT: mov z19.h, z2.h[5] ; CHECK-NEXT: strh w8, [sp, #10] ; CHECK-NEXT: fmov w8, s20 -; CHECK-NEXT: mov z20.h, z5.h[4] +; CHECK-NEXT: mov z20.h, z6.h[4] ; CHECK-NEXT: fmov w9, s19 ; CHECK-NEXT: strh w8, [sp, #8] ; CHECK-NEXT: fmov w8, s18 @@ -186,10 +186,10 @@ define void @zip_v32i16(ptr %a, ptr %b) { ; CHECK-NEXT: ldr q2, [sp, #32] ; CHECK-NEXT: strh w8, [sp, #6] ; CHECK-NEXT: fmov w8, s20 +; CHECK-NEXT: fmov w9, s18 ; CHECK-NEXT: add z2.h, z16.h, z2.h ; CHECK-NEXT: strh w8, [sp, #2] -; CHECK-NEXT: fmov w8, s18 -; CHECK-NEXT: strh w8, [sp] +; CHECK-NEXT: strh w9, [sp] ; CHECK-NEXT: ldr q4, [sp] ; CHECK-NEXT: stp q3, q2, [x0, #32] ; CHECK-NEXT: add z1.h, z17.h, z4.h @@ -471,9 +471,9 @@ define void @trn_v4f64(ptr %a, ptr %b) { define void @trn_v4f32(ptr %a, ptr %b) { ; CHECK-LABEL: trn_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldr q0, [x0] ; CHECK-NEXT: ldr q1, [x1] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: trn1 z2.s, z0.s, z1.s ; CHECK-NEXT: trn2 z0.s, z0.s, z1.s ; CHECK-NEXT: fadd z0.s, p0/m, z0.s, z2.s diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll index f2ba4a7cc356..ab7c42b3e9e3 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll @@ -8,8 +8,8 @@ target triple = "aarch64-unknown-linux-gnu" define i1 @ptest_v16i1(ptr %a, ptr %b) { ; CHECK-LABEL: ptest_v16i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q1, q0, [x0, #32] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q2, q3, [x0] ; CHECK-NEXT: fcmne p1.s, p0/z, z0.s, #0.0 ; CHECK-NEXT: fcmne p2.s, p0/z, z1.s, #0.0 @@ -20,7 +20,6 @@ define i1 @ptest_v16i1(ptr %a, ptr %b) { ; CHECK-NEXT: mov z2.s, p3/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z3.s, p0/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: ptrue p1.b, vl16 ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h @@ -31,7 +30,8 @@ define i1 @ptest_v16i1(ptr %a, ptr %b) { ; CHECK-NEXT: uzp1 z0.b, z1.b, z1.b ; CHECK-NEXT: uzp1 z1.b, z3.b, z3.b ; CHECK-NEXT: splice z1.b, p0, z1.b, z0.b -; CHECK-NEXT: umaxv b0, p1, z1.b +; CHECK-NEXT: ptrue p0.b, vl16 +; CHECK-NEXT: umaxv b0, p0, z1.b ; CHECK-NEXT: fmov w8, s0 ; CHECK-NEXT: and w0, w8, #0x1 ; CHECK-NEXT: ret @@ -45,41 +45,41 @@ define i1 @ptest_v16i1(ptr %a, ptr %b) { define i1 @ptest_or_v16i1(ptr %a, ptr %b) { ; CHECK-LABEL: ptest_or_v16i1: ; CHECK: // %bb.0: +; CHECK-NEXT: ldp q1, q0, [x0, #32] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ldp q0, q1, [x0] -; CHECK-NEXT: ldp q2, q3, [x0, #32] -; CHECK-NEXT: ldp q4, q5, [x1] -; CHECK-NEXT: ldp q6, q7, [x1, #32] -; CHECK-NEXT: fcmne p1.s, p0/z, z3.s, #0.0 -; CHECK-NEXT: fcmne p2.s, p0/z, z2.s, #0.0 -; CHECK-NEXT: fcmne p3.s, p0/z, z1.s, #0.0 -; CHECK-NEXT: fcmne p4.s, p0/z, z0.s, #0.0 -; CHECK-NEXT: fcmne p5.s, p0/z, z7.s, #0.0 -; CHECK-NEXT: fcmne p6.s, p0/z, z6.s, #0.0 -; CHECK-NEXT: fcmne p7.s, p0/z, z5.s, #0.0 -; CHECK-NEXT: fcmne p0.s, p0/z, z4.s, #0.0 +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: ldp q4, q5, [x1, #32] +; CHECK-NEXT: fcmne p1.s, p0/z, z0.s, #0.0 +; CHECK-NEXT: fcmne p2.s, p0/z, z1.s, #0.0 +; CHECK-NEXT: ldp q0, q1, [x1] +; CHECK-NEXT: fcmne p3.s, p0/z, z3.s, #0.0 +; CHECK-NEXT: fcmne p4.s, p0/z, z2.s, #0.0 +; CHECK-NEXT: fcmne p5.s, p0/z, z5.s, #0.0 +; CHECK-NEXT: fcmne p6.s, p0/z, z4.s, #0.0 +; CHECK-NEXT: fcmne p7.s, p0/z, z1.s, #0.0 +; CHECK-NEXT: fcmne p0.s, p0/z, z0.s, #0.0 ; CHECK-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z1.s, p2/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z2.s, p3/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z3.s, p4/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z4.s, p5/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z5.s, p6/z, #-1 // =0xffffffffffffffff +; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: mov z6.s, p7/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z7.s, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: ptrue p1.h, vl4 -; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h ; CHECK-NEXT: uzp1 z5.h, z5.h, z5.h +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h ; CHECK-NEXT: uzp1 z7.h, z7.h, z7.h +; CHECK-NEXT: splice z1.h, p0, z1.h, z0.h +; CHECK-NEXT: splice z3.h, p0, z3.h, z2.h +; CHECK-NEXT: splice z5.h, p0, z5.h, z4.h +; CHECK-NEXT: splice z7.h, p0, z7.h, z6.h ; CHECK-NEXT: ptrue p0.b, vl8 -; CHECK-NEXT: splice z1.h, p1, z1.h, z0.h -; CHECK-NEXT: splice z3.h, p1, z3.h, z2.h -; CHECK-NEXT: splice z5.h, p1, z5.h, z4.h -; CHECK-NEXT: splice z7.h, p1, z7.h, z6.h ; CHECK-NEXT: uzp1 z0.b, z1.b, z1.b ; CHECK-NEXT: uzp1 z1.b, z3.b, z3.b ; CHECK-NEXT: uzp1 z2.b, z5.b, z5.b @@ -112,41 +112,41 @@ declare i1 @llvm.vector.reduce.or.i1.v16i1(<16 x i1>) define i1 @ptest_and_v16i1(ptr %a, ptr %b) { ; CHECK-LABEL: ptest_and_v16i1: ; CHECK: // %bb.0: +; CHECK-NEXT: ldp q1, q0, [x0, #32] ; CHECK-NEXT: ptrue p0.s, vl4 -; CHECK-NEXT: ldp q0, q1, [x0] -; CHECK-NEXT: ldp q2, q3, [x0, #32] -; CHECK-NEXT: ldp q4, q5, [x1] -; CHECK-NEXT: ldp q6, q7, [x1, #32] -; CHECK-NEXT: fcmne p1.s, p0/z, z3.s, #0.0 -; CHECK-NEXT: fcmne p2.s, p0/z, z2.s, #0.0 -; CHECK-NEXT: fcmne p3.s, p0/z, z1.s, #0.0 -; CHECK-NEXT: fcmne p4.s, p0/z, z0.s, #0.0 -; CHECK-NEXT: fcmne p5.s, p0/z, z7.s, #0.0 -; CHECK-NEXT: fcmne p6.s, p0/z, z6.s, #0.0 -; CHECK-NEXT: fcmne p7.s, p0/z, z5.s, #0.0 -; CHECK-NEXT: fcmne p0.s, p0/z, z4.s, #0.0 +; CHECK-NEXT: ldp q2, q3, [x0] +; CHECK-NEXT: ldp q4, q5, [x1, #32] +; CHECK-NEXT: fcmne p1.s, p0/z, z0.s, #0.0 +; CHECK-NEXT: fcmne p2.s, p0/z, z1.s, #0.0 +; CHECK-NEXT: ldp q0, q1, [x1] +; CHECK-NEXT: fcmne p3.s, p0/z, z3.s, #0.0 +; CHECK-NEXT: fcmne p4.s, p0/z, z2.s, #0.0 +; CHECK-NEXT: fcmne p5.s, p0/z, z5.s, #0.0 +; CHECK-NEXT: fcmne p6.s, p0/z, z4.s, #0.0 +; CHECK-NEXT: fcmne p7.s, p0/z, z1.s, #0.0 +; CHECK-NEXT: fcmne p0.s, p0/z, z0.s, #0.0 ; CHECK-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z1.s, p2/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z2.s, p3/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z3.s, p4/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z4.s, p5/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z5.s, p6/z, #-1 // =0xffffffffffffffff +; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: mov z6.s, p7/z, #-1 // =0xffffffffffffffff ; CHECK-NEXT: mov z7.s, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: ptrue p1.h, vl4 -; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h ; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h ; CHECK-NEXT: uzp1 z5.h, z5.h, z5.h +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h ; CHECK-NEXT: uzp1 z7.h, z7.h, z7.h +; CHECK-NEXT: splice z1.h, p0, z1.h, z0.h +; CHECK-NEXT: splice z3.h, p0, z3.h, z2.h +; CHECK-NEXT: splice z5.h, p0, z5.h, z4.h +; CHECK-NEXT: splice z7.h, p0, z7.h, z6.h ; CHECK-NEXT: ptrue p0.b, vl8 -; CHECK-NEXT: splice z1.h, p1, z1.h, z0.h -; CHECK-NEXT: splice z3.h, p1, z3.h, z2.h -; CHECK-NEXT: splice z5.h, p1, z5.h, z4.h -; CHECK-NEXT: splice z7.h, p1, z7.h, z6.h ; CHECK-NEXT: uzp1 z0.b, z1.b, z1.b ; CHECK-NEXT: uzp1 z1.b, z3.b, z3.b ; CHECK-NEXT: uzp1 z2.b, z5.b, z5.b diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-rev.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-rev.ll index f686efff67b6..bfa931044bc5 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-rev.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-rev.ll @@ -49,8 +49,8 @@ define <16 x i8> @bitreverse_v16i8(<16 x i8> %op) { define void @bitreverse_v32i8(ptr %a) { ; CHECK-LABEL: bitreverse_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: rbit z0.b, p0/m, z0.b ; CHECK-NEXT: rbit z1.b, p0/m, z1.b ; CHECK-NEXT: stp q0, q1, [x0] @@ -101,8 +101,8 @@ define <8 x i16> @bitreverse_v8i16(<8 x i16> %op) { define void @bitreverse_v16i16(ptr %a) { ; CHECK-LABEL: bitreverse_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: rbit z0.h, p0/m, z0.h ; CHECK-NEXT: rbit z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -140,8 +140,8 @@ define <4 x i32> @bitreverse_v4i32(<4 x i32> %op) { define void @bitreverse_v8i32(ptr %a) { ; CHECK-LABEL: bitreverse_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: rbit z0.s, p0/m, z0.s ; CHECK-NEXT: rbit z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -179,8 +179,8 @@ define <2 x i64> @bitreverse_v2i64(<2 x i64> %op) { define void @bitreverse_v4i64(ptr %a) { ; CHECK-LABEL: bitreverse_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: rbit z0.d, p0/m, z0.d ; CHECK-NEXT: rbit z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] @@ -235,8 +235,8 @@ define <8 x i16> @bswap_v8i16(<8 x i16> %op) { define void @bswap_v16i16(ptr %a) { ; CHECK-LABEL: bswap_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: revb z0.h, p0/m, z0.h ; CHECK-NEXT: revb z1.h, p0/m, z1.h ; CHECK-NEXT: stp q0, q1, [x0] @@ -274,8 +274,8 @@ define <4 x i32> @bswap_v4i32(<4 x i32> %op) { define void @bswap_v8i32(ptr %a) { ; CHECK-LABEL: bswap_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: revb z0.s, p0/m, z0.s ; CHECK-NEXT: revb z1.s, p0/m, z1.s ; CHECK-NEXT: stp q0, q1, [x0] @@ -313,8 +313,8 @@ define <2 x i64> @bswap_v2i64(<2 x i64> %op) { define void @bswap_v4i64(ptr %a) { ; CHECK-LABEL: bswap_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: revb z0.d, p0/m, z0.d ; CHECK-NEXT: revb z1.d, p0/m, z1.d ; CHECK-NEXT: stp q0, q1, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-sdiv-pow2.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-sdiv-pow2.ll index 76bb465774d5..9dd42e7831e0 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-sdiv-pow2.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-sdiv-pow2.ll @@ -45,8 +45,8 @@ define <16 x i8> @sdiv_v16i8(<16 x i8> %op1) { define void @sdiv_v32i8(ptr %a) { ; CHECK-LABEL: sdiv_v32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.b, vl16 ; CHECK-NEXT: asrd z0.b, p0/m, z0.b, #5 ; CHECK-NEXT: asrd z1.b, p0/m, z1.b, #5 ; CHECK-NEXT: stp q0, q1, [x0] @@ -97,8 +97,8 @@ define <8 x i16> @sdiv_v8i16(<8 x i16> %op1) { define void @sdiv_v16i16(ptr %a) { ; CHECK-LABEL: sdiv_v16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.h, vl8 ; CHECK-NEXT: asrd z0.h, p0/m, z0.h, #5 ; CHECK-NEXT: asrd z1.h, p0/m, z1.h, #5 ; CHECK-NEXT: stp q0, q1, [x0] @@ -136,8 +136,8 @@ define <4 x i32> @sdiv_v4i32(<4 x i32> %op1) { define void @sdiv_v8i32(ptr %a) { ; CHECK-LABEL: sdiv_v8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: asrd z0.s, p0/m, z0.s, #5 ; CHECK-NEXT: asrd z1.s, p0/m, z1.s, #5 ; CHECK-NEXT: stp q0, q1, [x0] @@ -176,8 +176,8 @@ define <2 x i64> @sdiv_v2i64(<2 x i64> %op1) { define void @sdiv_v4i64(ptr %a) { ; CHECK-LABEL: sdiv_v4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: ldp q0, q1, [x0] +; CHECK-NEXT: ptrue p0.d, vl2 ; CHECK-NEXT: asrd z0.d, p0/m, z0.d, #5 ; CHECK-NEXT: asrd z1.d, p0/m, z1.d, #5 ; CHECK-NEXT: stp q0, q1, [x0] diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll index ff1f8699b91a..ad0d4ef0afef 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-shuffle.ll @@ -37,9 +37,9 @@ define void @interleave_store_without_splat(ptr %a, <4 x i32> %v1, <4 x i32> %v2 define void @interleave_store_legalization(ptr %a, <8 x i32> %v1, <8 x i32> %v2) { ; CHECK-LABEL: interleave_store_legalization: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: mov z5.d, z2.d ; CHECK-NEXT: // kill: def $q3 killed $q3 def $z2_z3 +; CHECK-NEXT: ptrue p0.s, vl4 ; CHECK-NEXT: mov x8, #8 // =0x8 ; CHECK-NEXT: mov z4.d, z0.d ; CHECK-NEXT: mov z2.d, z1.d diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-stores.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-stores.ll index 367ccbeeea81..06709ca3685c 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-stores.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-stores.ll @@ -8,8 +8,8 @@ target triple = "aarch64-unknown-linux-gnu" define void @store_v4i8(ptr %a) { ; CHECK-LABEL: store_v4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: mov z0.h, #0 // =0x0 +; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: st1b { z0.h }, p0, [x0] ; CHECK-NEXT: ret store <4 x i8> zeroinitializer, ptr %a @@ -49,8 +49,8 @@ define void @store_v32i8(ptr %a) { define void @store_v2i16(ptr %a) { ; CHECK-LABEL: store_v2i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: mov z0.s, #0 // =0x0 +; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: st1h { z0.s }, p0, [x0] ; CHECK-NEXT: ret store <2 x i16> zeroinitializer, ptr %a diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll index 4fef67831401..70219dd30f76 100644 --- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll +++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc.ll @@ -87,51 +87,51 @@ define void @trunc_v64i16_v64i8(ptr %in, ptr %out) nounwind { define void @trunc_v128i16_v128i8(ptr %in, ptr %out) nounwind { ; CHECK-LABEL: trunc_v128i16_v128i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ldp q0, q1, [x0, #192] +; CHECK-NEXT: ldp q2, q3, [x0, #192] ; CHECK-NEXT: ptrue p0.b, vl8 -; CHECK-NEXT: ldp q6, q7, [x0, #224] -; CHECK-NEXT: ldp q2, q3, [x0, #32] -; CHECK-NEXT: uzp1 z1.b, z1.b, z1.b -; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b -; CHECK-NEXT: uzp1 z7.b, z7.b, z7.b -; CHECK-NEXT: uzp1 z6.b, z6.b, z6.b -; CHECK-NEXT: ldp q4, q5, [x0] +; CHECK-NEXT: ldp q6, q7, [x0, #64] +; CHECK-NEXT: ldp q16, q17, [x0, #224] ; CHECK-NEXT: uzp1 z3.b, z3.b, z3.b -; CHECK-NEXT: ldp q16, q17, [x0, #64] ; CHECK-NEXT: uzp1 z2.b, z2.b, z2.b -; CHECK-NEXT: ldp q18, q19, [x0, #128] -; CHECK-NEXT: splice z0.b, p0, z0.b, z1.b ; CHECK-NEXT: ldp q20, q21, [x0, #160] -; CHECK-NEXT: splice z6.b, p0, z6.b, z7.b -; CHECK-NEXT: ldp q22, q23, [x0, #96] -; CHECK-NEXT: uzp1 z1.b, z17.b, z17.b -; CHECK-NEXT: uzp1 z19.b, z19.b, z19.b -; CHECK-NEXT: uzp1 z18.b, z18.b, z18.b +; CHECK-NEXT: uzp1 z7.b, z7.b, z7.b +; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: uzp1 z17.b, z17.b, z17.b +; CHECK-NEXT: ldp q4, q5, [x0, #96] ; CHECK-NEXT: uzp1 z16.b, z16.b, z16.b -; CHECK-NEXT: uzp1 z21.b, z21.b, z21.b +; CHECK-NEXT: ldp q18, q19, [x0, #128] +; CHECK-NEXT: splice z2.b, p0, z2.b, z3.b +; CHECK-NEXT: uzp1 z3.b, z21.b, z21.b ; CHECK-NEXT: uzp1 z20.b, z20.b, z20.b -; CHECK-NEXT: uzp1 z5.b, z5.b, z5.b -; CHECK-NEXT: uzp1 z7.b, z23.b, z23.b -; CHECK-NEXT: uzp1 z17.b, z22.b, z22.b +; CHECK-NEXT: uzp1 z6.b, z6.b, z6.b +; CHECK-NEXT: ldp q21, q22, [x0] +; CHECK-NEXT: splice z16.b, p0, z16.b, z17.b +; CHECK-NEXT: uzp1 z19.b, z19.b, z19.b +; CHECK-NEXT: uzp1 z18.b, z18.b, z18.b ; CHECK-NEXT: uzp1 z4.b, z4.b, z4.b -; CHECK-NEXT: splice z2.b, p0, z2.b, z3.b -; CHECK-NEXT: add z0.b, z0.b, z0.b +; CHECK-NEXT: splice z20.b, p0, z20.b, z3.b +; CHECK-NEXT: uzp1 z3.b, z5.b, z5.b +; CHECK-NEXT: splice z6.b, p0, z6.b, z7.b +; CHECK-NEXT: uzp1 z5.b, z22.b, z22.b +; CHECK-NEXT: uzp1 z7.b, z21.b, z21.b +; CHECK-NEXT: uzp1 z1.b, z1.b, z1.b +; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b ; CHECK-NEXT: splice z18.b, p0, z18.b, z19.b -; CHECK-NEXT: splice z16.b, p0, z16.b, z1.b -; CHECK-NEXT: add z1.b, z6.b, z6.b -; CHECK-NEXT: splice z20.b, p0, z20.b, z21.b -; CHECK-NEXT: splice z17.b, p0, z17.b, z7.b -; CHECK-NEXT: splice z4.b, p0, z4.b, z5.b -; CHECK-NEXT: stp q0, q1, [x1, #96] ; CHECK-NEXT: add z2.b, z2.b, z2.b +; CHECK-NEXT: splice z4.b, p0, z4.b, z3.b +; CHECK-NEXT: add z3.b, z16.b, z16.b +; CHECK-NEXT: splice z7.b, p0, z7.b, z5.b +; CHECK-NEXT: splice z0.b, p0, z0.b, z1.b +; CHECK-NEXT: add z1.b, z20.b, z20.b ; CHECK-NEXT: add z5.b, z18.b, z18.b -; CHECK-NEXT: add z0.b, z16.b, z16.b -; CHECK-NEXT: add z3.b, z20.b, z20.b -; CHECK-NEXT: add z1.b, z17.b, z17.b -; CHECK-NEXT: add z4.b, z4.b, z4.b -; CHECK-NEXT: stp q5, q3, [x1, #64] -; CHECK-NEXT: stp q4, q2, [x1] -; CHECK-NEXT: stp q0, q1, [x1, #32] +; CHECK-NEXT: stp q2, q3, [x1, #96] +; CHECK-NEXT: add z2.b, z6.b, z6.b +; CHECK-NEXT: add z3.b, z4.b, z4.b +; CHECK-NEXT: add z4.b, z7.b, z7.b +; CHECK-NEXT: add z0.b, z0.b, z0.b +; CHECK-NEXT: stp q5, q1, [x1, #64] +; CHECK-NEXT: stp q2, q3, [x1, #32] +; CHECK-NEXT: stp q4, q0, [x1] ; CHECK-NEXT: ret %a = load <128 x i16>, ptr %in %b = trunc <128 x i16> %a to <128 x i8> @@ -226,55 +226,55 @@ define void @trunc_v32i32_v32i8(ptr %in, ptr %out) nounwind { define void @trunc_v64i32_v64i8(ptr %in, ptr %out) nounwind { ; CHECK-LABEL: trunc_v64i32_v64i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ldp q0, q1, [x0, #64] -; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: ldp q2, q3, [x0, #160] -; CHECK-NEXT: ptrue p1.b, vl8 -; CHECK-NEXT: ldp q4, q5, [x0, #96] -; CHECK-NEXT: ldp q6, q7, [x0] -; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h -; CHECK-NEXT: ldp q16, q17, [x0, #128] +; CHECK-NEXT: ptrue p0.h, vl4 +; CHECK-NEXT: ldp q4, q5, [x0, #128] +; CHECK-NEXT: ldp q0, q1, [x0, #64] +; CHECK-NEXT: ldp q6, q7, [x0, #96] ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h -; CHECK-NEXT: ldp q18, q19, [x0, #192] ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h -; CHECK-NEXT: ldp q20, q21, [x0, #224] +; CHECK-NEXT: uzp1 z5.h, z5.h, z5.h +; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h +; CHECK-NEXT: ldp q16, q17, [x0] +; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK-NEXT: ldp q18, q19, [x0, #192] ; CHECK-NEXT: uzp1 z7.h, z7.h, z7.h +; CHECK-NEXT: ldp q20, q21, [x0, #224] +; CHECK-NEXT: splice z2.h, p0, z2.h, z3.h ; CHECK-NEXT: ldp q22, q23, [x0, #32] -; CHECK-NEXT: uzp1 z17.h, z17.h, z17.h -; CHECK-NEXT: uzp1 z16.h, z16.h, z16.h +; CHECK-NEXT: splice z4.h, p0, z4.h, z5.h ; CHECK-NEXT: uzp1 z19.h, z19.h, z19.h ; CHECK-NEXT: uzp1 z18.h, z18.h, z18.h -; CHECK-NEXT: uzp1 z21.h, z21.h, z21.h -; CHECK-NEXT: uzp1 z20.h, z20.h, z20.h +; CHECK-NEXT: uzp1 z17.h, z17.h, z17.h +; CHECK-NEXT: uzp1 z3.h, z21.h, z21.h +; CHECK-NEXT: uzp1 z5.h, z20.h, z20.h +; CHECK-NEXT: uzp1 z16.h, z16.h, z16.h +; CHECK-NEXT: uzp1 z20.h, z23.h, z23.h +; CHECK-NEXT: uzp1 z21.h, z22.h, z22.h ; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h -; CHECK-NEXT: uzp1 z23.h, z23.h, z23.h -; CHECK-NEXT: uzp1 z22.h, z22.h, z22.h -; CHECK-NEXT: uzp1 z5.h, z5.h, z5.h -; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h -; CHECK-NEXT: splice z2.h, p0, z2.h, z3.h -; CHECK-NEXT: splice z16.h, p0, z16.h, z17.h ; CHECK-NEXT: splice z18.h, p0, z18.h, z19.h -; CHECK-NEXT: splice z20.h, p0, z20.h, z21.h +; CHECK-NEXT: splice z5.h, p0, z5.h, z3.h +; CHECK-NEXT: splice z16.h, p0, z16.h, z17.h +; CHECK-NEXT: splice z21.h, p0, z21.h, z20.h ; CHECK-NEXT: splice z6.h, p0, z6.h, z7.h -; CHECK-NEXT: splice z22.h, p0, z22.h, z23.h -; CHECK-NEXT: splice z4.h, p0, z4.h, z5.h ; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h ; CHECK-NEXT: uzp1 z1.b, z2.b, z2.b -; CHECK-NEXT: uzp1 z2.b, z16.b, z16.b -; CHECK-NEXT: uzp1 z5.b, z18.b, z18.b -; CHECK-NEXT: uzp1 z3.b, z20.b, z20.b -; CHECK-NEXT: uzp1 z6.b, z6.b, z6.b -; CHECK-NEXT: uzp1 z7.b, z22.b, z22.b -; CHECK-NEXT: uzp1 z4.b, z4.b, z4.b +; CHECK-NEXT: uzp1 z2.b, z4.b, z4.b +; CHECK-NEXT: ptrue p0.b, vl8 +; CHECK-NEXT: uzp1 z4.b, z18.b, z18.b +; CHECK-NEXT: uzp1 z3.b, z5.b, z5.b +; CHECK-NEXT: uzp1 z7.b, z16.b, z16.b +; CHECK-NEXT: uzp1 z5.b, z21.b, z21.b +; CHECK-NEXT: splice z2.b, p0, z2.b, z1.b +; CHECK-NEXT: uzp1 z1.b, z6.b, z6.b ; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b -; CHECK-NEXT: splice z2.b, p1, z2.b, z1.b -; CHECK-NEXT: splice z5.b, p1, z5.b, z3.b -; CHECK-NEXT: splice z6.b, p1, z6.b, z7.b -; CHECK-NEXT: splice z0.b, p1, z0.b, z4.b +; CHECK-NEXT: splice z4.b, p0, z4.b, z3.b +; CHECK-NEXT: splice z7.b, p0, z7.b, z5.b +; CHECK-NEXT: splice z0.b, p0, z0.b, z1.b ; CHECK-NEXT: add z1.b, z2.b, z2.b -; CHECK-NEXT: add z2.b, z5.b, z5.b -; CHECK-NEXT: add z3.b, z6.b, z6.b +; CHECK-NEXT: add z2.b, z4.b, z4.b +; CHECK-NEXT: add z3.b, z7.b, z7.b ; CHECK-NEXT: add z0.b, z0.b, z0.b ; CHECK-NEXT: stp q1, q2, [x1, #32] ; CHECK-NEXT: stp q3, q0, [x1] @@ -368,51 +368,51 @@ define void @trunc_v32i32_v32i16(ptr %in, ptr %out) nounwind { define void @trunc_v64i32_v64i16(ptr %in, ptr %out) nounwind { ; CHECK-LABEL: trunc_v64i32_v64i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ldp q0, q1, [x0, #192] +; CHECK-NEXT: ldp q2, q3, [x0, #192] ; CHECK-NEXT: ptrue p0.h, vl4 -; CHECK-NEXT: ldp q6, q7, [x0, #224] -; CHECK-NEXT: ldp q2, q3, [x0, #32] -; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h -; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h -; CHECK-NEXT: uzp1 z7.h, z7.h, z7.h -; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h -; CHECK-NEXT: ldp q4, q5, [x0] +; CHECK-NEXT: ldp q6, q7, [x0, #64] +; CHECK-NEXT: ldp q16, q17, [x0, #224] ; CHECK-NEXT: uzp1 z3.h, z3.h, z3.h -; CHECK-NEXT: ldp q16, q17, [x0, #64] ; CHECK-NEXT: uzp1 z2.h, z2.h, z2.h -; CHECK-NEXT: ldp q18, q19, [x0, #128] -; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h ; CHECK-NEXT: ldp q20, q21, [x0, #160] -; CHECK-NEXT: splice z6.h, p0, z6.h, z7.h -; CHECK-NEXT: ldp q22, q23, [x0, #96] -; CHECK-NEXT: uzp1 z1.h, z17.h, z17.h -; CHECK-NEXT: uzp1 z19.h, z19.h, z19.h -; CHECK-NEXT: uzp1 z18.h, z18.h, z18.h +; CHECK-NEXT: uzp1 z7.h, z7.h, z7.h +; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: uzp1 z17.h, z17.h, z17.h +; CHECK-NEXT: ldp q4, q5, [x0, #96] ; CHECK-NEXT: uzp1 z16.h, z16.h, z16.h -; CHECK-NEXT: uzp1 z21.h, z21.h, z21.h +; CHECK-NEXT: ldp q18, q19, [x0, #128] +; CHECK-NEXT: splice z2.h, p0, z2.h, z3.h +; CHECK-NEXT: uzp1 z3.h, z21.h, z21.h ; CHECK-NEXT: uzp1 z20.h, z20.h, z20.h -; CHECK-NEXT: uzp1 z5.h, z5.h, z5.h -; CHECK-NEXT: uzp1 z7.h, z23.h, z23.h -; CHECK-NEXT: uzp1 z17.h, z22.h, z22.h +; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h +; CHECK-NEXT: ldp q21, q22, [x0] +; CHECK-NEXT: splice z16.h, p0, z16.h, z17.h +; CHECK-NEXT: uzp1 z19.h, z19.h, z19.h +; CHECK-NEXT: uzp1 z18.h, z18.h, z18.h ; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h -; CHECK-NEXT: splice z2.h, p0, z2.h, z3.h -; CHECK-NEXT: add z0.h, z0.h, z0.h +; CHECK-NEXT: splice z20.h, p0, z20.h, z3.h +; CHECK-NEXT: uzp1 z3.h, z5.h, z5.h +; CHECK-NEXT: splice z6.h, p0, z6.h, z7.h +; CHECK-NEXT: uzp1 z5.h, z22.h, z22.h +; CHECK-NEXT: uzp1 z7.h, z21.h, z21.h +; CHECK-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: splice z18.h, p0, z18.h, z19.h -; CHECK-NEXT: splice z16.h, p0, z16.h, z1.h -; CHECK-NEXT: add z1.h, z6.h, z6.h -; CHECK-NEXT: splice z20.h, p0, z20.h, z21.h -; CHECK-NEXT: splice z17.h, p0, z17.h, z7.h -; CHECK-NEXT: splice z4.h, p0, z4.h, z5.h -; CHECK-NEXT: stp q0, q1, [x1, #96] ; CHECK-NEXT: add z2.h, z2.h, z2.h +; CHECK-NEXT: splice z4.h, p0, z4.h, z3.h +; CHECK-NEXT: add z3.h, z16.h, z16.h +; CHECK-NEXT: splice z7.h, p0, z7.h, z5.h +; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h +; CHECK-NEXT: add z1.h, z20.h, z20.h ; CHECK-NEXT: add z5.h, z18.h, z18.h -; CHECK-NEXT: add z0.h, z16.h, z16.h -; CHECK-NEXT: add z3.h, z20.h, z20.h -; CHECK-NEXT: add z1.h, z17.h, z17.h -; CHECK-NEXT: add z4.h, z4.h, z4.h -; CHECK-NEXT: stp q5, q3, [x1, #64] -; CHECK-NEXT: stp q4, q2, [x1] -; CHECK-NEXT: stp q0, q1, [x1, #32] +; CHECK-NEXT: stp q2, q3, [x1, #96] +; CHECK-NEXT: add z2.h, z6.h, z6.h +; CHECK-NEXT: add z3.h, z4.h, z4.h +; CHECK-NEXT: add z4.h, z7.h, z7.h +; CHECK-NEXT: add z0.h, z0.h, z0.h +; CHECK-NEXT: stp q5, q1, [x1, #64] +; CHECK-NEXT: stp q2, q3, [x1, #32] +; CHECK-NEXT: stp q4, q0, [x1] ; CHECK-NEXT: ret %a = load <64 x i32>, ptr %in %b = trunc <64 x i32> %a to <64 x i16> @@ -535,19 +535,19 @@ define void @trunc_v32i64_v32i8(ptr %in, ptr %out) nounwind { ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: splice z2.s, p0, z2.s, z3.s ; CHECK-NEXT: splice z16.s, p0, z16.s, z17.s -; CHECK-NEXT: splice z18.s, p0, z18.s, z19.s ; CHECK-NEXT: splice z20.s, p0, z20.s, z21.s -; CHECK-NEXT: splice z6.s, p0, z6.s, z7.s +; CHECK-NEXT: splice z18.s, p0, z18.s, z19.s ; CHECK-NEXT: splice z22.s, p0, z22.s, z23.s +; CHECK-NEXT: splice z6.s, p0, z6.s, z7.s ; CHECK-NEXT: splice z4.s, p0, z4.s, z5.s ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s ; CHECK-NEXT: ptrue p0.h, vl4 ; CHECK-NEXT: uzp1 z1.h, z2.h, z2.h ; CHECK-NEXT: uzp1 z2.h, z16.h, z16.h -; CHECK-NEXT: uzp1 z5.h, z18.h, z18.h ; CHECK-NEXT: uzp1 z3.h, z20.h, z20.h -; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h +; CHECK-NEXT: uzp1 z5.h, z18.h, z18.h ; CHECK-NEXT: uzp1 z7.h, z22.h, z22.h +; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h ; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h ; CHECK-NEXT: splice z2.h, p0, z2.h, z1.h @@ -658,55 +658,55 @@ define void @trunc_v16i64_v16i16(ptr %in, ptr %out) nounwind { define void @trunc_v32i64_v32i16(ptr %in, ptr %out) nounwind { ; CHECK-LABEL: trunc_v32i64_v32i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ldp q0, q1, [x0, #64] -; CHECK-NEXT: ptrue p0.s, vl2 ; CHECK-NEXT: ldp q2, q3, [x0, #160] -; CHECK-NEXT: ptrue p1.h, vl4 -; CHECK-NEXT: ldp q4, q5, [x0, #96] -; CHECK-NEXT: ldp q6, q7, [x0] -; CHECK-NEXT: uzp1 z1.s, z1.s, z1.s -; CHECK-NEXT: ldp q16, q17, [x0, #128] +; CHECK-NEXT: ptrue p0.s, vl2 +; CHECK-NEXT: ldp q4, q5, [x0, #128] +; CHECK-NEXT: ldp q0, q1, [x0, #64] +; CHECK-NEXT: ldp q6, q7, [x0, #96] ; CHECK-NEXT: uzp1 z3.s, z3.s, z3.s -; CHECK-NEXT: ldp q18, q19, [x0, #192] ; CHECK-NEXT: uzp1 z2.s, z2.s, z2.s -; CHECK-NEXT: ldp q20, q21, [x0, #224] +; CHECK-NEXT: uzp1 z5.s, z5.s, z5.s +; CHECK-NEXT: uzp1 z4.s, z4.s, z4.s +; CHECK-NEXT: ldp q16, q17, [x0] +; CHECK-NEXT: uzp1 z1.s, z1.s, z1.s +; CHECK-NEXT: ldp q18, q19, [x0, #192] ; CHECK-NEXT: uzp1 z7.s, z7.s, z7.s +; CHECK-NEXT: ldp q20, q21, [x0, #224] +; CHECK-NEXT: splice z2.s, p0, z2.s, z3.s ; CHECK-NEXT: ldp q22, q23, [x0, #32] -; CHECK-NEXT: uzp1 z17.s, z17.s, z17.s -; CHECK-NEXT: uzp1 z16.s, z16.s, z16.s +; CHECK-NEXT: splice z4.s, p0, z4.s, z5.s ; CHECK-NEXT: uzp1 z19.s, z19.s, z19.s ; CHECK-NEXT: uzp1 z18.s, z18.s, z18.s -; CHECK-NEXT: uzp1 z21.s, z21.s, z21.s -; CHECK-NEXT: uzp1 z20.s, z20.s, z20.s +; CHECK-NEXT: uzp1 z17.s, z17.s, z17.s +; CHECK-NEXT: uzp1 z3.s, z21.s, z21.s +; CHECK-NEXT: uzp1 z5.s, z20.s, z20.s +; CHECK-NEXT: uzp1 z16.s, z16.s, z16.s +; CHECK-NEXT: uzp1 z20.s, z23.s, z23.s +; CHECK-NEXT: uzp1 z21.s, z22.s, z22.s ; CHECK-NEXT: uzp1 z6.s, z6.s, z6.s -; CHECK-NEXT: uzp1 z23.s, z23.s, z23.s -; CHECK-NEXT: uzp1 z22.s, z22.s, z22.s -; CHECK-NEXT: uzp1 z5.s, z5.s, z5.s -; CHECK-NEXT: uzp1 z4.s, z4.s, z4.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s -; CHECK-NEXT: splice z2.s, p0, z2.s, z3.s -; CHECK-NEXT: splice z16.s, p0, z16.s, z17.s ; CHECK-NEXT: splice z18.s, p0, z18.s, z19.s -; CHECK-NEXT: splice z20.s, p0, z20.s, z21.s +; CHECK-NEXT: splice z5.s, p0, z5.s, z3.s +; CHECK-NEXT: splice z16.s, p0, z16.s, z17.s +; CHECK-NEXT: splice z21.s, p0, z21.s, z20.s ; CHECK-NEXT: splice z6.s, p0, z6.s, z7.s -; CHECK-NEXT: splice z22.s, p0, z22.s, z23.s -; CHECK-NEXT: splice z4.s, p0, z4.s, z5.s ; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s ; CHECK-NEXT: uzp1 z1.h, z2.h, z2.h -; CHECK-NEXT: uzp1 z2.h, z16.h, z16.h -; CHECK-NEXT: uzp1 z5.h, z18.h, z18.h -; CHECK-NEXT: uzp1 z3.h, z20.h, z20.h -; CHECK-NEXT: uzp1 z6.h, z6.h, z6.h -; CHECK-NEXT: uzp1 z7.h, z22.h, z22.h -; CHECK-NEXT: uzp1 z4.h, z4.h, z4.h +; CHECK-NEXT: uzp1 z2.h, z4.h, z4.h +; CHECK-NEXT: ptrue p0.h, vl4 +; CHECK-NEXT: uzp1 z4.h, z18.h, z18.h +; CHECK-NEXT: uzp1 z3.h, z5.h, z5.h +; CHECK-NEXT: uzp1 z7.h, z16.h, z16.h +; CHECK-NEXT: uzp1 z5.h, z21.h, z21.h +; CHECK-NEXT: splice z2.h, p0, z2.h, z1.h +; CHECK-NEXT: uzp1 z1.h, z6.h, z6.h ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h -; CHECK-NEXT: splice z2.h, p1, z2.h, z1.h -; CHECK-NEXT: splice z5.h, p1, z5.h, z3.h -; CHECK-NEXT: splice z6.h, p1, z6.h, z7.h -; CHECK-NEXT: splice z0.h, p1, z0.h, z4.h +; CHECK-NEXT: splice z4.h, p0, z4.h, z3.h +; CHECK-NEXT: splice z7.h, p0, z7.h, z5.h +; CHECK-NEXT: splice z0.h, p0, z0.h, z1.h ; CHECK-NEXT: add z1.h, z2.h, z2.h -; CHECK-NEXT: add z2.h, z5.h, z5.h -; CHECK-NEXT: add z3.h, z6.h, z6.h +; CHECK-NEXT: add z2.h, z4.h, z4.h +; CHECK-NEXT: add z3.h, z7.h, z7.h ; CHECK-NEXT: add z0.h, z0.h, z0.h ; CHECK-NEXT: stp q1, q2, [x1, #32] ; CHECK-NEXT: stp q3, q0, [x1] @@ -800,51 +800,51 @@ define void @trunc_v16i64_v16i32(ptr %in, ptr %out) nounwind { define void @trunc_v32i64_v32i32(ptr %in, ptr %out) nounwind { ; CHECK-LABEL: trunc_v32i64_v32i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ldp q0, q1, [x0, #192] +; CHECK-NEXT: ldp q2, q3, [x0, #192] ; CHECK-NEXT: ptrue p0.s, vl2 -; CHECK-NEXT: ldp q6, q7, [x0, #224] -; CHECK-NEXT: ldp q2, q3, [x0, #32] -; CHECK-NEXT: uzp1 z1.s, z1.s, z1.s -; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s -; CHECK-NEXT: uzp1 z7.s, z7.s, z7.s -; CHECK-NEXT: uzp1 z6.s, z6.s, z6.s -; CHECK-NEXT: ldp q4, q5, [x0] +; CHECK-NEXT: ldp q6, q7, [x0, #64] +; CHECK-NEXT: ldp q16, q17, [x0, #224] ; CHECK-NEXT: uzp1 z3.s, z3.s, z3.s -; CHECK-NEXT: ldp q16, q17, [x0, #64] ; CHECK-NEXT: uzp1 z2.s, z2.s, z2.s -; CHECK-NEXT: ldp q18, q19, [x0, #128] -; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s ; CHECK-NEXT: ldp q20, q21, [x0, #160] -; CHECK-NEXT: splice z6.s, p0, z6.s, z7.s -; CHECK-NEXT: ldp q22, q23, [x0, #96] -; CHECK-NEXT: uzp1 z1.s, z17.s, z17.s -; CHECK-NEXT: uzp1 z19.s, z19.s, z19.s -; CHECK-NEXT: uzp1 z18.s, z18.s, z18.s +; CHECK-NEXT: uzp1 z7.s, z7.s, z7.s +; CHECK-NEXT: ldp q0, q1, [x0, #32] +; CHECK-NEXT: uzp1 z17.s, z17.s, z17.s +; CHECK-NEXT: ldp q4, q5, [x0, #96] ; CHECK-NEXT: uzp1 z16.s, z16.s, z16.s -; CHECK-NEXT: uzp1 z21.s, z21.s, z21.s +; CHECK-NEXT: ldp q18, q19, [x0, #128] +; CHECK-NEXT: splice z2.s, p0, z2.s, z3.s +; CHECK-NEXT: uzp1 z3.s, z21.s, z21.s ; CHECK-NEXT: uzp1 z20.s, z20.s, z20.s -; CHECK-NEXT: uzp1 z5.s, z5.s, z5.s -; CHECK-NEXT: uzp1 z7.s, z23.s, z23.s -; CHECK-NEXT: uzp1 z17.s, z22.s, z22.s +; CHECK-NEXT: uzp1 z6.s, z6.s, z6.s +; CHECK-NEXT: ldp q21, q22, [x0] +; CHECK-NEXT: splice z16.s, p0, z16.s, z17.s +; CHECK-NEXT: uzp1 z19.s, z19.s, z19.s +; CHECK-NEXT: uzp1 z18.s, z18.s, z18.s ; CHECK-NEXT: uzp1 z4.s, z4.s, z4.s -; CHECK-NEXT: splice z2.s, p0, z2.s, z3.s -; CHECK-NEXT: add z0.s, z0.s, z0.s +; CHECK-NEXT: splice z20.s, p0, z20.s, z3.s +; CHECK-NEXT: uzp1 z3.s, z5.s, z5.s +; CHECK-NEXT: splice z6.s, p0, z6.s, z7.s +; CHECK-NEXT: uzp1 z5.s, z22.s, z22.s +; CHECK-NEXT: uzp1 z7.s, z21.s, z21.s +; CHECK-NEXT: uzp1 z1.s, z1.s, z1.s +; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s ; CHECK-NEXT: splice z18.s, p0, z18.s, z19.s -; CHECK-NEXT: splice z16.s, p0, z16.s, z1.s -; CHECK-NEXT: add z1.s, z6.s, z6.s -; CHECK-NEXT: splice z20.s, p0, z20.s, z21.s -; CHECK-NEXT: splice z17.s, p0, z17.s, z7.s -; CHECK-NEXT: splice z4.s, p0, z4.s, z5.s -; CHECK-NEXT: stp q0, q1, [x1, #96] ; CHECK-NEXT: add z2.s, z2.s, z2.s +; CHECK-NEXT: splice z4.s, p0, z4.s, z3.s +; CHECK-NEXT: add z3.s, z16.s, z16.s +; CHECK-NEXT: splice z7.s, p0, z7.s, z5.s +; CHECK-NEXT: splice z0.s, p0, z0.s, z1.s +; CHECK-NEXT: add z1.s, z20.s, z20.s ; CHECK-NEXT: add z5.s, z18.s, z18.s -; CHECK-NEXT: add z0.s, z16.s, z16.s -; CHECK-NEXT: add z3.s, z20.s, z20.s -; CHECK-NEXT: add z1.s, z17.s, z17.s -; CHECK-NEXT: add z4.s, z4.s, z4.s -; CHECK-NEXT: stp q5, q3, [x1, #64] -; CHECK-NEXT: stp q4, q2, [x1] -; CHECK-NEXT: stp q0, q1, [x1, #32] +; CHECK-NEXT: stp q2, q3, [x1, #96] +; CHECK-NEXT: add z2.s, z6.s, z6.s +; CHECK-NEXT: add z3.s, z4.s, z4.s +; CHECK-NEXT: add z4.s, z7.s, z7.s +; CHECK-NEXT: add z0.s, z0.s, z0.s +; CHECK-NEXT: stp q5, q1, [x1, #64] +; CHECK-NEXT: stp q2, q3, [x1, #32] +; CHECK-NEXT: stp q4, q0, [x1] ; CHECK-NEXT: ret %a = load <32 x i64>, ptr %in %b = trunc <32 x i64> %a to <32 x i32> diff --git a/llvm/test/CodeGen/AArch64/sve-trunc.ll b/llvm/test/CodeGen/AArch64/sve-trunc.ll index 92dfc7396136..0ec6538947c7 100644 --- a/llvm/test/CodeGen/AArch64/sve-trunc.ll +++ b/llvm/test/CodeGen/AArch64/sve-trunc.ll @@ -61,8 +61,8 @@ entry: define @trunc_i64toi1( %in) { ; CHECK-LABEL: trunc_i64toi1: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z0.d, z0.d, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: ret entry: @@ -73,9 +73,9 @@ entry: define @trunc_i64toi1_split( %in) { ; CHECK-LABEL: trunc_i64toi1_split: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z1.d, z1.d, #0x1 ; CHECK-NEXT: and z0.d, z0.d, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cmpne p1.d, p0/z, z1.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 ; CHECK-NEXT: uzp1 p0.s, p0.s, p1.s @@ -88,11 +88,11 @@ entry: define @trunc_i64toi1_split2( %in) { ; CHECK-LABEL: trunc_i64toi1_split2: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z3.d, z3.d, #0x1 ; CHECK-NEXT: and z2.d, z2.d, #0x1 ; CHECK-NEXT: and z1.d, z1.d, #0x1 ; CHECK-NEXT: and z0.d, z0.d, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cmpne p1.d, p0/z, z3.d, #0 ; CHECK-NEXT: cmpne p2.d, p0/z, z2.d, #0 ; CHECK-NEXT: cmpne p3.d, p0/z, z1.d, #0 @@ -111,12 +111,12 @@ define @trunc_i64toi1_split3( %in) { ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: str p7, [sp, #4, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p6, [sp, #5, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p5, [sp, #6, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG ; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z7.d, z7.d, #0x1 ; CHECK-NEXT: and z6.d, z6.d, #0x1 ; CHECK-NEXT: and z5.d, z5.d, #0x1 @@ -125,23 +125,25 @@ define @trunc_i64toi1_split3( %in) { ; CHECK-NEXT: and z2.d, z2.d, #0x1 ; CHECK-NEXT: and z1.d, z1.d, #0x1 ; CHECK-NEXT: and z0.d, z0.d, #0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: cmpne p1.d, p0/z, z7.d, #0 ; CHECK-NEXT: cmpne p2.d, p0/z, z6.d, #0 ; CHECK-NEXT: cmpne p3.d, p0/z, z5.d, #0 ; CHECK-NEXT: cmpne p4.d, p0/z, z4.d, #0 ; CHECK-NEXT: cmpne p5.d, p0/z, z3.d, #0 ; CHECK-NEXT: cmpne p6.d, p0/z, z2.d, #0 -; CHECK-NEXT: uzp1 p1.s, p2.s, p1.s -; CHECK-NEXT: cmpne p2.d, p0/z, z1.d, #0 +; CHECK-NEXT: cmpne p7.d, p0/z, z1.d, #0 ; CHECK-NEXT: cmpne p0.d, p0/z, z0.d, #0 -; CHECK-NEXT: uzp1 p3.s, p4.s, p3.s -; CHECK-NEXT: uzp1 p4.s, p6.s, p5.s +; CHECK-NEXT: uzp1 p1.s, p2.s, p1.s +; CHECK-NEXT: uzp1 p2.s, p4.s, p3.s +; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: uzp1 p3.s, p6.s, p5.s ; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p0.s, p0.s, p2.s +; CHECK-NEXT: uzp1 p0.s, p0.s, p7.s +; CHECK-NEXT: ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: uzp1 p1.h, p2.h, p1.h ; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p1.h, p3.h, p1.h -; CHECK-NEXT: uzp1 p0.h, p0.h, p4.h -; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload +; CHECK-NEXT: uzp1 p0.h, p0.h, p3.h ; CHECK-NEXT: uzp1 p0.b, p0.b, p1.b ; CHECK-NEXT: addvl sp, sp, #1 ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload @@ -155,8 +157,8 @@ entry: define @trunc_i32toi1( %in) { ; CHECK-LABEL: trunc_i32toi1: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z0.s, z0.s, #0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 ; CHECK-NEXT: ret entry: @@ -167,8 +169,8 @@ entry: define @trunc_i16toi1( %in) { ; CHECK-LABEL: trunc_i16toi1: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and z0.h, z0.h, #0x1 +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 ; CHECK-NEXT: ret entry: @@ -179,8 +181,8 @@ entry: define @trunc_i8toi1( %in) { ; CHECK-LABEL: trunc_i8toi1: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: and z0.b, z0.b, #0x1 +; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: cmpne p0.b, p0/z, z0.b, #0 ; CHECK-NEXT: ret entry: @@ -191,8 +193,8 @@ entry: define @trunc_nxv1i32_to_nxv1i1( %in) { ; CHECK-LABEL: trunc_nxv1i32_to_nxv1i1: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z0.s, z0.s, #0x1 +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: cmpne p0.s, p0/z, z0.s, #0 ; CHECK-NEXT: punpklo p0.h, p0.b ; CHECK-NEXT: punpklo p0.h, p0.b @@ -204,8 +206,8 @@ define @trunc_nxv1i32_to_nxv1i1( %in) { define void @trunc_promoteIntRes( %0, ptr %ptr) { ; CHECK-LABEL: trunc_promoteIntRes: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z1.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: st1h { z0.s }, p0, [x0] ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/sve-umulo-sdnode.ll b/llvm/test/CodeGen/AArch64/sve-umulo-sdnode.ll index 36d64725742e..818f37c85ffd 100644 --- a/llvm/test/CodeGen/AArch64/sve-umulo-sdnode.ll +++ b/llvm/test/CodeGen/AArch64/sve-umulo-sdnode.ll @@ -6,9 +6,9 @@ declare { , } @llvm.umul.with.overflow.nxv2i8 define @umulo_nxv2i8( %x, %y) { ; CHECK-LABEL: umulo_nxv2i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z1.d, z1.d, #0xff ; CHECK-NEXT: and z0.d, z0.d, #0xff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: lsr z1.d, z0.d, #8 ; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 @@ -26,9 +26,9 @@ declare { , } @llvm.umul.with.overflow.nxv4i8 define @umulo_nxv4i8( %x, %y) { ; CHECK-LABEL: umulo_nxv4i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z1.s, z1.s, #0xff ; CHECK-NEXT: and z0.s, z0.s, #0xff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: lsr z1.s, z0.s, #8 ; CHECK-NEXT: cmpne p0.s, p0/z, z1.s, #0 @@ -46,9 +46,9 @@ declare { , } @llvm.umul.with.overflow.nxv8i8 define @umulo_nxv8i8( %x, %y) { ; CHECK-LABEL: umulo_nxv8i8: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: and z1.h, z1.h, #0xff ; CHECK-NEXT: and z0.h, z0.h, #0xff +; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: mul z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: lsr z1.h, z0.h, #8 ; CHECK-NEXT: cmpne p0.h, p0/z, z1.h, #0 @@ -119,9 +119,9 @@ define @umulo_nxv64i8( %x, , } @llvm.umul.with.overflow.nxv2i define @umulo_nxv2i16( %x, %y) { ; CHECK-LABEL: umulo_nxv2i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z1.d, z1.d, #0xffff ; CHECK-NEXT: and z0.d, z0.d, #0xffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: lsr z1.d, z0.d, #16 ; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 @@ -163,9 +163,9 @@ declare { , } @llvm.umul.with.overflow.nxv4i define @umulo_nxv4i16( %x, %y) { ; CHECK-LABEL: umulo_nxv4i16: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: and z1.s, z1.s, #0xffff ; CHECK-NEXT: and z0.s, z0.s, #0xffff +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: mul z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: lsr z1.s, z0.s, #16 ; CHECK-NEXT: cmpne p0.s, p0/z, z1.s, #0 @@ -236,9 +236,9 @@ define @umulo_nxv32i16( %x, , } @llvm.umul.with.overflow.nxv2i define @umulo_nxv2i32( %x, %y) { ; CHECK-LABEL: umulo_nxv2i32: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: and z1.d, z1.d, #0xffffffff ; CHECK-NEXT: and z0.d, z0.d, #0xffffffff +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: lsr z1.d, z0.d, #32 ; CHECK-NEXT: cmpne p0.d, p0/z, z1.d, #0 @@ -333,9 +333,9 @@ define @umulo_nxv16i32( %x, @umulo_nxv8i64( %x, %a, ptr %b) #0 { define void @uzp1_i8_invalid( %a, ptr %b) #0 { ; CHECK-LABEL: uzp1_i8_invalid: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b, vl128 ; CHECK-NEXT: uzp1 z0.b, z0.b, z0.b +; CHECK-NEXT: ptrue p0.b, vl128 ; CHECK-NEXT: st1b { z0.b }, p0, [x0] ; CHECK-NEXT: ret %a.bc = bitcast %a to @@ -141,8 +141,8 @@ define void @uzp1_i16_valid( %a, ptr %b) #0 { define void @uzp1_i16_invalid( %a, ptr %b) #0 { ; CHECK-LABEL: uzp1_i16_invalid: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-NEXT: ptrue p0.h, vl64 ; CHECK-NEXT: st1h { z0.h }, p0, [x0] ; CHECK-NEXT: ret %a.bc = bitcast %a to @@ -168,8 +168,8 @@ define void @uzp1_i32_valid( %a, ptr %b) #0 { define void @uzp1_i32_invalid( %a, ptr %b) #0 { ; CHECK-LABEL: uzp1_i32_invalid: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-NEXT: ptrue p0.s, vl32 ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret %a.bc = bitcast %a to @@ -182,8 +182,8 @@ define void @uzp1_i32_invalid( %a, ptr %b) #0 { define void @uzp1_invalid_all( %a, ptr %b) #0 { ; CHECK-LABEL: uzp1_invalid_all: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: st1w { z0.s }, p0, [x0] ; CHECK-NEXT: ret %a.bc = bitcast %a to diff --git a/llvm/test/CodeGen/AArch64/sve-vecreduce-dot.ll b/llvm/test/CodeGen/AArch64/sve-vecreduce-dot.ll index 194d1071301d..91f8f5c2c90d 100644 --- a/llvm/test/CodeGen/AArch64/sve-vecreduce-dot.ll +++ b/llvm/test/CodeGen/AArch64/sve-vecreduce-dot.ll @@ -8,11 +8,11 @@ define i32 @test( %bin.rdx, %bin.rdx2) { ; CHECK-NEXT: sunpklo z5.h, z0.b ; CHECK-NEXT: sunpkhi z0.h, z0.b ; CHECK-NEXT: sunpkhi z2.h, z2.b -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpklo z6.h, z1.b ; CHECK-NEXT: sunpkhi z1.h, z1.b ; CHECK-NEXT: sunpklo z7.h, z3.b ; CHECK-NEXT: sunpkhi z3.h, z3.b +; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: sunpkhi z24.s, z5.h ; CHECK-NEXT: sunpklo z5.s, z5.h ; CHECK-NEXT: sunpklo z25.s, z4.h diff --git a/llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll b/llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll index 898090340869..0bdaefdfc2a3 100644 --- a/llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll +++ b/llvm/test/CodeGen/AArch64/sve-vecreduce-fold.ll @@ -80,8 +80,8 @@ define i1 @reduce_and_insert_subvec_into_var( %in, @test_copysign_v4f32_v4f64( %a, ; CHECK_NO_EXTEND_ROUND-LABEL: test_copysign_v4f32_v4f64: ; CHECK_NO_EXTEND_ROUND: // %bb.0: ; CHECK_NO_EXTEND_ROUND-NEXT: ptrue p0.d +; CHECK_NO_EXTEND_ROUND-NEXT: mov z3.s, #0x7fffffff ; CHECK_NO_EXTEND_ROUND-NEXT: fcvt z2.s, p0/m, z2.d ; CHECK_NO_EXTEND_ROUND-NEXT: fcvt z1.s, p0/m, z1.d ; CHECK_NO_EXTEND_ROUND-NEXT: uzp1 z1.s, z1.s, z2.s -; CHECK_NO_EXTEND_ROUND-NEXT: mov z2.s, #0x7fffffff -; CHECK_NO_EXTEND_ROUND-NEXT: bsl z0.d, z0.d, z1.d, z2.d +; CHECK_NO_EXTEND_ROUND-NEXT: bsl z0.d, z0.d, z1.d, z3.d ; CHECK_NO_EXTEND_ROUND-NEXT: ret ; ; CHECK_EXTEND_ROUND-LABEL: test_copysign_v4f32_v4f64: @@ -107,9 +107,9 @@ declare @llvm.copysign.v2f64( %a, @test_copysign_v4f64_v4f32( %a, %b) #0 { ; CHECK_NO_EXTEND_ROUND-LABEL: test_copysign_v4f64_v4f32: ; CHECK_NO_EXTEND_ROUND: // %bb.0: -; CHECK_NO_EXTEND_ROUND-NEXT: ptrue p0.d ; CHECK_NO_EXTEND_ROUND-NEXT: uunpkhi z3.d, z2.s ; CHECK_NO_EXTEND_ROUND-NEXT: uunpklo z2.d, z2.s +; CHECK_NO_EXTEND_ROUND-NEXT: ptrue p0.d ; CHECK_NO_EXTEND_ROUND-NEXT: mov z4.d, #0x7fffffffffffffff ; CHECK_NO_EXTEND_ROUND-NEXT: fcvt z3.d, p0/m, z3.s ; CHECK_NO_EXTEND_ROUND-NEXT: fcvt z2.d, p0/m, z2.s @@ -119,9 +119,9 @@ define @test_copysign_v4f64_v4f32( %a ; ; CHECK_EXTEND_ROUND-LABEL: test_copysign_v4f64_v4f32: ; CHECK_EXTEND_ROUND: // %bb.0: -; CHECK_EXTEND_ROUND-NEXT: ptrue p0.d ; CHECK_EXTEND_ROUND-NEXT: uunpkhi z3.d, z2.s ; CHECK_EXTEND_ROUND-NEXT: uunpklo z2.d, z2.s +; CHECK_EXTEND_ROUND-NEXT: ptrue p0.d ; CHECK_EXTEND_ROUND-NEXT: mov z4.d, #0x7fffffffffffffff ; CHECK_EXTEND_ROUND-NEXT: fcvt z2.d, p0/m, z2.s ; CHECK_EXTEND_ROUND-NEXT: fcvt z3.d, p0/m, z3.s @@ -176,11 +176,11 @@ define @test_copysign_v4f16_v4f64( %a, @test_copysign_v8f16_v8f32( %a, %arg1){ ; CHECK-LABEL: wide_add_shift_add_rshrnb_b: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.b ; CHECK-NEXT: rshrnb z1.b, z1.h, #6 ; CHECK-NEXT: rshrnb z0.b, z0.h, #6 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: ld1b { z2.b }, p0/z, [x0, x1] ; CHECK-NEXT: uzp1 z0.b, z0.b, z1.b -; CHECK-NEXT: ld1b { z1.b }, p0/z, [x0, x1] -; CHECK-NEXT: add z0.b, z1.b, z0.b +; CHECK-NEXT: add z0.b, z2.b, z0.b ; CHECK-NEXT: st1b { z0.b }, p0, [x0, x1] ; CHECK-NEXT: ret %1 = add %arg1, splat (i16 32) @@ -141,12 +141,12 @@ define void @wide_add_shift_add_rshrnb_b(ptr %dest, i64 %index, %arg1){ ; CHECK-LABEL: wide_add_shift_add_rshrnb_h: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.h ; CHECK-NEXT: rshrnb z1.h, z1.s, #6 ; CHECK-NEXT: rshrnb z0.h, z0.s, #6 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: ld1h { z2.h }, p0/z, [x0, x1, lsl #1] ; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h -; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0, x1, lsl #1] -; CHECK-NEXT: add z0.h, z1.h, z0.h +; CHECK-NEXT: add z0.h, z2.h, z0.h ; CHECK-NEXT: st1h { z0.h }, p0, [x0, x1, lsl #1] ; CHECK-NEXT: ret %1 = add %arg1, splat (i32 32) @@ -162,12 +162,12 @@ define void @wide_add_shift_add_rshrnb_h(ptr %dest, i64 %index, %arg1){ ; CHECK-LABEL: wide_add_shift_add_rshrnb_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.s ; CHECK-NEXT: rshrnb z1.s, z1.d, #32 ; CHECK-NEXT: rshrnb z0.s, z0.d, #32 +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, x1, lsl #2] ; CHECK-NEXT: uzp1 z0.s, z0.s, z1.s -; CHECK-NEXT: ld1w { z1.s }, p0/z, [x0, x1, lsl #2] -; CHECK-NEXT: add z0.s, z1.s, z0.s +; CHECK-NEXT: add z0.s, z2.s, z0.s ; CHECK-NEXT: st1w { z0.s }, p0, [x0, x1, lsl #2] ; CHECK-NEXT: ret %1 = add %arg1, splat (i64 2147483648) @@ -188,11 +188,11 @@ define void @neg_wide_add_shift_add_rshrnb_d(ptr %dest, i64 %index, %arg1, splat (i64 140737488355328) diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-arith-imm.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-arith-imm.ll index 8bfcd088e1a8..500973d053f5 100644 --- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-arith-imm.ll +++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-arith-imm.ll @@ -437,8 +437,8 @@ define @uqsub_i32_ptrue_all_h( %a) #0 { define @uqsub_i32_ptrue_all_d( %a) #0 { ; CHECK-LABEL: uqsub_i32_ptrue_all_d: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: mov z1.s, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: uqsub z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %pg.d = tail call @llvm.aarch64.sve.ptrue.nxv2i1(i32 31) diff --git a/llvm/test/CodeGen/AArch64/sve2-rsh.ll b/llvm/test/CodeGen/AArch64/sve2-rsh.ll index 516ef3bd581e..9addd16f8929 100644 --- a/llvm/test/CodeGen/AArch64/sve2-rsh.ll +++ b/llvm/test/CodeGen/AArch64/sve2-rsh.ll @@ -18,8 +18,8 @@ define @neg_urshr_1( %x) { define @neg_urshr_2( %x, %y) { ; CHECK-LABEL: neg_urshr_2: ; CHECK: // %bb.0: -; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: add z0.d, z0.d, #32 // =0x20 +; CHECK-NEXT: ptrue p0.d ; CHECK-NEXT: lsr z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %add = add nuw nsw %x, splat (i64 32) diff --git a/llvm/test/CodeGen/AArch64/sve2-xar.ll b/llvm/test/CodeGen/AArch64/sve2-xar.ll index e297ade6b9ae..e5a240b7a53f 100644 --- a/llvm/test/CodeGen/AArch64/sve2-xar.ll +++ b/llvm/test/CodeGen/AArch64/sve2-xar.ll @@ -152,9 +152,9 @@ define @xar_nxv2i64_l_neg1( %x, , } @sel_x2_i8(target("aarch64.svc ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z5.d, z4.d ; CHECK-NEXT: mov z7.d, z2.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z4.d, z3.d ; CHECK-NEXT: mov z6.d, z1.d +; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: sel { z0.b, z1.b }, pn8, { z6.b, z7.b }, { z4.b, z5.b } ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 @@ -28,12 +28,12 @@ define { , } @sel_x2_i16(target("aarch64.sv ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z5.d, z4.d ; CHECK-NEXT: mov z7.d, z2.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z4.d, z3.d ; CHECK-NEXT: mov z6.d, z1.d +; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: sel { z0.h, z1.h }, pn8, { z6.h, z7.h }, { z4.h, z5.h } ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 @@ -48,12 +48,12 @@ define { , } @sel_x2_f16(target("aarch64. ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z5.d, z4.d ; CHECK-NEXT: mov z7.d, z2.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z4.d, z3.d ; CHECK-NEXT: mov z6.d, z1.d +; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: sel { z0.h, z1.h }, pn8, { z6.h, z7.h }, { z4.h, z5.h } ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 @@ -68,12 +68,12 @@ define { , } @sel_x2_bf16(target("aar ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z5.d, z4.d ; CHECK-NEXT: mov z7.d, z2.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z4.d, z3.d ; CHECK-NEXT: mov z6.d, z1.d +; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: sel { z0.h, z1.h }, pn8, { z6.h, z7.h }, { z4.h, z5.h } ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 @@ -88,12 +88,12 @@ define { , } @sel_x2_i32(target("aarch64.sv ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z5.d, z4.d ; CHECK-NEXT: mov z7.d, z2.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z4.d, z3.d ; CHECK-NEXT: mov z6.d, z1.d +; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: sel { z0.s, z1.s }, pn8, { z6.s, z7.s }, { z4.s, z5.s } ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 @@ -108,12 +108,12 @@ define { , } @sel_x2_f32(target("aarch6 ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z5.d, z4.d ; CHECK-NEXT: mov z7.d, z2.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z4.d, z3.d ; CHECK-NEXT: mov z6.d, z1.d +; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: sel { z0.s, z1.s }, pn8, { z6.s, z7.s }, { z4.s, z5.s } ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 @@ -128,12 +128,12 @@ define { , } @sel_x2_i64(target("aarch64.sv ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z5.d, z4.d ; CHECK-NEXT: mov z7.d, z2.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z4.d, z3.d ; CHECK-NEXT: mov z6.d, z1.d +; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: sel { z0.d, z1.d }, pn8, { z6.d, z7.d }, { z4.d, z5.d } ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 @@ -148,12 +148,12 @@ define { , } @sel_x2_f64(target("aarc ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z5.d, z4.d ; CHECK-NEXT: mov z7.d, z2.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z4.d, z3.d ; CHECK-NEXT: mov z6.d, z1.d +; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: sel { z0.d, z1.d }, pn8, { z6.d, z7.d }, { z4.d, z5.d } ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload ; CHECK-NEXT: addvl sp, sp, #1 diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-selx4.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-selx4.ll index df504362680b..3a21eaead5f7 100644 --- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-selx4.ll +++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-selx4.ll @@ -8,17 +8,17 @@ define { , , , , , , , , , , , , , , , , , , , , , , , , %unused, %zn0, %unused, %zn0, %unused, %zn0, %unused, %zn0, %unused, %zn0, %unused, %zn0, ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: st1h { z2.h, z3.h }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -117,9 +117,9 @@ define void @st1_x2_f32( %unused, %zn0, < ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: st1w { z2.s, z3.s }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -135,9 +135,9 @@ define void @st1_x2_f64( %unused, %zn0, ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: st1d { z2.d, z3.d }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -153,8 +153,8 @@ define void @st1_x4_i8( %unused, %zn0, %unused, %zn0, %unused, %zn0, %unused, %zn0, %unused, %zn0, %unused, %zn0, ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d @@ -273,8 +273,8 @@ define void @st1_x4_f32( %unused, %zn0, < ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d @@ -293,8 +293,8 @@ define void @st1_x4_f64( %unused, %zn0, ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d @@ -315,9 +315,9 @@ define void @stnt1_x2_i8( %unused, %zn0, %unused, %zn0, < ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: stnt1h { z2.h, z3.h }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -351,9 +351,9 @@ define void @stnt1_x2_i32( %unused, %zn0, < ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: stnt1w { z2.s, z3.s }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -369,9 +369,9 @@ define void @stnt1_x2_i64( %unused, %zn0, < ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: stnt1d { z2.d, z3.d }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -387,9 +387,9 @@ define void @stnt1_x2_f16( %unused, %zn0, ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: stnt1h { z2.h, z3.h }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -405,9 +405,9 @@ define void @stnt1_x2_bf16( %unused, %zn ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: stnt1h { z2.h, z3.h }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -423,9 +423,9 @@ define void @stnt1_x2_f32( %unused, %zn0, ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: stnt1w { z2.s, z3.s }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -441,9 +441,9 @@ define void @stnt1_x2_f64( %unused, %zn0 ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 +; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b -; CHECK-NEXT: mov z3.d, z2.d ; CHECK-NEXT: mov z2.d, z1.d ; CHECK-NEXT: stnt1d { z2.d, z3.d }, pn8, [x0] ; CHECK-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Folded Reload @@ -459,8 +459,8 @@ define void @stnt1_x4_i8( %unused, %zn0, %unused, %zn0, < ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d @@ -499,8 +499,8 @@ define void @stnt1_x4_i32( %unused, %zn0, < ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d @@ -519,8 +519,8 @@ define void @stnt1_x4_i64( %unused, %zn0, < ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d @@ -539,8 +539,8 @@ define void @stnt1_x4_f16( %unused, %zn0, ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d @@ -559,8 +559,8 @@ define void @stnt1_x4_bf16( %unused, %zn ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d @@ -579,8 +579,8 @@ define void @stnt1_x4_f32( %unused, %zn0, ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d @@ -599,8 +599,8 @@ define void @stnt1_x4_f64( %unused, %zn0 ; CHECK: // %bb.0: ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov z7.d, z4.d +; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: mov p8.b, p0.b ; CHECK-NEXT: mov z6.d, z3.d ; CHECK-NEXT: mov z5.d, z2.d diff --git a/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll b/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll index 30ff70088454..16521834090b 100644 --- a/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll +++ b/llvm/test/CodeGen/AArch64/uadd_sat_vec.ll @@ -147,11 +147,11 @@ define void @v8i8(ptr %px, ptr %py, ptr %pz) nounwind { define void @v4i8(ptr %px, ptr %py, ptr %pz) nounwind { ; CHECK-SD-LABEL: v4i8: ; CHECK-SD: // %bb.0: -; CHECK-SD-NEXT: ldr s1, [x0] -; CHECK-SD-NEXT: ldr s2, [x1] -; CHECK-SD-NEXT: movi d0, #0xff00ff00ff00ff -; CHECK-SD-NEXT: uaddl v1.8h, v1.8b, v2.8b -; CHECK-SD-NEXT: umin v0.4h, v1.4h, v0.4h +; CHECK-SD-NEXT: ldr s0, [x0] +; CHECK-SD-NEXT: ldr s1, [x1] +; CHECK-SD-NEXT: movi d2, #0xff00ff00ff00ff +; CHECK-SD-NEXT: uaddl v0.8h, v0.8b, v1.8b +; CHECK-SD-NEXT: umin v0.4h, v0.4h, v2.4h ; CHECK-SD-NEXT: uzp1 v0.8b, v0.8b, v0.8b ; CHECK-SD-NEXT: str s0, [x2] ; CHECK-SD-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/urem-seteq-vec-nonzero.ll b/llvm/test/CodeGen/AArch64/urem-seteq-vec-nonzero.ll index b31ce94cdaae..d5f1febaeb7d 100644 --- a/llvm/test/CodeGen/AArch64/urem-seteq-vec-nonzero.ll +++ b/llvm/test/CodeGen/AArch64/urem-seteq-vec-nonzero.ll @@ -52,11 +52,11 @@ define <4 x i1> @t32_6_part0(<4 x i32> %X) nounwind { ; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s ; CHECK-NEXT: dup v1.4s, w8 ; CHECK-NEXT: mul v0.4s, v0.4s, v1.4s -; CHECK-NEXT: shl v1.4s, v0.4s, #31 -; CHECK-NEXT: usra v1.4s, v0.4s, #1 -; CHECK-NEXT: movi v0.16b, #170 -; CHECK-NEXT: fneg v0.4s, v0.4s -; CHECK-NEXT: cmhs v0.4s, v0.4s, v1.4s +; CHECK-NEXT: movi v1.16b, #170 +; CHECK-NEXT: shl v2.4s, v0.4s, #31 +; CHECK-NEXT: fneg v1.4s, v1.4s +; CHECK-NEXT: usra v2.4s, v0.4s, #1 +; CHECK-NEXT: cmhs v0.4s, v1.4s, v2.4s ; CHECK-NEXT: xtn v0.4h, v0.4s ; CHECK-NEXT: ret %urem = urem <4 x i32> %X, diff --git a/llvm/test/CodeGen/AArch64/vec_uaddo.ll b/llvm/test/CodeGen/AArch64/vec_uaddo.ll index 00609b0df9b4..37c6374215d8 100644 --- a/llvm/test/CodeGen/AArch64/vec_uaddo.ll +++ b/llvm/test/CodeGen/AArch64/vec_uaddo.ll @@ -100,9 +100,9 @@ define <6 x i32> @uaddo_v6i32(<6 x i32> %a0, <6 x i32> %a1, ptr %p2) nounwind { ; CHECK-NEXT: mov v0.s[3], w3 ; CHECK-NEXT: cmhi v3.4s, v3.4s, v2.4s ; CHECK-NEXT: str d2, [x8, #16] -; CHECK-NEXT: add v1.4s, v0.4s, v1.4s ; CHECK-NEXT: mov w5, v3.s[1] ; CHECK-NEXT: fmov w4, s3 +; CHECK-NEXT: add v1.4s, v0.4s, v1.4s ; CHECK-NEXT: cmhi v0.4s, v0.4s, v1.4s ; CHECK-NEXT: str q1, [x8] ; CHECK-NEXT: mov w1, v0.s[1] @@ -248,10 +248,10 @@ define <4 x i32> @uaddo_v4i1(<4 x i1> %a0, <4 x i1> %a1, ptr %p2) nounwind { ; CHECK-NEXT: eor v2.8b, v0.8b, v1.8b ; CHECK-NEXT: and v0.8b, v0.8b, v1.8b ; CHECK-NEXT: adrp x8, .LCPI10_0 -; CHECK-NEXT: shl v2.4h, v2.4h, #15 -; CHECK-NEXT: ushll v0.4s, v0.4h, #0 -; CHECK-NEXT: cmlt v1.4h, v2.4h, #0 +; CHECK-NEXT: shl v1.4h, v2.4h, #15 ; CHECK-NEXT: ldr d2, [x8, :lo12:.LCPI10_0] +; CHECK-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-NEXT: cmlt v1.4h, v1.4h, #0 ; CHECK-NEXT: shl v0.4s, v0.4s, #31 ; CHECK-NEXT: and v1.8b, v1.8b, v2.8b ; CHECK-NEXT: cmlt v0.4s, v0.4s, #0 diff --git a/llvm/test/CodeGen/AArch64/vecreduce-add.ll b/llvm/test/CodeGen/AArch64/vecreduce-add.ll index 66ef436f48c6..3254c5ebe9c6 100644 --- a/llvm/test/CodeGen/AArch64/vecreduce-add.ll +++ b/llvm/test/CodeGen/AArch64/vecreduce-add.ll @@ -2177,65 +2177,65 @@ define i32 @test_udot_v48i8(ptr %p1, ptr %p2) { ; CHECK-GI-BASE-LABEL: test_udot_v48i8: ; CHECK-GI-BASE: // %bb.0: // %entry ; CHECK-GI-BASE-NEXT: ldp q0, q3, [x1] -; CHECK-GI-BASE-NEXT: ldr q6, [x1, #32] +; CHECK-GI-BASE-NEXT: ldr q6, [x0, #32] ; CHECK-GI-BASE-NEXT: ldp q1, q2, [x0] -; CHECK-GI-BASE-NEXT: ldr q17, [x0, #32] +; CHECK-GI-BASE-NEXT: ldr q7, [x1, #32] +; CHECK-GI-BASE-NEXT: ushll v20.8h, v6.8b, #0 +; CHECK-GI-BASE-NEXT: ushll2 v6.8h, v6.16b, #0 ; CHECK-GI-BASE-NEXT: ushll v4.8h, v0.8b, #0 ; CHECK-GI-BASE-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-GI-BASE-NEXT: ushll v7.8h, v3.8b, #0 +; CHECK-GI-BASE-NEXT: ushll v16.8h, v3.8b, #0 ; CHECK-GI-BASE-NEXT: ushll v5.8h, v1.8b, #0 ; CHECK-GI-BASE-NEXT: ushll2 v1.8h, v1.16b, #0 -; CHECK-GI-BASE-NEXT: ushll v16.8h, v2.8b, #0 +; CHECK-GI-BASE-NEXT: ushll v17.8h, v2.8b, #0 ; CHECK-GI-BASE-NEXT: ushll2 v3.8h, v3.16b, #0 ; CHECK-GI-BASE-NEXT: ushll2 v2.8h, v2.16b, #0 ; CHECK-GI-BASE-NEXT: umull v18.4s, v4.4h, v5.4h ; CHECK-GI-BASE-NEXT: umull2 v4.4s, v4.8h, v5.8h -; CHECK-GI-BASE-NEXT: umull2 v19.4s, v0.8h, v1.8h -; CHECK-GI-BASE-NEXT: umull v20.4s, v7.4h, v16.4h -; CHECK-GI-BASE-NEXT: umull v0.4s, v0.4h, v1.4h -; CHECK-GI-BASE-NEXT: ushll v5.8h, v6.8b, #0 -; CHECK-GI-BASE-NEXT: ushll v1.8h, v17.8b, #0 -; CHECK-GI-BASE-NEXT: umull2 v7.4s, v7.8h, v16.8h -; CHECK-GI-BASE-NEXT: ushll2 v6.8h, v6.16b, #0 -; CHECK-GI-BASE-NEXT: ushll2 v17.8h, v17.16b, #0 -; CHECK-GI-BASE-NEXT: addv s16, v18.4s -; CHECK-GI-BASE-NEXT: addv s4, v4.4s -; CHECK-GI-BASE-NEXT: umull v18.4s, v3.4h, v2.4h +; CHECK-GI-BASE-NEXT: umull v5.4s, v0.4h, v1.4h +; CHECK-GI-BASE-NEXT: umull2 v0.4s, v0.8h, v1.8h +; CHECK-GI-BASE-NEXT: umull v19.4s, v16.4h, v17.4h +; CHECK-GI-BASE-NEXT: ushll v1.8h, v7.8b, #0 +; CHECK-GI-BASE-NEXT: umull2 v16.4s, v16.8h, v17.8h +; CHECK-GI-BASE-NEXT: umull v17.4s, v3.4h, v2.4h ; CHECK-GI-BASE-NEXT: umull2 v2.4s, v3.8h, v2.8h -; CHECK-GI-BASE-NEXT: addv s3, v19.4s -; CHECK-GI-BASE-NEXT: umull v19.4s, v5.4h, v1.4h -; CHECK-GI-BASE-NEXT: umull2 v1.4s, v5.8h, v1.8h -; CHECK-GI-BASE-NEXT: addv s5, v20.4s +; CHECK-GI-BASE-NEXT: ushll2 v7.8h, v7.16b, #0 +; CHECK-GI-BASE-NEXT: addv s18, v18.4s +; CHECK-GI-BASE-NEXT: addv s4, v4.4s +; CHECK-GI-BASE-NEXT: addv s5, v5.4s ; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: addv s7, v7.4s -; CHECK-GI-BASE-NEXT: umull v20.4s, v6.4h, v17.4h -; CHECK-GI-BASE-NEXT: umull2 v6.4s, v6.8h, v17.8h -; CHECK-GI-BASE-NEXT: fmov w8, s16 -; CHECK-GI-BASE-NEXT: fmov w9, s4 -; CHECK-GI-BASE-NEXT: fmov w10, s3 -; CHECK-GI-BASE-NEXT: addv s3, v18.4s +; CHECK-GI-BASE-NEXT: addv s19, v19.4s +; CHECK-GI-BASE-NEXT: umull v3.4s, v1.4h, v20.4h ; CHECK-GI-BASE-NEXT: addv s2, v2.4s -; CHECK-GI-BASE-NEXT: fmov w11, s5 -; CHECK-GI-BASE-NEXT: addv s4, v19.4s +; CHECK-GI-BASE-NEXT: umull2 v1.4s, v1.8h, v20.8h +; CHECK-GI-BASE-NEXT: umull v20.4s, v7.4h, v6.4h +; CHECK-GI-BASE-NEXT: fmov w8, s18 +; CHECK-GI-BASE-NEXT: fmov w9, s4 +; CHECK-GI-BASE-NEXT: fmov w10, s5 +; CHECK-GI-BASE-NEXT: fmov w11, s0 +; CHECK-GI-BASE-NEXT: fmov w12, s19 +; CHECK-GI-BASE-NEXT: addv s4, v16.4s +; CHECK-GI-BASE-NEXT: addv s5, v17.4s +; CHECK-GI-BASE-NEXT: addv s3, v3.4s +; CHECK-GI-BASE-NEXT: umull2 v0.4s, v7.8h, v6.8h ; CHECK-GI-BASE-NEXT: add w8, w8, w9 -; CHECK-GI-BASE-NEXT: fmov w9, s0 -; CHECK-GI-BASE-NEXT: addv s0, v1.4s -; CHECK-GI-BASE-NEXT: addv s1, v20.4s -; CHECK-GI-BASE-NEXT: addv s5, v6.4s -; CHECK-GI-BASE-NEXT: add w10, w10, w11 -; CHECK-GI-BASE-NEXT: fmov w11, s3 +; CHECK-GI-BASE-NEXT: addv s1, v1.4s +; CHECK-GI-BASE-NEXT: add w9, w11, w12 +; CHECK-GI-BASE-NEXT: add w8, w8, w10 +; CHECK-GI-BASE-NEXT: fmov w10, s4 +; CHECK-GI-BASE-NEXT: fmov w11, s5 ; CHECK-GI-BASE-NEXT: fmov w12, s2 -; CHECK-GI-BASE-NEXT: add w8, w8, w9 -; CHECK-GI-BASE-NEXT: fmov w9, s7 -; CHECK-GI-BASE-NEXT: add w9, w10, w9 +; CHECK-GI-BASE-NEXT: addv s4, v20.4s +; CHECK-GI-BASE-NEXT: addv s0, v0.4s +; CHECK-GI-BASE-NEXT: add w9, w9, w10 ; CHECK-GI-BASE-NEXT: add w10, w11, w12 -; CHECK-GI-BASE-NEXT: fmov w11, s4 +; CHECK-GI-BASE-NEXT: fmov w11, s3 ; CHECK-GI-BASE-NEXT: add w8, w8, w9 ; CHECK-GI-BASE-NEXT: add w9, w10, w11 -; CHECK-GI-BASE-NEXT: fmov w10, s0 -; CHECK-GI-BASE-NEXT: fmov w11, s5 -; CHECK-GI-BASE-NEXT: add w9, w9, w10 ; CHECK-GI-BASE-NEXT: fmov w10, s1 +; CHECK-GI-BASE-NEXT: fmov w11, s0 +; CHECK-GI-BASE-NEXT: add w9, w9, w10 +; CHECK-GI-BASE-NEXT: fmov w10, s4 ; CHECK-GI-BASE-NEXT: add w8, w8, w9 ; CHECK-GI-BASE-NEXT: add w9, w10, w11 ; CHECK-GI-BASE-NEXT: add w0, w8, w9 @@ -2527,65 +2527,65 @@ define i32 @test_sdot_v48i8(ptr %p1, ptr %p2) { ; CHECK-GI-BASE-LABEL: test_sdot_v48i8: ; CHECK-GI-BASE: // %bb.0: // %entry ; CHECK-GI-BASE-NEXT: ldp q0, q3, [x1] -; CHECK-GI-BASE-NEXT: ldr q6, [x1, #32] +; CHECK-GI-BASE-NEXT: ldr q6, [x0, #32] ; CHECK-GI-BASE-NEXT: ldp q1, q2, [x0] -; CHECK-GI-BASE-NEXT: ldr q17, [x0, #32] +; CHECK-GI-BASE-NEXT: ldr q7, [x1, #32] +; CHECK-GI-BASE-NEXT: sshll v20.8h, v6.8b, #0 +; CHECK-GI-BASE-NEXT: sshll2 v6.8h, v6.16b, #0 ; CHECK-GI-BASE-NEXT: sshll v4.8h, v0.8b, #0 ; CHECK-GI-BASE-NEXT: sshll2 v0.8h, v0.16b, #0 -; CHECK-GI-BASE-NEXT: sshll v7.8h, v3.8b, #0 +; CHECK-GI-BASE-NEXT: sshll v16.8h, v3.8b, #0 ; CHECK-GI-BASE-NEXT: sshll v5.8h, v1.8b, #0 ; CHECK-GI-BASE-NEXT: sshll2 v1.8h, v1.16b, #0 -; CHECK-GI-BASE-NEXT: sshll v16.8h, v2.8b, #0 +; CHECK-GI-BASE-NEXT: sshll v17.8h, v2.8b, #0 ; CHECK-GI-BASE-NEXT: sshll2 v3.8h, v3.16b, #0 ; CHECK-GI-BASE-NEXT: sshll2 v2.8h, v2.16b, #0 ; CHECK-GI-BASE-NEXT: smull v18.4s, v4.4h, v5.4h ; CHECK-GI-BASE-NEXT: smull2 v4.4s, v4.8h, v5.8h -; CHECK-GI-BASE-NEXT: smull2 v19.4s, v0.8h, v1.8h -; CHECK-GI-BASE-NEXT: smull v20.4s, v7.4h, v16.4h -; CHECK-GI-BASE-NEXT: smull v0.4s, v0.4h, v1.4h -; CHECK-GI-BASE-NEXT: sshll v5.8h, v6.8b, #0 -; CHECK-GI-BASE-NEXT: sshll v1.8h, v17.8b, #0 -; CHECK-GI-BASE-NEXT: smull2 v7.4s, v7.8h, v16.8h -; CHECK-GI-BASE-NEXT: sshll2 v6.8h, v6.16b, #0 -; CHECK-GI-BASE-NEXT: sshll2 v17.8h, v17.16b, #0 -; CHECK-GI-BASE-NEXT: addv s16, v18.4s -; CHECK-GI-BASE-NEXT: addv s4, v4.4s -; CHECK-GI-BASE-NEXT: smull v18.4s, v3.4h, v2.4h +; CHECK-GI-BASE-NEXT: smull v5.4s, v0.4h, v1.4h +; CHECK-GI-BASE-NEXT: smull2 v0.4s, v0.8h, v1.8h +; CHECK-GI-BASE-NEXT: smull v19.4s, v16.4h, v17.4h +; CHECK-GI-BASE-NEXT: sshll v1.8h, v7.8b, #0 +; CHECK-GI-BASE-NEXT: smull2 v16.4s, v16.8h, v17.8h +; CHECK-GI-BASE-NEXT: smull v17.4s, v3.4h, v2.4h ; CHECK-GI-BASE-NEXT: smull2 v2.4s, v3.8h, v2.8h -; CHECK-GI-BASE-NEXT: addv s3, v19.4s -; CHECK-GI-BASE-NEXT: smull v19.4s, v5.4h, v1.4h -; CHECK-GI-BASE-NEXT: smull2 v1.4s, v5.8h, v1.8h -; CHECK-GI-BASE-NEXT: addv s5, v20.4s +; CHECK-GI-BASE-NEXT: sshll2 v7.8h, v7.16b, #0 +; CHECK-GI-BASE-NEXT: addv s18, v18.4s +; CHECK-GI-BASE-NEXT: addv s4, v4.4s +; CHECK-GI-BASE-NEXT: addv s5, v5.4s ; CHECK-GI-BASE-NEXT: addv s0, v0.4s -; CHECK-GI-BASE-NEXT: addv s7, v7.4s -; CHECK-GI-BASE-NEXT: smull v20.4s, v6.4h, v17.4h -; CHECK-GI-BASE-NEXT: smull2 v6.4s, v6.8h, v17.8h -; CHECK-GI-BASE-NEXT: fmov w8, s16 -; CHECK-GI-BASE-NEXT: fmov w9, s4 -; CHECK-GI-BASE-NEXT: fmov w10, s3 -; CHECK-GI-BASE-NEXT: addv s3, v18.4s +; CHECK-GI-BASE-NEXT: addv s19, v19.4s +; CHECK-GI-BASE-NEXT: smull v3.4s, v1.4h, v20.4h ; CHECK-GI-BASE-NEXT: addv s2, v2.4s -; CHECK-GI-BASE-NEXT: fmov w11, s5 -; CHECK-GI-BASE-NEXT: addv s4, v19.4s +; CHECK-GI-BASE-NEXT: smull2 v1.4s, v1.8h, v20.8h +; CHECK-GI-BASE-NEXT: smull v20.4s, v7.4h, v6.4h +; CHECK-GI-BASE-NEXT: fmov w8, s18 +; CHECK-GI-BASE-NEXT: fmov w9, s4 +; CHECK-GI-BASE-NEXT: fmov w10, s5 +; CHECK-GI-BASE-NEXT: fmov w11, s0 +; CHECK-GI-BASE-NEXT: fmov w12, s19 +; CHECK-GI-BASE-NEXT: addv s4, v16.4s +; CHECK-GI-BASE-NEXT: addv s5, v17.4s +; CHECK-GI-BASE-NEXT: addv s3, v3.4s +; CHECK-GI-BASE-NEXT: smull2 v0.4s, v7.8h, v6.8h ; CHECK-GI-BASE-NEXT: add w8, w8, w9 -; CHECK-GI-BASE-NEXT: fmov w9, s0 -; CHECK-GI-BASE-NEXT: addv s0, v1.4s -; CHECK-GI-BASE-NEXT: addv s1, v20.4s -; CHECK-GI-BASE-NEXT: addv s5, v6.4s -; CHECK-GI-BASE-NEXT: add w10, w10, w11 -; CHECK-GI-BASE-NEXT: fmov w11, s3 +; CHECK-GI-BASE-NEXT: addv s1, v1.4s +; CHECK-GI-BASE-NEXT: add w9, w11, w12 +; CHECK-GI-BASE-NEXT: add w8, w8, w10 +; CHECK-GI-BASE-NEXT: fmov w10, s4 +; CHECK-GI-BASE-NEXT: fmov w11, s5 ; CHECK-GI-BASE-NEXT: fmov w12, s2 -; CHECK-GI-BASE-NEXT: add w8, w8, w9 -; CHECK-GI-BASE-NEXT: fmov w9, s7 -; CHECK-GI-BASE-NEXT: add w9, w10, w9 +; CHECK-GI-BASE-NEXT: addv s4, v20.4s +; CHECK-GI-BASE-NEXT: addv s0, v0.4s +; CHECK-GI-BASE-NEXT: add w9, w9, w10 ; CHECK-GI-BASE-NEXT: add w10, w11, w12 -; CHECK-GI-BASE-NEXT: fmov w11, s4 +; CHECK-GI-BASE-NEXT: fmov w11, s3 ; CHECK-GI-BASE-NEXT: add w8, w8, w9 ; CHECK-GI-BASE-NEXT: add w9, w10, w11 -; CHECK-GI-BASE-NEXT: fmov w10, s0 -; CHECK-GI-BASE-NEXT: fmov w11, s5 -; CHECK-GI-BASE-NEXT: add w9, w9, w10 ; CHECK-GI-BASE-NEXT: fmov w10, s1 +; CHECK-GI-BASE-NEXT: fmov w11, s0 +; CHECK-GI-BASE-NEXT: add w9, w9, w10 +; CHECK-GI-BASE-NEXT: fmov w10, s4 ; CHECK-GI-BASE-NEXT: add w8, w8, w9 ; CHECK-GI-BASE-NEXT: add w9, w10, w11 ; CHECK-GI-BASE-NEXT: add w0, w8, w9 @@ -2640,13 +2640,13 @@ define i32 @test_udot_v8i8_multi_use(<8 x i8> %a, <8 x i8> %b) { ; CHECK-SD-DOT-LABEL: test_udot_v8i8_multi_use: ; CHECK-SD-DOT: // %bb.0: // %entry ; CHECK-SD-DOT-NEXT: movi v2.2d, #0000000000000000 +; CHECK-SD-DOT-NEXT: ushll v3.8h, v0.8b, #0 +; CHECK-SD-DOT-NEXT: ushll v4.8h, v1.8b, #0 ; CHECK-SD-DOT-NEXT: udot v2.2s, v1.8b, v0.8b -; CHECK-SD-DOT-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-SD-DOT-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-SD-DOT-NEXT: umull v0.4s, v1.4h, v0.4h -; CHECK-SD-DOT-NEXT: addp v2.2s, v2.2s, v2.2s +; CHECK-SD-DOT-NEXT: umull v0.4s, v4.4h, v3.4h +; CHECK-SD-DOT-NEXT: addp v1.2s, v2.2s, v2.2s ; CHECK-SD-DOT-NEXT: fmov w9, s0 -; CHECK-SD-DOT-NEXT: fmov w8, s2 +; CHECK-SD-DOT-NEXT: fmov w8, s1 ; CHECK-SD-DOT-NEXT: add w0, w8, w9 ; CHECK-SD-DOT-NEXT: ret ; @@ -3534,21 +3534,21 @@ entry: define i64 @add_pair_v4i8_v4i64_sext(<4 x i8> %x, <4 x i8> %y) { ; CHECK-SD-LABEL: add_pair_v4i8_v4i64_sext: ; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 ; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0 -; CHECK-SD-NEXT: ushll v2.2d, v0.2s, #0 -; CHECK-SD-NEXT: ushll v3.2d, v1.2s, #0 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: ushll v2.2d, v1.2s, #0 +; CHECK-SD-NEXT: ushll v3.2d, v0.2s, #0 ; CHECK-SD-NEXT: ushll2 v0.2d, v0.4s, #0 ; CHECK-SD-NEXT: ushll2 v1.2d, v1.4s, #0 -; CHECK-SD-NEXT: shl v2.2d, v2.2d, #56 ; CHECK-SD-NEXT: shl v3.2d, v3.2d, #56 +; CHECK-SD-NEXT: shl v2.2d, v2.2d, #56 ; CHECK-SD-NEXT: shl v0.2d, v0.2d, #56 ; CHECK-SD-NEXT: shl v1.2d, v1.2d, #56 -; CHECK-SD-NEXT: sshr v2.2d, v2.2d, #56 ; CHECK-SD-NEXT: sshr v3.2d, v3.2d, #56 -; CHECK-SD-NEXT: ssra v2.2d, v0.2d, #56 -; CHECK-SD-NEXT: ssra v3.2d, v1.2d, #56 -; CHECK-SD-NEXT: add v0.2d, v2.2d, v3.2d +; CHECK-SD-NEXT: sshr v2.2d, v2.2d, #56 +; CHECK-SD-NEXT: ssra v3.2d, v0.2d, #56 +; CHECK-SD-NEXT: ssra v2.2d, v1.2d, #56 +; CHECK-SD-NEXT: add v0.2d, v3.2d, v2.2d ; CHECK-SD-NEXT: addp d0, v0.2d ; CHECK-SD-NEXT: fmov x0, d0 ; CHECK-SD-NEXT: ret @@ -3816,37 +3816,37 @@ define i16 @add_v24i8_v24i16_zext(<24 x i8> %x) { ; CHECK-SD-NEXT: ldr b1, [sp, #64] ; CHECK-SD-NEXT: add x8, sp, #72 ; CHECK-SD-NEXT: ldr b2, [sp] -; CHECK-SD-NEXT: add x9, sp, #8 +; CHECK-SD-NEXT: add x9, sp, #80 ; CHECK-SD-NEXT: ld1 { v1.b }[1], [x8] -; CHECK-SD-NEXT: add x8, sp, #80 +; CHECK-SD-NEXT: add x8, sp, #8 ; CHECK-SD-NEXT: mov v0.b[1], w1 -; CHECK-SD-NEXT: ld1 { v2.b }[1], [x9] -; CHECK-SD-NEXT: add x9, sp, #16 -; CHECK-SD-NEXT: ld1 { v1.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #88 -; CHECK-SD-NEXT: ld1 { v2.b }[2], [x9] -; CHECK-SD-NEXT: add x9, sp, #24 +; CHECK-SD-NEXT: ld1 { v2.b }[1], [x8] +; CHECK-SD-NEXT: add x8, sp, #16 +; CHECK-SD-NEXT: ld1 { v1.b }[2], [x9] +; CHECK-SD-NEXT: add x9, sp, #88 +; CHECK-SD-NEXT: ld1 { v2.b }[2], [x8] +; CHECK-SD-NEXT: add x8, sp, #24 ; CHECK-SD-NEXT: mov v0.b[2], w2 -; CHECK-SD-NEXT: ld1 { v1.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #96 -; CHECK-SD-NEXT: ld1 { v2.b }[3], [x9] -; CHECK-SD-NEXT: add x9, sp, #32 +; CHECK-SD-NEXT: ld1 { v1.b }[3], [x9] +; CHECK-SD-NEXT: add x9, sp, #96 +; CHECK-SD-NEXT: ld1 { v2.b }[3], [x8] +; CHECK-SD-NEXT: add x8, sp, #32 ; CHECK-SD-NEXT: mov v0.b[3], w3 -; CHECK-SD-NEXT: ld1 { v1.b }[4], [x8] -; CHECK-SD-NEXT: add x8, sp, #104 -; CHECK-SD-NEXT: ld1 { v2.b }[4], [x9] -; CHECK-SD-NEXT: add x9, sp, #40 -; CHECK-SD-NEXT: ld1 { v1.b }[5], [x8] -; CHECK-SD-NEXT: add x8, sp, #112 +; CHECK-SD-NEXT: ld1 { v1.b }[4], [x9] +; CHECK-SD-NEXT: add x9, sp, #104 +; CHECK-SD-NEXT: ld1 { v2.b }[4], [x8] +; CHECK-SD-NEXT: add x8, sp, #40 +; CHECK-SD-NEXT: ld1 { v1.b }[5], [x9] +; CHECK-SD-NEXT: add x9, sp, #112 ; CHECK-SD-NEXT: mov v0.b[4], w4 -; CHECK-SD-NEXT: ld1 { v2.b }[5], [x9] -; CHECK-SD-NEXT: add x9, sp, #48 -; CHECK-SD-NEXT: ld1 { v1.b }[6], [x8] -; CHECK-SD-NEXT: add x8, sp, #120 -; CHECK-SD-NEXT: ld1 { v2.b }[6], [x9] -; CHECK-SD-NEXT: mov v0.b[5], w5 -; CHECK-SD-NEXT: ld1 { v1.b }[7], [x8] +; CHECK-SD-NEXT: ld1 { v2.b }[5], [x8] +; CHECK-SD-NEXT: add x8, sp, #48 +; CHECK-SD-NEXT: ld1 { v1.b }[6], [x9] +; CHECK-SD-NEXT: add x9, sp, #120 +; CHECK-SD-NEXT: ld1 { v2.b }[6], [x8] ; CHECK-SD-NEXT: add x8, sp, #56 +; CHECK-SD-NEXT: mov v0.b[5], w5 +; CHECK-SD-NEXT: ld1 { v1.b }[7], [x9] ; CHECK-SD-NEXT: ld1 { v2.b }[7], [x8] ; CHECK-SD-NEXT: mov v0.b[6], w6 ; CHECK-SD-NEXT: mov v0.b[7], w7 @@ -3942,37 +3942,37 @@ define i16 @add_v24i8_v24i16_sext(<24 x i8> %x) { ; CHECK-SD-NEXT: ldr b1, [sp, #64] ; CHECK-SD-NEXT: add x8, sp, #72 ; CHECK-SD-NEXT: ldr b2, [sp] -; CHECK-SD-NEXT: add x9, sp, #8 +; CHECK-SD-NEXT: add x9, sp, #80 ; CHECK-SD-NEXT: ld1 { v1.b }[1], [x8] -; CHECK-SD-NEXT: add x8, sp, #80 +; CHECK-SD-NEXT: add x8, sp, #8 ; CHECK-SD-NEXT: mov v0.b[1], w1 -; CHECK-SD-NEXT: ld1 { v2.b }[1], [x9] -; CHECK-SD-NEXT: add x9, sp, #16 -; CHECK-SD-NEXT: ld1 { v1.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #88 -; CHECK-SD-NEXT: ld1 { v2.b }[2], [x9] -; CHECK-SD-NEXT: add x9, sp, #24 +; CHECK-SD-NEXT: ld1 { v2.b }[1], [x8] +; CHECK-SD-NEXT: add x8, sp, #16 +; CHECK-SD-NEXT: ld1 { v1.b }[2], [x9] +; CHECK-SD-NEXT: add x9, sp, #88 +; CHECK-SD-NEXT: ld1 { v2.b }[2], [x8] +; CHECK-SD-NEXT: add x8, sp, #24 ; CHECK-SD-NEXT: mov v0.b[2], w2 -; CHECK-SD-NEXT: ld1 { v1.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #96 -; CHECK-SD-NEXT: ld1 { v2.b }[3], [x9] -; CHECK-SD-NEXT: add x9, sp, #32 +; CHECK-SD-NEXT: ld1 { v1.b }[3], [x9] +; CHECK-SD-NEXT: add x9, sp, #96 +; CHECK-SD-NEXT: ld1 { v2.b }[3], [x8] +; CHECK-SD-NEXT: add x8, sp, #32 ; CHECK-SD-NEXT: mov v0.b[3], w3 -; CHECK-SD-NEXT: ld1 { v1.b }[4], [x8] -; CHECK-SD-NEXT: add x8, sp, #104 -; CHECK-SD-NEXT: ld1 { v2.b }[4], [x9] -; CHECK-SD-NEXT: add x9, sp, #40 -; CHECK-SD-NEXT: ld1 { v1.b }[5], [x8] -; CHECK-SD-NEXT: add x8, sp, #112 +; CHECK-SD-NEXT: ld1 { v1.b }[4], [x9] +; CHECK-SD-NEXT: add x9, sp, #104 +; CHECK-SD-NEXT: ld1 { v2.b }[4], [x8] +; CHECK-SD-NEXT: add x8, sp, #40 +; CHECK-SD-NEXT: ld1 { v1.b }[5], [x9] +; CHECK-SD-NEXT: add x9, sp, #112 ; CHECK-SD-NEXT: mov v0.b[4], w4 -; CHECK-SD-NEXT: ld1 { v2.b }[5], [x9] -; CHECK-SD-NEXT: add x9, sp, #48 -; CHECK-SD-NEXT: ld1 { v1.b }[6], [x8] -; CHECK-SD-NEXT: add x8, sp, #120 -; CHECK-SD-NEXT: ld1 { v2.b }[6], [x9] -; CHECK-SD-NEXT: mov v0.b[5], w5 -; CHECK-SD-NEXT: ld1 { v1.b }[7], [x8] +; CHECK-SD-NEXT: ld1 { v2.b }[5], [x8] +; CHECK-SD-NEXT: add x8, sp, #48 +; CHECK-SD-NEXT: ld1 { v1.b }[6], [x9] +; CHECK-SD-NEXT: add x9, sp, #120 +; CHECK-SD-NEXT: ld1 { v2.b }[6], [x8] ; CHECK-SD-NEXT: add x8, sp, #56 +; CHECK-SD-NEXT: mov v0.b[5], w5 +; CHECK-SD-NEXT: ld1 { v1.b }[7], [x9] ; CHECK-SD-NEXT: ld1 { v2.b }[7], [x8] ; CHECK-SD-NEXT: mov v0.b[6], w6 ; CHECK-SD-NEXT: mov v0.b[7], w7 @@ -4069,48 +4069,48 @@ define i32 @add_v24i8_v24i32_zext(<24 x i8> %x) { ; CHECK-SD-BASE-NEXT: ldr b1, [sp, #64] ; CHECK-SD-BASE-NEXT: add x8, sp, #72 ; CHECK-SD-BASE-NEXT: ldr b2, [sp] -; CHECK-SD-BASE-NEXT: add x9, sp, #8 +; CHECK-SD-BASE-NEXT: add x9, sp, #80 ; CHECK-SD-BASE-NEXT: ld1 { v1.b }[1], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #80 +; CHECK-SD-BASE-NEXT: add x8, sp, #8 ; CHECK-SD-BASE-NEXT: mov v0.b[1], w1 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[1], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #16 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[2], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #88 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[2], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #24 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[1], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #16 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[2], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #88 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[2], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #24 ; CHECK-SD-BASE-NEXT: mov v0.b[2], w2 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[3], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #96 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[3], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #32 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[3], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #96 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[3], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #32 ; CHECK-SD-BASE-NEXT: mov v0.b[3], w3 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[4], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #104 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[4], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #40 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[5], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #112 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[4], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #104 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[4], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #40 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[5], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #112 ; CHECK-SD-BASE-NEXT: mov v0.b[4], w4 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[5], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #48 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[6], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #120 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[6], [x9] -; CHECK-SD-BASE-NEXT: mov v0.b[5], w5 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[7], [x8] +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[5], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #48 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[6], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #120 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[6], [x8] ; CHECK-SD-BASE-NEXT: add x8, sp, #56 +; CHECK-SD-BASE-NEXT: mov v0.b[5], w5 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[7], [x9] ; CHECK-SD-BASE-NEXT: ld1 { v2.b }[7], [x8] ; CHECK-SD-BASE-NEXT: mov v0.b[6], w6 ; CHECK-SD-BASE-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-SD-BASE-NEXT: ushll v2.8h, v2.8b, #0 ; CHECK-SD-BASE-NEXT: mov v0.b[7], w7 ; CHECK-SD-BASE-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-SD-BASE-NEXT: uaddl2 v3.4s, v0.8h, v1.8h ; CHECK-SD-BASE-NEXT: uaddl v0.4s, v0.4h, v1.4h -; CHECK-SD-BASE-NEXT: ushll v1.8h, v2.8b, #0 -; CHECK-SD-BASE-NEXT: uaddw2 v2.4s, v3.4s, v1.8h -; CHECK-SD-BASE-NEXT: uaddw v0.4s, v0.4s, v1.4h -; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-SD-BASE-NEXT: uaddw2 v1.4s, v3.4s, v2.8h +; CHECK-SD-BASE-NEXT: uaddw v0.4s, v0.4s, v2.4h +; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-SD-BASE-NEXT: addv s0, v0.4s ; CHECK-SD-BASE-NEXT: fmov w0, s0 ; CHECK-SD-BASE-NEXT: ret @@ -4147,9 +4147,9 @@ define i32 @add_v24i8_v24i32_zext(<24 x i8> %x) { ; CHECK-SD-DOT-NEXT: udot v4.2s, v1.8b, v5.8b ; CHECK-SD-DOT-NEXT: mov v0.b[7], w7 ; CHECK-SD-DOT-NEXT: addp v1.2s, v4.2s, v4.2s +; CHECK-SD-DOT-NEXT: fmov w9, s1 ; CHECK-SD-DOT-NEXT: ld1 { v0.b }[8], [x8] ; CHECK-SD-DOT-NEXT: add x8, sp, #8 -; CHECK-SD-DOT-NEXT: fmov w9, s1 ; CHECK-SD-DOT-NEXT: ld1 { v0.b }[9], [x8] ; CHECK-SD-DOT-NEXT: add x8, sp, #16 ; CHECK-SD-DOT-NEXT: ld1 { v0.b }[10], [x8] @@ -4342,48 +4342,48 @@ define i32 @add_v24i8_v24i32_sext(<24 x i8> %x) { ; CHECK-SD-BASE-NEXT: ldr b1, [sp, #64] ; CHECK-SD-BASE-NEXT: add x8, sp, #72 ; CHECK-SD-BASE-NEXT: ldr b2, [sp] -; CHECK-SD-BASE-NEXT: add x9, sp, #8 +; CHECK-SD-BASE-NEXT: add x9, sp, #80 ; CHECK-SD-BASE-NEXT: ld1 { v1.b }[1], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #80 +; CHECK-SD-BASE-NEXT: add x8, sp, #8 ; CHECK-SD-BASE-NEXT: mov v0.b[1], w1 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[1], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #16 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[2], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #88 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[2], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #24 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[1], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #16 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[2], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #88 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[2], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #24 ; CHECK-SD-BASE-NEXT: mov v0.b[2], w2 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[3], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #96 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[3], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #32 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[3], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #96 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[3], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #32 ; CHECK-SD-BASE-NEXT: mov v0.b[3], w3 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[4], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #104 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[4], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #40 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[5], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #112 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[4], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #104 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[4], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #40 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[5], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #112 ; CHECK-SD-BASE-NEXT: mov v0.b[4], w4 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[5], [x9] -; CHECK-SD-BASE-NEXT: add x9, sp, #48 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[6], [x8] -; CHECK-SD-BASE-NEXT: add x8, sp, #120 -; CHECK-SD-BASE-NEXT: ld1 { v2.b }[6], [x9] -; CHECK-SD-BASE-NEXT: mov v0.b[5], w5 -; CHECK-SD-BASE-NEXT: ld1 { v1.b }[7], [x8] +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[5], [x8] +; CHECK-SD-BASE-NEXT: add x8, sp, #48 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[6], [x9] +; CHECK-SD-BASE-NEXT: add x9, sp, #120 +; CHECK-SD-BASE-NEXT: ld1 { v2.b }[6], [x8] ; CHECK-SD-BASE-NEXT: add x8, sp, #56 +; CHECK-SD-BASE-NEXT: mov v0.b[5], w5 +; CHECK-SD-BASE-NEXT: ld1 { v1.b }[7], [x9] ; CHECK-SD-BASE-NEXT: ld1 { v2.b }[7], [x8] ; CHECK-SD-BASE-NEXT: mov v0.b[6], w6 ; CHECK-SD-BASE-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-SD-BASE-NEXT: sshll v2.8h, v2.8b, #0 ; CHECK-SD-BASE-NEXT: mov v0.b[7], w7 ; CHECK-SD-BASE-NEXT: sshll v0.8h, v0.8b, #0 ; CHECK-SD-BASE-NEXT: saddl2 v3.4s, v0.8h, v1.8h ; CHECK-SD-BASE-NEXT: saddl v0.4s, v0.4h, v1.4h -; CHECK-SD-BASE-NEXT: sshll v1.8h, v2.8b, #0 -; CHECK-SD-BASE-NEXT: saddw2 v2.4s, v3.4s, v1.8h -; CHECK-SD-BASE-NEXT: saddw v0.4s, v0.4s, v1.4h -; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-SD-BASE-NEXT: saddw2 v1.4s, v3.4s, v2.8h +; CHECK-SD-BASE-NEXT: saddw v0.4s, v0.4s, v2.4h +; CHECK-SD-BASE-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-SD-BASE-NEXT: addv s0, v0.4s ; CHECK-SD-BASE-NEXT: fmov w0, s0 ; CHECK-SD-BASE-NEXT: ret @@ -4420,9 +4420,9 @@ define i32 @add_v24i8_v24i32_sext(<24 x i8> %x) { ; CHECK-SD-DOT-NEXT: sdot v4.2s, v1.8b, v5.8b ; CHECK-SD-DOT-NEXT: mov v0.b[7], w7 ; CHECK-SD-DOT-NEXT: addp v1.2s, v4.2s, v4.2s +; CHECK-SD-DOT-NEXT: fmov w9, s1 ; CHECK-SD-DOT-NEXT: ld1 { v0.b }[8], [x8] ; CHECK-SD-DOT-NEXT: add x8, sp, #8 -; CHECK-SD-DOT-NEXT: fmov w9, s1 ; CHECK-SD-DOT-NEXT: ld1 { v0.b }[9], [x8] ; CHECK-SD-DOT-NEXT: add x8, sp, #16 ; CHECK-SD-DOT-NEXT: ld1 { v0.b }[10], [x8] @@ -4611,23 +4611,23 @@ entry: define i32 @full(ptr %p1, i32 noundef %s1, ptr %p2, i32 noundef %s2) { ; CHECK-SD-BASE-LABEL: full: ; CHECK-SD-BASE: // %bb.0: // %entry -; CHECK-SD-BASE-NEXT: ldr d0, [x2] -; CHECK-SD-BASE-NEXT: ldr d1, [x0] ; CHECK-SD-BASE-NEXT: // kill: def $w3 killed $w3 def $x3 ; CHECK-SD-BASE-NEXT: // kill: def $w1 killed $w1 def $x1 ; CHECK-SD-BASE-NEXT: sxtw x8, w3 ; CHECK-SD-BASE-NEXT: sxtw x9, w1 -; CHECK-SD-BASE-NEXT: uabdl v0.8h, v1.8b, v0.8b -; CHECK-SD-BASE-NEXT: add x11, x2, x8 +; CHECK-SD-BASE-NEXT: ldr d0, [x0] +; CHECK-SD-BASE-NEXT: ldr d1, [x2] ; CHECK-SD-BASE-NEXT: add x10, x0, x9 -; CHECK-SD-BASE-NEXT: ldr d2, [x11] -; CHECK-SD-BASE-NEXT: add x11, x11, x8 +; CHECK-SD-BASE-NEXT: add x11, x2, x8 +; CHECK-SD-BASE-NEXT: uabdl v0.8h, v0.8b, v1.8b ; CHECK-SD-BASE-NEXT: ldr d1, [x10] +; CHECK-SD-BASE-NEXT: ldr d2, [x11] ; CHECK-SD-BASE-NEXT: add x10, x10, x9 -; CHECK-SD-BASE-NEXT: uaddlp v0.4s, v0.8h +; CHECK-SD-BASE-NEXT: add x11, x11, x8 ; CHECK-SD-BASE-NEXT: uabdl v1.8h, v1.8b, v2.8b ; CHECK-SD-BASE-NEXT: ldr d2, [x11] ; CHECK-SD-BASE-NEXT: add x11, x11, x8 +; CHECK-SD-BASE-NEXT: uaddlp v0.4s, v0.8h ; CHECK-SD-BASE-NEXT: uadalp v0.4s, v1.8h ; CHECK-SD-BASE-NEXT: ldr d1, [x10] ; CHECK-SD-BASE-NEXT: add x10, x10, x9 @@ -4723,98 +4723,98 @@ define i32 @full(ptr %p1, i32 noundef %s1, ptr %p2, i32 noundef %s2) { ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: // kill: def $w1 killed $w1 def $x1 ; CHECK-GI-NEXT: // kill: def $w3 killed $w3 def $x3 -; CHECK-GI-NEXT: sxtw x8, w1 -; CHECK-GI-NEXT: sxtw x9, w3 +; CHECK-GI-NEXT: sxtw x8, w3 +; CHECK-GI-NEXT: sxtw x9, w1 ; CHECK-GI-NEXT: ldr d0, [x0] ; CHECK-GI-NEXT: ldr d1, [x2] -; CHECK-GI-NEXT: add x10, x0, x8 -; CHECK-GI-NEXT: add x11, x2, x9 +; CHECK-GI-NEXT: add x10, x0, x9 +; CHECK-GI-NEXT: add x11, x2, x8 ; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 ; CHECK-GI-NEXT: ldr d2, [x10] -; CHECK-GI-NEXT: ldr d3, [x11] +; CHECK-GI-NEXT: add x10, x10, x9 +; CHECK-GI-NEXT: add x12, x11, x8 ; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-GI-NEXT: add x10, x10, x8 -; CHECK-GI-NEXT: add x11, x11, x9 +; CHECK-GI-NEXT: ldr d3, [x11] +; CHECK-GI-NEXT: ldr d4, [x10] +; CHECK-GI-NEXT: ldr d5, [x12] +; CHECK-GI-NEXT: add x10, x10, x9 +; CHECK-GI-NEXT: add x11, x12, x8 ; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #0 ; CHECK-GI-NEXT: ushll v3.8h, v3.8b, #0 -; CHECK-GI-NEXT: ldr d4, [x10] -; CHECK-GI-NEXT: ldr d5, [x11] -; CHECK-GI-NEXT: add x10, x10, x8 -; CHECK-GI-NEXT: add x11, x11, x9 +; CHECK-GI-NEXT: ushll v4.8h, v4.8b, #0 +; CHECK-GI-NEXT: ushll v5.8h, v5.8b, #0 ; CHECK-GI-NEXT: uabdl v6.4s, v0.4h, v1.4h ; CHECK-GI-NEXT: uabdl2 v0.4s, v0.8h, v1.8h ; CHECK-GI-NEXT: ldr d1, [x10] -; CHECK-GI-NEXT: ushll v4.8h, v4.8b, #0 -; CHECK-GI-NEXT: ushll v5.8h, v5.8b, #0 ; CHECK-GI-NEXT: ldr d7, [x11] +; CHECK-GI-NEXT: add x10, x10, x9 +; CHECK-GI-NEXT: add x11, x11, x8 ; CHECK-GI-NEXT: uabdl v16.4s, v2.4h, v3.4h ; CHECK-GI-NEXT: uabdl2 v2.4s, v2.8h, v3.8h -; CHECK-GI-NEXT: ushll v3.8h, v1.8b, #0 -; CHECK-GI-NEXT: ushll v7.8h, v7.8b, #0 -; CHECK-GI-NEXT: add x10, x10, x8 -; CHECK-GI-NEXT: add x11, x11, x9 -; CHECK-GI-NEXT: uabdl v1.4s, v4.4h, v5.4h +; CHECK-GI-NEXT: uabdl v3.4s, v4.4h, v5.4h ; CHECK-GI-NEXT: uabdl2 v4.4s, v4.8h, v5.8h +; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-GI-NEXT: ushll v7.8h, v7.8b, #0 ; CHECK-GI-NEXT: ldr d5, [x10] -; CHECK-GI-NEXT: add v2.4s, v16.4s, v2.4s -; CHECK-GI-NEXT: ldr d16, [x11] +; CHECK-GI-NEXT: ldr d17, [x11] +; CHECK-GI-NEXT: add x10, x10, x9 +; CHECK-GI-NEXT: add x11, x11, x8 ; CHECK-GI-NEXT: add v0.4s, v6.4s, v0.4s -; CHECK-GI-NEXT: uabdl v6.4s, v3.4h, v7.4h -; CHECK-GI-NEXT: uabdl2 v3.4s, v3.8h, v7.8h ; CHECK-GI-NEXT: ushll v5.8h, v5.8b, #0 -; CHECK-GI-NEXT: add x10, x10, x8 -; CHECK-GI-NEXT: ushll v7.8h, v16.8b, #0 -; CHECK-GI-NEXT: add x11, x11, x9 -; CHECK-GI-NEXT: ldr d16, [x10] -; CHECK-GI-NEXT: ldr d17, [x11] -; CHECK-GI-NEXT: add v1.4s, v1.4s, v4.4s -; CHECK-GI-NEXT: add x10, x10, x8 -; CHECK-GI-NEXT: add x11, x11, x9 -; CHECK-GI-NEXT: add v3.4s, v6.4s, v3.4s -; CHECK-GI-NEXT: ushll v16.8h, v16.8b, #0 ; CHECK-GI-NEXT: ushll v17.8h, v17.8b, #0 -; CHECK-GI-NEXT: uabdl v22.4s, v5.4h, v7.4h -; CHECK-GI-NEXT: uabdl2 v5.4s, v5.8h, v7.8h +; CHECK-GI-NEXT: add v2.4s, v16.4s, v2.4s +; CHECK-GI-NEXT: add v3.4s, v3.4s, v4.4s +; CHECK-GI-NEXT: uabdl v4.4s, v1.4h, v7.4h +; CHECK-GI-NEXT: uabdl2 v1.4s, v1.8h, v7.8h +; CHECK-GI-NEXT: ldr d7, [x10] +; CHECK-GI-NEXT: ldr d16, [x11] +; CHECK-GI-NEXT: add x10, x10, x9 +; CHECK-GI-NEXT: add x11, x11, x8 ; CHECK-GI-NEXT: ldr d18, [x10] +; CHECK-GI-NEXT: ldr d20, [x10, x9] ; CHECK-GI-NEXT: ldr d19, [x11] -; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: ldr d21, [x11, x8] +; CHECK-GI-NEXT: uabdl v6.4s, v5.4h, v17.4h +; CHECK-GI-NEXT: ushll v7.8h, v7.8b, #0 +; CHECK-GI-NEXT: ushll v16.8h, v16.8b, #0 +; CHECK-GI-NEXT: uabdl2 v5.4s, v5.8h, v17.8h +; CHECK-GI-NEXT: ushll v17.8h, v18.8b, #0 +; CHECK-GI-NEXT: ushll v18.8h, v19.8b, #0 +; CHECK-GI-NEXT: add v1.4s, v4.4s, v1.4s +; CHECK-GI-NEXT: ushll v4.8h, v20.8b, #0 +; CHECK-GI-NEXT: ushll v19.8h, v21.8b, #0 ; CHECK-GI-NEXT: addv s2, v2.4s -; CHECK-GI-NEXT: addv s1, v1.4s -; CHECK-GI-NEXT: ushll v18.8h, v18.8b, #0 -; CHECK-GI-NEXT: ushll v19.8h, v19.8b, #0 -; CHECK-GI-NEXT: uabdl v4.4s, v16.4h, v17.4h -; CHECK-GI-NEXT: uabdl2 v16.4s, v16.8h, v17.8h -; CHECK-GI-NEXT: add v5.4s, v22.4s, v5.4s -; CHECK-GI-NEXT: ldr d20, [x10, x8] -; CHECK-GI-NEXT: ldr d21, [x11, x9] +; CHECK-GI-NEXT: addv s0, v0.4s ; CHECK-GI-NEXT: addv s3, v3.4s +; CHECK-GI-NEXT: uabdl v20.4s, v7.4h, v16.4h +; CHECK-GI-NEXT: uabdl2 v7.4s, v7.8h, v16.8h +; CHECK-GI-NEXT: add v5.4s, v6.4s, v5.4s +; CHECK-GI-NEXT: uabdl v6.4s, v17.4h, v18.4h +; CHECK-GI-NEXT: uabdl2 v16.4s, v17.8h, v18.8h +; CHECK-GI-NEXT: uabdl v17.4s, v4.4h, v19.4h +; CHECK-GI-NEXT: uabdl2 v4.4s, v4.8h, v19.8h ; CHECK-GI-NEXT: fmov w8, s2 +; CHECK-GI-NEXT: addv s1, v1.4s ; CHECK-GI-NEXT: fmov w9, s0 -; CHECK-GI-NEXT: ushll v7.8h, v20.8b, #0 -; CHECK-GI-NEXT: ushll v20.8h, v21.8b, #0 -; CHECK-GI-NEXT: uabdl v6.4s, v18.4h, v19.4h -; CHECK-GI-NEXT: uabdl2 v17.4s, v18.8h, v19.8h -; CHECK-GI-NEXT: add v4.4s, v4.4s, v16.4s -; CHECK-GI-NEXT: addv s5, v5.4s -; CHECK-GI-NEXT: fmov w10, s1 +; CHECK-GI-NEXT: fmov w10, s3 +; CHECK-GI-NEXT: add v7.4s, v20.4s, v7.4s +; CHECK-GI-NEXT: add v0.4s, v17.4s, v4.4s +; CHECK-GI-NEXT: addv s4, v5.4s +; CHECK-GI-NEXT: add v2.4s, v6.4s, v16.4s ; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: fmov w9, s3 -; CHECK-GI-NEXT: uabdl v18.4s, v7.4h, v20.4h -; CHECK-GI-NEXT: uabdl2 v7.4s, v7.8h, v20.8h -; CHECK-GI-NEXT: add v6.4s, v6.4s, v17.4s +; CHECK-GI-NEXT: fmov w9, s1 ; CHECK-GI-NEXT: add w8, w10, w8 -; CHECK-GI-NEXT: addv s0, v4.4s +; CHECK-GI-NEXT: addv s3, v7.4s +; CHECK-GI-NEXT: addv s1, v2.4s +; CHECK-GI-NEXT: addv s0, v0.4s ; CHECK-GI-NEXT: add w8, w9, w8 -; CHECK-GI-NEXT: fmov w9, s5 -; CHECK-GI-NEXT: add v7.4s, v18.4s, v7.4s -; CHECK-GI-NEXT: addv s1, v6.4s +; CHECK-GI-NEXT: fmov w9, s4 ; CHECK-GI-NEXT: add w8, w9, w8 -; CHECK-GI-NEXT: fmov w9, s0 -; CHECK-GI-NEXT: addv s2, v7.4s +; CHECK-GI-NEXT: fmov w9, s3 ; CHECK-GI-NEXT: add w8, w9, w8 ; CHECK-GI-NEXT: fmov w9, s1 ; CHECK-GI-NEXT: add w8, w9, w8 -; CHECK-GI-NEXT: fmov w9, s2 +; CHECK-GI-NEXT: fmov w9, s0 ; CHECK-GI-NEXT: add w0, w9, w8 ; CHECK-GI-NEXT: ret entry: diff --git a/llvm/test/CodeGen/AArch64/vector-fcopysign.ll b/llvm/test/CodeGen/AArch64/vector-fcopysign.ll index de26676b5c73..063b23275c61 100644 --- a/llvm/test/CodeGen/AArch64/vector-fcopysign.ll +++ b/llvm/test/CodeGen/AArch64/vector-fcopysign.ll @@ -264,15 +264,15 @@ define <4 x bfloat> @test_copysign_v4bf16_v4bf16(<4 x bfloat> %a, <4 x bfloat> % define <4 x bfloat> @test_copysign_v4bf16_v4f32(<4 x bfloat> %a, <4 x float> %b) #0 { ; CHECK-LABEL: test_copysign_v4bf16_v4f32: ; CHECK: ; %bb.0: -; CHECK-NEXT: movi.4s v2, #127, msl #8 -; CHECK-NEXT: movi.4s v3, #1 +; CHECK-NEXT: movi.4s v2, #1 +; CHECK-NEXT: movi.4s v3, #127, msl #8 ; CHECK-NEXT: ushr.4s v4, v1, #16 -; CHECK-NEXT: add.4s v2, v1, v2 -; CHECK-NEXT: and.16b v3, v4, v3 -; CHECK-NEXT: add.4s v2, v3, v2 -; CHECK-NEXT: fcmeq.4s v3, v1, v1 +; CHECK-NEXT: and.16b v2, v4, v2 +; CHECK-NEXT: add.4s v3, v1, v3 +; CHECK-NEXT: fcmeq.4s v4, v1, v1 ; CHECK-NEXT: orr.4s v1, #64, lsl #16 -; CHECK-NEXT: bit.16b v1, v2, v3 +; CHECK-NEXT: add.4s v2, v2, v3 +; CHECK-NEXT: bit.16b v1, v2, v4 ; CHECK-NEXT: mvni.4h v2, #128, lsl #8 ; CHECK-NEXT: shrn.4h v1, v1, #16 ; CHECK-NEXT: bif.8b v0, v1, v2 @@ -286,16 +286,16 @@ define <4 x bfloat> @test_copysign_v4bf16_v4f64(<4 x bfloat> %a, <4 x double> %b ; CHECK-LABEL: test_copysign_v4bf16_v4f64: ; CHECK: ; %bb.0: ; CHECK-NEXT: fcvtxn v1.2s, v1.2d -; CHECK-NEXT: movi.4s v3, #1 +; CHECK-NEXT: movi.4s v3, #127, msl #8 ; CHECK-NEXT: fcvtxn2 v1.4s, v2.2d -; CHECK-NEXT: movi.4s v2, #127, msl #8 +; CHECK-NEXT: movi.4s v2, #1 ; CHECK-NEXT: ushr.4s v4, v1, #16 -; CHECK-NEXT: add.4s v2, v1, v2 -; CHECK-NEXT: and.16b v3, v4, v3 -; CHECK-NEXT: add.4s v2, v3, v2 -; CHECK-NEXT: fcmeq.4s v3, v1, v1 +; CHECK-NEXT: add.4s v3, v1, v3 +; CHECK-NEXT: and.16b v2, v4, v2 +; CHECK-NEXT: fcmeq.4s v4, v1, v1 ; CHECK-NEXT: orr.4s v1, #64, lsl #16 -; CHECK-NEXT: bit.16b v1, v2, v3 +; CHECK-NEXT: add.4s v2, v2, v3 +; CHECK-NEXT: bit.16b v1, v2, v4 ; CHECK-NEXT: mvni.4h v2, #128, lsl #8 ; CHECK-NEXT: shrn.4h v1, v1, #16 ; CHECK-NEXT: bif.8b v0, v1, v2 @@ -322,22 +322,22 @@ define <8 x bfloat> @test_copysign_v8bf16_v8bf16(<8 x bfloat> %a, <8 x bfloat> % define <8 x bfloat> @test_copysign_v8bf16_v8f32(<8 x bfloat> %a, <8 x float> %b) #0 { ; CHECK-LABEL: test_copysign_v8bf16_v8f32: ; CHECK: ; %bb.0: -; CHECK-NEXT: movi.4s v3, #127, msl #8 -; CHECK-NEXT: movi.4s v4, #1 +; CHECK-NEXT: movi.4s v3, #1 +; CHECK-NEXT: movi.4s v4, #127, msl #8 ; CHECK-NEXT: ushr.4s v5, v2, #16 ; CHECK-NEXT: ushr.4s v6, v1, #16 -; CHECK-NEXT: add.4s v7, v2, v3 -; CHECK-NEXT: add.4s v3, v1, v3 -; CHECK-NEXT: and.16b v5, v5, v4 -; CHECK-NEXT: and.16b v4, v6, v4 +; CHECK-NEXT: and.16b v5, v5, v3 +; CHECK-NEXT: add.4s v7, v2, v4 +; CHECK-NEXT: and.16b v3, v6, v3 +; CHECK-NEXT: add.4s v4, v1, v4 ; CHECK-NEXT: fcmeq.4s v6, v2, v2 ; CHECK-NEXT: orr.4s v2, #64, lsl #16 ; CHECK-NEXT: add.4s v5, v5, v7 -; CHECK-NEXT: add.4s v3, v4, v3 -; CHECK-NEXT: fcmeq.4s v4, v1, v1 +; CHECK-NEXT: fcmeq.4s v7, v1, v1 ; CHECK-NEXT: orr.4s v1, #64, lsl #16 +; CHECK-NEXT: add.4s v3, v3, v4 ; CHECK-NEXT: bit.16b v2, v5, v6 -; CHECK-NEXT: bit.16b v1, v3, v4 +; CHECK-NEXT: bit.16b v1, v3, v7 ; CHECK-NEXT: uzp2.8h v1, v1, v2 ; CHECK-NEXT: mvni.8h v2, #128, lsl #8 ; CHECK-NEXT: bif.16b v0, v1, v2 diff --git a/llvm/test/CodeGen/AArch64/vector-gep.ll b/llvm/test/CodeGen/AArch64/vector-gep.ll index 30317dce85e6..c7858416e179 100644 --- a/llvm/test/CodeGen/AArch64/vector-gep.ll +++ b/llvm/test/CodeGen/AArch64/vector-gep.ll @@ -13,11 +13,11 @@ define <2 x ptr> @vector_gep(<2 x ptr> %0) { ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: Lloh0: ; CHECK-NEXT: adrp x8, lCPI0_0@PAGE +; CHECK-NEXT: movi v2.2d, #0x000000ffffffff ; CHECK-NEXT: Lloh1: ; CHECK-NEXT: ldr q1, [x8, lCPI0_0@PAGEOFF] ; CHECK-NEXT: add v0.2d, v0.2d, v1.2d -; CHECK-NEXT: movi v1.2d, #0x000000ffffffff -; CHECK-NEXT: and v0.16b, v0.16b, v1.16b +; CHECK-NEXT: and v0.16b, v0.16b, v2.16b ; CHECK-NEXT: ret ; CHECK-NEXT: .loh AdrpLdr Lloh0, Lloh1 entry: diff --git a/llvm/test/CodeGen/AArch64/vselect-constants.ll b/llvm/test/CodeGen/AArch64/vselect-constants.ll index a32147eebd75..5e6ff1e0740c 100644 --- a/llvm/test/CodeGen/AArch64/vselect-constants.ll +++ b/llvm/test/CodeGen/AArch64/vselect-constants.ll @@ -370,10 +370,9 @@ define @signbit_mask_xor_nxv16i8( %a, %a, zeroinitializer %xor = xor %a, %b diff --git a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll index 08ad34c7b03b..599bd811d7d5 100644 --- a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll +++ b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll @@ -1697,14 +1697,14 @@ define void @zext_v8i8_to_v8i64_with_add_in_sequence_in_loop(ptr %src, ptr %dst) ; CHECK-NEXT: ldp q18, q16, [x10, #96] ; CHECK-NEXT: uaddw.2d v2, v17, v2 ; CHECK-NEXT: stp q4, q5, [x10, #32] -; CHECK-NEXT: ldp q17, q5, [x10, #64] -; CHECK-NEXT: uaddw2.2d v16, v16, v7 +; CHECK-NEXT: uaddw2.2d v5, v16, v7 +; CHECK-NEXT: ldp q16, q4, [x10, #64] ; CHECK-NEXT: uaddw.2d v7, v18, v7 ; CHECK-NEXT: stp q2, q6, [x10] -; CHECK-NEXT: uaddw2.2d v4, v5, v3 -; CHECK-NEXT: uaddw.2d v3, v17, v3 -; CHECK-NEXT: stp q7, q16, [x10, #96] -; CHECK-NEXT: stp q3, q4, [x10, #64] +; CHECK-NEXT: uaddw2.2d v4, v4, v3 +; CHECK-NEXT: uaddw.2d v2, v16, v3 +; CHECK-NEXT: stp q7, q5, [x10, #96] +; CHECK-NEXT: stp q2, q4, [x10, #64] ; CHECK-NEXT: b.ne LBB17_1 ; CHECK-NEXT: ; %bb.2: ; %exit ; CHECK-NEXT: ret @@ -1729,15 +1729,15 @@ define void @zext_v8i8_to_v8i64_with_add_in_sequence_in_loop(ptr %src, ptr %dst) ; CHECK-BE-NEXT: ld1 { v3.8b }, [x10] ; CHECK-BE-NEXT: add x10, x1, x8 ; CHECK-BE-NEXT: add x8, x8, #128 -; CHECK-BE-NEXT: add x15, x10, #96 ; CHECK-BE-NEXT: add x11, x10, #32 ; CHECK-BE-NEXT: add x14, x10, #64 +; CHECK-BE-NEXT: add x15, x10, #96 ; CHECK-BE-NEXT: tbl v4.16b, { v2.16b }, v1.16b ; CHECK-BE-NEXT: tbl v2.16b, { v2.16b }, v0.16b -; CHECK-BE-NEXT: ld1 { v16.2d }, [x15] -; CHECK-BE-NEXT: tbl v5.16b, { v3.16b }, v1.16b +; CHECK-BE-NEXT: ld1 { v5.2d }, [x10] +; CHECK-BE-NEXT: tbl v6.16b, { v3.16b }, v1.16b ; CHECK-BE-NEXT: tbl v3.16b, { v3.16b }, v0.16b -; CHECK-BE-NEXT: ld1 { v6.2d }, [x10] +; CHECK-BE-NEXT: ld1 { v16.2d }, [x15] ; CHECK-BE-NEXT: ld1 { v19.2d }, [x14] ; CHECK-BE-NEXT: ld1 { v21.2d }, [x11] ; CHECK-BE-NEXT: add x12, x10, #48 @@ -1747,11 +1747,12 @@ define void @zext_v8i8_to_v8i64_with_add_in_sequence_in_loop(ptr %src, ptr %dst) ; CHECK-BE-NEXT: rev32 v7.8b, v4.8b ; CHECK-BE-NEXT: ext v4.16b, v4.16b, v4.16b, #8 ; CHECK-BE-NEXT: rev32 v17.8b, v2.8b -; CHECK-BE-NEXT: ext v18.16b, v5.16b, v5.16b, #8 +; CHECK-BE-NEXT: ext v18.16b, v6.16b, v6.16b, #8 ; CHECK-BE-NEXT: ext v20.16b, v3.16b, v3.16b, #8 ; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #8 -; CHECK-BE-NEXT: rev32 v5.8b, v5.8b +; CHECK-BE-NEXT: rev32 v6.8b, v6.8b ; CHECK-BE-NEXT: rev32 v3.8b, v3.8b +; CHECK-BE-NEXT: ld1 { v22.2d }, [x12] ; CHECK-BE-NEXT: cmp x8, #1024 ; CHECK-BE-NEXT: rev32 v4.8b, v4.8b ; CHECK-BE-NEXT: uaddw v7.2d, v16.2d, v7.2s @@ -1760,22 +1761,21 @@ define void @zext_v8i8_to_v8i64_with_add_in_sequence_in_loop(ptr %src, ptr %dst) ; CHECK-BE-NEXT: rev32 v20.8b, v20.8b ; CHECK-BE-NEXT: rev32 v2.8b, v2.8b ; CHECK-BE-NEXT: uaddw v17.2d, v19.2d, v17.2s -; CHECK-BE-NEXT: ld1 { v19.2d }, [x12] -; CHECK-BE-NEXT: uaddw v5.2d, v21.2d, v5.2s -; CHECK-BE-NEXT: ld1 { v21.2d }, [x13] -; CHECK-BE-NEXT: uaddw v3.2d, v6.2d, v3.2s -; CHECK-BE-NEXT: ld1 { v6.2d }, [x17] -; CHECK-BE-NEXT: uaddw v4.2d, v16.2d, v4.2s +; CHECK-BE-NEXT: ld1 { v19.2d }, [x13] +; CHECK-BE-NEXT: uaddw v6.2d, v21.2d, v6.2s +; CHECK-BE-NEXT: uaddw v3.2d, v5.2d, v3.2s +; CHECK-BE-NEXT: ld1 { v5.2d }, [x17] ; CHECK-BE-NEXT: st1 { v7.2d }, [x15] -; CHECK-BE-NEXT: uaddw v7.2d, v19.2d, v18.2s -; CHECK-BE-NEXT: uaddw v16.2d, v21.2d, v20.2s -; CHECK-BE-NEXT: uaddw v2.2d, v6.2d, v2.2s -; CHECK-BE-NEXT: st1 { v17.2d }, [x14] -; CHECK-BE-NEXT: st1 { v5.2d }, [x11] +; CHECK-BE-NEXT: uaddw v4.2d, v16.2d, v4.2s +; CHECK-BE-NEXT: st1 { v6.2d }, [x11] +; CHECK-BE-NEXT: uaddw v6.2d, v22.2d, v18.2s ; CHECK-BE-NEXT: st1 { v3.2d }, [x10] +; CHECK-BE-NEXT: uaddw v3.2d, v19.2d, v20.2s +; CHECK-BE-NEXT: uaddw v2.2d, v5.2d, v2.2s +; CHECK-BE-NEXT: st1 { v17.2d }, [x14] ; CHECK-BE-NEXT: st1 { v4.2d }, [x16] -; CHECK-BE-NEXT: st1 { v7.2d }, [x12] -; CHECK-BE-NEXT: st1 { v16.2d }, [x13] +; CHECK-BE-NEXT: st1 { v6.2d }, [x12] +; CHECK-BE-NEXT: st1 { v3.2d }, [x13] ; CHECK-BE-NEXT: st1 { v2.2d }, [x17] ; CHECK-BE-NEXT: b.ne .LBB17_1 ; CHECK-BE-NEXT: // %bb.2: // %exit diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-neon-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-neon-instructions.s index 73eadc268bf2..f703392f3e9d 100644 --- a/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-neon-instructions.s +++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-neon-instructions.s @@ -1070,14 +1070,14 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: [6]: HasSideEffects (U) # CHECK: [1] [2] [3] [4] [5] [6] Instructions: -# CHECK-NEXT: 1 4 0.50 abs d29, d24 -# CHECK-NEXT: 1 4 0.50 abs v0.16b, v0.16b -# CHECK-NEXT: 1 4 0.50 abs v0.2d, v0.2d -# CHECK-NEXT: 1 4 0.50 abs v0.2s, v0.2s -# CHECK-NEXT: 1 4 0.50 abs v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 abs v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 abs v0.8b, v0.8b -# CHECK-NEXT: 1 4 0.50 abs v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 abs d29, d24 +# CHECK-NEXT: 1 3 0.50 abs v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 abs v0.2d, v0.2d +# CHECK-NEXT: 1 3 0.50 abs v0.2s, v0.2s +# CHECK-NEXT: 1 3 0.50 abs v0.4h, v0.4h +# CHECK-NEXT: 1 3 0.50 abs v0.4s, v0.4s +# CHECK-NEXT: 1 3 0.50 abs v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 abs v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 add d17, d31, d29 # CHECK-NEXT: 1 3 0.50 add v0.8b, v0.8b, v0.8b # CHECK-NEXT: 1 4 0.50 addhn v0.2s, v0.2d, v0.2d @@ -1086,8 +1086,8 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 addhn2 v0.16b, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 addhn2 v0.4s, v0.2d, v0.2d # CHECK-NEXT: 1 4 0.50 addhn2 v0.8h, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 addp v0.2d, v0.2d, v0.2d -# CHECK-NEXT: 1 4 0.50 addp v0.8b, v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 addp v0.2d, v0.2d, v0.2d +# CHECK-NEXT: 1 3 0.50 addp v0.8b, v0.8b, v0.8b # CHECK-NEXT: 1 3 0.50 and v0.8b, v0.8b, v0.8b # CHECK-NEXT: 1 3 0.50 bic v0.4h, #15, lsl #8 # CHECK-NEXT: 1 3 0.50 bic v0.8b, v0.8b, v0.8b @@ -1441,13 +1441,13 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 mvni v0.2s, #0 # CHECK-NEXT: 1 3 0.50 mvni v0.4s, #16, msl #16 # CHECK-NEXT: 1 3 0.50 neg d29, d24 -# CHECK-NEXT: 1 4 0.50 neg v0.16b, v0.16b -# CHECK-NEXT: 1 4 0.50 neg v0.2d, v0.2d +# CHECK-NEXT: 1 3 0.50 neg v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 neg v0.2d, v0.2d # CHECK-NEXT: 1 3 0.50 neg v0.2s, v0.2s # CHECK-NEXT: 1 3 0.50 neg v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 neg v0.4s, v0.4s +# CHECK-NEXT: 1 3 0.50 neg v0.4s, v0.4s # CHECK-NEXT: 1 3 0.50 neg v0.8b, v0.8b -# CHECK-NEXT: 1 4 0.50 neg v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 neg v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 mvn v0.16b, v0.16b # CHECK-NEXT: 1 3 0.50 mvn v0.8b, v0.8b # CHECK-NEXT: 1 3 0.50 orn v0.16b, v0.16b, v0.16b @@ -1457,12 +1457,12 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 pmul v0.8b, v0.8b, v0.8b # CHECK-NEXT: 1 4 0.50 pmull v0.8h, v0.8b, v0.8b # CHECK-NEXT: 1 4 0.50 pmull2 v0.8h, v0.16b, v0.16b -# CHECK-NEXT: 1 4 0.50 raddhn v0.2s, v0.2d, v0.2d -# CHECK-NEXT: 1 4 0.50 raddhn v0.4h, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 raddhn v0.8b, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 raddhn2 v0.16b, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 raddhn2 v0.4s, v0.2d, v0.2d -# CHECK-NEXT: 1 4 0.50 raddhn2 v0.8h, v0.4s, v0.4s +# CHECK-NEXT: 1 8 0.50 raddhn v0.2s, v0.2d, v0.2d +# CHECK-NEXT: 1 8 0.50 raddhn v0.4h, v0.4s, v0.4s +# CHECK-NEXT: 1 8 0.50 raddhn v0.8b, v0.8h, v0.8h +# CHECK-NEXT: 1 8 0.50 raddhn2 v0.16b, v0.8h, v0.8h +# CHECK-NEXT: 1 8 0.50 raddhn2 v0.4s, v0.2d, v0.2d +# CHECK-NEXT: 1 8 0.50 raddhn2 v0.8h, v0.4s, v0.4s # CHECK-NEXT: 1 4 0.50 rbit v0.16b, v0.16b # CHECK-NEXT: 1 4 0.50 rbit v0.8b, v0.8b # CHECK-NEXT: 1 4 0.50 rev16 v21.8b, v1.8b @@ -1483,19 +1483,19 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 rshrn2 v0.16b, v0.8h, #3 # CHECK-NEXT: 1 4 0.50 rshrn2 v0.4s, v0.2d, #3 # CHECK-NEXT: 1 4 0.50 rshrn2 v0.8h, v0.4s, #3 -# CHECK-NEXT: 1 4 0.50 rsubhn v0.2s, v0.2d, v0.2d -# CHECK-NEXT: 1 4 0.50 rsubhn v0.4h, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 rsubhn v0.8b, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 rsubhn2 v0.16b, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 rsubhn2 v0.4s, v0.2d, v0.2d -# CHECK-NEXT: 1 4 0.50 rsubhn2 v0.8h, v0.4s, v0.4s -# CHECK-NEXT: 1 8 0.50 saba v0.16b, v0.16b, v0.16b -# CHECK-NEXT: 1 8 0.50 sabal v0.2d, v0.2s, v0.2s -# CHECK-NEXT: 1 8 0.50 sabal v0.4s, v0.4h, v0.4h -# CHECK-NEXT: 1 8 0.50 sabal v0.8h, v0.8b, v0.8b -# CHECK-NEXT: 1 8 0.50 sabal2 v0.2d, v0.4s, v0.4s -# CHECK-NEXT: 1 8 0.50 sabal2 v0.4s, v0.8h, v0.8h -# CHECK-NEXT: 1 8 0.50 sabal2 v0.8h, v0.16b, v0.16b +# CHECK-NEXT: 1 8 0.50 rsubhn v0.2s, v0.2d, v0.2d +# CHECK-NEXT: 1 8 0.50 rsubhn v0.4h, v0.4s, v0.4s +# CHECK-NEXT: 1 8 0.50 rsubhn v0.8b, v0.8h, v0.8h +# CHECK-NEXT: 1 8 0.50 rsubhn2 v0.16b, v0.8h, v0.8h +# CHECK-NEXT: 1 8 0.50 rsubhn2 v0.4s, v0.2d, v0.2d +# CHECK-NEXT: 1 8 0.50 rsubhn2 v0.8h, v0.4s, v0.4s +# CHECK-NEXT: 1 6 0.50 saba v0.16b, v0.16b, v0.16b +# CHECK-NEXT: 1 6 0.50 sabal v0.2d, v0.2s, v0.2s +# CHECK-NEXT: 1 6 0.50 sabal v0.4s, v0.4h, v0.4h +# CHECK-NEXT: 1 6 0.50 sabal v0.8h, v0.8b, v0.8b +# CHECK-NEXT: 1 6 0.50 sabal2 v0.2d, v0.4s, v0.4s +# CHECK-NEXT: 1 6 0.50 sabal2 v0.4s, v0.8h, v0.8h +# CHECK-NEXT: 1 6 0.50 sabal2 v0.8h, v0.16b, v0.16b # CHECK-NEXT: 1 3 0.50 sabd v0.4h, v0.4h, v0.4h # CHECK-NEXT: 1 3 0.50 sabdl v0.2d, v0.2s, v0.2s # CHECK-NEXT: 1 3 0.50 sabdl v0.4s, v0.4h, v0.4h @@ -1503,30 +1503,30 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 sabdl2 v0.2d, v0.4s, v0.4s # CHECK-NEXT: 1 3 0.50 sabdl2 v0.4s, v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 sabdl2 v0.8h, v0.16b, v0.16b -# CHECK-NEXT: 1 8 1.00 sadalp v0.1d, v0.2s -# CHECK-NEXT: 1 8 1.00 sadalp v0.2d, v0.4s -# CHECK-NEXT: 1 8 1.00 sadalp v0.2s, v0.4h -# CHECK-NEXT: 1 8 1.00 sadalp v0.4h, v0.8b -# CHECK-NEXT: 1 8 1.00 sadalp v0.4s, v0.8h -# CHECK-NEXT: 1 8 1.00 sadalp v0.8h, v0.16b -# CHECK-NEXT: 1 4 0.50 saddl v0.2d, v0.2s, v0.2s -# CHECK-NEXT: 1 4 0.50 saddl v0.4s, v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 saddl v0.8h, v0.8b, v0.8b -# CHECK-NEXT: 1 4 0.50 saddl2 v0.2d, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 saddl2 v0.4s, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 saddl2 v0.8h, v0.16b, v0.16b -# CHECK-NEXT: 1 4 0.50 saddlp v0.1d, v0.2s -# CHECK-NEXT: 1 4 0.50 saddlp v0.2d, v0.4s -# CHECK-NEXT: 1 4 0.50 saddlp v0.2s, v0.4h -# CHECK-NEXT: 1 4 0.50 saddlp v0.4h, v0.8b -# CHECK-NEXT: 1 4 0.50 saddlp v0.4s, v0.8h -# CHECK-NEXT: 1 4 0.50 saddlp v0.8h, v0.16b -# CHECK-NEXT: 1 4 0.50 saddw v0.2d, v0.2d, v0.2s -# CHECK-NEXT: 1 4 0.50 saddw v0.4s, v0.4s, v0.4h -# CHECK-NEXT: 1 4 0.50 saddw v0.8h, v0.8h, v0.8b -# CHECK-NEXT: 1 4 0.50 saddw2 v0.2d, v0.2d, v0.4s -# CHECK-NEXT: 1 4 0.50 saddw2 v0.4s, v0.4s, v0.8h -# CHECK-NEXT: 1 4 0.50 saddw2 v0.8h, v0.8h, v0.16b +# CHECK-NEXT: 1 7 1.00 sadalp v0.1d, v0.2s +# CHECK-NEXT: 1 7 1.00 sadalp v0.2d, v0.4s +# CHECK-NEXT: 1 7 1.00 sadalp v0.2s, v0.4h +# CHECK-NEXT: 1 7 1.00 sadalp v0.4h, v0.8b +# CHECK-NEXT: 1 7 1.00 sadalp v0.4s, v0.8h +# CHECK-NEXT: 1 7 1.00 sadalp v0.8h, v0.16b +# CHECK-NEXT: 1 3 0.50 saddl v0.2d, v0.2s, v0.2s +# CHECK-NEXT: 1 3 0.50 saddl v0.4s, v0.4h, v0.4h +# CHECK-NEXT: 1 3 0.50 saddl v0.8h, v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 saddl2 v0.2d, v0.4s, v0.4s +# CHECK-NEXT: 1 3 0.50 saddl2 v0.4s, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 saddl2 v0.8h, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 saddlp v0.1d, v0.2s +# CHECK-NEXT: 1 3 0.50 saddlp v0.2d, v0.4s +# CHECK-NEXT: 1 3 0.50 saddlp v0.2s, v0.4h +# CHECK-NEXT: 1 3 0.50 saddlp v0.4h, v0.8b +# CHECK-NEXT: 1 3 0.50 saddlp v0.4s, v0.8h +# CHECK-NEXT: 1 3 0.50 saddlp v0.8h, v0.16b +# CHECK-NEXT: 1 3 0.50 saddw v0.2d, v0.2d, v0.2s +# CHECK-NEXT: 1 3 0.50 saddw v0.4s, v0.4s, v0.4h +# CHECK-NEXT: 1 3 0.50 saddw v0.8h, v0.8h, v0.8b +# CHECK-NEXT: 1 3 0.50 saddw2 v0.2d, v0.2d, v0.4s +# CHECK-NEXT: 1 3 0.50 saddw2 v0.4s, v0.4s, v0.8h +# CHECK-NEXT: 1 3 0.50 saddw2 v0.8h, v0.8h, v0.16b # CHECK-NEXT: 1 4 0.50 scvtf d21, d12 # CHECK-NEXT: 1 4 0.50 scvtf d21, d12, #64 # CHECK-NEXT: 1 4 0.50 scvtf s22, s13 @@ -1573,18 +1573,18 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 sli v0.4s, v0.4s, #3 # CHECK-NEXT: 1 4 0.50 sli v0.8b, v0.8b, #3 # CHECK-NEXT: 1 4 0.50 sli v0.8h, v0.8h, #3 -# CHECK-NEXT: 1 4 0.50 smax v0.2s, v0.2s, v0.2s -# CHECK-NEXT: 1 4 0.50 smax v0.4h, v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 smax v0.8b, v0.8b, v0.8b -# CHECK-NEXT: 1 4 0.50 smaxp v0.2s, v0.2s, v0.2s -# CHECK-NEXT: 1 4 0.50 smaxp v0.4h, v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 smaxp v0.8b, v0.8b, v0.8b -# CHECK-NEXT: 1 4 0.50 smin v0.16b, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 smax v0.2s, v0.2s, v0.2s +# CHECK-NEXT: 1 3 0.50 smax v0.4h, v0.4h, v0.4h +# CHECK-NEXT: 1 3 0.50 smax v0.8b, v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 smaxp v0.2s, v0.2s, v0.2s +# CHECK-NEXT: 1 3 0.50 smaxp v0.4h, v0.4h, v0.4h +# CHECK-NEXT: 1 3 0.50 smaxp v0.8b, v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 smin v0.16b, v0.16b, v0.16b # CHECK-NEXT: 1 4 0.50 smin v0.4s, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 smin v0.8h, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 sminp v0.16b, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 smin v0.8h, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 sminp v0.16b, v0.16b, v0.16b # CHECK-NEXT: 1 4 0.50 sminp v0.4s, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 sminp v0.8h, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 sminp v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 smlal v0.2d, v0.2s, v0.2s # CHECK-NEXT: 1 4 0.50 smlal v0.4s, v0.4h, v0.4h # CHECK-NEXT: 1 4 0.50 smlal v0.8h, v0.8b, v0.8b @@ -1777,14 +1777,14 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 srshr v0.4s, v0.4s, #3 # CHECK-NEXT: 1 3 0.50 srshr v0.8b, v0.8b, #3 # CHECK-NEXT: 1 3 0.50 srshr v0.8h, v0.8h, #3 -# CHECK-NEXT: 1 8 1.00 srsra d15, d11, #19 -# CHECK-NEXT: 1 8 1.00 srsra v0.16b, v0.16b, #3 -# CHECK-NEXT: 1 8 1.00 srsra v0.2d, v0.2d, #3 -# CHECK-NEXT: 1 8 1.00 srsra v0.2s, v0.2s, #3 -# CHECK-NEXT: 1 8 1.00 srsra v0.4h, v0.4h, #3 -# CHECK-NEXT: 1 8 1.00 srsra v0.4s, v0.4s, #3 -# CHECK-NEXT: 1 8 1.00 srsra v0.8b, v0.8b, #3 -# CHECK-NEXT: 1 8 1.00 srsra v0.8h, v0.8h, #3 +# CHECK-NEXT: 1 7 1.00 srsra d15, d11, #19 +# CHECK-NEXT: 1 7 1.00 srsra v0.16b, v0.16b, #3 +# CHECK-NEXT: 1 7 1.00 srsra v0.2d, v0.2d, #3 +# CHECK-NEXT: 1 7 1.00 srsra v0.2s, v0.2s, #3 +# CHECK-NEXT: 1 7 1.00 srsra v0.4h, v0.4h, #3 +# CHECK-NEXT: 1 7 1.00 srsra v0.4s, v0.4s, #3 +# CHECK-NEXT: 1 7 1.00 srsra v0.8b, v0.8b, #3 +# CHECK-NEXT: 1 7 1.00 srsra v0.8h, v0.8h, #3 # CHECK-NEXT: 1 3 0.50 sshl d31, d31, d31 # CHECK-NEXT: 1 3 0.50 sshl v0.2d, v0.2d, v0.2d # CHECK-NEXT: 1 3 0.50 sshl v0.2s, v0.2s, v0.2s @@ -1800,26 +1800,26 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 sshr v0.4s, v0.4s, #3 # CHECK-NEXT: 1 3 0.50 sshr v0.8b, v0.8b, #3 # CHECK-NEXT: 1 3 0.50 sshr v0.8h, v0.8h, #3 -# CHECK-NEXT: 1 8 1.00 ssra d18, d12, #21 -# CHECK-NEXT: 1 8 1.00 ssra v0.16b, v0.16b, #3 -# CHECK-NEXT: 1 8 1.00 ssra v0.2d, v0.2d, #3 -# CHECK-NEXT: 1 8 1.00 ssra v0.2s, v0.2s, #3 -# CHECK-NEXT: 1 8 1.00 ssra v0.4h, v0.4h, #3 -# CHECK-NEXT: 1 8 1.00 ssra v0.4s, v0.4s, #3 -# CHECK-NEXT: 1 8 1.00 ssra v0.8b, v0.8b, #3 -# CHECK-NEXT: 1 8 1.00 ssra v0.8h, v0.8h, #3 -# CHECK-NEXT: 1 4 0.50 ssubl v0.2d, v0.2s, v0.2s -# CHECK-NEXT: 1 4 0.50 ssubl v0.4s, v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 ssubl v0.8h, v0.8b, v0.8b -# CHECK-NEXT: 1 4 0.50 ssubl2 v0.2d, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 ssubl2 v0.4s, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 ssubl2 v0.8h, v0.16b, v0.16b -# CHECK-NEXT: 1 4 0.50 ssubw v0.2d, v0.2d, v0.2s -# CHECK-NEXT: 1 4 0.50 ssubw v0.4s, v0.4s, v0.4h -# CHECK-NEXT: 1 4 0.50 ssubw v0.8h, v0.8h, v0.8b -# CHECK-NEXT: 1 4 0.50 ssubw2 v0.2d, v0.2d, v0.4s -# CHECK-NEXT: 1 4 0.50 ssubw2 v0.4s, v0.4s, v0.8h -# CHECK-NEXT: 1 4 0.50 ssubw2 v0.8h, v0.8h, v0.16b +# CHECK-NEXT: 1 3 0.50 ssra d18, d12, #21 +# CHECK-NEXT: 1 3 0.50 ssra v0.16b, v0.16b, #3 +# CHECK-NEXT: 1 3 0.50 ssra v0.2d, v0.2d, #3 +# CHECK-NEXT: 1 3 0.50 ssra v0.2s, v0.2s, #3 +# CHECK-NEXT: 1 3 0.50 ssra v0.4h, v0.4h, #3 +# CHECK-NEXT: 1 3 0.50 ssra v0.4s, v0.4s, #3 +# CHECK-NEXT: 1 3 0.50 ssra v0.8b, v0.8b, #3 +# CHECK-NEXT: 1 3 0.50 ssra v0.8h, v0.8h, #3 +# CHECK-NEXT: 1 3 0.50 ssubl v0.2d, v0.2s, v0.2s +# CHECK-NEXT: 1 3 0.50 ssubl v0.4s, v0.4h, v0.4h +# CHECK-NEXT: 1 3 0.50 ssubl v0.8h, v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 ssubl2 v0.2d, v0.4s, v0.4s +# CHECK-NEXT: 1 3 0.50 ssubl2 v0.4s, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 ssubl2 v0.8h, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 ssubw v0.2d, v0.2d, v0.2s +# CHECK-NEXT: 1 3 0.50 ssubw v0.4s, v0.4s, v0.4h +# CHECK-NEXT: 1 3 0.50 ssubw v0.8h, v0.8h, v0.8b +# CHECK-NEXT: 1 3 0.50 ssubw2 v0.2d, v0.2d, v0.4s +# CHECK-NEXT: 1 3 0.50 ssubw2 v0.4s, v0.4s, v0.8h +# CHECK-NEXT: 1 3 0.50 ssubw2 v0.8h, v0.8h, v0.16b # CHECK-NEXT: 1 4 1.00 * st1 { v0.16b }, [x0] # CHECK-NEXT: 2 5 2.00 * st1 { v0.2d, v1.2d, v2.2d }, [x0], #48 # CHECK-NEXT: 1 5 4.00 * st1 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0] @@ -1843,7 +1843,7 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 5 2.00 * st4 { v0.b, v1.b, v2.b, v3.b }[9], [x0] # CHECK-NEXT: 2 5 2.00 * st4 { v0.b, v1.b, v2.b, v3.b }[9], [x0], x5 # CHECK-NEXT: 1 3 0.50 sub d15, d5, d16 -# CHECK-NEXT: 1 4 0.50 sub v0.2d, v0.2d, v0.2d +# CHECK-NEXT: 1 3 0.50 sub v0.2d, v0.2d, v0.2d # CHECK-NEXT: 1 4 0.50 suqadd b19, b14 # CHECK-NEXT: 1 4 0.50 suqadd d18, d22 # CHECK-NEXT: 1 4 0.50 suqadd h20, h15 @@ -1885,13 +1885,13 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 trn2 v0.4s, v0.4s, v0.4s # CHECK-NEXT: 1 4 0.50 trn2 v0.8b, v0.8b, v0.8b # CHECK-NEXT: 1 4 0.50 trn2 v0.8h, v0.8h, v0.8h -# CHECK-NEXT: 1 8 0.50 uaba v0.8b, v0.8b, v0.8b -# CHECK-NEXT: 1 8 0.50 uabal v0.2d, v0.2s, v0.2s -# CHECK-NEXT: 1 8 0.50 uabal v0.4s, v0.4h, v0.4h -# CHECK-NEXT: 1 8 0.50 uabal v0.8h, v0.8b, v0.8b -# CHECK-NEXT: 1 8 0.50 uabal2 v0.2d, v0.4s, v0.4s -# CHECK-NEXT: 1 8 0.50 uabal2 v0.4s, v0.8h, v0.8h -# CHECK-NEXT: 1 8 0.50 uabal2 v0.8h, v0.16b, v0.16b +# CHECK-NEXT: 1 6 0.50 uaba v0.8b, v0.8b, v0.8b +# CHECK-NEXT: 1 6 0.50 uabal v0.2d, v0.2s, v0.2s +# CHECK-NEXT: 1 6 0.50 uabal v0.4s, v0.4h, v0.4h +# CHECK-NEXT: 1 6 0.50 uabal v0.8h, v0.8b, v0.8b +# CHECK-NEXT: 1 6 0.50 uabal2 v0.2d, v0.4s, v0.4s +# CHECK-NEXT: 1 6 0.50 uabal2 v0.4s, v0.8h, v0.8h +# CHECK-NEXT: 1 6 0.50 uabal2 v0.8h, v0.16b, v0.16b # CHECK-NEXT: 1 3 0.50 uabd v0.4h, v0.4h, v0.4h # CHECK-NEXT: 1 3 0.50 uabdl v0.2d, v0.2s, v0.2s # CHECK-NEXT: 1 3 0.50 uabdl v0.4s, v0.4h, v0.4h @@ -1899,30 +1899,30 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 uabdl2 v0.2d, v0.4s, v0.4s # CHECK-NEXT: 1 3 0.50 uabdl2 v0.4s, v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 uabdl2 v0.8h, v0.16b, v0.16b -# CHECK-NEXT: 1 8 1.00 uadalp v0.1d, v0.2s -# CHECK-NEXT: 1 8 1.00 uadalp v0.2d, v0.4s -# CHECK-NEXT: 1 8 1.00 uadalp v0.2s, v0.4h -# CHECK-NEXT: 1 8 1.00 uadalp v0.4h, v0.8b -# CHECK-NEXT: 1 8 1.00 uadalp v0.4s, v0.8h -# CHECK-NEXT: 1 8 1.00 uadalp v0.8h, v0.16b -# CHECK-NEXT: 1 4 0.50 uaddl v0.2d, v0.2s, v0.2s -# CHECK-NEXT: 1 4 0.50 uaddl v0.4s, v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 uaddl v0.8h, v0.8b, v0.8b -# CHECK-NEXT: 1 4 0.50 uaddl2 v0.2d, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 uaddl2 v0.4s, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 uaddl2 v0.8h, v0.16b, v0.16b -# CHECK-NEXT: 1 4 0.50 uaddlp v0.1d, v0.2s -# CHECK-NEXT: 1 4 0.50 uaddlp v0.2d, v0.4s -# CHECK-NEXT: 1 4 0.50 uaddlp v0.2s, v0.4h -# CHECK-NEXT: 1 4 0.50 uaddlp v0.4h, v0.8b -# CHECK-NEXT: 1 4 0.50 uaddlp v0.4s, v0.8h -# CHECK-NEXT: 1 4 0.50 uaddlp v0.8h, v0.16b -# CHECK-NEXT: 1 4 0.50 uaddw v0.2d, v0.2d, v0.2s -# CHECK-NEXT: 1 4 0.50 uaddw v0.4s, v0.4s, v0.4h -# CHECK-NEXT: 1 4 0.50 uaddw v0.8h, v0.8h, v0.8b -# CHECK-NEXT: 1 4 0.50 uaddw2 v0.2d, v0.2d, v0.4s -# CHECK-NEXT: 1 4 0.50 uaddw2 v0.4s, v0.4s, v0.8h -# CHECK-NEXT: 1 4 0.50 uaddw2 v0.8h, v0.8h, v0.16b +# CHECK-NEXT: 1 7 1.00 uadalp v0.1d, v0.2s +# CHECK-NEXT: 1 7 1.00 uadalp v0.2d, v0.4s +# CHECK-NEXT: 1 7 1.00 uadalp v0.2s, v0.4h +# CHECK-NEXT: 1 7 1.00 uadalp v0.4h, v0.8b +# CHECK-NEXT: 1 7 1.00 uadalp v0.4s, v0.8h +# CHECK-NEXT: 1 7 1.00 uadalp v0.8h, v0.16b +# CHECK-NEXT: 1 3 0.50 uaddl v0.2d, v0.2s, v0.2s +# CHECK-NEXT: 1 3 0.50 uaddl v0.4s, v0.4h, v0.4h +# CHECK-NEXT: 1 3 0.50 uaddl v0.8h, v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 uaddl2 v0.2d, v0.4s, v0.4s +# CHECK-NEXT: 1 3 0.50 uaddl2 v0.4s, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 uaddl2 v0.8h, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 uaddlp v0.1d, v0.2s +# CHECK-NEXT: 1 3 0.50 uaddlp v0.2d, v0.4s +# CHECK-NEXT: 1 3 0.50 uaddlp v0.2s, v0.4h +# CHECK-NEXT: 1 3 0.50 uaddlp v0.4h, v0.8b +# CHECK-NEXT: 1 3 0.50 uaddlp v0.4s, v0.8h +# CHECK-NEXT: 1 3 0.50 uaddlp v0.8h, v0.16b +# CHECK-NEXT: 1 3 0.50 uaddw v0.2d, v0.2d, v0.2s +# CHECK-NEXT: 1 3 0.50 uaddw v0.4s, v0.4s, v0.4h +# CHECK-NEXT: 1 3 0.50 uaddw v0.8h, v0.8h, v0.8b +# CHECK-NEXT: 1 3 0.50 uaddw2 v0.2d, v0.2d, v0.4s +# CHECK-NEXT: 1 3 0.50 uaddw2 v0.4s, v0.4s, v0.8h +# CHECK-NEXT: 1 3 0.50 uaddw2 v0.8h, v0.8h, v0.16b # CHECK-NEXT: 1 4 0.50 ucvtf d21, d14 # CHECK-NEXT: 1 4 0.50 ucvtf d21, d14, #64 # CHECK-NEXT: 1 4 0.50 ucvtf s22, s13 @@ -1935,21 +1935,21 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 ucvtf v0.4s, v0.4s # CHECK-NEXT: 1 4 0.50 ucvtf v0.4s, v0.4s, #3 # CHECK-NEXT: 1 4 0.50 ucvtf v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 uhadd v0.16b, v0.16b, v0.16b -# CHECK-NEXT: 1 4 0.50 uhadd v0.8h, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 uhsub v0.4s, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 umax v0.16b, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 uhadd v0.16b, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 uhadd v0.8h, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 uhsub v0.4s, v0.4s, v0.4s +# CHECK-NEXT: 1 3 0.50 umax v0.16b, v0.16b, v0.16b # CHECK-NEXT: 1 4 0.50 umax v0.4s, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 umax v0.8h, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 umaxp v0.16b, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 umax v0.8h, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 umaxp v0.16b, v0.16b, v0.16b # CHECK-NEXT: 1 4 0.50 umaxp v0.4s, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 umaxp v0.8h, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 umin v0.2s, v0.2s, v0.2s -# CHECK-NEXT: 1 4 0.50 umin v0.4h, v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 umin v0.8b, v0.8b, v0.8b -# CHECK-NEXT: 1 4 0.50 uminp v0.2s, v0.2s, v0.2s -# CHECK-NEXT: 1 4 0.50 uminp v0.4h, v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 uminp v0.8b, v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 umaxp v0.8h, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 umin v0.2s, v0.2s, v0.2s +# CHECK-NEXT: 1 3 0.50 umin v0.4h, v0.4h, v0.4h +# CHECK-NEXT: 1 3 0.50 umin v0.8b, v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 uminp v0.2s, v0.2s, v0.2s +# CHECK-NEXT: 1 3 0.50 uminp v0.4h, v0.4h, v0.4h +# CHECK-NEXT: 1 3 0.50 uminp v0.8b, v0.8b, v0.8b # CHECK-NEXT: 1 4 0.50 umlal v0.2d, v0.2s, v0.2s # CHECK-NEXT: 1 4 0.50 umlal v0.4s, v0.4h, v0.4h # CHECK-NEXT: 1 4 0.50 umlal v0.8h, v0.8b, v0.8b @@ -2024,9 +2024,9 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 uqxtn2 v0.8h, v0.4s # CHECK-NEXT: 1 4 0.50 urecpe v0.2s, v0.2s # CHECK-NEXT: 1 4 0.50 urecpe v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 urhadd v0.16b, v0.16b, v0.16b -# CHECK-NEXT: 1 4 0.50 urhadd v0.4s, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 urhadd v0.8h, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 urhadd v0.16b, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 urhadd v0.4s, v0.4s, v0.4s +# CHECK-NEXT: 1 3 0.50 urhadd v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 urshl d8, d7, d4 # CHECK-NEXT: 1 3 0.50 urshl v0.16b, v0.16b, v0.16b # CHECK-NEXT: 1 3 0.50 urshl v0.2d, v0.2d, v0.2d @@ -2042,14 +2042,14 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 3 0.50 urshr v0.8h, v0.8h, #3 # CHECK-NEXT: 1 12 9.00 ursqrte v0.2s, v0.2s # CHECK-NEXT: 1 12 9.00 ursqrte v0.4s, v0.4s -# CHECK-NEXT: 1 8 1.00 ursra d18, d10, #13 -# CHECK-NEXT: 1 8 1.00 ursra v0.16b, v0.16b, #3 -# CHECK-NEXT: 1 8 1.00 ursra v0.2d, v0.2d, #3 -# CHECK-NEXT: 1 8 1.00 ursra v0.2s, v0.2s, #3 -# CHECK-NEXT: 1 8 1.00 ursra v0.4h, v0.4h, #3 -# CHECK-NEXT: 1 8 1.00 ursra v0.4s, v0.4s, #3 -# CHECK-NEXT: 1 8 1.00 ursra v0.8b, v0.8b, #3 -# CHECK-NEXT: 1 8 1.00 ursra v0.8h, v0.8h, #3 +# CHECK-NEXT: 1 7 1.00 ursra d18, d10, #13 +# CHECK-NEXT: 1 7 1.00 ursra v0.16b, v0.16b, #3 +# CHECK-NEXT: 1 7 1.00 ursra v0.2d, v0.2d, #3 +# CHECK-NEXT: 1 7 1.00 ursra v0.2s, v0.2s, #3 +# CHECK-NEXT: 1 7 1.00 ursra v0.4h, v0.4h, #3 +# CHECK-NEXT: 1 7 1.00 ursra v0.4s, v0.4s, #3 +# CHECK-NEXT: 1 7 1.00 ursra v0.8b, v0.8b, #3 +# CHECK-NEXT: 1 7 1.00 ursra v0.8h, v0.8h, #3 # CHECK-NEXT: 1 3 0.50 ushl d0, d0, d0 # CHECK-NEXT: 1 3 0.50 ushl v0.16b, v0.16b, v0.16b # CHECK-NEXT: 1 3 0.50 ushl v0.4s, v0.4s, v0.4s @@ -2075,26 +2075,26 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: 1 4 0.50 usqadd v0.4s, v0.4s # CHECK-NEXT: 1 4 0.50 usqadd v0.8b, v0.8b # CHECK-NEXT: 1 4 0.50 usqadd v0.8h, v0.8h -# CHECK-NEXT: 1 8 1.00 usra d20, d13, #61 -# CHECK-NEXT: 1 8 1.00 usra v0.16b, v0.16b, #3 -# CHECK-NEXT: 1 8 1.00 usra v0.2d, v0.2d, #3 -# CHECK-NEXT: 1 8 1.00 usra v0.2s, v0.2s, #3 -# CHECK-NEXT: 1 8 1.00 usra v0.4h, v0.4h, #3 -# CHECK-NEXT: 1 8 1.00 usra v0.4s, v0.4s, #3 -# CHECK-NEXT: 1 8 1.00 usra v0.8b, v0.8b, #3 -# CHECK-NEXT: 1 8 1.00 usra v0.8h, v0.8h, #3 -# CHECK-NEXT: 1 4 0.50 usubl v0.2d, v0.2s, v0.2s -# CHECK-NEXT: 1 4 0.50 usubl v0.4s, v0.4h, v0.4h -# CHECK-NEXT: 1 4 0.50 usubl v0.8h, v0.8b, v0.8b -# CHECK-NEXT: 1 4 0.50 usubl2 v0.2d, v0.4s, v0.4s -# CHECK-NEXT: 1 4 0.50 usubl2 v0.4s, v0.8h, v0.8h -# CHECK-NEXT: 1 4 0.50 usubl2 v0.8h, v0.16b, v0.16b -# CHECK-NEXT: 1 4 0.50 usubw v0.2d, v0.2d, v0.2s -# CHECK-NEXT: 1 4 0.50 usubw v0.4s, v0.4s, v0.4h -# CHECK-NEXT: 1 4 0.50 usubw v0.8h, v0.8h, v0.8b -# CHECK-NEXT: 1 4 0.50 usubw2 v0.2d, v0.2d, v0.4s -# CHECK-NEXT: 1 4 0.50 usubw2 v0.4s, v0.4s, v0.8h -# CHECK-NEXT: 1 4 0.50 usubw2 v0.8h, v0.8h, v0.16b +# CHECK-NEXT: 1 3 0.50 usra d20, d13, #61 +# CHECK-NEXT: 1 3 0.50 usra v0.16b, v0.16b, #3 +# CHECK-NEXT: 1 3 0.50 usra v0.2d, v0.2d, #3 +# CHECK-NEXT: 1 3 0.50 usra v0.2s, v0.2s, #3 +# CHECK-NEXT: 1 3 0.50 usra v0.4h, v0.4h, #3 +# CHECK-NEXT: 1 3 0.50 usra v0.4s, v0.4s, #3 +# CHECK-NEXT: 1 3 0.50 usra v0.8b, v0.8b, #3 +# CHECK-NEXT: 1 3 0.50 usra v0.8h, v0.8h, #3 +# CHECK-NEXT: 1 3 0.50 usubl v0.2d, v0.2s, v0.2s +# CHECK-NEXT: 1 3 0.50 usubl v0.4s, v0.4h, v0.4h +# CHECK-NEXT: 1 3 0.50 usubl v0.8h, v0.8b, v0.8b +# CHECK-NEXT: 1 3 0.50 usubl2 v0.2d, v0.4s, v0.4s +# CHECK-NEXT: 1 3 0.50 usubl2 v0.4s, v0.8h, v0.8h +# CHECK-NEXT: 1 3 0.50 usubl2 v0.8h, v0.16b, v0.16b +# CHECK-NEXT: 1 3 0.50 usubw v0.2d, v0.2d, v0.2s +# CHECK-NEXT: 1 3 0.50 usubw v0.4s, v0.4s, v0.4h +# CHECK-NEXT: 1 3 0.50 usubw v0.8h, v0.8h, v0.8b +# CHECK-NEXT: 1 3 0.50 usubw2 v0.2d, v0.2d, v0.4s +# CHECK-NEXT: 1 3 0.50 usubw2 v0.4s, v0.4s, v0.8h +# CHECK-NEXT: 1 3 0.50 usubw2 v0.8h, v0.8h, v0.16b # CHECK-NEXT: 1 4 0.50 uzp1 v0.16b, v0.16b, v0.16b # CHECK-NEXT: 1 4 0.50 uzp1 v0.2d, v0.2d, v0.2d # CHECK-NEXT: 1 4 0.50 uzp1 v0.2s, v0.2s, v0.2s @@ -2148,7 +2148,7 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK: Resource pressure per iteration: # CHECK-NEXT: [0] [1.0] [1.1] [2] [3] [4] [5] [6] [7] [8] [9] [10.0] [10.1] [11] -# CHECK-NEXT: - - - - - 39.00 91.00 - - 509.00 509.00 3.00 3.00 197.00 +# CHECK-NEXT: - - - - - 39.00 91.00 - - 501.00 501.00 3.00 3.00 197.00 # CHECK: Resource pressure by instruction: # CHECK-NEXT: [0] [1.0] [1.1] [2] [3] [4] [5] [6] [7] [8] [9] [10.0] [10.1] [11] Instructions: @@ -2882,14 +2882,14 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - sshr v0.4s, v0.4s, #3 # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - sshr v0.8b, v0.8b, #3 # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - sshr v0.8h, v0.8h, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - ssra d18, d12, #21 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - ssra v0.16b, v0.16b, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - ssra v0.2d, v0.2d, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - ssra v0.2s, v0.2s, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - ssra v0.4h, v0.4h, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - ssra v0.4s, v0.4s, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - ssra v0.8b, v0.8b, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - ssra v0.8h, v0.8h, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssra d18, d12, #21 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssra v0.16b, v0.16b, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssra v0.2d, v0.2d, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssra v0.2s, v0.2s, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssra v0.4h, v0.4h, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssra v0.4s, v0.4s, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssra v0.8b, v0.8b, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssra v0.8h, v0.8h, #3 # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssubl v0.2d, v0.2s, v0.2s # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssubl v0.4s, v0.4h, v0.4h # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - ssubl v0.8h, v0.8b, v0.8b @@ -3157,14 +3157,14 @@ zip2 v0.8h, v0.8h, v0.8h # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usqadd v0.4s, v0.4s # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usqadd v0.8b, v0.8b # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usqadd v0.8h, v0.8h -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - usra d20, d13, #61 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - usra v0.16b, v0.16b, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - usra v0.2d, v0.2d, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - usra v0.2s, v0.2s, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - usra v0.4h, v0.4h, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - usra v0.4s, v0.4s, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - usra v0.8b, v0.8b, #3 -# CHECK-NEXT: - - - - - - - - - 1.00 1.00 - - - usra v0.8h, v0.8h, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usra d20, d13, #61 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usra v0.16b, v0.16b, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usra v0.2d, v0.2d, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usra v0.2s, v0.2s, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usra v0.4h, v0.4h, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usra v0.4s, v0.4s, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usra v0.8b, v0.8b, #3 +# CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usra v0.8h, v0.8h, #3 # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usubl v0.2d, v0.2s, v0.2s # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usubl v0.4s, v0.4h, v0.4h # CHECK-NEXT: - - - - - - - - - 0.50 0.50 - - - usubl v0.8h, v0.8b, v0.8b diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-sve-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-sve-instructions.s index a8fb8b669838..d8051e7ecb4f 100644 --- a/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-sve-instructions.s +++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-sve-instructions.s @@ -3476,12 +3476,12 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 add z31.s, p7/m, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 add z31.s, z31.s, #65280 # CHECK-NEXT: 1 3 0.50 add z31.s, z31.s, z31.s -# CHECK-NEXT: 1 4 0.50 addhnb z0.b, z1.h, z31.h -# CHECK-NEXT: 1 4 0.50 addhnb z0.h, z1.s, z31.s -# CHECK-NEXT: 1 4 0.50 addhnb z0.s, z1.d, z31.d -# CHECK-NEXT: 1 4 0.50 addhnt z0.b, z1.h, z31.h -# CHECK-NEXT: 1 4 0.50 addhnt z0.h, z1.s, z31.s -# CHECK-NEXT: 1 4 0.50 addhnt z0.s, z1.d, z31.d +# CHECK-NEXT: 1 8 0.50 addhnb z0.b, z1.h, z31.h +# CHECK-NEXT: 1 8 0.50 addhnb z0.h, z1.s, z31.s +# CHECK-NEXT: 1 8 0.50 addhnb z0.s, z1.d, z31.d +# CHECK-NEXT: 1 8 0.50 addhnt z0.b, z1.h, z31.h +# CHECK-NEXT: 1 8 0.50 addhnt z0.h, z1.s, z31.s +# CHECK-NEXT: 1 8 0.50 addhnt z0.s, z1.d, z31.d # CHECK-NEXT: 1 3 0.50 addp z0.b, p0/m, z0.b, z1.b # CHECK-NEXT: 1 3 0.50 addp z0.h, p0/m, z0.h, z1.h # CHECK-NEXT: 1 3 0.50 addp z29.s, p7/m, z29.s, z30.s @@ -3516,7 +3516,7 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 aesimc z31.b, z31.b # CHECK-NEXT: 1 3 0.50 aesmc z0.b, z0.b # CHECK-NEXT: 1 3 0.50 aesmc z31.b, z31.b -# CHECK-NEXT: 1 6 1.00 and p0.b, p0/z, p0.b, p1.b +# CHECK-NEXT: 1 2 1.00 and p0.b, p0/z, p0.b, p1.b # CHECK-NEXT: 1 3 0.50 and z0.d, z0.d, #0x6 # CHECK-NEXT: 1 3 0.50 and z0.d, z0.d, #0xfffffffffffffff9 # CHECK-NEXT: 1 3 0.50 and z0.d, z0.d, z0.d @@ -3531,7 +3531,7 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 and z31.s, p7/m, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 and z5.b, z5.b, #0x6 # CHECK-NEXT: 1 3 0.50 and z5.b, z5.b, #0xf9 -# CHECK-NEXT: 1 6 1.00 ands p0.b, p0/z, p0.b, p1.b +# CHECK-NEXT: 1 2 1.00 ands p0.b, p0/z, p0.b, p1.b # CHECK-NEXT: 1 4 1.00 andv b0, p7, z31.b # CHECK-NEXT: 1 4 1.00 andv d0, p7, z31.d # CHECK-NEXT: 1 4 1.00 andv h0, p7, z31.h @@ -3574,7 +3574,7 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 asrr z0.d, p0/m, z0.d, z0.d # CHECK-NEXT: 1 3 0.50 asrr z0.h, p0/m, z0.h, z0.h # CHECK-NEXT: 1 3 0.50 asrr z0.s, p0/m, z0.s, z0.s -# CHECK-NEXT: 1 3 0.50 bcax z29.d, z29.d, z30.d, z31.d +# CHECK-NEXT: 1 4 0.50 bcax z29.d, z29.d, z30.d, z31.d # CHECK-NEXT: 1 14 13.00 bdep z0.b, z1.b, z31.b # CHECK-NEXT: 1 70 69.00 bdep z0.d, z1.d, z31.d # CHECK-NEXT: 1 22 21.00 bdep z0.h, z1.h, z31.h @@ -3603,34 +3603,34 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 70 69.00 bgrp z0.d, z1.d, z31.d # CHECK-NEXT: 1 22 21.00 bgrp z0.h, z1.h, z31.h # CHECK-NEXT: 1 38 37.00 bgrp z0.s, z1.s, z31.s -# CHECK-NEXT: 1 6 1.00 bic p0.b, p0/z, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 bic p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 bic p0.b, p0/z, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 bic p15.b, p15/z, p15.b, p15.b # CHECK-NEXT: 1 3 0.50 bic z0.d, z0.d, z0.d # CHECK-NEXT: 1 3 0.50 bic z23.d, z13.d, z8.d # CHECK-NEXT: 1 3 0.50 bic z31.b, p7/m, z31.b, z31.b # CHECK-NEXT: 1 3 0.50 bic z31.d, p7/m, z31.d, z31.d # CHECK-NEXT: 1 3 0.50 bic z31.h, p7/m, z31.h, z31.h # CHECK-NEXT: 1 3 0.50 bic z31.s, p7/m, z31.s, z31.s -# CHECK-NEXT: 1 6 1.00 bics p0.b, p0/z, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 bics p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 brka p0.b, p15/m, p15.b -# CHECK-NEXT: 1 6 1.00 brka p0.b, p15/z, p15.b -# CHECK-NEXT: 1 6 1.00 brkas p0.b, p15/z, p15.b -# CHECK-NEXT: 1 6 1.00 brkb p0.b, p15/m, p15.b -# CHECK-NEXT: 1 6 1.00 brkb p0.b, p15/z, p15.b -# CHECK-NEXT: 1 6 1.00 brkbs p0.b, p15/z, p15.b -# CHECK-NEXT: 1 6 1.00 brkn p0.b, p15/z, p1.b, p0.b -# CHECK-NEXT: 1 6 1.00 brkn p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 brkns p0.b, p15/z, p1.b, p0.b -# CHECK-NEXT: 1 6 1.00 brkns p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 brkpa p0.b, p15/z, p1.b, p2.b -# CHECK-NEXT: 1 6 1.00 brkpa p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 brkpas p0.b, p15/z, p1.b, p2.b -# CHECK-NEXT: 1 6 1.00 brkpas p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 brkpb p0.b, p15/z, p1.b, p2.b -# CHECK-NEXT: 1 6 1.00 brkpb p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 brkpbs p0.b, p15/z, p1.b, p2.b -# CHECK-NEXT: 1 6 1.00 brkpbs p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 bics p0.b, p0/z, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 bics p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 brka p0.b, p15/m, p15.b +# CHECK-NEXT: 1 2 1.00 brka p0.b, p15/z, p15.b +# CHECK-NEXT: 1 2 1.00 brkas p0.b, p15/z, p15.b +# CHECK-NEXT: 1 2 1.00 brkb p0.b, p15/m, p15.b +# CHECK-NEXT: 1 2 1.00 brkb p0.b, p15/z, p15.b +# CHECK-NEXT: 1 2 1.00 brkbs p0.b, p15/z, p15.b +# CHECK-NEXT: 1 2 1.00 brkn p0.b, p15/z, p1.b, p0.b +# CHECK-NEXT: 1 2 1.00 brkn p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 brkns p0.b, p15/z, p1.b, p0.b +# CHECK-NEXT: 1 2 1.00 brkns p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 brkpa p0.b, p15/z, p1.b, p2.b +# CHECK-NEXT: 1 2 1.00 brkpa p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 4 1.00 brkpas p0.b, p15/z, p1.b, p2.b +# CHECK-NEXT: 1 4 1.00 brkpas p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 brkpb p0.b, p15/z, p1.b, p2.b +# CHECK-NEXT: 1 2 1.00 brkpb p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 4 1.00 brkpbs p0.b, p15/z, p1.b, p2.b +# CHECK-NEXT: 1 4 1.00 brkpbs p15.b, p15/z, p15.b, p15.b # CHECK-NEXT: 1 3 0.50 bsl z0.d, z0.d, z1.d, z2.d # CHECK-NEXT: 1 3 0.50 bsl1n z0.d, z0.d, z1.d, z2.d # CHECK-NEXT: 1 3 0.50 bsl2n z0.d, z0.d, z1.d, z2.d @@ -3704,163 +3704,163 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 cmla z31.h, z31.h, z31.h, #180 # CHECK-NEXT: 1 4 0.50 cmla z31.s, z30.s, z7.s[0], #180 # CHECK-NEXT: 1 4 0.50 cmla z31.s, z31.s, z31.s, #180 -# CHECK-NEXT: 1 3 0.50 cmpeq p0.b, p0/z, z0.b, #-16 -# CHECK-NEXT: 1 3 0.50 cmpeq p0.b, p0/z, z0.b, #15 -# CHECK-NEXT: 1 3 0.50 cmpeq p0.b, p0/z, z0.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmpeq p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmpeq p0.d, p0/z, z0.d, #-16 -# CHECK-NEXT: 1 3 0.50 cmpeq p0.d, p0/z, z0.d, #15 -# CHECK-NEXT: 1 3 0.50 cmpeq p0.d, p0/z, z0.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmpeq p0.h, p0/z, z0.h, #-16 -# CHECK-NEXT: 1 3 0.50 cmpeq p0.h, p0/z, z0.h, #15 -# CHECK-NEXT: 1 3 0.50 cmpeq p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmpeq p0.h, p0/z, z0.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmpeq p0.s, p0/z, z0.s, #-16 -# CHECK-NEXT: 1 3 0.50 cmpeq p0.s, p0/z, z0.s, #15 -# CHECK-NEXT: 1 3 0.50 cmpeq p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmpeq p0.s, p0/z, z0.s, z0.s -# CHECK-NEXT: 1 3 0.50 cmpge p0.b, p0/z, z0.b, #-16 -# CHECK-NEXT: 1 3 0.50 cmpge p0.b, p0/z, z0.b, #15 -# CHECK-NEXT: 1 3 0.50 cmpge p0.b, p0/z, z0.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmpge p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmpge p0.b, p0/z, z1.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmpge p0.d, p0/z, z0.d, #-16 -# CHECK-NEXT: 1 3 0.50 cmpge p0.d, p0/z, z0.d, #15 -# CHECK-NEXT: 1 3 0.50 cmpge p0.d, p0/z, z0.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmpge p0.d, p0/z, z1.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmpge p0.h, p0/z, z0.h, #-16 -# CHECK-NEXT: 1 3 0.50 cmpge p0.h, p0/z, z0.h, #15 -# CHECK-NEXT: 1 3 0.50 cmpge p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmpge p0.h, p0/z, z0.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmpge p0.h, p0/z, z1.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmpge p0.s, p0/z, z0.s, #-16 -# CHECK-NEXT: 1 3 0.50 cmpge p0.s, p0/z, z0.s, #15 -# CHECK-NEXT: 1 3 0.50 cmpge p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmpge p0.s, p0/z, z0.s, z0.s -# CHECK-NEXT: 1 3 0.50 cmpge p0.s, p0/z, z1.s, z0.s -# CHECK-NEXT: 1 3 0.50 cmpgt p0.b, p0/z, z0.b, #-16 -# CHECK-NEXT: 1 3 0.50 cmpgt p0.b, p0/z, z0.b, #15 -# CHECK-NEXT: 1 3 0.50 cmpgt p0.b, p0/z, z0.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmpgt p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmpgt p0.b, p0/z, z1.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmpgt p0.d, p0/z, z0.d, #-16 -# CHECK-NEXT: 1 3 0.50 cmpgt p0.d, p0/z, z0.d, #15 -# CHECK-NEXT: 1 3 0.50 cmpgt p0.d, p0/z, z0.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmpgt p0.d, p0/z, z1.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmpgt p0.h, p0/z, z0.h, #-16 -# CHECK-NEXT: 1 3 0.50 cmpgt p0.h, p0/z, z0.h, #15 -# CHECK-NEXT: 1 3 0.50 cmpgt p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmpgt p0.h, p0/z, z0.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmpgt p0.h, p0/z, z1.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmpgt p0.s, p0/z, z0.s, #-16 -# CHECK-NEXT: 1 3 0.50 cmpgt p0.s, p0/z, z0.s, #15 -# CHECK-NEXT: 1 3 0.50 cmpgt p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmpgt p0.s, p0/z, z0.s, z0.s -# CHECK-NEXT: 1 3 0.50 cmpgt p0.s, p0/z, z1.s, z0.s -# CHECK-NEXT: 1 3 0.50 cmphi p0.b, p0/z, z0.b, #0 -# CHECK-NEXT: 1 3 0.50 cmphi p0.b, p0/z, z0.b, #127 -# CHECK-NEXT: 1 3 0.50 cmphi p0.b, p0/z, z0.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmphi p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmphi p0.b, p0/z, z1.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmphi p0.d, p0/z, z0.d, #0 -# CHECK-NEXT: 1 3 0.50 cmphi p0.d, p0/z, z0.d, #127 -# CHECK-NEXT: 1 3 0.50 cmphi p0.d, p0/z, z0.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmphi p0.d, p0/z, z1.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmphi p0.h, p0/z, z0.h, #0 -# CHECK-NEXT: 1 3 0.50 cmphi p0.h, p0/z, z0.h, #127 -# CHECK-NEXT: 1 3 0.50 cmphi p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmphi p0.h, p0/z, z0.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmphi p0.h, p0/z, z1.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmphi p0.s, p0/z, z0.s, #0 -# CHECK-NEXT: 1 3 0.50 cmphi p0.s, p0/z, z0.s, #127 -# CHECK-NEXT: 1 3 0.50 cmphi p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmphi p0.s, p0/z, z0.s, z0.s -# CHECK-NEXT: 1 3 0.50 cmphi p0.s, p0/z, z1.s, z0.s -# CHECK-NEXT: 1 3 0.50 cmphs p0.b, p0/z, z0.b, #0 -# CHECK-NEXT: 1 3 0.50 cmphs p0.b, p0/z, z0.b, #127 -# CHECK-NEXT: 1 3 0.50 cmphs p0.b, p0/z, z0.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmphs p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmphs p0.b, p0/z, z1.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmphs p0.d, p0/z, z0.d, #0 -# CHECK-NEXT: 1 3 0.50 cmphs p0.d, p0/z, z0.d, #127 -# CHECK-NEXT: 1 3 0.50 cmphs p0.d, p0/z, z0.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmphs p0.d, p0/z, z1.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmphs p0.h, p0/z, z0.h, #0 -# CHECK-NEXT: 1 3 0.50 cmphs p0.h, p0/z, z0.h, #127 -# CHECK-NEXT: 1 3 0.50 cmphs p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmphs p0.h, p0/z, z0.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmphs p0.h, p0/z, z1.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmphs p0.s, p0/z, z0.s, #0 -# CHECK-NEXT: 1 3 0.50 cmphs p0.s, p0/z, z0.s, #127 -# CHECK-NEXT: 1 3 0.50 cmphs p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmphs p0.s, p0/z, z0.s, z0.s -# CHECK-NEXT: 1 3 0.50 cmphs p0.s, p0/z, z1.s, z0.s -# CHECK-NEXT: 1 3 0.50 cmple p0.b, p0/z, z0.b, #-16 -# CHECK-NEXT: 1 3 0.50 cmple p0.b, p0/z, z0.b, #15 -# CHECK-NEXT: 1 3 0.50 cmple p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmple p0.d, p0/z, z0.d, #-16 -# CHECK-NEXT: 1 3 0.50 cmple p0.d, p0/z, z0.d, #15 -# CHECK-NEXT: 1 3 0.50 cmple p0.h, p0/z, z0.h, #-16 -# CHECK-NEXT: 1 3 0.50 cmple p0.h, p0/z, z0.h, #15 -# CHECK-NEXT: 1 3 0.50 cmple p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmple p0.s, p0/z, z0.s, #-16 -# CHECK-NEXT: 1 3 0.50 cmple p0.s, p0/z, z0.s, #15 -# CHECK-NEXT: 1 3 0.50 cmple p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmplo p0.b, p0/z, z0.b, #0 -# CHECK-NEXT: 1 3 0.50 cmplo p0.b, p0/z, z0.b, #127 -# CHECK-NEXT: 1 3 0.50 cmplo p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmplo p0.d, p0/z, z0.d, #0 -# CHECK-NEXT: 1 3 0.50 cmplo p0.d, p0/z, z0.d, #127 -# CHECK-NEXT: 1 3 0.50 cmplo p0.h, p0/z, z0.h, #0 -# CHECK-NEXT: 1 3 0.50 cmplo p0.h, p0/z, z0.h, #127 -# CHECK-NEXT: 1 3 0.50 cmplo p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmplo p0.s, p0/z, z0.s, #0 -# CHECK-NEXT: 1 3 0.50 cmplo p0.s, p0/z, z0.s, #127 -# CHECK-NEXT: 1 3 0.50 cmplo p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmpls p0.b, p0/z, z0.b, #0 -# CHECK-NEXT: 1 3 0.50 cmpls p0.b, p0/z, z0.b, #127 -# CHECK-NEXT: 1 3 0.50 cmpls p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmpls p0.d, p0/z, z0.d, #0 -# CHECK-NEXT: 1 3 0.50 cmpls p0.d, p0/z, z0.d, #127 -# CHECK-NEXT: 1 3 0.50 cmpls p0.h, p0/z, z0.h, #0 -# CHECK-NEXT: 1 3 0.50 cmpls p0.h, p0/z, z0.h, #127 -# CHECK-NEXT: 1 3 0.50 cmpls p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmpls p0.s, p0/z, z0.s, #0 -# CHECK-NEXT: 1 3 0.50 cmpls p0.s, p0/z, z0.s, #127 -# CHECK-NEXT: 1 3 0.50 cmpls p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmplt p0.b, p0/z, z0.b, #-16 -# CHECK-NEXT: 1 3 0.50 cmplt p0.b, p0/z, z0.b, #15 -# CHECK-NEXT: 1 3 0.50 cmplt p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmplt p0.d, p0/z, z0.d, #-16 -# CHECK-NEXT: 1 3 0.50 cmplt p0.d, p0/z, z0.d, #15 -# CHECK-NEXT: 1 3 0.50 cmplt p0.h, p0/z, z0.h, #-16 -# CHECK-NEXT: 1 3 0.50 cmplt p0.h, p0/z, z0.h, #15 -# CHECK-NEXT: 1 3 0.50 cmplt p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmplt p0.s, p0/z, z0.s, #-16 -# CHECK-NEXT: 1 3 0.50 cmplt p0.s, p0/z, z0.s, #15 -# CHECK-NEXT: 1 3 0.50 cmplt p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmpne p0.b, p0/z, z0.b, #-16 -# CHECK-NEXT: 1 3 0.50 cmpne p0.b, p0/z, z0.b, #15 -# CHECK-NEXT: 1 3 0.50 cmpne p0.b, p0/z, z0.b, z0.b -# CHECK-NEXT: 1 3 0.50 cmpne p0.b, p0/z, z0.b, z0.d -# CHECK-NEXT: 1 3 0.50 cmpne p0.d, p0/z, z0.d, #-16 -# CHECK-NEXT: 1 3 0.50 cmpne p0.d, p0/z, z0.d, #15 -# CHECK-NEXT: 1 3 0.50 cmpne p0.d, p0/z, z0.d, z0.d -# CHECK-NEXT: 1 3 0.50 cmpne p0.h, p0/z, z0.h, #-16 -# CHECK-NEXT: 1 3 0.50 cmpne p0.h, p0/z, z0.h, #15 -# CHECK-NEXT: 1 3 0.50 cmpne p0.h, p0/z, z0.h, z0.d -# CHECK-NEXT: 1 3 0.50 cmpne p0.h, p0/z, z0.h, z0.h -# CHECK-NEXT: 1 3 0.50 cmpne p0.s, p0/z, z0.s, #-16 -# CHECK-NEXT: 1 3 0.50 cmpne p0.s, p0/z, z0.s, #15 -# CHECK-NEXT: 1 3 0.50 cmpne p0.s, p0/z, z0.s, z0.d -# CHECK-NEXT: 1 3 0.50 cmpne p0.s, p0/z, z0.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmpeq p0.b, p0/z, z0.b, #-16 +# CHECK-NEXT: 1 4 0.50 cmpeq p0.b, p0/z, z0.b, #15 +# CHECK-NEXT: 1 4 0.50 cmpeq p0.b, p0/z, z0.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmpeq p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmpeq p0.d, p0/z, z0.d, #-16 +# CHECK-NEXT: 1 4 0.50 cmpeq p0.d, p0/z, z0.d, #15 +# CHECK-NEXT: 1 4 0.50 cmpeq p0.d, p0/z, z0.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmpeq p0.h, p0/z, z0.h, #-16 +# CHECK-NEXT: 1 4 0.50 cmpeq p0.h, p0/z, z0.h, #15 +# CHECK-NEXT: 1 4 0.50 cmpeq p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmpeq p0.h, p0/z, z0.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmpeq p0.s, p0/z, z0.s, #-16 +# CHECK-NEXT: 1 4 0.50 cmpeq p0.s, p0/z, z0.s, #15 +# CHECK-NEXT: 1 4 0.50 cmpeq p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmpeq p0.s, p0/z, z0.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmpge p0.b, p0/z, z0.b, #-16 +# CHECK-NEXT: 1 4 0.50 cmpge p0.b, p0/z, z0.b, #15 +# CHECK-NEXT: 1 4 0.50 cmpge p0.b, p0/z, z0.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmpge p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmpge p0.b, p0/z, z1.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmpge p0.d, p0/z, z0.d, #-16 +# CHECK-NEXT: 1 4 0.50 cmpge p0.d, p0/z, z0.d, #15 +# CHECK-NEXT: 1 4 0.50 cmpge p0.d, p0/z, z0.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmpge p0.d, p0/z, z1.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmpge p0.h, p0/z, z0.h, #-16 +# CHECK-NEXT: 1 4 0.50 cmpge p0.h, p0/z, z0.h, #15 +# CHECK-NEXT: 1 4 0.50 cmpge p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmpge p0.h, p0/z, z0.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmpge p0.h, p0/z, z1.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmpge p0.s, p0/z, z0.s, #-16 +# CHECK-NEXT: 1 4 0.50 cmpge p0.s, p0/z, z0.s, #15 +# CHECK-NEXT: 1 4 0.50 cmpge p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmpge p0.s, p0/z, z0.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmpge p0.s, p0/z, z1.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmpgt p0.b, p0/z, z0.b, #-16 +# CHECK-NEXT: 1 4 0.50 cmpgt p0.b, p0/z, z0.b, #15 +# CHECK-NEXT: 1 4 0.50 cmpgt p0.b, p0/z, z0.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmpgt p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmpgt p0.b, p0/z, z1.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmpgt p0.d, p0/z, z0.d, #-16 +# CHECK-NEXT: 1 4 0.50 cmpgt p0.d, p0/z, z0.d, #15 +# CHECK-NEXT: 1 4 0.50 cmpgt p0.d, p0/z, z0.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmpgt p0.d, p0/z, z1.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmpgt p0.h, p0/z, z0.h, #-16 +# CHECK-NEXT: 1 4 0.50 cmpgt p0.h, p0/z, z0.h, #15 +# CHECK-NEXT: 1 4 0.50 cmpgt p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmpgt p0.h, p0/z, z0.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmpgt p0.h, p0/z, z1.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmpgt p0.s, p0/z, z0.s, #-16 +# CHECK-NEXT: 1 4 0.50 cmpgt p0.s, p0/z, z0.s, #15 +# CHECK-NEXT: 1 4 0.50 cmpgt p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmpgt p0.s, p0/z, z0.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmpgt p0.s, p0/z, z1.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmphi p0.b, p0/z, z0.b, #0 +# CHECK-NEXT: 1 4 0.50 cmphi p0.b, p0/z, z0.b, #127 +# CHECK-NEXT: 1 4 0.50 cmphi p0.b, p0/z, z0.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmphi p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmphi p0.b, p0/z, z1.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmphi p0.d, p0/z, z0.d, #0 +# CHECK-NEXT: 1 4 0.50 cmphi p0.d, p0/z, z0.d, #127 +# CHECK-NEXT: 1 4 0.50 cmphi p0.d, p0/z, z0.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmphi p0.d, p0/z, z1.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmphi p0.h, p0/z, z0.h, #0 +# CHECK-NEXT: 1 4 0.50 cmphi p0.h, p0/z, z0.h, #127 +# CHECK-NEXT: 1 4 0.50 cmphi p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmphi p0.h, p0/z, z0.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmphi p0.h, p0/z, z1.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmphi p0.s, p0/z, z0.s, #0 +# CHECK-NEXT: 1 4 0.50 cmphi p0.s, p0/z, z0.s, #127 +# CHECK-NEXT: 1 4 0.50 cmphi p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmphi p0.s, p0/z, z0.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmphi p0.s, p0/z, z1.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmphs p0.b, p0/z, z0.b, #0 +# CHECK-NEXT: 1 4 0.50 cmphs p0.b, p0/z, z0.b, #127 +# CHECK-NEXT: 1 4 0.50 cmphs p0.b, p0/z, z0.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmphs p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmphs p0.b, p0/z, z1.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmphs p0.d, p0/z, z0.d, #0 +# CHECK-NEXT: 1 4 0.50 cmphs p0.d, p0/z, z0.d, #127 +# CHECK-NEXT: 1 4 0.50 cmphs p0.d, p0/z, z0.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmphs p0.d, p0/z, z1.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmphs p0.h, p0/z, z0.h, #0 +# CHECK-NEXT: 1 4 0.50 cmphs p0.h, p0/z, z0.h, #127 +# CHECK-NEXT: 1 4 0.50 cmphs p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmphs p0.h, p0/z, z0.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmphs p0.h, p0/z, z1.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmphs p0.s, p0/z, z0.s, #0 +# CHECK-NEXT: 1 4 0.50 cmphs p0.s, p0/z, z0.s, #127 +# CHECK-NEXT: 1 4 0.50 cmphs p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmphs p0.s, p0/z, z0.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmphs p0.s, p0/z, z1.s, z0.s +# CHECK-NEXT: 1 4 0.50 cmple p0.b, p0/z, z0.b, #-16 +# CHECK-NEXT: 1 4 0.50 cmple p0.b, p0/z, z0.b, #15 +# CHECK-NEXT: 1 4 0.50 cmple p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmple p0.d, p0/z, z0.d, #-16 +# CHECK-NEXT: 1 4 0.50 cmple p0.d, p0/z, z0.d, #15 +# CHECK-NEXT: 1 4 0.50 cmple p0.h, p0/z, z0.h, #-16 +# CHECK-NEXT: 1 4 0.50 cmple p0.h, p0/z, z0.h, #15 +# CHECK-NEXT: 1 4 0.50 cmple p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmple p0.s, p0/z, z0.s, #-16 +# CHECK-NEXT: 1 4 0.50 cmple p0.s, p0/z, z0.s, #15 +# CHECK-NEXT: 1 4 0.50 cmple p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmplo p0.b, p0/z, z0.b, #0 +# CHECK-NEXT: 1 4 0.50 cmplo p0.b, p0/z, z0.b, #127 +# CHECK-NEXT: 1 4 0.50 cmplo p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmplo p0.d, p0/z, z0.d, #0 +# CHECK-NEXT: 1 4 0.50 cmplo p0.d, p0/z, z0.d, #127 +# CHECK-NEXT: 1 4 0.50 cmplo p0.h, p0/z, z0.h, #0 +# CHECK-NEXT: 1 4 0.50 cmplo p0.h, p0/z, z0.h, #127 +# CHECK-NEXT: 1 4 0.50 cmplo p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmplo p0.s, p0/z, z0.s, #0 +# CHECK-NEXT: 1 4 0.50 cmplo p0.s, p0/z, z0.s, #127 +# CHECK-NEXT: 1 4 0.50 cmplo p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmpls p0.b, p0/z, z0.b, #0 +# CHECK-NEXT: 1 4 0.50 cmpls p0.b, p0/z, z0.b, #127 +# CHECK-NEXT: 1 4 0.50 cmpls p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmpls p0.d, p0/z, z0.d, #0 +# CHECK-NEXT: 1 4 0.50 cmpls p0.d, p0/z, z0.d, #127 +# CHECK-NEXT: 1 4 0.50 cmpls p0.h, p0/z, z0.h, #0 +# CHECK-NEXT: 1 4 0.50 cmpls p0.h, p0/z, z0.h, #127 +# CHECK-NEXT: 1 4 0.50 cmpls p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmpls p0.s, p0/z, z0.s, #0 +# CHECK-NEXT: 1 4 0.50 cmpls p0.s, p0/z, z0.s, #127 +# CHECK-NEXT: 1 4 0.50 cmpls p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmplt p0.b, p0/z, z0.b, #-16 +# CHECK-NEXT: 1 4 0.50 cmplt p0.b, p0/z, z0.b, #15 +# CHECK-NEXT: 1 4 0.50 cmplt p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmplt p0.d, p0/z, z0.d, #-16 +# CHECK-NEXT: 1 4 0.50 cmplt p0.d, p0/z, z0.d, #15 +# CHECK-NEXT: 1 4 0.50 cmplt p0.h, p0/z, z0.h, #-16 +# CHECK-NEXT: 1 4 0.50 cmplt p0.h, p0/z, z0.h, #15 +# CHECK-NEXT: 1 4 0.50 cmplt p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmplt p0.s, p0/z, z0.s, #-16 +# CHECK-NEXT: 1 4 0.50 cmplt p0.s, p0/z, z0.s, #15 +# CHECK-NEXT: 1 4 0.50 cmplt p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmpne p0.b, p0/z, z0.b, #-16 +# CHECK-NEXT: 1 4 0.50 cmpne p0.b, p0/z, z0.b, #15 +# CHECK-NEXT: 1 4 0.50 cmpne p0.b, p0/z, z0.b, z0.b +# CHECK-NEXT: 1 4 0.50 cmpne p0.b, p0/z, z0.b, z0.d +# CHECK-NEXT: 1 4 0.50 cmpne p0.d, p0/z, z0.d, #-16 +# CHECK-NEXT: 1 4 0.50 cmpne p0.d, p0/z, z0.d, #15 +# CHECK-NEXT: 1 4 0.50 cmpne p0.d, p0/z, z0.d, z0.d +# CHECK-NEXT: 1 4 0.50 cmpne p0.h, p0/z, z0.h, #-16 +# CHECK-NEXT: 1 4 0.50 cmpne p0.h, p0/z, z0.h, #15 +# CHECK-NEXT: 1 4 0.50 cmpne p0.h, p0/z, z0.h, z0.d +# CHECK-NEXT: 1 4 0.50 cmpne p0.h, p0/z, z0.h, z0.h +# CHECK-NEXT: 1 4 0.50 cmpne p0.s, p0/z, z0.s, #-16 +# CHECK-NEXT: 1 4 0.50 cmpne p0.s, p0/z, z0.s, #15 +# CHECK-NEXT: 1 4 0.50 cmpne p0.s, p0/z, z0.s, z0.d +# CHECK-NEXT: 1 4 0.50 cmpne p0.s, p0/z, z0.s, z0.s # CHECK-NEXT: 1 3 0.50 cnot z31.b, p7/m, z31.b # CHECK-NEXT: 1 3 0.50 cnot z31.d, p7/m, z31.d # CHECK-NEXT: 1 3 0.50 cnot z31.h, p7/m, z31.h # CHECK-NEXT: 1 3 0.50 cnot z31.s, p7/m, z31.s -# CHECK-NEXT: 1 4 0.50 cnt z31.b, p7/m, z31.b +# CHECK-NEXT: 1 3 0.50 cnt z31.b, p7/m, z31.b # CHECK-NEXT: 1 12 0.50 cnt z31.d, p7/m, z31.d -# CHECK-NEXT: 1 4 0.50 cnt z31.h, p7/m, z31.h +# CHECK-NEXT: 1 3 0.50 cnt z31.h, p7/m, z31.h # CHECK-NEXT: 1 8 0.50 cnt z31.s, p7/m, z31.s # CHECK-NEXT: 1 1 0.33 cntb x0 # CHECK-NEXT: 1 1 0.33 cntb x0, #28 @@ -3874,10 +3874,10 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 1 0.33 cnth x0, #28 # CHECK-NEXT: 1 1 0.33 cnth x0, all, mul #16 # CHECK-NEXT: 1 1 0.33 cnth x0, pow2 -# CHECK-NEXT: 1 6 1.00 cntp x0, p15, p0.b -# CHECK-NEXT: 1 6 1.00 cntp x0, p15, p0.d -# CHECK-NEXT: 1 6 1.00 cntp x0, p15, p0.h -# CHECK-NEXT: 1 6 1.00 cntp x0, p15, p0.s +# CHECK-NEXT: 1 4 1.00 cntp x0, p15, p0.b +# CHECK-NEXT: 1 4 1.00 cntp x0, p15, p0.d +# CHECK-NEXT: 1 4 1.00 cntp x0, p15, p0.h +# CHECK-NEXT: 1 4 1.00 cntp x0, p15, p0.s # CHECK-NEXT: 1 1 0.33 cntw x0 # CHECK-NEXT: 1 1 0.33 cntw x0, #28 # CHECK-NEXT: 1 1 0.33 cntw x0, all, mul #16 @@ -3892,42 +3892,42 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 1 0.33 ctermne wzr, w30 # CHECK-NEXT: 1 1 0.33 ctermne x30, xzr # CHECK-NEXT: 1 1 0.33 ctermne xzr, x30 -# CHECK-NEXT: 1 1 0.33 decb x0 -# CHECK-NEXT: 1 1 0.33 decb x0, #14 -# CHECK-NEXT: 1 1 0.33 decb x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 decb x0, pow2 -# CHECK-NEXT: 1 1 0.33 decb x0, vl1 -# CHECK-NEXT: 1 1 0.33 decd x0 -# CHECK-NEXT: 1 1 0.33 decd x0, #14 -# CHECK-NEXT: 1 1 0.33 decd x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 decd x0, pow2 -# CHECK-NEXT: 1 1 0.33 decd x0, vl1 -# CHECK-NEXT: 1 1 0.33 dech x0 -# CHECK-NEXT: 1 1 0.33 dech x0, #14 -# CHECK-NEXT: 1 1 0.33 dech x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 dech x0, pow2 -# CHECK-NEXT: 1 1 0.33 dech x0, vl1 -# CHECK-NEXT: 1 6 1.00 decp x0, p0.b -# CHECK-NEXT: 1 6 1.00 decp x0, p0.d -# CHECK-NEXT: 1 6 1.00 decp x0, p0.h -# CHECK-NEXT: 1 6 1.00 decp x0, p0.s -# CHECK-NEXT: 1 6 1.00 decp xzr, p15.b -# CHECK-NEXT: 1 6 1.00 decp xzr, p15.d -# CHECK-NEXT: 1 6 1.00 decp xzr, p15.h -# CHECK-NEXT: 1 6 1.00 decp xzr, p15.s +# CHECK-NEXT: 1 3 0.33 decb x0 +# CHECK-NEXT: 1 3 0.33 decb x0, #14 +# CHECK-NEXT: 1 3 0.33 decb x0, all, mul #16 +# CHECK-NEXT: 1 3 0.33 decb x0, pow2 +# CHECK-NEXT: 1 3 0.33 decb x0, vl1 +# CHECK-NEXT: 1 3 0.33 decd x0 +# CHECK-NEXT: 1 3 0.33 decd x0, #14 +# CHECK-NEXT: 1 3 0.33 decd x0, all, mul #16 +# CHECK-NEXT: 1 3 0.33 decd x0, pow2 +# CHECK-NEXT: 1 3 0.33 decd x0, vl1 +# CHECK-NEXT: 1 3 0.33 dech x0 +# CHECK-NEXT: 1 3 0.33 dech x0, #14 +# CHECK-NEXT: 1 3 0.33 dech x0, all, mul #16 +# CHECK-NEXT: 1 3 0.33 dech x0, pow2 +# CHECK-NEXT: 1 3 0.33 dech x0, vl1 +# CHECK-NEXT: 1 4 1.00 decp x0, p0.b +# CHECK-NEXT: 1 4 1.00 decp x0, p0.d +# CHECK-NEXT: 1 4 1.00 decp x0, p0.h +# CHECK-NEXT: 1 4 1.00 decp x0, p0.s +# CHECK-NEXT: 1 4 1.00 decp xzr, p15.b +# CHECK-NEXT: 1 4 1.00 decp xzr, p15.d +# CHECK-NEXT: 1 4 1.00 decp xzr, p15.h +# CHECK-NEXT: 1 4 1.00 decp xzr, p15.s # CHECK-NEXT: 1 4 0.50 decp z31.d, p15.d # CHECK-NEXT: 1 4 0.50 decp z31.h, p15.h # CHECK-NEXT: 1 4 0.50 decp z31.s, p15.s -# CHECK-NEXT: 1 1 0.33 decw x0 -# CHECK-NEXT: 1 1 0.33 decw x0, #14 -# CHECK-NEXT: 1 1 0.33 decw x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 decw x0, pow2 -# CHECK-NEXT: 1 1 0.33 decw x0, vl1 +# CHECK-NEXT: 1 3 0.33 decw x0 +# CHECK-NEXT: 1 3 0.33 decw x0, #14 +# CHECK-NEXT: 1 3 0.33 decw x0, all, mul #16 +# CHECK-NEXT: 1 3 0.33 decw x0, pow2 +# CHECK-NEXT: 1 3 0.33 decw x0, vl1 # CHECK-NEXT: 1 4 0.50 dupm z0.d, #0xfffffffffffffff9 # CHECK-NEXT: 1 4 0.50 dupm z0.s, #0xfffffff9 # CHECK-NEXT: 1 4 0.50 dupm z23.h, #0xfff9 # CHECK-NEXT: 1 4 0.50 dupm z5.b, #0xf9 -# CHECK-NEXT: 1 6 1.00 eor p0.b, p0/z, p0.b, p1.b +# CHECK-NEXT: 1 2 1.00 eor p0.b, p0/z, p0.b, p1.b # CHECK-NEXT: 1 3 0.50 eor z0.d, z0.d, #0x6 # CHECK-NEXT: 1 3 0.50 eor z0.d, z0.d, #0xfffffffffffffff9 # CHECK-NEXT: 1 3 0.50 eor z0.d, z0.d, z0.d @@ -3942,12 +3942,12 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 eor z31.s, p7/m, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 eor z5.b, z5.b, #0x6 # CHECK-NEXT: 1 3 0.50 eor z5.b, z5.b, #0xf9 -# CHECK-NEXT: 1 3 0.50 eor3 z29.d, z29.d, z30.d, z31.d +# CHECK-NEXT: 1 4 0.50 eor3 z29.d, z29.d, z30.d, z31.d # CHECK-NEXT: 1 4 0.50 eorbt z0.b, z1.b, z31.b # CHECK-NEXT: 1 4 0.50 eorbt z0.d, z1.d, z31.d # CHECK-NEXT: 1 4 0.50 eorbt z0.h, z1.h, z31.h # CHECK-NEXT: 1 4 0.50 eorbt z0.s, z1.s, z31.s -# CHECK-NEXT: 1 6 1.00 eors p0.b, p0/z, p0.b, p1.b +# CHECK-NEXT: 1 2 1.00 eors p0.b, p0/z, p0.b, p1.b # CHECK-NEXT: 1 4 0.50 eortb z0.b, z1.b, z31.b # CHECK-NEXT: 1 4 0.50 eortb z0.d, z1.d, z31.d # CHECK-NEXT: 1 4 0.50 eortb z0.h, z1.h, z31.h @@ -4303,49 +4303,49 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 ftsmul z0.d, z1.d, z31.d # CHECK-NEXT: 1 4 0.50 ftsmul z0.h, z1.h, z31.h # CHECK-NEXT: 1 4 0.50 ftsmul z0.s, z1.s, z31.s -# CHECK-NEXT: 1 4 0.50 ftssel z0.d, z1.d, z31.d -# CHECK-NEXT: 1 4 0.50 ftssel z0.h, z1.h, z31.h -# CHECK-NEXT: 1 4 0.50 ftssel z0.s, z1.s, z31.s +# CHECK-NEXT: 1 3 0.50 ftssel z0.d, z1.d, z31.d +# CHECK-NEXT: 1 3 0.50 ftssel z0.h, z1.h, z31.h +# CHECK-NEXT: 1 3 0.50 ftssel z0.s, z1.s, z31.s # CHECK-NEXT: 1 8 2.00 histcnt z0.s, p0/z, z1.s, z2.s # CHECK-NEXT: 1 8 2.00 histcnt z29.d, p7/z, z30.d, z31.d # CHECK-NEXT: 1 8 2.00 histseg z0.b, z1.b, z31.b -# CHECK-NEXT: 1 1 0.33 incb x0 -# CHECK-NEXT: 1 1 0.33 incb x0, #14 -# CHECK-NEXT: 1 1 0.33 incb x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 incb x0, pow2 -# CHECK-NEXT: 1 1 0.33 incb x0, vl1 -# CHECK-NEXT: 1 1 0.33 incd x0 -# CHECK-NEXT: 1 1 0.33 incd x0, #14 -# CHECK-NEXT: 1 1 0.33 incd x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 incd x0, pow2 -# CHECK-NEXT: 1 1 0.33 incd x0, vl1 -# CHECK-NEXT: 1 4 0.50 incd z0.d -# CHECK-NEXT: 1 4 0.50 incd z0.d, all, mul #16 -# CHECK-NEXT: 1 1 0.33 inch x0 -# CHECK-NEXT: 1 1 0.33 inch x0, #14 -# CHECK-NEXT: 1 1 0.33 inch x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 inch x0, pow2 -# CHECK-NEXT: 1 1 0.33 inch x0, vl1 -# CHECK-NEXT: 1 4 0.50 inch z0.h -# CHECK-NEXT: 1 4 0.50 inch z0.h, all, mul #16 -# CHECK-NEXT: 1 6 1.00 incp x0, p0.b -# CHECK-NEXT: 1 6 1.00 incp x0, p0.d -# CHECK-NEXT: 1 6 1.00 incp x0, p0.h -# CHECK-NEXT: 1 6 1.00 incp x0, p0.s -# CHECK-NEXT: 1 6 1.00 incp xzr, p15.b -# CHECK-NEXT: 1 6 1.00 incp xzr, p15.d -# CHECK-NEXT: 1 6 1.00 incp xzr, p15.h -# CHECK-NEXT: 1 6 1.00 incp xzr, p15.s +# CHECK-NEXT: 1 3 0.33 incb x0 +# CHECK-NEXT: 1 3 0.33 incb x0, #14 +# CHECK-NEXT: 1 3 0.33 incb x0, all, mul #16 +# CHECK-NEXT: 1 3 0.33 incb x0, pow2 +# CHECK-NEXT: 1 3 0.33 incb x0, vl1 +# CHECK-NEXT: 1 3 0.33 incd x0 +# CHECK-NEXT: 1 3 0.33 incd x0, #14 +# CHECK-NEXT: 1 3 0.33 incd x0, all, mul #16 +# CHECK-NEXT: 1 3 0.33 incd x0, pow2 +# CHECK-NEXT: 1 3 0.33 incd x0, vl1 +# CHECK-NEXT: 1 3 0.50 incd z0.d +# CHECK-NEXT: 1 3 0.50 incd z0.d, all, mul #16 +# CHECK-NEXT: 1 3 0.33 inch x0 +# CHECK-NEXT: 1 3 0.33 inch x0, #14 +# CHECK-NEXT: 1 3 0.33 inch x0, all, mul #16 +# CHECK-NEXT: 1 3 0.33 inch x0, pow2 +# CHECK-NEXT: 1 3 0.33 inch x0, vl1 +# CHECK-NEXT: 1 3 0.50 inch z0.h +# CHECK-NEXT: 1 3 0.50 inch z0.h, all, mul #16 +# CHECK-NEXT: 1 4 1.00 incp x0, p0.b +# CHECK-NEXT: 1 4 1.00 incp x0, p0.d +# CHECK-NEXT: 1 4 1.00 incp x0, p0.h +# CHECK-NEXT: 1 4 1.00 incp x0, p0.s +# CHECK-NEXT: 1 4 1.00 incp xzr, p15.b +# CHECK-NEXT: 1 4 1.00 incp xzr, p15.d +# CHECK-NEXT: 1 4 1.00 incp xzr, p15.h +# CHECK-NEXT: 1 4 1.00 incp xzr, p15.s # CHECK-NEXT: 1 4 0.50 incp z31.d, p15.d # CHECK-NEXT: 1 4 0.50 incp z31.h, p15.h # CHECK-NEXT: 1 4 0.50 incp z31.s, p15.s -# CHECK-NEXT: 1 1 0.33 incw x0 -# CHECK-NEXT: 1 1 0.33 incw x0, #14 -# CHECK-NEXT: 1 1 0.33 incw x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 incw x0, pow2 -# CHECK-NEXT: 1 1 0.33 incw x0, vl1 -# CHECK-NEXT: 1 4 0.50 incw z0.s -# CHECK-NEXT: 1 4 0.50 incw z0.s, all, mul #16 +# CHECK-NEXT: 1 3 0.33 incw x0 +# CHECK-NEXT: 1 3 0.33 incw x0, #14 +# CHECK-NEXT: 1 3 0.33 incw x0, all, mul #16 +# CHECK-NEXT: 1 3 0.33 incw x0, pow2 +# CHECK-NEXT: 1 3 0.33 incw x0, vl1 +# CHECK-NEXT: 1 3 0.50 incw z0.s +# CHECK-NEXT: 1 3 0.50 incw z0.s, all, mul #16 # CHECK-NEXT: 1 4 0.50 index z0.b, #0, #0 # CHECK-NEXT: 1 4 0.50 index z0.d, #0, #0 # CHECK-NEXT: 1 4 0.50 index z0.h, #0, #0 @@ -4412,8 +4412,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 * ld1b { z0.d }, p0/z, [x0] # CHECK-NEXT: 1 7 7.00 * ld1b { z0.d }, p0/z, [z0.d] # CHECK-NEXT: 1 3 0.50 * ld1b { z0.h }, p0/z, [x0] -# CHECK-NEXT: 1 9 4.50 * ld1b { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: 1 9 4.50 * ld1b { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: 1 7 3.50 * ld1b { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * ld1b { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: 1 3 0.50 * ld1b { z0.s }, p0/z, [x0] # CHECK-NEXT: 1 9 9.00 * ld1b { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 3 0.50 * ld1b { z21.b }, p5/z, [x10, #5, mul vl] @@ -4450,8 +4450,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 * ld1h { z0.d }, p0/z, [x0] # CHECK-NEXT: 1 7 7.00 * ld1h { z0.d }, p0/z, [z0.d] # CHECK-NEXT: 1 3 0.50 * ld1h { z0.h }, p0/z, [x0] -# CHECK-NEXT: 1 9 4.50 * ld1h { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: 1 9 4.50 * ld1h { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: 1 7 3.50 * ld1h { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * ld1h { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: 1 3 0.50 * ld1h { z0.s }, p0/z, [x0] # CHECK-NEXT: 1 9 9.00 * ld1h { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 3 0.50 * ld1h { z21.d }, p5/z, [x10, #5, mul vl] @@ -4467,8 +4467,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * ld1h { z31.d }, p7/z, [z31.d, #62] # CHECK-NEXT: 1 3 0.50 * ld1h { z31.h }, p7/z, [sp, #-1, mul vl] # CHECK-NEXT: 1 3 0.50 * ld1h { z31.s }, p7/z, [sp, #-1, mul vl] -# CHECK-NEXT: 1 9 4.50 * ld1h { z31.s }, p7/z, [sp, z31.s, sxtw #1] -# CHECK-NEXT: 1 9 4.50 * ld1h { z31.s }, p7/z, [sp, z31.s, uxtw #1] +# CHECK-NEXT: 1 7 3.50 * ld1h { z31.s }, p7/z, [sp, z31.s, sxtw #1] +# CHECK-NEXT: 1 7 3.50 * ld1h { z31.s }, p7/z, [sp, z31.s, uxtw #1] # CHECK-NEXT: 1 9 9.00 * ld1h { z31.s }, p7/z, [z31.s, #62] # CHECK-NEXT: 1 3 0.50 * ld1h { z5.h }, p3/z, [sp, x16, lsl #1] # CHECK-NEXT: 1 3 0.50 * ld1h { z5.h }, p3/z, [x17, x16, lsl #1] @@ -4529,7 +4529,7 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 * ld1sb { z0.h }, p0/z, [sp, x0] # CHECK-NEXT: 1 3 0.50 * ld1sb { z0.h }, p0/z, [x0, x0] # CHECK-NEXT: 1 3 0.50 * ld1sb { z0.h }, p0/z, [x0] -# CHECK-NEXT: 1 9 4.50 * ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw] # CHECK-NEXT: 1 3 0.50 * ld1sb { z0.s }, p0/z, [x0] # CHECK-NEXT: 1 9 9.00 * ld1sb { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 3 0.50 * ld1sb { z21.d }, p5/z, [x10, #5, mul vl] @@ -4549,8 +4549,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1] # CHECK-NEXT: 1 3 0.50 * ld1sh { z0.d }, p0/z, [x0] # CHECK-NEXT: 1 7 7.00 * ld1sh { z0.d }, p0/z, [z0.d] -# CHECK-NEXT: 1 9 4.50 * ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: 1 9 4.50 * ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: 1 7 3.50 * ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: 1 3 0.50 * ld1sh { z0.s }, p0/z, [x0] # CHECK-NEXT: 1 9 9.00 * ld1sh { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 3 0.50 * ld1sh { z21.d }, p5/z, [x10, #5, mul vl] @@ -4565,8 +4565,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * ld1sh { z31.d }, p7/z, [sp, z31.d] # CHECK-NEXT: 1 7 7.00 * ld1sh { z31.d }, p7/z, [z31.d, #62] # CHECK-NEXT: 1 3 0.50 * ld1sh { z31.s }, p7/z, [sp, #-1, mul vl] -# CHECK-NEXT: 1 9 4.50 * ld1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1] -# CHECK-NEXT: 1 9 4.50 * ld1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1] +# CHECK-NEXT: 1 7 3.50 * ld1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1] +# CHECK-NEXT: 1 7 3.50 * ld1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1] # CHECK-NEXT: 1 9 9.00 * ld1sh { z31.s }, p7/z, [z31.s, #62] # CHECK-NEXT: 1 7 7.00 * ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw #2] # CHECK-NEXT: 1 7 7.00 * ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw #2] @@ -4585,8 +4585,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * ld1w { z0.d }, p0/z, [x0, z0.d, uxtw #2] # CHECK-NEXT: 1 3 0.50 * ld1w { z0.d }, p0/z, [x0] # CHECK-NEXT: 1 7 7.00 * ld1w { z0.d }, p0/z, [z0.d] -# CHECK-NEXT: 1 9 4.50 * ld1w { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: 1 9 4.50 * ld1w { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: 1 7 3.50 * ld1w { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * ld1w { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: 1 3 0.50 * ld1w { z0.s }, p0/z, [x0] # CHECK-NEXT: 1 9 9.00 * ld1w { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 3 0.50 * ld1w { z21.d }, p5/z, [x10, #5, mul vl] @@ -4601,8 +4601,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * ld1w { z31.d }, p7/z, [sp, z31.d] # CHECK-NEXT: 1 7 7.00 * ld1w { z31.d }, p7/z, [z31.d, #124] # CHECK-NEXT: 1 3 0.50 * ld1w { z31.s }, p7/z, [sp, #-1, mul vl] -# CHECK-NEXT: 1 9 4.50 * ld1w { z31.s }, p7/z, [sp, z31.s, sxtw #2] -# CHECK-NEXT: 1 9 4.50 * ld1w { z31.s }, p7/z, [sp, z31.s, uxtw #2] +# CHECK-NEXT: 1 7 3.50 * ld1w { z31.s }, p7/z, [sp, z31.s, sxtw #2] +# CHECK-NEXT: 1 7 3.50 * ld1w { z31.s }, p7/z, [sp, z31.s, uxtw #2] # CHECK-NEXT: 1 9 9.00 * ld1w { z31.s }, p7/z, [z31.s, #124] # CHECK-NEXT: 1 3 2.00 * ld2b { z0.b, z1.b }, p0/z, [x0, x0] # CHECK-NEXT: 1 3 1.00 * ld2b { z0.b, z1.b }, p0/z, [x0] @@ -4668,8 +4668,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * U ldff1b { z0.d }, p0/z, [z0.d] # CHECK-NEXT: 1 3 0.50 * U ldff1b { z0.h }, p0/z, [x0, x0] # CHECK-NEXT: 1 3 0.50 * U ldff1b { z0.s }, p0/z, [x0, x0] -# CHECK-NEXT: 1 9 4.50 * U ldff1b { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: 1 9 4.50 * U ldff1b { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1b { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1b { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: 1 9 9.00 * U ldff1b { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 7 7.00 * U ldff1b { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: 1 7 7.00 * U ldff1b { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -4696,8 +4696,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * U ldff1h { z0.d }, p0/z, [z0.d] # CHECK-NEXT: 1 3 0.50 * U ldff1h { z0.h }, p0/z, [x0, x0, lsl #1] # CHECK-NEXT: 1 3 0.50 * U ldff1h { z0.s }, p0/z, [x0, x0, lsl #1] -# CHECK-NEXT: 1 9 4.50 * U ldff1h { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: 1 9 4.50 * U ldff1h { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1h { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1h { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: 1 9 9.00 * U ldff1h { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 7 7.00 * U ldff1h { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: 1 7 7.00 * U ldff1h { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -4706,16 +4706,16 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 * U ldff1h { z31.d }, p7/z, [sp] # CHECK-NEXT: 1 7 7.00 * U ldff1h { z31.d }, p7/z, [z31.d, #62] # CHECK-NEXT: 1 3 0.50 * U ldff1h { z31.h }, p7/z, [sp] -# CHECK-NEXT: 1 9 4.50 * U ldff1h { z31.s }, p7/z, [sp, z31.s, sxtw #1] -# CHECK-NEXT: 1 9 4.50 * U ldff1h { z31.s }, p7/z, [sp, z31.s, uxtw #1] +# CHECK-NEXT: 1 7 3.50 * U ldff1h { z31.s }, p7/z, [sp, z31.s, sxtw #1] +# CHECK-NEXT: 1 7 3.50 * U ldff1h { z31.s }, p7/z, [sp, z31.s, uxtw #1] # CHECK-NEXT: 1 3 0.50 * U ldff1h { z31.s }, p7/z, [sp] # CHECK-NEXT: 1 9 9.00 * U ldff1h { z31.s }, p7/z, [z31.s, #62] # CHECK-NEXT: 1 3 0.50 * U ldff1sb { z0.d }, p0/z, [x0, x0] # CHECK-NEXT: 1 7 7.00 * U ldff1sb { z0.d }, p0/z, [z0.d] # CHECK-NEXT: 1 3 0.50 * U ldff1sb { z0.h }, p0/z, [x0, x0] # CHECK-NEXT: 1 3 0.50 * U ldff1sb { z0.s }, p0/z, [x0, x0] -# CHECK-NEXT: 1 9 4.50 * U ldff1sb { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: 1 9 4.50 * U ldff1sb { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1sb { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1sb { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: 1 9 9.00 * U ldff1sb { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 7 7.00 * U ldff1sb { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: 1 7 7.00 * U ldff1sb { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -4730,8 +4730,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * U ldff1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1] # CHECK-NEXT: 1 7 7.00 * U ldff1sh { z0.d }, p0/z, [z0.d] # CHECK-NEXT: 1 3 0.50 * U ldff1sh { z0.s }, p0/z, [x0, x0, lsl #1] -# CHECK-NEXT: 1 9 4.50 * U ldff1sh { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: 1 9 4.50 * U ldff1sh { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1sh { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1sh { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: 1 9 9.00 * U ldff1sh { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 7 7.00 * U ldff1sh { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: 1 7 7.00 * U ldff1sh { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -4739,8 +4739,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * U ldff1sh { z31.d }, p7/z, [sp, z31.d] # CHECK-NEXT: 1 3 0.50 * U ldff1sh { z31.d }, p7/z, [sp] # CHECK-NEXT: 1 7 7.00 * U ldff1sh { z31.d }, p7/z, [z31.d, #62] -# CHECK-NEXT: 1 9 4.50 * U ldff1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1] -# CHECK-NEXT: 1 9 4.50 * U ldff1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1] +# CHECK-NEXT: 1 7 3.50 * U ldff1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1] +# CHECK-NEXT: 1 7 3.50 * U ldff1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1] # CHECK-NEXT: 1 3 0.50 * U ldff1sh { z31.s }, p7/z, [sp] # CHECK-NEXT: 1 9 9.00 * U ldff1sh { z31.s }, p7/z, [z31.s, #62] # CHECK-NEXT: 1 3 0.50 * U ldff1sw { z0.d }, p0/z, [x0, x0, lsl #2] @@ -4758,8 +4758,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * U ldff1w { z0.d }, p0/z, [x0, z0.d, uxtw #2] # CHECK-NEXT: 1 7 7.00 * U ldff1w { z0.d }, p0/z, [z0.d] # CHECK-NEXT: 1 3 0.50 * U ldff1w { z0.s }, p0/z, [x0, x0, lsl #2] -# CHECK-NEXT: 1 9 4.50 * U ldff1w { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: 1 9 4.50 * U ldff1w { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1w { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: 1 7 3.50 * U ldff1w { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: 1 9 9.00 * U ldff1w { z0.s }, p0/z, [z0.s] # CHECK-NEXT: 1 7 7.00 * U ldff1w { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: 1 7 7.00 * U ldff1w { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -4767,8 +4767,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 7.00 * U ldff1w { z31.d }, p7/z, [sp, z31.d] # CHECK-NEXT: 1 3 0.50 * U ldff1w { z31.d }, p7/z, [sp] # CHECK-NEXT: 1 7 7.00 * U ldff1w { z31.d }, p7/z, [z31.d, #124] -# CHECK-NEXT: 1 9 4.50 * U ldff1w { z31.s }, p7/z, [sp, z31.s, sxtw #2] -# CHECK-NEXT: 1 9 4.50 * U ldff1w { z31.s }, p7/z, [sp, z31.s, uxtw #2] +# CHECK-NEXT: 1 7 3.50 * U ldff1w { z31.s }, p7/z, [sp, z31.s, sxtw #2] +# CHECK-NEXT: 1 7 3.50 * U ldff1w { z31.s }, p7/z, [sp, z31.s, uxtw #2] # CHECK-NEXT: 1 3 0.50 * U ldff1w { z31.s }, p7/z, [sp] # CHECK-NEXT: 1 9 9.00 * U ldff1w { z31.s }, p7/z, [z31.s, #124] # CHECK-NEXT: 1 3 0.50 * U ldnf1b { z0.b }, p0/z, [x0] @@ -4959,12 +4959,12 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 mls z0.h, z1.h, z7.h[7] # CHECK-NEXT: 1 4 0.50 mls z0.s, p7/m, z1.s, z31.s # CHECK-NEXT: 1 4 0.50 mls z0.s, z1.s, z7.s[3] -# CHECK-NEXT: 1 6 1.00 mov p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 mov p0.b, p0/m, p0.b -# CHECK-NEXT: 1 6 1.00 mov p0.b, p0/z, p0.b -# CHECK-NEXT: 1 6 1.00 mov p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 mov p15.b, p15/m, p15.b -# CHECK-NEXT: 1 6 1.00 mov p15.b, p15/z, p15.b +# CHECK-NEXT: 1 2 1.00 mov p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 mov p0.b, p0/m, p0.b +# CHECK-NEXT: 1 2 1.00 mov p0.b, p0/z, p0.b +# CHECK-NEXT: 1 2 1.00 mov p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 mov p15.b, p15/m, p15.b +# CHECK-NEXT: 1 2 1.00 mov p15.b, p15/z, p15.b # CHECK-NEXT: 1 3 0.50 mov z0.b, #127 # CHECK-NEXT: 1 3 0.50 mov z0.b, b0 # CHECK-NEXT: 1 3 0.50 mov z0.b, p0/m, b0 @@ -5062,10 +5062,10 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 mov z5.h, #-6 # CHECK-NEXT: 1 3 0.50 mov z5.q, z17.q[3] # CHECK-NEXT: 1 3 0.50 mov z5.s, #-6 -# CHECK-NEXT: 1 6 1.00 movs p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 movs p0.b, p0/z, p0.b -# CHECK-NEXT: 1 6 1.00 movs p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 movs p15.b, p15/z, p15.b +# CHECK-NEXT: 1 2 1.00 movs p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 movs p0.b, p0/z, p0.b +# CHECK-NEXT: 1 2 1.00 movs p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 movs p15.b, p15/z, p15.b # CHECK-NEXT: 1 1 1.00 U mrs x3, ID_AA64ZFR0_EL1 # CHECK-NEXT: 1 1 1.00 U mrs x3, ZCR_EL1 # CHECK-NEXT: 1 1 1.00 U mrs x3, ZCR_EL12 @@ -5098,10 +5098,10 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 mul z31.h, z31.h, #127 # CHECK-NEXT: 1 4 0.50 mul z31.s, z31.s, #-128 # CHECK-NEXT: 1 4 0.50 mul z31.s, z31.s, #127 -# CHECK-NEXT: 1 6 1.00 nand p0.b, p0/z, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 nand p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 nands p0.b, p0/z, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 nands p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 nand p0.b, p0/z, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 nand p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 nands p0.b, p0/z, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 nands p15.b, p15/z, p15.b, p15.b # CHECK-NEXT: 1 3 0.50 nbsl z0.d, z0.d, z1.d, z2.d # CHECK-NEXT: 1 3 0.50 neg z0.b, p0/m, z0.b # CHECK-NEXT: 1 3 0.50 neg z0.d, p0/m, z0.d @@ -5115,23 +5115,23 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 1.00 nmatch p0.h, p0/z, z0.h, z0.h # CHECK-NEXT: 1 7 1.00 nmatch p15.b, p7/z, z30.b, z31.b # CHECK-NEXT: 1 7 1.00 nmatch p15.h, p7/z, z30.h, z31.h -# CHECK-NEXT: 1 6 1.00 nor p0.b, p0/z, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 nor p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 nors p0.b, p0/z, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 nors p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 not p0.b, p0/z, p0.b -# CHECK-NEXT: 1 6 1.00 not p15.b, p15/z, p15.b +# CHECK-NEXT: 1 2 1.00 nor p0.b, p0/z, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 nor p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 nors p0.b, p0/z, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 nors p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 not p0.b, p0/z, p0.b +# CHECK-NEXT: 1 2 1.00 not p15.b, p15/z, p15.b # CHECK-NEXT: 1 3 0.50 not z31.b, p7/m, z31.b # CHECK-NEXT: 1 3 0.50 not z31.d, p7/m, z31.d # CHECK-NEXT: 1 3 0.50 not z31.h, p7/m, z31.h # CHECK-NEXT: 1 3 0.50 not z31.s, p7/m, z31.s -# CHECK-NEXT: 1 6 1.00 nots p0.b, p0/z, p0.b -# CHECK-NEXT: 1 6 1.00 nots p15.b, p15/z, p15.b -# CHECK-NEXT: 1 6 1.00 orn p0.b, p0/z, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 orn p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 orns p0.b, p0/z, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 orns p15.b, p15/z, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 orr p0.b, p0/z, p0.b, p1.b +# CHECK-NEXT: 1 2 1.00 nots p0.b, p0/z, p0.b +# CHECK-NEXT: 1 2 1.00 nots p15.b, p15/z, p15.b +# CHECK-NEXT: 1 2 1.00 orn p0.b, p0/z, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 orn p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 orns p0.b, p0/z, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 orns p15.b, p15/z, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 orr p0.b, p0/z, p0.b, p1.b # CHECK-NEXT: 1 3 0.50 orr z0.d, z0.d, #0x6 # CHECK-NEXT: 1 3 0.50 orr z0.d, z0.d, #0xfffffffffffffff9 # CHECK-NEXT: 1 3 0.50 orr z0.s, z0.s, #0x6 @@ -5145,27 +5145,27 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 orr z31.s, p7/m, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 orr z5.b, z5.b, #0x6 # CHECK-NEXT: 1 3 0.50 orr z5.b, z5.b, #0xf9 -# CHECK-NEXT: 1 6 1.00 orrs p0.b, p0/z, p0.b, p1.b +# CHECK-NEXT: 1 2 1.00 orrs p0.b, p0/z, p0.b, p1.b # CHECK-NEXT: 1 4 1.00 orv b0, p7, z31.b # CHECK-NEXT: 1 4 1.00 orv d0, p7, z31.d # CHECK-NEXT: 1 4 1.00 orv h0, p7, z31.h # CHECK-NEXT: 1 4 1.00 orv s0, p7, z31.s -# CHECK-NEXT: 1 6 1.00 pfalse p15.b -# CHECK-NEXT: 1 6 1.00 pfirst p0.b, p15, p0.b -# CHECK-NEXT: 1 6 1.00 pfirst p15.b, p15, p15.b +# CHECK-NEXT: 1 2 1.00 pfalse p15.b +# CHECK-NEXT: 1 2 1.00 pfirst p0.b, p15, p0.b +# CHECK-NEXT: 1 2 1.00 pfirst p15.b, p15, p15.b # CHECK-NEXT: 1 4 0.50 pmul z0.b, z1.b, z2.b # CHECK-NEXT: 1 4 0.50 pmul z29.b, z30.b, z31.b -# CHECK-NEXT: 1 6 1.00 pmullb z0.h, z1.b, z2.b -# CHECK-NEXT: 1 6 1.00 pmullb z29.q, z30.d, z31.d -# CHECK-NEXT: 1 6 1.00 pmullb z31.d, z31.s, z31.s -# CHECK-NEXT: 1 6 1.00 pmullt z0.h, z1.b, z2.b -# CHECK-NEXT: 1 6 1.00 pmullt z29.q, z30.d, z31.d -# CHECK-NEXT: 1 6 1.00 pmullt z31.d, z31.s, z31.s -# CHECK-NEXT: 1 6 1.00 pnext p0.b, p15, p0.b -# CHECK-NEXT: 1 6 1.00 pnext p0.d, p15, p0.d -# CHECK-NEXT: 1 6 1.00 pnext p0.h, p15, p0.h -# CHECK-NEXT: 1 6 1.00 pnext p0.s, p15, p0.s -# CHECK-NEXT: 1 6 1.00 pnext p15.b, p15, p15.b +# CHECK-NEXT: 1 9 1.00 pmullb z0.h, z1.b, z2.b +# CHECK-NEXT: 1 9 1.00 pmullb z29.q, z30.d, z31.d +# CHECK-NEXT: 1 9 1.00 pmullb z31.d, z31.s, z31.s +# CHECK-NEXT: 1 9 1.00 pmullt z0.h, z1.b, z2.b +# CHECK-NEXT: 1 9 1.00 pmullt z29.q, z30.d, z31.d +# CHECK-NEXT: 1 9 1.00 pmullt z31.d, z31.s, z31.s +# CHECK-NEXT: 1 2 1.00 pnext p0.b, p15, p0.b +# CHECK-NEXT: 1 2 1.00 pnext p0.d, p15, p0.d +# CHECK-NEXT: 1 2 1.00 pnext p0.h, p15, p0.h +# CHECK-NEXT: 1 2 1.00 pnext p0.s, p15, p0.s +# CHECK-NEXT: 1 2 1.00 pnext p15.b, p15, p15.b # CHECK-NEXT: 1 0 0.50 * * U prfb #14, p0, [x0] # CHECK-NEXT: 1 0 0.50 * * U prfb #15, p0, [x0] # CHECK-NEXT: 1 0 0.50 * * U prfb #6, p0, [x0] @@ -5274,97 +5274,97 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 0 0.50 * * U prfw pstl2strm, p0, [x0] # CHECK-NEXT: 1 0 0.50 * * U prfw pstl3keep, p0, [x0] # CHECK-NEXT: 1 0 0.50 * * U prfw pstl3strm, p0, [x0] -# CHECK-NEXT: 1 6 1.00 ptest p15, p0.b -# CHECK-NEXT: 1 6 1.00 ptest p15, p15.b -# CHECK-NEXT: 1 6 1.00 ptrue p0.b, pow2 -# CHECK-NEXT: 1 6 1.00 ptrue p0.d, pow2 -# CHECK-NEXT: 1 6 1.00 ptrue p0.h, pow2 -# CHECK-NEXT: 1 6 1.00 ptrue p0.s, pow2 -# CHECK-NEXT: 1 6 1.00 ptrue p15.b -# CHECK-NEXT: 1 6 1.00 ptrue p15.d -# CHECK-NEXT: 1 6 1.00 ptrue p15.h -# CHECK-NEXT: 1 6 1.00 ptrue p15.s -# CHECK-NEXT: 1 6 1.00 ptrue p7.s -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #14 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #15 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #16 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #17 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #18 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #19 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #20 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #21 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #22 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #23 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #24 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #25 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #26 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #27 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, #28 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, mul3 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, mul4 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl1 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl128 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl16 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl2 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl256 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl3 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl32 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl4 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl5 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl6 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl64 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl7 -# CHECK-NEXT: 1 6 1.00 ptrue p7.s, vl8 -# CHECK-NEXT: 1 6 1.00 ptrues p0.b, pow2 -# CHECK-NEXT: 1 6 1.00 ptrues p0.d, pow2 -# CHECK-NEXT: 1 6 1.00 ptrues p0.h, pow2 -# CHECK-NEXT: 1 6 1.00 ptrues p0.s, pow2 -# CHECK-NEXT: 1 6 1.00 ptrues p15.b -# CHECK-NEXT: 1 6 1.00 ptrues p15.d -# CHECK-NEXT: 1 6 1.00 ptrues p15.h -# CHECK-NEXT: 1 6 1.00 ptrues p15.s -# CHECK-NEXT: 1 6 1.00 ptrues p7.s -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #14 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #15 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #16 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #17 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #18 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #19 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #20 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #21 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #22 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #23 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #24 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #25 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #26 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #27 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, #28 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, mul3 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, mul4 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl1 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl128 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl16 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl2 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl256 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl3 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl32 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl4 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl5 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl6 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl64 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl7 -# CHECK-NEXT: 1 6 1.00 ptrues p7.s, vl8 -# CHECK-NEXT: 1 6 1.00 punpkhi p0.h, p0.b -# CHECK-NEXT: 1 6 1.00 punpkhi p15.h, p15.b -# CHECK-NEXT: 1 6 1.00 punpklo p0.h, p0.b -# CHECK-NEXT: 1 6 1.00 punpklo p15.h, p15.b -# CHECK-NEXT: 1 4 0.50 raddhnb z0.b, z1.h, z31.h -# CHECK-NEXT: 1 4 0.50 raddhnb z0.h, z1.s, z31.s -# CHECK-NEXT: 1 4 0.50 raddhnb z0.s, z1.d, z31.d -# CHECK-NEXT: 1 4 0.50 raddhnt z0.b, z1.h, z31.h -# CHECK-NEXT: 1 4 0.50 raddhnt z0.h, z1.s, z31.s -# CHECK-NEXT: 1 4 0.50 raddhnt z0.s, z1.d, z31.d -# CHECK-NEXT: 1 8 1.00 rax1 z0.d, z1.d, z31.d +# CHECK-NEXT: 1 2 1.00 ptest p15, p0.b +# CHECK-NEXT: 1 2 1.00 ptest p15, p15.b +# CHECK-NEXT: 1 2 1.00 ptrue p0.b, pow2 +# CHECK-NEXT: 1 2 1.00 ptrue p0.d, pow2 +# CHECK-NEXT: 1 2 1.00 ptrue p0.h, pow2 +# CHECK-NEXT: 1 2 1.00 ptrue p0.s, pow2 +# CHECK-NEXT: 1 2 1.00 ptrue p15.b +# CHECK-NEXT: 1 2 1.00 ptrue p15.d +# CHECK-NEXT: 1 2 1.00 ptrue p15.h +# CHECK-NEXT: 1 2 1.00 ptrue p15.s +# CHECK-NEXT: 1 2 1.00 ptrue p7.s +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #14 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #15 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #16 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #17 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #18 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #19 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #20 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #21 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #22 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #23 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #24 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #25 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #26 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #27 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, #28 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, mul3 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, mul4 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl1 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl128 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl16 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl2 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl256 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl3 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl32 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl4 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl5 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl6 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl64 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl7 +# CHECK-NEXT: 1 2 1.00 ptrue p7.s, vl8 +# CHECK-NEXT: 1 2 1.00 ptrues p0.b, pow2 +# CHECK-NEXT: 1 2 1.00 ptrues p0.d, pow2 +# CHECK-NEXT: 1 2 1.00 ptrues p0.h, pow2 +# CHECK-NEXT: 1 2 1.00 ptrues p0.s, pow2 +# CHECK-NEXT: 1 2 1.00 ptrues p15.b +# CHECK-NEXT: 1 2 1.00 ptrues p15.d +# CHECK-NEXT: 1 2 1.00 ptrues p15.h +# CHECK-NEXT: 1 2 1.00 ptrues p15.s +# CHECK-NEXT: 1 2 1.00 ptrues p7.s +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #14 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #15 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #16 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #17 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #18 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #19 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #20 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #21 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #22 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #23 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #24 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #25 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #26 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #27 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, #28 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, mul3 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, mul4 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl1 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl128 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl16 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl2 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl256 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl3 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl32 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl4 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl5 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl6 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl64 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl7 +# CHECK-NEXT: 1 2 1.00 ptrues p7.s, vl8 +# CHECK-NEXT: 1 2 1.00 punpkhi p0.h, p0.b +# CHECK-NEXT: 1 2 1.00 punpkhi p15.h, p15.b +# CHECK-NEXT: 1 2 1.00 punpklo p0.h, p0.b +# CHECK-NEXT: 1 2 1.00 punpklo p15.h, p15.b +# CHECK-NEXT: 1 8 0.50 raddhnb z0.b, z1.h, z31.h +# CHECK-NEXT: 1 8 0.50 raddhnb z0.h, z1.s, z31.s +# CHECK-NEXT: 1 8 0.50 raddhnb z0.s, z1.d, z31.d +# CHECK-NEXT: 1 8 0.50 raddhnt z0.b, z1.h, z31.h +# CHECK-NEXT: 1 8 0.50 raddhnt z0.h, z1.s, z31.s +# CHECK-NEXT: 1 8 0.50 raddhnt z0.s, z1.d, z31.d +# CHECK-NEXT: 1 9 1.00 rax1 z0.d, z1.d, z31.d # CHECK-NEXT: 1 3 0.50 rbit z0.b, p7/m, z31.b # CHECK-NEXT: 1 3 0.50 rbit z0.d, p7/m, z31.d # CHECK-NEXT: 1 3 0.50 rbit z0.h, p7/m, z31.h @@ -5379,16 +5379,16 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 1 0.33 rdvl x21, #-32 # CHECK-NEXT: 1 1 0.33 rdvl x23, #31 # CHECK-NEXT: 1 1 0.33 rdvl xzr, #-1 -# CHECK-NEXT: 1 4 0.50 rev z0.b, z31.b -# CHECK-NEXT: 1 4 0.50 rev z0.d, z31.d -# CHECK-NEXT: 1 4 0.50 rev z0.h, z31.h -# CHECK-NEXT: 1 4 0.50 rev z0.s, z31.s -# CHECK-NEXT: 1 4 0.50 revb z0.d, p7/m, z31.d -# CHECK-NEXT: 1 4 0.50 revb z0.h, p7/m, z31.h -# CHECK-NEXT: 1 4 0.50 revb z0.s, p7/m, z31.s -# CHECK-NEXT: 1 4 0.50 revh z0.d, p7/m, z31.d -# CHECK-NEXT: 1 4 0.50 revh z0.s, p7/m, z31.s -# CHECK-NEXT: 1 4 0.50 revw z0.d, p7/m, z31.d +# CHECK-NEXT: 1 3 0.50 rev z0.b, z31.b +# CHECK-NEXT: 1 3 0.50 rev z0.d, z31.d +# CHECK-NEXT: 1 3 0.50 rev z0.h, z31.h +# CHECK-NEXT: 1 3 0.50 rev z0.s, z31.s +# CHECK-NEXT: 1 3 0.50 revb z0.d, p7/m, z31.d +# CHECK-NEXT: 1 3 0.50 revb z0.h, p7/m, z31.h +# CHECK-NEXT: 1 3 0.50 revb z0.s, p7/m, z31.s +# CHECK-NEXT: 1 3 0.50 revh z0.d, p7/m, z31.d +# CHECK-NEXT: 1 3 0.50 revh z0.s, p7/m, z31.s +# CHECK-NEXT: 1 3 0.50 revw z0.d, p7/m, z31.d # CHECK-NEXT: 1 4 0.50 rshrnb z0.b, z0.h, #1 # CHECK-NEXT: 1 4 0.50 rshrnb z0.h, z0.s, #1 # CHECK-NEXT: 1 4 0.50 rshrnb z0.s, z0.d, #1 @@ -5401,22 +5401,22 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 rshrnt z31.b, z31.h, #8 # CHECK-NEXT: 1 4 0.50 rshrnt z31.h, z31.s, #16 # CHECK-NEXT: 1 4 0.50 rshrnt z31.s, z31.d, #32 -# CHECK-NEXT: 1 4 0.50 rsubhnb z0.b, z1.h, z31.h -# CHECK-NEXT: 1 4 0.50 rsubhnb z0.h, z1.s, z31.s -# CHECK-NEXT: 1 4 0.50 rsubhnb z0.s, z1.d, z31.d -# CHECK-NEXT: 1 4 0.50 rsubhnt z0.b, z1.h, z31.h -# CHECK-NEXT: 1 4 0.50 rsubhnt z0.h, z1.s, z31.s -# CHECK-NEXT: 1 4 0.50 rsubhnt z0.s, z1.d, z31.d -# CHECK-NEXT: 1 8 1.00 saba z0.b, z1.b, z31.b -# CHECK-NEXT: 1 8 1.00 saba z0.d, z1.d, z31.d -# CHECK-NEXT: 1 8 1.00 saba z0.h, z1.h, z31.h -# CHECK-NEXT: 1 8 1.00 saba z0.s, z1.s, z31.s -# CHECK-NEXT: 1 8 1.00 sabalb z0.d, z1.s, z31.s -# CHECK-NEXT: 1 8 1.00 sabalb z0.h, z1.b, z31.b -# CHECK-NEXT: 1 8 1.00 sabalb z0.s, z1.h, z31.h -# CHECK-NEXT: 1 8 1.00 sabalt z0.d, z1.s, z31.s -# CHECK-NEXT: 1 8 1.00 sabalt z0.h, z1.b, z31.b -# CHECK-NEXT: 1 8 1.00 sabalt z0.s, z1.h, z31.h +# CHECK-NEXT: 1 8 0.50 rsubhnb z0.b, z1.h, z31.h +# CHECK-NEXT: 1 8 0.50 rsubhnb z0.h, z1.s, z31.s +# CHECK-NEXT: 1 8 0.50 rsubhnb z0.s, z1.d, z31.d +# CHECK-NEXT: 1 8 0.50 rsubhnt z0.b, z1.h, z31.h +# CHECK-NEXT: 1 8 0.50 rsubhnt z0.h, z1.s, z31.s +# CHECK-NEXT: 1 8 0.50 rsubhnt z0.s, z1.d, z31.d +# CHECK-NEXT: 1 6 1.00 saba z0.b, z1.b, z31.b +# CHECK-NEXT: 1 6 1.00 saba z0.d, z1.d, z31.d +# CHECK-NEXT: 1 6 1.00 saba z0.h, z1.h, z31.h +# CHECK-NEXT: 1 6 1.00 saba z0.s, z1.s, z31.s +# CHECK-NEXT: 1 6 1.00 sabalb z0.d, z1.s, z31.s +# CHECK-NEXT: 1 6 1.00 sabalb z0.h, z1.b, z31.b +# CHECK-NEXT: 1 6 1.00 sabalb z0.s, z1.h, z31.h +# CHECK-NEXT: 1 6 1.00 sabalt z0.d, z1.s, z31.s +# CHECK-NEXT: 1 6 1.00 sabalt z0.h, z1.b, z31.b +# CHECK-NEXT: 1 6 1.00 sabalt z0.s, z1.h, z31.h # CHECK-NEXT: 1 3 0.50 sabd z31.b, p7/m, z31.b, z31.b # CHECK-NEXT: 1 3 0.50 sabd z31.d, p7/m, z31.d, z31.d # CHECK-NEXT: 1 3 0.50 sabd z31.h, p7/m, z31.h, z31.h @@ -5430,24 +5430,24 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 1.00 sadalp z0.h, p0/m, z1.b # CHECK-NEXT: 1 7 1.00 sadalp z29.s, p0/m, z30.h # CHECK-NEXT: 1 7 1.00 sadalp z30.d, p7/m, z31.s -# CHECK-NEXT: 1 3 0.50 saddlb z0.h, z1.b, z2.b -# CHECK-NEXT: 1 3 0.50 saddlb z29.s, z30.h, z31.h -# CHECK-NEXT: 1 3 0.50 saddlb z31.d, z31.s, z31.s -# CHECK-NEXT: 1 3 0.50 saddlbt z0.d, z1.s, z31.s -# CHECK-NEXT: 1 3 0.50 saddlbt z0.h, z1.b, z31.b -# CHECK-NEXT: 1 3 0.50 saddlbt z0.s, z1.h, z31.h -# CHECK-NEXT: 1 3 0.50 saddlt z0.h, z1.b, z2.b -# CHECK-NEXT: 1 3 0.50 saddlt z29.s, z30.h, z31.h -# CHECK-NEXT: 1 3 0.50 saddlt z31.d, z31.s, z31.s +# CHECK-NEXT: 1 4 0.50 saddlb z0.h, z1.b, z2.b +# CHECK-NEXT: 1 4 0.50 saddlb z29.s, z30.h, z31.h +# CHECK-NEXT: 1 4 0.50 saddlb z31.d, z31.s, z31.s +# CHECK-NEXT: 1 4 0.50 saddlbt z0.d, z1.s, z31.s +# CHECK-NEXT: 1 4 0.50 saddlbt z0.h, z1.b, z31.b +# CHECK-NEXT: 1 4 0.50 saddlbt z0.s, z1.h, z31.h +# CHECK-NEXT: 1 4 0.50 saddlt z0.h, z1.b, z2.b +# CHECK-NEXT: 1 4 0.50 saddlt z29.s, z30.h, z31.h +# CHECK-NEXT: 1 4 0.50 saddlt z31.d, z31.s, z31.s # CHECK-NEXT: 1 4 1.00 saddv d0, p7, z31.b # CHECK-NEXT: 1 4 1.00 saddv d0, p7, z31.h # CHECK-NEXT: 1 4 1.00 saddv d0, p7, z31.s -# CHECK-NEXT: 1 3 0.50 saddwb z0.h, z1.h, z2.b -# CHECK-NEXT: 1 3 0.50 saddwb z29.s, z30.s, z31.h -# CHECK-NEXT: 1 3 0.50 saddwb z31.d, z31.d, z31.s -# CHECK-NEXT: 1 3 0.50 saddwt z0.h, z1.h, z2.b -# CHECK-NEXT: 1 3 0.50 saddwt z29.s, z30.s, z31.h -# CHECK-NEXT: 1 3 0.50 saddwt z31.d, z31.d, z31.s +# CHECK-NEXT: 1 4 0.50 saddwb z0.h, z1.h, z2.b +# CHECK-NEXT: 1 4 0.50 saddwb z29.s, z30.s, z31.h +# CHECK-NEXT: 1 4 0.50 saddwb z31.d, z31.d, z31.s +# CHECK-NEXT: 1 4 0.50 saddwt z0.h, z1.h, z2.b +# CHECK-NEXT: 1 4 0.50 saddwt z29.s, z30.s, z31.h +# CHECK-NEXT: 1 4 0.50 saddwt z31.d, z31.d, z31.s # CHECK-NEXT: 1 4 0.50 sbclb z0.d, z1.d, z31.d # CHECK-NEXT: 1 4 0.50 sbclb z0.s, z1.s, z31.s # CHECK-NEXT: 1 4 0.50 sbclt z0.d, z1.d, z31.d @@ -5504,8 +5504,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 sli z31.d, z31.d, #63 # CHECK-NEXT: 1 3 0.50 sli z31.h, z31.h, #15 # CHECK-NEXT: 1 3 0.50 sli z31.s, z31.s, #31 -# CHECK-NEXT: 1 8 1.00 sm4e z0.s, z0.s, z31.s -# CHECK-NEXT: 1 8 1.00 sm4ekey z0.s, z1.s, z31.s +# CHECK-NEXT: 1 9 1.00 sm4e z0.s, z0.s, z31.s +# CHECK-NEXT: 1 9 1.00 sm4ekey z0.s, z1.s, z31.s # CHECK-NEXT: 1 3 0.50 smax z0.b, z0.b, #-128 # CHECK-NEXT: 1 3 0.50 smax z0.d, z0.d, #-128 # CHECK-NEXT: 1 3 0.50 smax z0.h, z0.h, #-128 @@ -5624,61 +5624,61 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 sqcadd z31.d, z31.d, z31.d, #270 # CHECK-NEXT: 1 4 0.50 sqcadd z31.h, z31.h, z31.h, #270 # CHECK-NEXT: 1 4 0.50 sqcadd z31.s, z31.s, z31.s, #270 -# CHECK-NEXT: 1 1 0.33 sqdecb x0 -# CHECK-NEXT: 1 1 0.33 sqdecb x0, #14 -# CHECK-NEXT: 1 1 0.33 sqdecb x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdecb x0, pow2 -# CHECK-NEXT: 1 1 0.33 sqdecb x0, vl1 -# CHECK-NEXT: 1 1 0.33 sqdecb x0, w0 -# CHECK-NEXT: 1 1 0.33 sqdecb x0, w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdecb x0, w0, pow2 -# CHECK-NEXT: 1 1 0.33 sqdecb x0, w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdecd x0 -# CHECK-NEXT: 1 1 0.33 sqdecd x0, #14 -# CHECK-NEXT: 1 1 0.33 sqdecd x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdecd x0, pow2 -# CHECK-NEXT: 1 1 0.33 sqdecd x0, vl1 -# CHECK-NEXT: 1 1 0.33 sqdecd x0, w0 -# CHECK-NEXT: 1 1 0.33 sqdecd x0, w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdecd x0, w0, pow2 -# CHECK-NEXT: 1 1 0.33 sqdecd x0, w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdecb x0 +# CHECK-NEXT: 1 4 0.33 sqdecb x0, #14 +# CHECK-NEXT: 1 4 0.33 sqdecb x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdecb x0, pow2 +# CHECK-NEXT: 1 4 0.33 sqdecb x0, vl1 +# CHECK-NEXT: 1 4 0.33 sqdecb x0, w0 +# CHECK-NEXT: 1 4 0.33 sqdecb x0, w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdecb x0, w0, pow2 +# CHECK-NEXT: 1 4 0.33 sqdecb x0, w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdecd x0 +# CHECK-NEXT: 1 4 0.33 sqdecd x0, #14 +# CHECK-NEXT: 1 4 0.33 sqdecd x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdecd x0, pow2 +# CHECK-NEXT: 1 4 0.33 sqdecd x0, vl1 +# CHECK-NEXT: 1 4 0.33 sqdecd x0, w0 +# CHECK-NEXT: 1 4 0.33 sqdecd x0, w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdecd x0, w0, pow2 +# CHECK-NEXT: 1 4 0.33 sqdecd x0, w0, pow2, mul #16 # CHECK-NEXT: 1 4 0.50 sqdecd z0.d # CHECK-NEXT: 1 4 0.50 sqdecd z0.d, all, mul #16 # CHECK-NEXT: 1 4 0.50 sqdecd z0.d, pow2 # CHECK-NEXT: 1 4 0.50 sqdecd z0.d, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdech x0 -# CHECK-NEXT: 1 1 0.33 sqdech x0, #14 -# CHECK-NEXT: 1 1 0.33 sqdech x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdech x0, pow2 -# CHECK-NEXT: 1 1 0.33 sqdech x0, vl1 -# CHECK-NEXT: 1 1 0.33 sqdech x0, w0 -# CHECK-NEXT: 1 1 0.33 sqdech x0, w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdech x0, w0, pow2 -# CHECK-NEXT: 1 1 0.33 sqdech x0, w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdech x0 +# CHECK-NEXT: 1 4 0.33 sqdech x0, #14 +# CHECK-NEXT: 1 4 0.33 sqdech x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdech x0, pow2 +# CHECK-NEXT: 1 4 0.33 sqdech x0, vl1 +# CHECK-NEXT: 1 4 0.33 sqdech x0, w0 +# CHECK-NEXT: 1 4 0.33 sqdech x0, w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdech x0, w0, pow2 +# CHECK-NEXT: 1 4 0.33 sqdech x0, w0, pow2, mul #16 # CHECK-NEXT: 1 4 0.50 sqdech z0.h # CHECK-NEXT: 1 4 0.50 sqdech z0.h, all, mul #16 # CHECK-NEXT: 1 4 0.50 sqdech z0.h, pow2 # CHECK-NEXT: 1 4 0.50 sqdech z0.h, pow2, mul #16 -# CHECK-NEXT: 1 8 1.00 sqdecp x0, p0.b -# CHECK-NEXT: 1 8 1.00 sqdecp x0, p0.d -# CHECK-NEXT: 1 8 1.00 sqdecp x0, p0.h -# CHECK-NEXT: 1 8 1.00 sqdecp x0, p0.s -# CHECK-NEXT: 1 8 1.00 sqdecp xzr, p15.b, wzr -# CHECK-NEXT: 1 8 1.00 sqdecp xzr, p15.d, wzr -# CHECK-NEXT: 1 8 1.00 sqdecp xzr, p15.h, wzr -# CHECK-NEXT: 1 8 1.00 sqdecp xzr, p15.s, wzr +# CHECK-NEXT: 1 9 1.00 sqdecp x0, p0.b +# CHECK-NEXT: 1 9 1.00 sqdecp x0, p0.d +# CHECK-NEXT: 1 9 1.00 sqdecp x0, p0.h +# CHECK-NEXT: 1 9 1.00 sqdecp x0, p0.s +# CHECK-NEXT: 1 9 1.00 sqdecp xzr, p15.b, wzr +# CHECK-NEXT: 1 9 1.00 sqdecp xzr, p15.d, wzr +# CHECK-NEXT: 1 9 1.00 sqdecp xzr, p15.h, wzr +# CHECK-NEXT: 1 9 1.00 sqdecp xzr, p15.s, wzr # CHECK-NEXT: 1 4 0.50 sqdecp z0.d, p0.d # CHECK-NEXT: 1 4 0.50 sqdecp z0.h, p0.h # CHECK-NEXT: 1 4 0.50 sqdecp z0.s, p0.s -# CHECK-NEXT: 1 1 0.33 sqdecw x0 -# CHECK-NEXT: 1 1 0.33 sqdecw x0, #14 -# CHECK-NEXT: 1 1 0.33 sqdecw x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdecw x0, pow2 -# CHECK-NEXT: 1 1 0.33 sqdecw x0, vl1 -# CHECK-NEXT: 1 1 0.33 sqdecw x0, w0 -# CHECK-NEXT: 1 1 0.33 sqdecw x0, w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqdecw x0, w0, pow2 -# CHECK-NEXT: 1 1 0.33 sqdecw x0, w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdecw x0 +# CHECK-NEXT: 1 4 0.33 sqdecw x0, #14 +# CHECK-NEXT: 1 4 0.33 sqdecw x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdecw x0, pow2 +# CHECK-NEXT: 1 4 0.33 sqdecw x0, vl1 +# CHECK-NEXT: 1 4 0.33 sqdecw x0, w0 +# CHECK-NEXT: 1 4 0.33 sqdecw x0, w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqdecw x0, w0, pow2 +# CHECK-NEXT: 1 4 0.33 sqdecw x0, w0, pow2, mul #16 # CHECK-NEXT: 1 4 0.50 sqdecw z0.s # CHECK-NEXT: 1 4 0.50 sqdecw z0.s, all, mul #16 # CHECK-NEXT: 1 4 0.50 sqdecw z0.s, pow2 @@ -5726,61 +5726,61 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 sqdmullt z0.s, z1.h, z7.h[7] # CHECK-NEXT: 1 4 0.50 sqdmullt z29.s, z30.h, z31.h # CHECK-NEXT: 1 4 0.50 sqdmullt z31.d, z31.s, z31.s -# CHECK-NEXT: 1 1 0.33 sqincb x0 -# CHECK-NEXT: 1 1 0.33 sqincb x0, #14 -# CHECK-NEXT: 1 1 0.33 sqincb x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqincb x0, pow2 -# CHECK-NEXT: 1 1 0.33 sqincb x0, vl1 -# CHECK-NEXT: 1 1 0.33 sqincb x0, w0 -# CHECK-NEXT: 1 1 0.33 sqincb x0, w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqincb x0, w0, pow2 -# CHECK-NEXT: 1 1 0.33 sqincb x0, w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 sqincd x0 -# CHECK-NEXT: 1 1 0.33 sqincd x0, #14 -# CHECK-NEXT: 1 1 0.33 sqincd x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqincd x0, pow2 -# CHECK-NEXT: 1 1 0.33 sqincd x0, vl1 -# CHECK-NEXT: 1 1 0.33 sqincd x0, w0 -# CHECK-NEXT: 1 1 0.33 sqincd x0, w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqincd x0, w0, pow2 -# CHECK-NEXT: 1 1 0.33 sqincd x0, w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 sqincb x0 +# CHECK-NEXT: 1 4 0.33 sqincb x0, #14 +# CHECK-NEXT: 1 4 0.33 sqincb x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqincb x0, pow2 +# CHECK-NEXT: 1 4 0.33 sqincb x0, vl1 +# CHECK-NEXT: 1 4 0.33 sqincb x0, w0 +# CHECK-NEXT: 1 4 0.33 sqincb x0, w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqincb x0, w0, pow2 +# CHECK-NEXT: 1 4 0.33 sqincb x0, w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 sqincd x0 +# CHECK-NEXT: 1 4 0.33 sqincd x0, #14 +# CHECK-NEXT: 1 4 0.33 sqincd x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqincd x0, pow2 +# CHECK-NEXT: 1 4 0.33 sqincd x0, vl1 +# CHECK-NEXT: 1 4 0.33 sqincd x0, w0 +# CHECK-NEXT: 1 4 0.33 sqincd x0, w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqincd x0, w0, pow2 +# CHECK-NEXT: 1 4 0.33 sqincd x0, w0, pow2, mul #16 # CHECK-NEXT: 1 4 0.50 sqincd z0.d # CHECK-NEXT: 1 4 0.50 sqincd z0.d, all, mul #16 # CHECK-NEXT: 1 4 0.50 sqincd z0.d, pow2 # CHECK-NEXT: 1 4 0.50 sqincd z0.d, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 sqinch x0 -# CHECK-NEXT: 1 1 0.33 sqinch x0, #14 -# CHECK-NEXT: 1 1 0.33 sqinch x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqinch x0, pow2 -# CHECK-NEXT: 1 1 0.33 sqinch x0, vl1 -# CHECK-NEXT: 1 1 0.33 sqinch x0, w0 -# CHECK-NEXT: 1 1 0.33 sqinch x0, w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqinch x0, w0, pow2 -# CHECK-NEXT: 1 1 0.33 sqinch x0, w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 sqinch x0 +# CHECK-NEXT: 1 4 0.33 sqinch x0, #14 +# CHECK-NEXT: 1 4 0.33 sqinch x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqinch x0, pow2 +# CHECK-NEXT: 1 4 0.33 sqinch x0, vl1 +# CHECK-NEXT: 1 4 0.33 sqinch x0, w0 +# CHECK-NEXT: 1 4 0.33 sqinch x0, w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqinch x0, w0, pow2 +# CHECK-NEXT: 1 4 0.33 sqinch x0, w0, pow2, mul #16 # CHECK-NEXT: 1 4 0.50 sqinch z0.h # CHECK-NEXT: 1 4 0.50 sqinch z0.h, all, mul #16 # CHECK-NEXT: 1 4 0.50 sqinch z0.h, pow2 # CHECK-NEXT: 1 4 0.50 sqinch z0.h, pow2, mul #16 -# CHECK-NEXT: 1 8 1.00 sqincp x0, p0.b -# CHECK-NEXT: 1 8 1.00 sqincp x0, p0.d -# CHECK-NEXT: 1 8 1.00 sqincp x0, p0.h -# CHECK-NEXT: 1 8 1.00 sqincp x0, p0.s -# CHECK-NEXT: 1 8 1.00 sqincp xzr, p15.b, wzr -# CHECK-NEXT: 1 8 1.00 sqincp xzr, p15.d, wzr -# CHECK-NEXT: 1 8 1.00 sqincp xzr, p15.h, wzr -# CHECK-NEXT: 1 8 1.00 sqincp xzr, p15.s, wzr +# CHECK-NEXT: 1 9 1.00 sqincp x0, p0.b +# CHECK-NEXT: 1 9 1.00 sqincp x0, p0.d +# CHECK-NEXT: 1 9 1.00 sqincp x0, p0.h +# CHECK-NEXT: 1 9 1.00 sqincp x0, p0.s +# CHECK-NEXT: 1 9 1.00 sqincp xzr, p15.b, wzr +# CHECK-NEXT: 1 9 1.00 sqincp xzr, p15.d, wzr +# CHECK-NEXT: 1 9 1.00 sqincp xzr, p15.h, wzr +# CHECK-NEXT: 1 9 1.00 sqincp xzr, p15.s, wzr # CHECK-NEXT: 1 4 0.50 sqincp z0.d, p0.d # CHECK-NEXT: 1 4 0.50 sqincp z0.h, p0.h # CHECK-NEXT: 1 4 0.50 sqincp z0.s, p0.s -# CHECK-NEXT: 1 1 0.33 sqincw x0 -# CHECK-NEXT: 1 1 0.33 sqincw x0, #14 -# CHECK-NEXT: 1 1 0.33 sqincw x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqincw x0, pow2 -# CHECK-NEXT: 1 1 0.33 sqincw x0, vl1 -# CHECK-NEXT: 1 1 0.33 sqincw x0, w0 -# CHECK-NEXT: 1 1 0.33 sqincw x0, w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 sqincw x0, w0, pow2 -# CHECK-NEXT: 1 1 0.33 sqincw x0, w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 sqincw x0 +# CHECK-NEXT: 1 4 0.33 sqincw x0, #14 +# CHECK-NEXT: 1 4 0.33 sqincw x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqincw x0, pow2 +# CHECK-NEXT: 1 4 0.33 sqincw x0, vl1 +# CHECK-NEXT: 1 4 0.33 sqincw x0, w0 +# CHECK-NEXT: 1 4 0.33 sqincw x0, w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 sqincw x0, w0, pow2 +# CHECK-NEXT: 1 4 0.33 sqincw x0, w0, pow2, mul #16 # CHECK-NEXT: 1 4 0.50 sqincw z0.s # CHECK-NEXT: 1 4 0.50 sqincw z0.s, all, mul #16 # CHECK-NEXT: 1 4 0.50 sqincw z0.s, pow2 @@ -6001,24 +6001,24 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 ssra z31.d, z31.d, #64 # CHECK-NEXT: 1 4 0.50 ssra z31.h, z31.h, #16 # CHECK-NEXT: 1 4 0.50 ssra z31.s, z31.s, #32 -# CHECK-NEXT: 1 3 0.50 ssublb z0.h, z1.b, z2.b -# CHECK-NEXT: 1 3 0.50 ssublb z29.s, z30.h, z31.h -# CHECK-NEXT: 1 3 0.50 ssublb z31.d, z31.s, z31.s -# CHECK-NEXT: 1 3 0.50 ssublbt z0.d, z1.s, z31.s -# CHECK-NEXT: 1 3 0.50 ssublbt z0.h, z1.b, z31.b -# CHECK-NEXT: 1 3 0.50 ssublbt z0.s, z1.h, z31.h -# CHECK-NEXT: 1 3 0.50 ssublt z0.h, z1.b, z2.b -# CHECK-NEXT: 1 3 0.50 ssublt z29.s, z30.h, z31.h -# CHECK-NEXT: 1 3 0.50 ssublt z31.d, z31.s, z31.s -# CHECK-NEXT: 1 3 0.50 ssubltb z0.d, z1.s, z31.s -# CHECK-NEXT: 1 3 0.50 ssubltb z0.h, z1.b, z31.b -# CHECK-NEXT: 1 3 0.50 ssubltb z0.s, z1.h, z31.h -# CHECK-NEXT: 1 3 0.50 ssubwb z0.h, z1.h, z2.b -# CHECK-NEXT: 1 3 0.50 ssubwb z29.s, z30.s, z31.h -# CHECK-NEXT: 1 3 0.50 ssubwb z31.d, z31.d, z31.s -# CHECK-NEXT: 1 3 0.50 ssubwt z0.h, z1.h, z2.b -# CHECK-NEXT: 1 3 0.50 ssubwt z29.s, z30.s, z31.h -# CHECK-NEXT: 1 3 0.50 ssubwt z31.d, z31.d, z31.s +# CHECK-NEXT: 1 4 0.50 ssublb z0.h, z1.b, z2.b +# CHECK-NEXT: 1 4 0.50 ssublb z29.s, z30.h, z31.h +# CHECK-NEXT: 1 4 0.50 ssublb z31.d, z31.s, z31.s +# CHECK-NEXT: 1 4 0.50 ssublbt z0.d, z1.s, z31.s +# CHECK-NEXT: 1 4 0.50 ssublbt z0.h, z1.b, z31.b +# CHECK-NEXT: 1 4 0.50 ssublbt z0.s, z1.h, z31.h +# CHECK-NEXT: 1 4 0.50 ssublt z0.h, z1.b, z2.b +# CHECK-NEXT: 1 4 0.50 ssublt z29.s, z30.h, z31.h +# CHECK-NEXT: 1 4 0.50 ssublt z31.d, z31.s, z31.s +# CHECK-NEXT: 1 4 0.50 ssubltb z0.d, z1.s, z31.s +# CHECK-NEXT: 1 4 0.50 ssubltb z0.h, z1.b, z31.b +# CHECK-NEXT: 1 4 0.50 ssubltb z0.s, z1.h, z31.h +# CHECK-NEXT: 1 4 0.50 ssubwb z0.h, z1.h, z2.b +# CHECK-NEXT: 1 4 0.50 ssubwb z29.s, z30.s, z31.h +# CHECK-NEXT: 1 4 0.50 ssubwb z31.d, z31.d, z31.s +# CHECK-NEXT: 1 4 0.50 ssubwt z0.h, z1.h, z2.b +# CHECK-NEXT: 1 4 0.50 ssubwt z29.s, z30.s, z31.h +# CHECK-NEXT: 1 4 0.50 ssubwt z31.d, z31.d, z31.s # CHECK-NEXT: 1 1 1.00 * st1b { z0.b }, p0, [x0, x0] # CHECK-NEXT: 1 1 1.00 * st1b { z0.b }, p0, [x0] # CHECK-NEXT: 1 1 1.00 * st1b { z0.d }, p0, [x0, x0] @@ -6250,12 +6250,12 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 sub z31.s, p7/m, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 sub z31.s, z31.s, #65280 # CHECK-NEXT: 1 3 0.50 sub z31.s, z31.s, z31.s -# CHECK-NEXT: 1 4 0.50 subhnb z0.b, z1.h, z31.h -# CHECK-NEXT: 1 4 0.50 subhnb z0.h, z1.s, z31.s -# CHECK-NEXT: 1 4 0.50 subhnb z0.s, z1.d, z31.d -# CHECK-NEXT: 1 4 0.50 subhnt z0.b, z1.h, z31.h -# CHECK-NEXT: 1 4 0.50 subhnt z0.h, z1.s, z31.s -# CHECK-NEXT: 1 4 0.50 subhnt z0.s, z1.d, z31.d +# CHECK-NEXT: 1 8 0.50 subhnb z0.b, z1.h, z31.h +# CHECK-NEXT: 1 8 0.50 subhnb z0.h, z1.s, z31.s +# CHECK-NEXT: 1 8 0.50 subhnb z0.s, z1.d, z31.d +# CHECK-NEXT: 1 8 0.50 subhnt z0.b, z1.h, z31.h +# CHECK-NEXT: 1 8 0.50 subhnt z0.h, z1.s, z31.s +# CHECK-NEXT: 1 8 0.50 subhnt z0.s, z1.d, z31.d # CHECK-NEXT: 1 3 0.50 subr z0.b, p0/m, z0.b, z0.b # CHECK-NEXT: 1 3 0.50 subr z0.b, z0.b, #0 # CHECK-NEXT: 1 3 0.50 subr z0.d, p0/m, z0.d, z0.d @@ -6305,32 +6305,32 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 tbx z31.d, z31.d, z31.d # CHECK-NEXT: 1 4 0.50 tbx z31.h, z31.h, z31.h # CHECK-NEXT: 1 4 0.50 tbx z31.s, z31.s, z31.s -# CHECK-NEXT: 1 6 1.00 trn1 p15.b, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 trn1 p15.d, p15.d, p15.d -# CHECK-NEXT: 1 6 1.00 trn1 p15.h, p15.h, p15.h -# CHECK-NEXT: 1 6 1.00 trn1 p15.s, p15.s, p15.s -# CHECK-NEXT: 1 4 0.50 trn1 z31.b, z31.b, z31.b -# CHECK-NEXT: 1 4 0.50 trn1 z31.d, z31.d, z31.d -# CHECK-NEXT: 1 4 0.50 trn1 z31.h, z31.h, z31.h -# CHECK-NEXT: 1 4 0.50 trn1 z31.s, z31.s, z31.s -# CHECK-NEXT: 1 6 1.00 trn2 p15.b, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 trn2 p15.d, p15.d, p15.d -# CHECK-NEXT: 1 6 1.00 trn2 p15.h, p15.h, p15.h -# CHECK-NEXT: 1 6 1.00 trn2 p15.s, p15.s, p15.s -# CHECK-NEXT: 1 4 0.50 trn2 z31.b, z31.b, z31.b -# CHECK-NEXT: 1 4 0.50 trn2 z31.d, z31.d, z31.d -# CHECK-NEXT: 1 4 0.50 trn2 z31.h, z31.h, z31.h -# CHECK-NEXT: 1 4 0.50 trn2 z31.s, z31.s, z31.s -# CHECK-NEXT: 1 8 1.00 uaba z0.b, z1.b, z31.b -# CHECK-NEXT: 1 8 1.00 uaba z0.d, z1.d, z31.d -# CHECK-NEXT: 1 8 1.00 uaba z0.h, z1.h, z31.h -# CHECK-NEXT: 1 8 1.00 uaba z0.s, z1.s, z31.s -# CHECK-NEXT: 1 8 1.00 uabalb z0.d, z1.s, z31.s -# CHECK-NEXT: 1 8 1.00 uabalb z0.h, z1.b, z31.b -# CHECK-NEXT: 1 8 1.00 uabalb z0.s, z1.h, z31.h -# CHECK-NEXT: 1 8 1.00 uabalt z0.d, z1.s, z31.s -# CHECK-NEXT: 1 8 1.00 uabalt z0.h, z1.b, z31.b -# CHECK-NEXT: 1 8 1.00 uabalt z0.s, z1.h, z31.h +# CHECK-NEXT: 1 2 1.00 trn1 p15.b, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 trn1 p15.d, p15.d, p15.d +# CHECK-NEXT: 1 2 1.00 trn1 p15.h, p15.h, p15.h +# CHECK-NEXT: 1 2 1.00 trn1 p15.s, p15.s, p15.s +# CHECK-NEXT: 1 3 0.50 trn1 z31.b, z31.b, z31.b +# CHECK-NEXT: 1 3 0.50 trn1 z31.d, z31.d, z31.d +# CHECK-NEXT: 1 3 0.50 trn1 z31.h, z31.h, z31.h +# CHECK-NEXT: 1 3 0.50 trn1 z31.s, z31.s, z31.s +# CHECK-NEXT: 1 2 1.00 trn2 p15.b, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 trn2 p15.d, p15.d, p15.d +# CHECK-NEXT: 1 2 1.00 trn2 p15.h, p15.h, p15.h +# CHECK-NEXT: 1 2 1.00 trn2 p15.s, p15.s, p15.s +# CHECK-NEXT: 1 3 0.50 trn2 z31.b, z31.b, z31.b +# CHECK-NEXT: 1 3 0.50 trn2 z31.d, z31.d, z31.d +# CHECK-NEXT: 1 3 0.50 trn2 z31.h, z31.h, z31.h +# CHECK-NEXT: 1 3 0.50 trn2 z31.s, z31.s, z31.s +# CHECK-NEXT: 1 6 1.00 uaba z0.b, z1.b, z31.b +# CHECK-NEXT: 1 6 1.00 uaba z0.d, z1.d, z31.d +# CHECK-NEXT: 1 6 1.00 uaba z0.h, z1.h, z31.h +# CHECK-NEXT: 1 6 1.00 uaba z0.s, z1.s, z31.s +# CHECK-NEXT: 1 6 1.00 uabalb z0.d, z1.s, z31.s +# CHECK-NEXT: 1 6 1.00 uabalb z0.h, z1.b, z31.b +# CHECK-NEXT: 1 6 1.00 uabalb z0.s, z1.h, z31.h +# CHECK-NEXT: 1 6 1.00 uabalt z0.d, z1.s, z31.s +# CHECK-NEXT: 1 6 1.00 uabalt z0.h, z1.b, z31.b +# CHECK-NEXT: 1 6 1.00 uabalt z0.s, z1.h, z31.h # CHECK-NEXT: 1 3 0.50 uabd z31.b, p7/m, z31.b, z31.b # CHECK-NEXT: 1 3 0.50 uabd z31.d, p7/m, z31.d, z31.d # CHECK-NEXT: 1 3 0.50 uabd z31.h, p7/m, z31.h, z31.h @@ -6344,22 +6344,22 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 7 1.00 uadalp z0.h, p0/m, z1.b # CHECK-NEXT: 1 7 1.00 uadalp z29.s, p0/m, z30.h # CHECK-NEXT: 1 7 1.00 uadalp z30.d, p7/m, z31.s -# CHECK-NEXT: 1 3 0.50 uaddlb z0.h, z1.b, z2.b -# CHECK-NEXT: 1 3 0.50 uaddlb z29.s, z30.h, z31.h -# CHECK-NEXT: 1 3 0.50 uaddlb z31.d, z31.s, z31.s -# CHECK-NEXT: 1 3 0.50 uaddlt z0.h, z1.b, z2.b -# CHECK-NEXT: 1 3 0.50 uaddlt z29.s, z30.h, z31.h -# CHECK-NEXT: 1 3 0.50 uaddlt z31.d, z31.s, z31.s +# CHECK-NEXT: 1 4 0.50 uaddlb z0.h, z1.b, z2.b +# CHECK-NEXT: 1 4 0.50 uaddlb z29.s, z30.h, z31.h +# CHECK-NEXT: 1 4 0.50 uaddlb z31.d, z31.s, z31.s +# CHECK-NEXT: 1 4 0.50 uaddlt z0.h, z1.b, z2.b +# CHECK-NEXT: 1 4 0.50 uaddlt z29.s, z30.h, z31.h +# CHECK-NEXT: 1 4 0.50 uaddlt z31.d, z31.s, z31.s # CHECK-NEXT: 1 4 1.00 uaddv d0, p7, z31.b # CHECK-NEXT: 1 4 1.00 uaddv d0, p7, z31.d # CHECK-NEXT: 1 4 1.00 uaddv d0, p7, z31.h # CHECK-NEXT: 1 4 1.00 uaddv d0, p7, z31.s -# CHECK-NEXT: 1 3 0.50 uaddwb z0.h, z1.h, z2.b -# CHECK-NEXT: 1 3 0.50 uaddwb z29.s, z30.s, z31.h -# CHECK-NEXT: 1 3 0.50 uaddwb z31.d, z31.d, z31.s -# CHECK-NEXT: 1 3 0.50 uaddwt z0.h, z1.h, z2.b -# CHECK-NEXT: 1 3 0.50 uaddwt z29.s, z30.s, z31.h -# CHECK-NEXT: 1 3 0.50 uaddwt z31.d, z31.d, z31.s +# CHECK-NEXT: 1 4 0.50 uaddwb z0.h, z1.h, z2.b +# CHECK-NEXT: 1 4 0.50 uaddwb z29.s, z30.s, z31.h +# CHECK-NEXT: 1 4 0.50 uaddwb z31.d, z31.d, z31.s +# CHECK-NEXT: 1 4 0.50 uaddwt z0.h, z1.h, z2.b +# CHECK-NEXT: 1 4 0.50 uaddwt z29.s, z30.s, z31.h +# CHECK-NEXT: 1 4 0.50 uaddwt z31.d, z31.d, z31.s # CHECK-NEXT: 1 4 0.50 ucvtf z0.d, p0/m, z0.d # CHECK-NEXT: 1 4 0.50 ucvtf z0.d, p0/m, z0.s # CHECK-NEXT: 1 4 0.50 ucvtf z0.h, p0/m, z0.d @@ -6473,120 +6473,120 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 uqadd z31.d, z31.d, #65280 # CHECK-NEXT: 1 4 0.50 uqadd z31.h, z31.h, #65280 # CHECK-NEXT: 1 4 0.50 uqadd z31.s, z31.s, #65280 -# CHECK-NEXT: 1 1 0.33 uqdecb w0 -# CHECK-NEXT: 1 1 0.33 uqdecb w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdecb w0, pow2 -# CHECK-NEXT: 1 1 0.33 uqdecb w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdecb x0 -# CHECK-NEXT: 1 1 0.33 uqdecb x0, #14 -# CHECK-NEXT: 1 1 0.33 uqdecb x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdecb x0, pow2 -# CHECK-NEXT: 1 1 0.33 uqdecb x0, vl1 -# CHECK-NEXT: 1 1 0.33 uqdecd w0 -# CHECK-NEXT: 1 1 0.33 uqdecd w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdecd w0, pow2 -# CHECK-NEXT: 1 1 0.33 uqdecd w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdecd x0 -# CHECK-NEXT: 1 1 0.33 uqdecd x0, #14 -# CHECK-NEXT: 1 1 0.33 uqdecd x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdecd x0, pow2 -# CHECK-NEXT: 1 1 0.33 uqdecd x0, vl1 +# CHECK-NEXT: 1 4 0.33 uqdecb w0 +# CHECK-NEXT: 1 4 0.33 uqdecb w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdecb w0, pow2 +# CHECK-NEXT: 1 4 0.33 uqdecb w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdecb x0 +# CHECK-NEXT: 1 4 0.33 uqdecb x0, #14 +# CHECK-NEXT: 1 4 0.33 uqdecb x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdecb x0, pow2 +# CHECK-NEXT: 1 4 0.33 uqdecb x0, vl1 +# CHECK-NEXT: 1 4 0.33 uqdecd w0 +# CHECK-NEXT: 1 4 0.33 uqdecd w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdecd w0, pow2 +# CHECK-NEXT: 1 4 0.33 uqdecd w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdecd x0 +# CHECK-NEXT: 1 4 0.33 uqdecd x0, #14 +# CHECK-NEXT: 1 4 0.33 uqdecd x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdecd x0, pow2 +# CHECK-NEXT: 1 4 0.33 uqdecd x0, vl1 # CHECK-NEXT: 1 4 0.50 uqdecd z0.d # CHECK-NEXT: 1 4 0.50 uqdecd z0.d, all, mul #16 # CHECK-NEXT: 1 4 0.50 uqdecd z0.d, pow2 # CHECK-NEXT: 1 4 0.50 uqdecd z0.d, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdech w0 -# CHECK-NEXT: 1 1 0.33 uqdech w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdech w0, pow2 -# CHECK-NEXT: 1 1 0.33 uqdech w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdech x0 -# CHECK-NEXT: 1 1 0.33 uqdech x0, #14 -# CHECK-NEXT: 1 1 0.33 uqdech x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdech x0, pow2 -# CHECK-NEXT: 1 1 0.33 uqdech x0, vl1 +# CHECK-NEXT: 1 4 0.33 uqdech w0 +# CHECK-NEXT: 1 4 0.33 uqdech w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdech w0, pow2 +# CHECK-NEXT: 1 4 0.33 uqdech w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdech x0 +# CHECK-NEXT: 1 4 0.33 uqdech x0, #14 +# CHECK-NEXT: 1 4 0.33 uqdech x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdech x0, pow2 +# CHECK-NEXT: 1 4 0.33 uqdech x0, vl1 # CHECK-NEXT: 1 4 0.50 uqdech z0.h # CHECK-NEXT: 1 4 0.50 uqdech z0.h, all, mul #16 # CHECK-NEXT: 1 4 0.50 uqdech z0.h, pow2 # CHECK-NEXT: 1 4 0.50 uqdech z0.h, pow2, mul #16 -# CHECK-NEXT: 1 8 1.00 uqdecp wzr, p15.b -# CHECK-NEXT: 1 8 1.00 uqdecp wzr, p15.d -# CHECK-NEXT: 1 8 1.00 uqdecp wzr, p15.h -# CHECK-NEXT: 1 8 1.00 uqdecp wzr, p15.s -# CHECK-NEXT: 1 8 1.00 uqdecp x0, p0.b -# CHECK-NEXT: 1 8 1.00 uqdecp x0, p0.d -# CHECK-NEXT: 1 8 1.00 uqdecp x0, p0.h -# CHECK-NEXT: 1 8 1.00 uqdecp x0, p0.s +# CHECK-NEXT: 1 9 1.00 uqdecp wzr, p15.b +# CHECK-NEXT: 1 9 1.00 uqdecp wzr, p15.d +# CHECK-NEXT: 1 9 1.00 uqdecp wzr, p15.h +# CHECK-NEXT: 1 9 1.00 uqdecp wzr, p15.s +# CHECK-NEXT: 1 9 1.00 uqdecp x0, p0.b +# CHECK-NEXT: 1 9 1.00 uqdecp x0, p0.d +# CHECK-NEXT: 1 9 1.00 uqdecp x0, p0.h +# CHECK-NEXT: 1 9 1.00 uqdecp x0, p0.s # CHECK-NEXT: 1 4 0.50 uqdecp z0.d, p0.d # CHECK-NEXT: 1 4 0.50 uqdecp z0.h, p0.h # CHECK-NEXT: 1 4 0.50 uqdecp z0.s, p0.s -# CHECK-NEXT: 1 1 0.33 uqdecw w0 -# CHECK-NEXT: 1 1 0.33 uqdecw w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdecw w0, pow2 -# CHECK-NEXT: 1 1 0.33 uqdecw w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdecw x0 -# CHECK-NEXT: 1 1 0.33 uqdecw x0, #14 -# CHECK-NEXT: 1 1 0.33 uqdecw x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqdecw x0, pow2 -# CHECK-NEXT: 1 1 0.33 uqdecw x0, vl1 +# CHECK-NEXT: 1 4 0.33 uqdecw w0 +# CHECK-NEXT: 1 4 0.33 uqdecw w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdecw w0, pow2 +# CHECK-NEXT: 1 4 0.33 uqdecw w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdecw x0 +# CHECK-NEXT: 1 4 0.33 uqdecw x0, #14 +# CHECK-NEXT: 1 4 0.33 uqdecw x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqdecw x0, pow2 +# CHECK-NEXT: 1 4 0.33 uqdecw x0, vl1 # CHECK-NEXT: 1 4 0.50 uqdecw z0.s # CHECK-NEXT: 1 4 0.50 uqdecw z0.s, all, mul #16 # CHECK-NEXT: 1 4 0.50 uqdecw z0.s, pow2 # CHECK-NEXT: 1 4 0.50 uqdecw z0.s, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincb w0 -# CHECK-NEXT: 1 1 0.33 uqincb w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincb w0, pow2 -# CHECK-NEXT: 1 1 0.33 uqincb w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincb x0 -# CHECK-NEXT: 1 1 0.33 uqincb x0, #14 -# CHECK-NEXT: 1 1 0.33 uqincb x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincb x0, pow2 -# CHECK-NEXT: 1 1 0.33 uqincb x0, vl1 -# CHECK-NEXT: 1 1 0.33 uqincd w0 -# CHECK-NEXT: 1 1 0.33 uqincd w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincd w0, pow2 -# CHECK-NEXT: 1 1 0.33 uqincd w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincd x0 -# CHECK-NEXT: 1 1 0.33 uqincd x0, #14 -# CHECK-NEXT: 1 1 0.33 uqincd x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincd x0, pow2 -# CHECK-NEXT: 1 1 0.33 uqincd x0, vl1 +# CHECK-NEXT: 1 4 0.33 uqincb w0 +# CHECK-NEXT: 1 4 0.33 uqincb w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqincb w0, pow2 +# CHECK-NEXT: 1 4 0.33 uqincb w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 uqincb x0 +# CHECK-NEXT: 1 4 0.33 uqincb x0, #14 +# CHECK-NEXT: 1 4 0.33 uqincb x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqincb x0, pow2 +# CHECK-NEXT: 1 4 0.33 uqincb x0, vl1 +# CHECK-NEXT: 1 4 0.33 uqincd w0 +# CHECK-NEXT: 1 4 0.33 uqincd w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqincd w0, pow2 +# CHECK-NEXT: 1 4 0.33 uqincd w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 uqincd x0 +# CHECK-NEXT: 1 4 0.33 uqincd x0, #14 +# CHECK-NEXT: 1 4 0.33 uqincd x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqincd x0, pow2 +# CHECK-NEXT: 1 4 0.33 uqincd x0, vl1 # CHECK-NEXT: 1 4 0.50 uqincd z0.d # CHECK-NEXT: 1 4 0.50 uqincd z0.d, all, mul #16 # CHECK-NEXT: 1 4 0.50 uqincd z0.d, pow2 # CHECK-NEXT: 1 4 0.50 uqincd z0.d, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqinch w0 -# CHECK-NEXT: 1 1 0.33 uqinch w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqinch w0, pow2 -# CHECK-NEXT: 1 1 0.33 uqinch w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqinch x0 -# CHECK-NEXT: 1 1 0.33 uqinch x0, #14 -# CHECK-NEXT: 1 1 0.33 uqinch x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqinch x0, pow2 -# CHECK-NEXT: 1 1 0.33 uqinch x0, vl1 +# CHECK-NEXT: 1 4 0.33 uqinch w0 +# CHECK-NEXT: 1 4 0.33 uqinch w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqinch w0, pow2 +# CHECK-NEXT: 1 4 0.33 uqinch w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 uqinch x0 +# CHECK-NEXT: 1 4 0.33 uqinch x0, #14 +# CHECK-NEXT: 1 4 0.33 uqinch x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqinch x0, pow2 +# CHECK-NEXT: 1 4 0.33 uqinch x0, vl1 # CHECK-NEXT: 1 4 0.50 uqinch z0.h # CHECK-NEXT: 1 4 0.50 uqinch z0.h, all, mul #16 # CHECK-NEXT: 1 4 0.50 uqinch z0.h, pow2 # CHECK-NEXT: 1 4 0.50 uqinch z0.h, pow2, mul #16 -# CHECK-NEXT: 1 8 1.00 uqincp wzr, p15.b -# CHECK-NEXT: 1 8 1.00 uqincp wzr, p15.d -# CHECK-NEXT: 1 8 1.00 uqincp wzr, p15.h -# CHECK-NEXT: 1 8 1.00 uqincp wzr, p15.s -# CHECK-NEXT: 1 8 1.00 uqincp x0, p0.b -# CHECK-NEXT: 1 8 1.00 uqincp x0, p0.d -# CHECK-NEXT: 1 8 1.00 uqincp x0, p0.h -# CHECK-NEXT: 1 8 1.00 uqincp x0, p0.s +# CHECK-NEXT: 1 9 1.00 uqincp wzr, p15.b +# CHECK-NEXT: 1 9 1.00 uqincp wzr, p15.d +# CHECK-NEXT: 1 9 1.00 uqincp wzr, p15.h +# CHECK-NEXT: 1 9 1.00 uqincp wzr, p15.s +# CHECK-NEXT: 1 9 1.00 uqincp x0, p0.b +# CHECK-NEXT: 1 9 1.00 uqincp x0, p0.d +# CHECK-NEXT: 1 9 1.00 uqincp x0, p0.h +# CHECK-NEXT: 1 9 1.00 uqincp x0, p0.s # CHECK-NEXT: 1 4 0.50 uqincp z0.d, p0.d # CHECK-NEXT: 1 4 0.50 uqincp z0.h, p0.h # CHECK-NEXT: 1 4 0.50 uqincp z0.s, p0.s -# CHECK-NEXT: 1 1 0.33 uqincw w0 -# CHECK-NEXT: 1 1 0.33 uqincw w0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincw w0, pow2 -# CHECK-NEXT: 1 1 0.33 uqincw w0, pow2, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincw x0 -# CHECK-NEXT: 1 1 0.33 uqincw x0, #14 -# CHECK-NEXT: 1 1 0.33 uqincw x0, all, mul #16 -# CHECK-NEXT: 1 1 0.33 uqincw x0, pow2 -# CHECK-NEXT: 1 1 0.33 uqincw x0, vl1 +# CHECK-NEXT: 1 4 0.33 uqincw w0 +# CHECK-NEXT: 1 4 0.33 uqincw w0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqincw w0, pow2 +# CHECK-NEXT: 1 4 0.33 uqincw w0, pow2, mul #16 +# CHECK-NEXT: 1 4 0.33 uqincw x0 +# CHECK-NEXT: 1 4 0.33 uqincw x0, #14 +# CHECK-NEXT: 1 4 0.33 uqincw x0, all, mul #16 +# CHECK-NEXT: 1 4 0.33 uqincw x0, pow2 +# CHECK-NEXT: 1 4 0.33 uqincw x0, vl1 # CHECK-NEXT: 1 4 0.50 uqincw z0.s # CHECK-NEXT: 1 4 0.50 uqincw z0.s, all, mul #16 # CHECK-NEXT: 1 4 0.50 uqincw z0.s, pow2 @@ -6723,18 +6723,18 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 4 0.50 usra z31.d, z31.d, #64 # CHECK-NEXT: 1 4 0.50 usra z31.h, z31.h, #16 # CHECK-NEXT: 1 4 0.50 usra z31.s, z31.s, #32 -# CHECK-NEXT: 1 3 0.50 usublb z0.h, z1.b, z2.b -# CHECK-NEXT: 1 3 0.50 usublb z29.s, z30.h, z31.h -# CHECK-NEXT: 1 3 0.50 usublb z31.d, z31.s, z31.s -# CHECK-NEXT: 1 3 0.50 usublt z0.h, z1.b, z2.b -# CHECK-NEXT: 1 3 0.50 usublt z29.s, z30.h, z31.h -# CHECK-NEXT: 1 3 0.50 usublt z31.d, z31.s, z31.s -# CHECK-NEXT: 1 3 0.50 usubwb z0.h, z1.h, z2.b -# CHECK-NEXT: 1 3 0.50 usubwb z29.s, z30.s, z31.h -# CHECK-NEXT: 1 3 0.50 usubwb z31.d, z31.d, z31.s -# CHECK-NEXT: 1 3 0.50 usubwt z0.h, z1.h, z2.b -# CHECK-NEXT: 1 3 0.50 usubwt z29.s, z30.s, z31.h -# CHECK-NEXT: 1 3 0.50 usubwt z31.d, z31.d, z31.s +# CHECK-NEXT: 1 4 0.50 usublb z0.h, z1.b, z2.b +# CHECK-NEXT: 1 4 0.50 usublb z29.s, z30.h, z31.h +# CHECK-NEXT: 1 4 0.50 usublb z31.d, z31.s, z31.s +# CHECK-NEXT: 1 4 0.50 usublt z0.h, z1.b, z2.b +# CHECK-NEXT: 1 4 0.50 usublt z29.s, z30.h, z31.h +# CHECK-NEXT: 1 4 0.50 usublt z31.d, z31.s, z31.s +# CHECK-NEXT: 1 4 0.50 usubwb z0.h, z1.h, z2.b +# CHECK-NEXT: 1 4 0.50 usubwb z29.s, z30.s, z31.h +# CHECK-NEXT: 1 4 0.50 usubwb z31.d, z31.d, z31.s +# CHECK-NEXT: 1 4 0.50 usubwt z0.h, z1.h, z2.b +# CHECK-NEXT: 1 4 0.50 usubwt z29.s, z30.s, z31.h +# CHECK-NEXT: 1 4 0.50 usubwt z31.d, z31.d, z31.s # CHECK-NEXT: 1 4 0.50 uunpkhi z31.d, z31.s # CHECK-NEXT: 1 4 0.50 uunpkhi z31.h, z31.b # CHECK-NEXT: 1 4 0.50 uunpkhi z31.s, z31.h @@ -6753,82 +6753,82 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: 1 3 0.50 uxth z31.s, p7/m, z31.s # CHECK-NEXT: 1 3 0.50 uxtw z0.d, p0/m, z0.d # CHECK-NEXT: 1 3 0.50 uxtw z31.d, p7/m, z31.d -# CHECK-NEXT: 1 6 1.00 uzp1 p15.b, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 uzp1 p15.d, p15.d, p15.d -# CHECK-NEXT: 1 6 1.00 uzp1 p15.h, p15.h, p15.h -# CHECK-NEXT: 1 6 1.00 uzp1 p15.s, p15.s, p15.s -# CHECK-NEXT: 1 4 0.50 uzp1 z31.b, z31.b, z31.b -# CHECK-NEXT: 1 4 0.50 uzp1 z31.d, z31.d, z31.d -# CHECK-NEXT: 1 4 0.50 uzp1 z31.h, z31.h, z31.h -# CHECK-NEXT: 1 4 0.50 uzp1 z31.s, z31.s, z31.s -# CHECK-NEXT: 1 6 1.00 uzp2 p15.b, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 uzp2 p15.d, p15.d, p15.d -# CHECK-NEXT: 1 6 1.00 uzp2 p15.h, p15.h, p15.h -# CHECK-NEXT: 1 6 1.00 uzp2 p15.s, p15.s, p15.s -# CHECK-NEXT: 1 4 0.50 uzp2 z31.b, z31.b, z31.b -# CHECK-NEXT: 1 4 0.50 uzp2 z31.d, z31.d, z31.d -# CHECK-NEXT: 1 4 0.50 uzp2 z31.h, z31.h, z31.h -# CHECK-NEXT: 1 4 0.50 uzp2 z31.s, z31.s, z31.s -# CHECK-NEXT: 1 6 1.00 whilege p15.b, w0, wzr -# CHECK-NEXT: 1 6 1.00 whilege p15.b, wzr, w0 -# CHECK-NEXT: 1 6 1.00 whilege p15.b, x0, xzr -# CHECK-NEXT: 1 6 1.00 whilege p15.b, xzr, x0 -# CHECK-NEXT: 1 6 1.00 whilege p15.d, w0, wzr -# CHECK-NEXT: 1 6 1.00 whilege p15.d, x0, xzr -# CHECK-NEXT: 1 6 1.00 whilege p15.h, w0, wzr -# CHECK-NEXT: 1 6 1.00 whilege p15.h, x0, xzr -# CHECK-NEXT: 1 6 1.00 whilege p15.s, w0, wzr -# CHECK-NEXT: 1 6 1.00 whilege p15.s, x0, xzr -# CHECK-NEXT: 1 6 1.00 whilerw p15.b, x30, x30 -# CHECK-NEXT: 1 6 1.00 whilerw p15.d, x30, x30 -# CHECK-NEXT: 1 6 1.00 whilerw p15.h, x30, x30 -# CHECK-NEXT: 1 6 1.00 whilerw p15.s, x30, x30 -# CHECK-NEXT: 1 6 1.00 whilewr p15.b, x30, x30 -# CHECK-NEXT: 1 6 1.00 whilewr p15.d, x30, x30 -# CHECK-NEXT: 1 6 1.00 whilewr p15.h, x30, x30 -# CHECK-NEXT: 1 6 1.00 whilewr p15.s, x30, x30 +# CHECK-NEXT: 1 2 1.00 uzp1 p15.b, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 uzp1 p15.d, p15.d, p15.d +# CHECK-NEXT: 1 2 1.00 uzp1 p15.h, p15.h, p15.h +# CHECK-NEXT: 1 2 1.00 uzp1 p15.s, p15.s, p15.s +# CHECK-NEXT: 1 3 0.50 uzp1 z31.b, z31.b, z31.b +# CHECK-NEXT: 1 3 0.50 uzp1 z31.d, z31.d, z31.d +# CHECK-NEXT: 1 3 0.50 uzp1 z31.h, z31.h, z31.h +# CHECK-NEXT: 1 3 0.50 uzp1 z31.s, z31.s, z31.s +# CHECK-NEXT: 1 2 1.00 uzp2 p15.b, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 uzp2 p15.d, p15.d, p15.d +# CHECK-NEXT: 1 2 1.00 uzp2 p15.h, p15.h, p15.h +# CHECK-NEXT: 1 2 1.00 uzp2 p15.s, p15.s, p15.s +# CHECK-NEXT: 1 3 0.50 uzp2 z31.b, z31.b, z31.b +# CHECK-NEXT: 1 3 0.50 uzp2 z31.d, z31.d, z31.d +# CHECK-NEXT: 1 3 0.50 uzp2 z31.h, z31.h, z31.h +# CHECK-NEXT: 1 3 0.50 uzp2 z31.s, z31.s, z31.s +# CHECK-NEXT: 1 2 1.00 whilege p15.b, w0, wzr +# CHECK-NEXT: 1 2 1.00 whilege p15.b, wzr, w0 +# CHECK-NEXT: 1 2 1.00 whilege p15.b, x0, xzr +# CHECK-NEXT: 1 2 1.00 whilege p15.b, xzr, x0 +# CHECK-NEXT: 1 2 1.00 whilege p15.d, w0, wzr +# CHECK-NEXT: 1 2 1.00 whilege p15.d, x0, xzr +# CHECK-NEXT: 1 2 1.00 whilege p15.h, w0, wzr +# CHECK-NEXT: 1 2 1.00 whilege p15.h, x0, xzr +# CHECK-NEXT: 1 2 1.00 whilege p15.s, w0, wzr +# CHECK-NEXT: 1 2 1.00 whilege p15.s, x0, xzr +# CHECK-NEXT: 1 2 1.00 whilerw p15.b, x30, x30 +# CHECK-NEXT: 1 2 1.00 whilerw p15.d, x30, x30 +# CHECK-NEXT: 1 2 1.00 whilerw p15.h, x30, x30 +# CHECK-NEXT: 1 2 1.00 whilerw p15.s, x30, x30 +# CHECK-NEXT: 1 2 1.00 whilewr p15.b, x30, x30 +# CHECK-NEXT: 1 2 1.00 whilewr p15.d, x30, x30 +# CHECK-NEXT: 1 2 1.00 whilewr p15.h, x30, x30 +# CHECK-NEXT: 1 2 1.00 whilewr p15.s, x30, x30 # CHECK-NEXT: 1 1 0.33 * U wrffr p0.b # CHECK-NEXT: 1 1 0.33 * U wrffr p15.b -# CHECK-NEXT: 1 3 0.50 xar z0.b, z0.b, z1.b, #1 -# CHECK-NEXT: 1 3 0.50 xar z0.d, z0.d, z1.d, #1 -# CHECK-NEXT: 1 3 0.50 xar z0.h, z0.h, z1.h, #1 -# CHECK-NEXT: 1 3 0.50 xar z0.s, z0.s, z1.s, #1 -# CHECK-NEXT: 1 3 0.50 xar z31.b, z31.b, z30.b, #8 -# CHECK-NEXT: 1 3 0.50 xar z31.d, z31.d, z30.d, #64 -# CHECK-NEXT: 1 3 0.50 xar z31.h, z31.h, z30.h, #16 -# CHECK-NEXT: 1 3 0.50 xar z31.s, z31.s, z30.s, #32 -# CHECK-NEXT: 1 6 1.00 zip1 p0.b, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 zip1 p0.d, p0.d, p0.d -# CHECK-NEXT: 1 6 1.00 zip1 p0.h, p0.h, p0.h -# CHECK-NEXT: 1 6 1.00 zip1 p0.s, p0.s, p0.s -# CHECK-NEXT: 1 6 1.00 zip1 p15.b, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 zip1 p15.d, p15.d, p15.d -# CHECK-NEXT: 1 6 1.00 zip1 p15.h, p15.h, p15.h -# CHECK-NEXT: 1 6 1.00 zip1 p15.s, p15.s, p15.s -# CHECK-NEXT: 1 4 0.50 zip1 z0.b, z0.b, z0.b -# CHECK-NEXT: 1 4 0.50 zip1 z0.d, z0.d, z0.d -# CHECK-NEXT: 1 4 0.50 zip1 z0.h, z0.h, z0.h -# CHECK-NEXT: 1 4 0.50 zip1 z0.s, z0.s, z0.s -# CHECK-NEXT: 1 4 0.50 zip1 z31.b, z31.b, z31.b -# CHECK-NEXT: 1 4 0.50 zip1 z31.d, z31.d, z31.d -# CHECK-NEXT: 1 4 0.50 zip1 z31.h, z31.h, z31.h -# CHECK-NEXT: 1 4 0.50 zip1 z31.s, z31.s, z31.s -# CHECK-NEXT: 1 6 1.00 zip2 p0.b, p0.b, p0.b -# CHECK-NEXT: 1 6 1.00 zip2 p0.d, p0.d, p0.d -# CHECK-NEXT: 1 6 1.00 zip2 p0.h, p0.h, p0.h -# CHECK-NEXT: 1 6 1.00 zip2 p0.s, p0.s, p0.s -# CHECK-NEXT: 1 6 1.00 zip2 p15.b, p15.b, p15.b -# CHECK-NEXT: 1 6 1.00 zip2 p15.d, p15.d, p15.d -# CHECK-NEXT: 1 6 1.00 zip2 p15.h, p15.h, p15.h -# CHECK-NEXT: 1 6 1.00 zip2 p15.s, p15.s, p15.s -# CHECK-NEXT: 1 4 0.50 zip2 z0.b, z0.b, z0.b -# CHECK-NEXT: 1 4 0.50 zip2 z0.d, z0.d, z0.d -# CHECK-NEXT: 1 4 0.50 zip2 z0.h, z0.h, z0.h -# CHECK-NEXT: 1 4 0.50 zip2 z0.s, z0.s, z0.s -# CHECK-NEXT: 1 4 0.50 zip2 z31.b, z31.b, z31.b -# CHECK-NEXT: 1 4 0.50 zip2 z31.d, z31.d, z31.d -# CHECK-NEXT: 1 4 0.50 zip2 z31.h, z31.h, z31.h -# CHECK-NEXT: 1 4 0.50 zip2 z31.s, z31.s, z31.s +# CHECK-NEXT: 1 4 0.50 xar z0.b, z0.b, z1.b, #1 +# CHECK-NEXT: 1 4 0.50 xar z0.d, z0.d, z1.d, #1 +# CHECK-NEXT: 1 4 0.50 xar z0.h, z0.h, z1.h, #1 +# CHECK-NEXT: 1 4 0.50 xar z0.s, z0.s, z1.s, #1 +# CHECK-NEXT: 1 4 0.50 xar z31.b, z31.b, z30.b, #8 +# CHECK-NEXT: 1 4 0.50 xar z31.d, z31.d, z30.d, #64 +# CHECK-NEXT: 1 4 0.50 xar z31.h, z31.h, z30.h, #16 +# CHECK-NEXT: 1 4 0.50 xar z31.s, z31.s, z30.s, #32 +# CHECK-NEXT: 1 2 1.00 zip1 p0.b, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 zip1 p0.d, p0.d, p0.d +# CHECK-NEXT: 1 2 1.00 zip1 p0.h, p0.h, p0.h +# CHECK-NEXT: 1 2 1.00 zip1 p0.s, p0.s, p0.s +# CHECK-NEXT: 1 2 1.00 zip1 p15.b, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 zip1 p15.d, p15.d, p15.d +# CHECK-NEXT: 1 2 1.00 zip1 p15.h, p15.h, p15.h +# CHECK-NEXT: 1 2 1.00 zip1 p15.s, p15.s, p15.s +# CHECK-NEXT: 1 3 0.50 zip1 z0.b, z0.b, z0.b +# CHECK-NEXT: 1 3 0.50 zip1 z0.d, z0.d, z0.d +# CHECK-NEXT: 1 3 0.50 zip1 z0.h, z0.h, z0.h +# CHECK-NEXT: 1 3 0.50 zip1 z0.s, z0.s, z0.s +# CHECK-NEXT: 1 3 0.50 zip1 z31.b, z31.b, z31.b +# CHECK-NEXT: 1 3 0.50 zip1 z31.d, z31.d, z31.d +# CHECK-NEXT: 1 3 0.50 zip1 z31.h, z31.h, z31.h +# CHECK-NEXT: 1 3 0.50 zip1 z31.s, z31.s, z31.s +# CHECK-NEXT: 1 2 1.00 zip2 p0.b, p0.b, p0.b +# CHECK-NEXT: 1 2 1.00 zip2 p0.d, p0.d, p0.d +# CHECK-NEXT: 1 2 1.00 zip2 p0.h, p0.h, p0.h +# CHECK-NEXT: 1 2 1.00 zip2 p0.s, p0.s, p0.s +# CHECK-NEXT: 1 2 1.00 zip2 p15.b, p15.b, p15.b +# CHECK-NEXT: 1 2 1.00 zip2 p15.d, p15.d, p15.d +# CHECK-NEXT: 1 2 1.00 zip2 p15.h, p15.h, p15.h +# CHECK-NEXT: 1 2 1.00 zip2 p15.s, p15.s, p15.s +# CHECK-NEXT: 1 3 0.50 zip2 z0.b, z0.b, z0.b +# CHECK-NEXT: 1 3 0.50 zip2 z0.d, z0.d, z0.d +# CHECK-NEXT: 1 3 0.50 zip2 z0.h, z0.h, z0.h +# CHECK-NEXT: 1 3 0.50 zip2 z0.s, z0.s, z0.s +# CHECK-NEXT: 1 3 0.50 zip2 z31.b, z31.b, z31.b +# CHECK-NEXT: 1 3 0.50 zip2 z31.d, z31.d, z31.d +# CHECK-NEXT: 1 3 0.50 zip2 z31.h, z31.h, z31.h +# CHECK-NEXT: 1 3 0.50 zip2 z31.s, z31.s, z31.s # CHECK: Resources: # CHECK-NEXT: [0] - CortexA510UnitALU0 @@ -6848,7 +6848,7 @@ zip2 z31.s, z31.s, z31.s # CHECK: Resource pressure per iteration: # CHECK-NEXT: [0] [1.0] [1.1] [2] [3] [4] [5] [6] [7] [8] [9] [10.0] [10.1] [11] -# CHECK-NEXT: 79.00 75.00 75.00 9.00 - 240.00 3698.00 - - 1290.00 924.00 199.50 199.50 670.00 +# CHECK-NEXT: 79.00 75.00 75.00 9.00 - 209.00 3667.00 - - 1290.00 924.00 199.50 199.50 670.00 # CHECK: Resource pressure by instruction: # CHECK-NEXT: [0] [1.0] [1.1] [2] [3] [4] [5] [6] [7] [8] [9] [10.0] [10.1] [11] Instructions: @@ -7844,8 +7844,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1b { z0.d }, p0/z, [x0] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1b { z0.d }, p0/z, [z0.d] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1b { z0.h }, p0/z, [x0] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1b { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1b { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1b { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1b { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1b { z0.s }, p0/z, [x0] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ld1b { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1b { z21.b }, p5/z, [x10, #5, mul vl] @@ -7882,8 +7882,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1h { z0.d }, p0/z, [x0] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1h { z0.d }, p0/z, [z0.d] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1h { z0.h }, p0/z, [x0] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1h { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1h { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1h { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1h { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1h { z0.s }, p0/z, [x0] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ld1h { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1h { z21.d }, p5/z, [x10, #5, mul vl] @@ -7899,8 +7899,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1h { z31.d }, p7/z, [z31.d, #62] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1h { z31.h }, p7/z, [sp, #-1, mul vl] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1h { z31.s }, p7/z, [sp, #-1, mul vl] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1h { z31.s }, p7/z, [sp, z31.s, sxtw #1] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1h { z31.s }, p7/z, [sp, z31.s, uxtw #1] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1h { z31.s }, p7/z, [sp, z31.s, sxtw #1] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1h { z31.s }, p7/z, [sp, z31.s, uxtw #1] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ld1h { z31.s }, p7/z, [z31.s, #62] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1h { z5.h }, p3/z, [sp, x16, lsl #1] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1h { z5.h }, p3/z, [x17, x16, lsl #1] @@ -7961,7 +7961,7 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1sb { z0.h }, p0/z, [sp, x0] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1sb { z0.h }, p0/z, [x0, x0] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1sb { z0.h }, p0/z, [x0] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1sb { z0.s }, p0/z, [x0] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ld1sb { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1sb { z21.d }, p5/z, [x10, #5, mul vl] @@ -7981,8 +7981,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1sh { z0.d }, p0/z, [x0] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1sh { z0.d }, p0/z, [z0.d] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1sh { z0.s }, p0/z, [x0] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ld1sh { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1sh { z21.d }, p5/z, [x10, #5, mul vl] @@ -7997,8 +7997,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1sh { z31.d }, p7/z, [sp, z31.d] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1sh { z31.d }, p7/z, [z31.d, #62] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1sh { z31.s }, p7/z, [sp, #-1, mul vl] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ld1sh { z31.s }, p7/z, [z31.s, #62] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw #2] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw #2] @@ -8017,8 +8017,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1w { z0.d }, p0/z, [x0, z0.d, uxtw #2] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1w { z0.d }, p0/z, [x0] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1w { z0.d }, p0/z, [z0.d] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1w { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1w { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1w { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1w { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1w { z0.s }, p0/z, [x0] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ld1w { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1w { z21.d }, p5/z, [x10, #5, mul vl] @@ -8033,8 +8033,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1w { z31.d }, p7/z, [sp, z31.d] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ld1w { z31.d }, p7/z, [z31.d, #124] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ld1w { z31.s }, p7/z, [sp, #-1, mul vl] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1w { z31.s }, p7/z, [sp, z31.s, sxtw #2] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ld1w { z31.s }, p7/z, [sp, z31.s, uxtw #2] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1w { z31.s }, p7/z, [sp, z31.s, sxtw #2] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ld1w { z31.s }, p7/z, [sp, z31.s, uxtw #2] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ld1w { z31.s }, p7/z, [z31.s, #124] # CHECK-NEXT: - - - - - - 2.00 - - - - - - - ld2b { z0.b, z1.b }, p0/z, [x0, x0] # CHECK-NEXT: - - - - - - 1.00 - - - - - - - ld2b { z0.b, z1.b }, p0/z, [x0] @@ -8100,8 +8100,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1b { z0.d }, p0/z, [z0.d] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1b { z0.h }, p0/z, [x0, x0] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1b { z0.s }, p0/z, [x0, x0] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1b { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1b { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1b { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1b { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ldff1b { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1b { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1b { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -8128,8 +8128,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1h { z0.d }, p0/z, [z0.d] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1h { z0.h }, p0/z, [x0, x0, lsl #1] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1h { z0.s }, p0/z, [x0, x0, lsl #1] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1h { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1h { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1h { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1h { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ldff1h { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1h { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1h { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -8138,16 +8138,16 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1h { z31.d }, p7/z, [sp] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1h { z31.d }, p7/z, [z31.d, #62] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1h { z31.h }, p7/z, [sp] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1h { z31.s }, p7/z, [sp, z31.s, sxtw #1] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1h { z31.s }, p7/z, [sp, z31.s, uxtw #1] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1h { z31.s }, p7/z, [sp, z31.s, sxtw #1] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1h { z31.s }, p7/z, [sp, z31.s, uxtw #1] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1h { z31.s }, p7/z, [sp] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ldff1h { z31.s }, p7/z, [z31.s, #62] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1sb { z0.d }, p0/z, [x0, x0] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1sb { z0.d }, p0/z, [z0.d] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1sb { z0.h }, p0/z, [x0, x0] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1sb { z0.s }, p0/z, [x0, x0] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1sb { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1sb { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1sb { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1sb { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ldff1sb { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1sb { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1sb { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -8162,8 +8162,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1sh { z0.d }, p0/z, [z0.d] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1sh { z0.s }, p0/z, [x0, x0, lsl #1] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1sh { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1sh { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1sh { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1sh { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ldff1sh { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1sh { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1sh { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -8171,8 +8171,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1sh { z31.d }, p7/z, [sp, z31.d] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1sh { z31.d }, p7/z, [sp] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1sh { z31.d }, p7/z, [z31.d, #62] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1sh { z31.s }, p7/z, [sp, z31.s, sxtw #1] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1sh { z31.s }, p7/z, [sp, z31.s, uxtw #1] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1sh { z31.s }, p7/z, [sp] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ldff1sh { z31.s }, p7/z, [z31.s, #62] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1sw { z0.d }, p0/z, [x0, x0, lsl #2] @@ -8190,8 +8190,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1w { z0.d }, p0/z, [x0, z0.d, uxtw #2] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1w { z0.d }, p0/z, [z0.d] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1w { z0.s }, p0/z, [x0, x0, lsl #2] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1w { z0.s }, p0/z, [x0, z0.s, sxtw] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1w { z0.s }, p0/z, [x0, z0.s, uxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1w { z0.s }, p0/z, [x0, z0.s, sxtw] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1w { z0.s }, p0/z, [x0, z0.s, uxtw] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ldff1w { z0.s }, p0/z, [z0.s] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1w { z21.d }, p5/z, [x10, z21.d, sxtw] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1w { z21.d }, p5/z, [x10, z21.d, uxtw] @@ -8199,8 +8199,8 @@ zip2 z31.s, z31.s, z31.s # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1w { z31.d }, p7/z, [sp, z31.d] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1w { z31.d }, p7/z, [sp] # CHECK-NEXT: - - - - - - 7.00 - - - - - - - ldff1w { z31.d }, p7/z, [z31.d, #124] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1w { z31.s }, p7/z, [sp, z31.s, sxtw #2] -# CHECK-NEXT: - - - - - 4.50 4.50 - - - - - - - ldff1w { z31.s }, p7/z, [sp, z31.s, uxtw #2] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1w { z31.s }, p7/z, [sp, z31.s, sxtw #2] +# CHECK-NEXT: - - - - - 3.50 3.50 - - - - - - - ldff1w { z31.s }, p7/z, [sp, z31.s, uxtw #2] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldff1w { z31.s }, p7/z, [sp] # CHECK-NEXT: - - - - - - 9.00 - - - - - - - ldff1w { z31.s }, p7/z, [z31.s, #124] # CHECK-NEXT: - - - - - 0.50 0.50 - - - - - - - ldnf1b { z0.b }, p0/z, [x0] -- GitLab From 4572a2db2c2c48820e79fb65f3810348371db4f1 Mon Sep 17 00:00:00 2001 From: David Green Date: Wed, 17 Apr 2024 19:44:17 +0100 Subject: [PATCH 053/714] [AArch64] Add some test cases for LD2/LD3/LD4 shuffles. NFC --- .../CostModel/AArch64/shuffle-load.ll | 620 ++++++++++++++++++ 1 file changed, 620 insertions(+) diff --git a/llvm/test/Analysis/CostModel/AArch64/shuffle-load.ll b/llvm/test/Analysis/CostModel/AArch64/shuffle-load.ll index ec95a313e11e..106f2f9edc2e 100644 --- a/llvm/test/Analysis/CostModel/AArch64/shuffle-load.ll +++ b/llvm/test/Analysis/CostModel/AArch64/shuffle-load.ll @@ -401,3 +401,623 @@ entry: %lane = shufflevector <2 x i64> %tmp1, <2 x i64> undef, <2 x i32> zeroinitializer ret <2 x i64> %lane } + +define void @vld2(ptr %p) { +; CHECK-LABEL: 'vld2' +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i8 = load <4 x i8>, ptr %p, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8_1 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = load <8 x i8>, ptr %p, align 8 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i8_1 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = load <16 x i8>, ptr %p, align 16 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i8_1 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i8 = load <32 x i8>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i8_0 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i8_1 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = load <4 x i16>, ptr %p, align 8 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i16_0 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i16_1 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = load <8 x i16>, ptr %p, align 16 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16_1 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = load <16 x i16>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i16_0 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i16_1 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v32i16 = load <32 x i16>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i16_0 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i16_1 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = load <4 x i32>, ptr %p, align 16 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i32_1 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = load <8 x i32>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i32_0 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i32_1 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16i32 = load <16 x i32>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16i32_0 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16i32_1 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v32i32 = load <32 x i32>, ptr %p, align 128 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v32i32_0 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v32i32_1 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i64 = load <4 x i64>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i64 = load <8 x i64>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_0 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_1 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i64 = load <16 x i64>, ptr %p, align 128 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_0 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_1 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i64 = load <32 x i64>, ptr %p, align 256 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_0 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_1 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; CODESIZE-LABEL: 'vld2' +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = load <4 x i8>, ptr %p, align 4 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8_0 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8_1 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = load <8 x i8>, ptr %p, align 8 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i8_0 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i8_1 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = load <16 x i8>, ptr %p, align 16 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i8_0 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i8_1 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i8 = load <32 x i8>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i8_0 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i8_1 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = load <4 x i16>, ptr %p, align 8 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i16_0 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i16_1 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = load <8 x i16>, ptr %p, align 16 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16_0 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16_1 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = load <16 x i16>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i16_0 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i16_1 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v32i16 = load <32 x i16>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i16_0 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v32i16_1 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = load <4 x i32>, ptr %p, align 16 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i32_0 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i32_1 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = load <8 x i32>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i32_0 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i32_1 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16i32 = load <16 x i32>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16i32_0 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16i32_1 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v32i32 = load <32 x i32>, ptr %p, align 128 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v32i32_0 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v32i32_1 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i64 = load <4 x i64>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i64 = load <8 x i64>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_0 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_1 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i64 = load <16 x i64>, ptr %p, align 128 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_0 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_1 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i64 = load <32 x i64>, ptr %p, align 256 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_0 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_1 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %v4i8 = load <4 x i8>, ptr %p + %v4i8_0 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> + %v4i8_1 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> + %v8i8 = load <8 x i8>, ptr %p + %v8i8_0 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> + %v8i8_1 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> + %v16i8 = load <16 x i8>, ptr %p + %v16i8_0 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> + %v16i8_1 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> + %v32i8 = load <32 x i8>, ptr %p + %v32i8_0 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> + %v32i8_1 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> + + %v4i16 = load <4 x i16>, ptr %p + %v4i16_0 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> + %v4i16_1 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> + %v8i16 = load <8 x i16>, ptr %p + %v8i16_0 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> + %v8i16_1 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> + %v16i16 = load <16 x i16>, ptr %p + %v16i16_0 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> + %v16i16_1 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> + %v32i16 = load <32 x i16>, ptr %p + %v32i16_0 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> + %v32i16_1 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> + + %v4i32 = load <4 x i32>, ptr %p + %v4i32_0 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> + %v4i32_1 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> + %v8i32 = load <8 x i32>, ptr %p + %v8i32_0 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> + %v8i32_1 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> + %v16i32 = load <16 x i32>, ptr %p + %v16i32_0 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> + %v16i32_1 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> + %v32i32 = load <32 x i32>, ptr %p + %v32i32_0 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> + %v32i32_1 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> + + %v2i64 = load <4 x i64>, ptr %p + %v2i64_0 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> + %v2i64_1 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> + %v4i64 = load <8 x i64>, ptr %p + %v4i64_0 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> + %v4i64_1 = shufflevector <8 x i64> %v4i64, <8 x i64> undef, <4 x i32> + %v8i64 = load <16 x i64>, ptr %p + %v8i64_0 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> + %v8i64_1 = shufflevector <16 x i64> %v8i64, <16 x i64> undef, <8 x i32> + %v16i64 = load <32 x i64>, ptr %p + %v16i64_0 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> + %v16i64_1 = shufflevector <32 x i64> %v16i64, <32 x i64> undef, <16 x i32> + + ret void +} + + +define void @vld3(ptr %p) { +; CHECK-LABEL: 'vld3' +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i8 = load <6 x i8>, ptr %p, align 8 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_0 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_1 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_2 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = load <12 x i8>, ptr %p, align 16 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_0 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_1 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_2 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = load <24 x i8>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_0 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_1 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_2 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16i8 = load <48 x i8>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_0 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i8_1 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_2 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = load <6 x i16>, ptr %p, align 16 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_0 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_1 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_2 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = load <12 x i16>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_0 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_1 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_2 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i16 = load <24 x i16>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_0 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_1 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_2 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i16 = load <48 x i16>, ptr %p, align 128 +; CHECK-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_0 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i16_1 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_2 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i32 = load <6 x i32>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i32_1 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_2 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i32 = load <12 x i32>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_0 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i32_1 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_2 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32 = load <24 x i32>, ptr %p, align 128 +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_0 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32_1 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_2 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i32 = load <48 x i32>, ptr %p, align 256 +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_0 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i32_1 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_2 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2i64 = load <6 x i64>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_2 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i64 = load <12 x i64>, ptr %p, align 128 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_0 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_1 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_2 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i64 = load <24 x i64>, ptr %p, align 256 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_0 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_1 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_2 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i64 = load <48 x i64>, ptr %p, align 512 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_0 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_1 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_2 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; CODESIZE-LABEL: 'vld3' +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i8 = load <6 x i8>, ptr %p, align 8 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_0 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_1 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_2 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = load <12 x i8>, ptr %p, align 16 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_0 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_1 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_2 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = load <24 x i8>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_0 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_1 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_2 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16i8 = load <48 x i8>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_0 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i8_1 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_2 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = load <6 x i16>, ptr %p, align 16 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_0 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_1 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_2 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = load <12 x i16>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_0 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_1 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_2 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i16 = load <24 x i16>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_0 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_1 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_2 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i16 = load <48 x i16>, ptr %p, align 128 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_0 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i16_1 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_2 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i32 = load <6 x i32>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i32_1 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_2 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i32 = load <12 x i32>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_0 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i32_1 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_2 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32 = load <24 x i32>, ptr %p, align 128 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_0 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32_1 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_2 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i32 = load <48 x i32>, ptr %p, align 256 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_0 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i32_1 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_2 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2i64 = load <6 x i64>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_2 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i64 = load <12 x i64>, ptr %p, align 128 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_0 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_1 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_2 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i64 = load <24 x i64>, ptr %p, align 256 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_0 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_1 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_2 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i64 = load <48 x i64>, ptr %p, align 512 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_0 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_1 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_2 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %v2i8 = load <6 x i8>, ptr %p + %v2i8_0 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> + %v2i8_1 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> + %v2i8_2 = shufflevector <6 x i8> %v2i8, <6 x i8> undef, <2 x i32> + %v4i8 = load <12 x i8>, ptr %p + %v4i8_0 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> + %v4i8_1 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> + %v4i8_2 = shufflevector <12 x i8> %v4i8, <12 x i8> undef, <4 x i32> + %v8i8 = load <24 x i8>, ptr %p + %v8i8_0 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> + %v8i8_1 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> + %v8i8_2 = shufflevector <24 x i8> %v8i8, <24 x i8> undef, <8 x i32> + %v16i8 = load <48 x i8>, ptr %p + %v16i8_0 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> + %v16i8_1 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> + %v16i8_2 = shufflevector <48 x i8> %v16i8, <48 x i8> undef, <16 x i32> + + %v2i16 = load <6 x i16>, ptr %p + %v2i16_0 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> + %v2i16_1 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> + %v2i16_2 = shufflevector <6 x i16> %v2i16, <6 x i16> undef, <2 x i32> + %v4i16 = load <12 x i16>, ptr %p + %v4i16_0 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> + %v4i16_1 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> + %v4i16_2 = shufflevector <12 x i16> %v4i16, <12 x i16> undef, <4 x i32> + %v8i16 = load <24 x i16>, ptr %p + %v8i16_0 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> + %v8i16_1 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> + %v8i16_2 = shufflevector <24 x i16> %v8i16, <24 x i16> undef, <8 x i32> + %v16i16 = load <48 x i16>, ptr %p + %v16i16_0 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> + %v16i16_1 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> + %v16i16_2 = shufflevector <48 x i16> %v16i16, <48 x i16> undef, <16 x i32> + + %v2i32 = load <6 x i32>, ptr %p + %v2i32_0 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> + %v2i32_1 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> + %v2i32_2 = shufflevector <6 x i32> %v2i32, <6 x i32> undef, <2 x i32> + %v4i32 = load <12 x i32>, ptr %p + %v4i32_0 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> + %v4i32_1 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> + %v4i32_2 = shufflevector <12 x i32> %v4i32, <12 x i32> undef, <4 x i32> + %v8i32 = load <24 x i32>, ptr %p + %v8i32_0 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> + %v8i32_1 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> + %v8i32_2 = shufflevector <24 x i32> %v8i32, <24 x i32> undef, <8 x i32> + %v16i32 = load <48 x i32>, ptr %p + %v16i32_0 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> + %v16i32_1 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> + %v16i32_2 = shufflevector <48 x i32> %v16i32, <48 x i32> undef, <16 x i32> + + %v2i64 = load <6 x i64>, ptr %p + %v2i64_0 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> + %v2i64_1 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> + %v2i64_2 = shufflevector <6 x i64> %v2i64, <6 x i64> undef, <2 x i32> + %v4i64 = load <12 x i64>, ptr %p + %v4i64_0 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> + %v4i64_1 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> + %v4i64_2 = shufflevector <12 x i64> %v4i64, <12 x i64> undef, <4 x i32> + %v8i64 = load <24 x i64>, ptr %p + %v8i64_0 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> + %v8i64_1 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> + %v8i64_2 = shufflevector <24 x i64> %v8i64, <24 x i64> undef, <8 x i32> + %v16i64 = load <48 x i64>, ptr %p + %v16i64_0 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> + %v16i64_1 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> + %v16i64_2 = shufflevector <48 x i64> %v16i64, <48 x i64> undef, <16 x i32> + + ret void +} + +define void @vld4(ptr %p) { +; CHECK-LABEL: 'vld4' +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i8 = load <8 x i8>, ptr %p, align 8 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_0 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_1 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_2 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_3 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = load <16 x i8>, ptr %p, align 16 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_0 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_1 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_2 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_3 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = load <32 x i8>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_0 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_1 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_2 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_3 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16i8 = load <64 x i8>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_0 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_1 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_2 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_3 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = load <8 x i16>, ptr %p, align 16 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_0 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_1 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_2 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_3 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = load <16 x i16>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_0 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_1 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_2 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_3 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i16 = load <32 x i16>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_0 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_1 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_2 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_3 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i16 = load <64 x i16>, ptr %p, align 128 +; CHECK-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_0 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_1 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_2 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_3 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i32 = load <8 x i32>, ptr %p, align 32 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_2 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i32_3 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i32 = load <16 x i32>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_0 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_1 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_2 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_3 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32 = load <32 x i32>, ptr %p, align 128 +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_0 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_1 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_2 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_3 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i32 = load <64 x i32>, ptr %p, align 256 +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_0 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_1 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_2 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_3 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2i64 = load <8 x i64>, ptr %p, align 64 +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_2 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_3 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i64 = load <16 x i64>, ptr %p, align 128 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_0 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_1 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_2 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_3 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i64 = load <32 x i64>, ptr %p, align 256 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_0 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_1 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_2 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_3 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i64 = load <64 x i64>, ptr %p, align 512 +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_0 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_1 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_2 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_3 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; CODESIZE-LABEL: 'vld4' +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i8 = load <8 x i8>, ptr %p, align 8 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_0 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_1 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_2 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i8_3 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = load <16 x i8>, ptr %p, align 16 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_0 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_1 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_2 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i8_3 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = load <32 x i8>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_0 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_1 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_2 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %v8i8_3 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16i8 = load <64 x i8>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_0 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_1 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_2 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16i8_3 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = load <8 x i16>, ptr %p, align 16 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_0 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_1 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_2 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v2i16_3 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = load <16 x i16>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_0 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_1 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_2 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %v4i16_3 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i16 = load <32 x i16>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_0 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_1 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_2 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %v8i16_3 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i16 = load <64 x i16>, ptr %p, align 128 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_0 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_1 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_2 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16i16_3 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i32 = load <8 x i32>, ptr %p, align 32 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_0 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_1 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i32_2 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2i32_3 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i32 = load <16 x i32>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_0 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_1 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_2 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_3 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32 = load <32 x i32>, ptr %p, align 128 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_0 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_1 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_2 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_3 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i32 = load <64 x i32>, ptr %p, align 256 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_0 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_1 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_2 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_3 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v2i64 = load <8 x i64>, ptr %p, align 64 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_0 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_1 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_2 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_3 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i64 = load <16 x i64>, ptr %p, align 128 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_0 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_1 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_2 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_3 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i64 = load <32 x i64>, ptr %p, align 256 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_0 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_1 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_2 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_3 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i64 = load <64 x i64>, ptr %p, align 512 +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_0 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_1 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_2 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i64_3 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> +; CODESIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void +; + %v2i8 = load <8 x i8>, ptr %p + %v2i8_0 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> + %v2i8_1 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> + %v2i8_2 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> + %v2i8_3 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> + %v4i8 = load <16 x i8>, ptr %p + %v4i8_0 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> + %v4i8_1 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> + %v4i8_2 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> + %v4i8_3 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> + %v8i8 = load <32 x i8>, ptr %p + %v8i8_0 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> + %v8i8_1 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> + %v8i8_2 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> + %v8i8_3 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> + %v16i8 = load <64 x i8>, ptr %p + %v16i8_0 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> + %v16i8_1 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> + %v16i8_2 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> + %v16i8_3 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> + + %v2i16 = load <8 x i16>, ptr %p + %v2i16_0 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> + %v2i16_1 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> + %v2i16_2 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> + %v2i16_3 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> + %v4i16 = load <16 x i16>, ptr %p + %v4i16_0 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> + %v4i16_1 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> + %v4i16_2 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> + %v4i16_3 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> + %v8i16 = load <32 x i16>, ptr %p + %v8i16_0 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> + %v8i16_1 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> + %v8i16_2 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> + %v8i16_3 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> + %v16i16 = load <64 x i16>, ptr %p + %v16i16_0 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> + %v16i16_1 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> + %v16i16_2 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> + %v16i16_3 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> + + %v2i32 = load <8 x i32>, ptr %p + %v2i32_0 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> + %v2i32_1 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> + %v2i32_2 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> + %v2i32_3 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> + %v4i32 = load <16 x i32>, ptr %p + %v4i32_0 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> + %v4i32_1 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> + %v4i32_2 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> + %v4i32_3 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> + %v8i32 = load <32 x i32>, ptr %p + %v8i32_0 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> + %v8i32_1 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> + %v8i32_2 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> + %v8i32_3 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> + %v16i32 = load <64 x i32>, ptr %p + %v16i32_0 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> + %v16i32_1 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> + %v16i32_2 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> + %v16i32_3 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> + + %v2i64 = load <8 x i64>, ptr %p + %v2i64_0 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> + %v2i64_1 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> + %v2i64_2 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> + %v2i64_3 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> + %v4i64 = load <16 x i64>, ptr %p + %v4i64_0 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> + %v4i64_1 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> + %v4i64_2 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> + %v4i64_3 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> + %v8i64 = load <32 x i64>, ptr %p + %v8i64_0 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> + %v8i64_1 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> + %v8i64_2 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> + %v8i64_3 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> + %v16i64 = load <64 x i64>, ptr %p + %v16i64_0 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> + %v16i64_1 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> + %v16i64_2 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> + %v16i64_3 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> + + ret void +} + -- GitLab From 5a0942cd7423069e78fdfb9743a13aedfa7bdee0 Mon Sep 17 00:00:00 2001 From: Youngsuk Kim Date: Wed, 17 Apr 2024 14:45:33 -0400 Subject: [PATCH 054/714] [llvm][NVPTX] Don't emit unused var 'temp_param_reg' (NFC) (#89004) Don't emit unused variable 'temp_param_reg' which has been around since ae556d3ef72dfe5f40a337b7071f42b7bf5b66a4 . --- llvm/lib/Target/NVPTX/NVPTXInstrInfo.td | 3 +-- llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll | 2 -- llvm/test/CodeGen/NVPTX/i8x4-instructions.ll | 3 --- llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll | 1 - .../update_llc_test_checks/Inputs/nvptx-basic.ll.expected | 1 - 5 files changed, 1 insertion(+), 9 deletions(-) diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td index 4daf47cfd4df..cd8546005c02 100644 --- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td +++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td @@ -3785,8 +3785,7 @@ class Pseudo pattern> def Callseq_Start : NVPTXInst<(outs), (ins i32imm:$amt1, i32imm:$amt2), - "\\{ // callseq $amt1, $amt2\n" - "\t.reg .b32 temp_param_reg;", + "\\{ // callseq $amt1, $amt2", [(callseq_start timm:$amt1, timm:$amt2)]>; def Callseq_End : NVPTXInst<(outs), (ins i32imm:$amt1, i32imm:$amt2), diff --git a/llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll b/llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll index 09297fb819ce..ce81957f2a39 100644 --- a/llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll +++ b/llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll @@ -17,7 +17,6 @@ ; CHECK-32-NEXT: alloca.u32 %r[[ALLOCA:[0-9]]], %r[[SIZE3]], 16; ; CHECK-32-NEXT: cvta.local.u32 %r[[ALLOCA]], %r[[ALLOCA]]; ; CHECK-32-NEXT: { // callseq 0, 0 -; CHECK-32-NEXT: .reg .b32 temp_param_reg; ; CHECK-32-NEXT: .param .b32 param0; ; CHECK-32-NEXT: st.param.b32 [param0+0], %r[[ALLOCA]]; @@ -27,7 +26,6 @@ ; CHECK-64-NEXT: alloca.u64 %rd[[ALLOCA:[0-9]]], %rd[[SIZE3]], 16; ; CHECK-64-NEXT: cvta.local.u64 %rd[[ALLOCA]], %rd[[ALLOCA]]; ; CHECK-64-NEXT: { // callseq 0, 0 -; CHECK-64-NEXT: .reg .b32 temp_param_reg; ; CHECK-64-NEXT: .param .b64 param0; ; CHECK-64-NEXT: st.param.b64 [param0+0], %rd[[ALLOCA]]; diff --git a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll index 6895699a1dfe..96a4359d0ec4 100644 --- a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll +++ b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll @@ -827,7 +827,6 @@ define <4 x i8> @test_call(<4 x i8> %a, <4 x i8> %b) #0 { ; CHECK-NEXT: ld.param.u32 %r2, [test_call_param_1]; ; CHECK-NEXT: ld.param.u32 %r1, [test_call_param_0]; ; CHECK-NEXT: { // callseq 0, 0 -; CHECK-NEXT: .reg .b32 temp_param_reg; ; CHECK-NEXT: .param .align 4 .b8 param0[4]; ; CHECK-NEXT: st.param.b32 [param0+0], %r1; ; CHECK-NEXT: .param .align 4 .b8 param1[4]; @@ -856,7 +855,6 @@ define <4 x i8> @test_call_flipped(<4 x i8> %a, <4 x i8> %b) #0 { ; CHECK-NEXT: ld.param.u32 %r2, [test_call_flipped_param_1]; ; CHECK-NEXT: ld.param.u32 %r1, [test_call_flipped_param_0]; ; CHECK-NEXT: { // callseq 1, 0 -; CHECK-NEXT: .reg .b32 temp_param_reg; ; CHECK-NEXT: .param .align 4 .b8 param0[4]; ; CHECK-NEXT: st.param.b32 [param0+0], %r2; ; CHECK-NEXT: .param .align 4 .b8 param1[4]; @@ -885,7 +883,6 @@ define <4 x i8> @test_tailcall_flipped(<4 x i8> %a, <4 x i8> %b) #0 { ; CHECK-NEXT: ld.param.u32 %r2, [test_tailcall_flipped_param_1]; ; CHECK-NEXT: ld.param.u32 %r1, [test_tailcall_flipped_param_0]; ; CHECK-NEXT: { // callseq 2, 0 -; CHECK-NEXT: .reg .b32 temp_param_reg; ; CHECK-NEXT: .param .align 4 .b8 param0[4]; ; CHECK-NEXT: st.param.b32 [param0+0], %r2; ; CHECK-NEXT: .param .align 4 .b8 param1[4]; diff --git a/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll b/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll index 8d7608ead38e..de367dfa4acb 100644 --- a/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll +++ b/llvm/test/DebugInfo/NVPTX/dbg-declare-alloca.ll @@ -9,7 +9,6 @@ ; CHECK: add.u64 %rd1, %SP, 0; ; CHECK: .loc 1 5 3 // t.c:5:3 ; CHECK: { // callseq 0, 0 -; CHECK: .reg .b32 temp_param_reg; ; CHECK: .param .b64 param0; ; CHECK: st.param.b64 [param0+0], %rd1; ; CHECK: call.uni diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected index a65f140b4601..3ac63d070933 100644 --- a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected +++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected @@ -30,7 +30,6 @@ define dso_local void @caller_St8x4(ptr nocapture noundef readonly byval(%struct ; CHECK-NEXT: ld.u64 %rd7, [%SP+24]; ; CHECK-NEXT: ld.u64 %rd8, [%SP+16]; ; CHECK-NEXT: { // callseq 0, 0 -; CHECK-NEXT: .reg .b32 temp_param_reg; ; CHECK-NEXT: .param .align 16 .b8 param0[32]; ; CHECK-NEXT: st.param.v2.b64 [param0+0], {%rd6, %rd5}; ; CHECK-NEXT: st.param.v2.b64 [param0+16], {%rd8, %rd7}; -- GitLab From 800f1050e190430f217e1fd0db9414dacc835e11 Mon Sep 17 00:00:00 2001 From: Shilei Tian Date: Wed, 17 Apr 2024 14:53:01 -0400 Subject: [PATCH 055/714] [GitHub] Add a new mapping for `offload` subproject (#89118) Fix #89071. --- .github/new-prs-labeler.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/new-prs-labeler.yml b/.github/new-prs-labeler.yml index 1502d64a7d3e..9cf64417d3cb 100644 --- a/.github/new-prs-labeler.yml +++ b/.github/new-prs-labeler.yml @@ -944,3 +944,6 @@ openmp:libomptarget: bazel: - utils/bazel/** + +offload: + - offload/** -- GitLab From 6f7976c883da592f2cf6bfadef152e4203c00445 Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Wed, 17 Apr 2024 20:55:51 +0200 Subject: [PATCH 056/714] [libc++][TZDB] Adds sys_info formatter. (#85896) Implements parts of: - P0355 Extending to Calendars and Time Zones - P1361 Integration of chrono with text formatting --- libcxx/docs/Status/FormatPaper.csv | 2 +- libcxx/include/__chrono/convert_to_tm.h | 5 + libcxx/include/__chrono/formatter.h | 45 +++++- libcxx/include/__chrono/ostream.h | 20 +++ libcxx/include/__chrono/sys_info.h | 2 +- libcxx/include/chrono | 5 + .../time.zone.info.sys/ostream.pass.cpp | 74 ++++++++++ .../time/time.syn/formatter.sys_info.pass.cpp | 137 ++++++++++++++++++ .../time.zone.info.sys/ostream.pass.cpp | 52 +++++++ .../concept.formattable.compile.pass.cpp | 4 +- libcxx/test/support/test_macros.h | 8 + 11 files changed, 347 insertions(+), 7 deletions(-) create mode 100644 libcxx/test/libcxx/time/time.zone/time.zone.info/time.zone.info.sys/ostream.pass.cpp create mode 100644 libcxx/test/std/time/time.syn/formatter.sys_info.pass.cpp create mode 100644 libcxx/test/std/time/time.zone/time.zone.info/time.zone.info.sys/ostream.pass.cpp diff --git a/libcxx/docs/Status/FormatPaper.csv b/libcxx/docs/Status/FormatPaper.csv index e9d407e79e25..8ace18815f53 100644 --- a/libcxx/docs/Status/FormatPaper.csv +++ b/libcxx/docs/Status/FormatPaper.csv @@ -24,7 +24,7 @@ Section,Description,Dependencies,Assignee,Status,First released version `[time.syn] `_,"Formatter ``chrono::year_month_weekday``",,Mark de Wever,|Complete|,16.0 `[time.syn] `_,"Formatter ``chrono::year_month_weekday_last``",,Mark de Wever,|Complete|,16.0 `[time.syn] `_,"Formatter ``chrono::hh_mm_ss>``",,Mark de Wever,|Complete|,17.0 -`[time.syn] `_,"Formatter ``chrono::sys_info``",A ```` implementation,Mark de Wever,, +`[time.syn] `_,"Formatter ``chrono::sys_info``",,Mark de Wever,|Complete|,19.0 `[time.syn] `_,"Formatter ``chrono::local_info``",A ```` implementation,Mark de Wever,, `[time.syn] `_,"Formatter ``chrono::zoned_time``",A ```` implementation,Mark de Wever,, diff --git a/libcxx/include/__chrono/convert_to_tm.h b/libcxx/include/__chrono/convert_to_tm.h index 1301cd6f1f1a..d2c5cf922ba6 100644 --- a/libcxx/include/__chrono/convert_to_tm.h +++ b/libcxx/include/__chrono/convert_to_tm.h @@ -20,6 +20,7 @@ #include <__chrono/month_weekday.h> #include <__chrono/monthday.h> #include <__chrono/statically_widen.h> +#include <__chrono/sys_info.h> #include <__chrono/system_clock.h> #include <__chrono/time_point.h> #include <__chrono/weekday.h> @@ -171,6 +172,10 @@ _LIBCPP_HIDE_FROM_ABI _Tm __convert_to_tm(const _ChronoT& __value) { if (__value.hours().count() > std::numeric_limits::max()) std::__throw_format_error("Formatting hh_mm_ss, encountered an hour overflow"); __result.tm_hour = __value.hours().count(); +# if !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) + } else if constexpr (same_as<_ChronoT, chrono::sys_info>) { + // Has no time information. +# endif } else static_assert(sizeof(_ChronoT) == 0, "Add the missing type specialization"); diff --git a/libcxx/include/__chrono/formatter.h b/libcxx/include/__chrono/formatter.h index f76e7b2ea0e8..0196deb8c1ff 100644 --- a/libcxx/include/__chrono/formatter.h +++ b/libcxx/include/__chrono/formatter.h @@ -24,6 +24,7 @@ #include <__chrono/ostream.h> #include <__chrono/parser_std_format_spec.h> #include <__chrono/statically_widen.h> +#include <__chrono/sys_info.h> #include <__chrono/system_clock.h> #include <__chrono/time_point.h> #include <__chrono/weekday.h> @@ -186,10 +187,11 @@ __format_zone_offset(basic_stringstream<_CharT>& __sstr, chrono::seconds __offse chrono::hh_mm_ss __hms{__offset}; std::ostreambuf_iterator<_CharT> __out_it{__sstr}; + // Note HMS does not allow formatting hours > 23, but the offset is not limited to 24H. + std::format_to(__out_it, _LIBCPP_STATICALLY_WIDEN(_CharT, "{:02}"), __hms.hours().count()); if (__modifier) - std::format_to(__out_it, _LIBCPP_STATICALLY_WIDEN(_CharT, "{:%H:%M}"), __hms); - else - std::format_to(__out_it, _LIBCPP_STATICALLY_WIDEN(_CharT, "{:%H%M}"), __hms); + __sstr << _CharT(':'); + std::format_to(__out_it, _LIBCPP_STATICALLY_WIDEN(_CharT, "{:02}"), __hms.minutes().count()); } // Helper to store the time zone information needed for formatting. @@ -202,7 +204,12 @@ struct _LIBCPP_HIDE_FROM_ABI __time_zone { template _LIBCPP_HIDE_FROM_ABI __time_zone __convert_to_time_zone([[maybe_unused]] const _Tp& __value) { - return {"UTC", chrono::seconds{0}}; +# if !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) + if constexpr (same_as<_Tp, chrono::sys_info>) + return {__value.abbrev, __value.offset}; + else +# endif + return {"UTC", chrono::seconds{0}}; } template @@ -409,6 +416,10 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool __weekday_ok(const _Tp& __value) { return __value.weekday().ok(); else if constexpr (__is_hh_mm_ss<_Tp>) return true; +# if !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) + else if constexpr (same_as<_Tp, chrono::sys_info>) + return true; +# endif else static_assert(sizeof(_Tp) == 0, "Add the missing type specialization"); } @@ -449,6 +460,10 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool __weekday_name_ok(const _Tp& __value) { return __value.weekday().ok(); else if constexpr (__is_hh_mm_ss<_Tp>) return true; +# if !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) + else if constexpr (same_as<_Tp, chrono::sys_info>) + return true; +# endif else static_assert(sizeof(_Tp) == 0, "Add the missing type specialization"); } @@ -489,6 +504,10 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool __date_ok(const _Tp& __value) { return __value.ok(); else if constexpr (__is_hh_mm_ss<_Tp>) return true; +# if !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) + else if constexpr (same_as<_Tp, chrono::sys_info>) + return true; +# endif else static_assert(sizeof(_Tp) == 0, "Add the missing type specialization"); } @@ -529,6 +548,10 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool __month_name_ok(const _Tp& __value) { return __value.month().ok(); else if constexpr (__is_hh_mm_ss<_Tp>) return true; +# if !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) + else if constexpr (same_as<_Tp, chrono::sys_info>) + return true; +# endif else static_assert(sizeof(_Tp) == 0, "Add the missing type specialization"); } @@ -858,6 +881,20 @@ public: return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__time); } }; + +# if !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) +template <__fmt_char_type _CharT> +struct formatter : public __formatter_chrono<_CharT> { +public: + using _Base = __formatter_chrono<_CharT>; + + template + _LIBCPP_HIDE_FROM_ABI constexpr typename _ParseContext::iterator parse(_ParseContext& __ctx) { + return _Base::__parse(__ctx, __format_spec::__fields_chrono, __format_spec::__flags::__time_zone); + } +}; +# endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) + #endif // if _LIBCPP_STD_VER >= 20 _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__chrono/ostream.h b/libcxx/include/__chrono/ostream.h index b687ef8059d5..9476406d7610 100644 --- a/libcxx/include/__chrono/ostream.h +++ b/libcxx/include/__chrono/ostream.h @@ -19,6 +19,7 @@ #include <__chrono/month_weekday.h> #include <__chrono/monthday.h> #include <__chrono/statically_widen.h> +#include <__chrono/sys_info.h> #include <__chrono/system_clock.h> #include <__chrono/weekday.h> #include <__chrono/year.h> @@ -262,6 +263,25 @@ operator<<(basic_ostream<_CharT, _Traits>& __os, const hh_mm_ss<_Duration> __hms return __os << std::format(__os.getloc(), _LIBCPP_STATICALLY_WIDEN(_CharT, "{:L%T}"), __hms); } +# if !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) + +template +_LIBCPP_HIDE_FROM_ABI basic_ostream<_CharT, _Traits>& +operator<<(basic_ostream<_CharT, _Traits>& __os, const sys_info& __info) { + // __info.abbrev is always std::basic_string. + // Since these strings typically are short the conversion should be cheap. + std::basic_string<_CharT> __abbrev{__info.abbrev.begin(), __info.abbrev.end()}; + return __os << std::format( + _LIBCPP_STATICALLY_WIDEN(_CharT, "[{:%F %T}, {:%F %T}) {:%T} {:%Q%q} \"{}\""), + __info.begin, + __info.end, + hh_mm_ss{__info.offset}, + __info.save, + __abbrev); +} + +# endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) + } // namespace chrono #endif // if _LIBCPP_STD_VER >= 20 diff --git a/libcxx/include/__chrono/sys_info.h b/libcxx/include/__chrono/sys_info.h index 794d22f2ccc1..461d5322d413 100644 --- a/libcxx/include/__chrono/sys_info.h +++ b/libcxx/include/__chrono/sys_info.h @@ -42,7 +42,7 @@ struct sys_info { } // namespace chrono -# endif //_LIBCPP_STD_VER >= 20 +# endif // _LIBCPP_STD_VER >= 20 _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/chrono b/libcxx/include/chrono index 513ae52006e8..f0275ec1eba7 100644 --- a/libcxx/include/chrono +++ b/libcxx/include/chrono @@ -733,6 +733,10 @@ struct sys_info { string abbrev; }; +template // C++20 + basic_ostream& + operator<<(basic_ostream& os, const sys_info& si); + // 25.10.5, class time_zone // C++20 enum class choose {earliest, latest}; class time_zone { @@ -829,6 +833,7 @@ namespace std { template struct formatter; // C++20 template struct formatter>, charT>; // C++20 + template struct formatter; // C++20 } // namespace std namespace chrono { diff --git a/libcxx/test/libcxx/time/time.zone/time.zone.info/time.zone.info.sys/ostream.pass.cpp b/libcxx/test/libcxx/time/time.zone/time.zone.info/time.zone.info.sys/ostream.pass.cpp new file mode 100644 index 000000000000..faa7d855c8de --- /dev/null +++ b/libcxx/test/libcxx/time/time.zone/time.zone.info/time.zone.info.sys/ostream.pass.cpp @@ -0,0 +1,74 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17 +// UNSUPPORTED: no-localization + +// TODO FMT This test should not require std::to_chars(floating-point) +// XFAIL: availability-fp_to_chars-missing + +// XFAIL: libcpp-has-no-incomplete-tzdb + +// + +// template +// basic_ostream& +// operator<<(basic_ostream& os, const sys_info& r); + +// [time.zone.info.sys] +// 7 Effects: Streams out the sys_info object r in an unspecified format. +// 8 Returns: os. +// +// Tests the output produced by this function. + +#include +#include +#include +#include + +#include "assert_macros.h" +#include "test_macros.h" +#include "make_string.h" +#include "concat_macros.h" + +#define SV(S) MAKE_STRING_VIEW(CharT, S) + +template +static void test(std::basic_string_view expected, std::chrono::sys_info&& info) { + std::basic_stringstream sstr; + sstr << info; + std::basic_string output = sstr.str(); + + TEST_REQUIRE(expected == output, + TEST_WRITE_CONCATENATED("\nExpected output ", expected, "\nActual output ", output, '\n')); +} + +template +static void test() { + using namespace std::literals::chrono_literals; + namespace tz = std::chrono; + + test(SV("[-10484-10-16 15:30:08, 14423-03-17 15:30:07) 00:00:00 0min \"TZ\""), + tz::sys_info{tz::sys_seconds::min(), tz::sys_seconds::max(), 0s, 0min, "TZ"}); + + test(SV("[1970-01-01 00:00:00, 2038-12-31 00:00:00) 12:23:45 -67min \"DMY\""), + tz::sys_info{static_cast(tz::year_month_day{1970y, tz::January, 1d}), + static_cast(tz::year_month_day{2038y, tz::December, 31d}), + 12h + 23min + 45s, + -67min, + "DMY"}); +} + +int main(int, const char**) { + test(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test(); +#endif + + return 0; +} diff --git a/libcxx/test/std/time/time.syn/formatter.sys_info.pass.cpp b/libcxx/test/std/time/time.syn/formatter.sys_info.pass.cpp new file mode 100644 index 000000000000..0a41424cfdcb --- /dev/null +++ b/libcxx/test/std/time/time.syn/formatter.sys_info.pass.cpp @@ -0,0 +1,137 @@ +//===----------------------------------------------------------------------===// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17 +// UNSUPPORTED: no-localization + +// TODO FMT This test should not require std::to_chars(floating-point) +// XFAIL: availability-fp_to_chars-missing + +// XFAIL: libcpp-has-no-incomplete-tzdb + +// REQUIRES: locale.fr_FR.UTF-8 +// REQUIRES: locale.ja_JP.UTF-8 + +// +// +// template struct formatter; + +#include +#include + +#include +#include +#include +#include +#include + +#include "formatter_tests.h" +#include "make_string.h" +#include "platform_support.h" // locale name macros +#include "test_macros.h" + +template +static void test_no_chrono_specs() { +// This test libc++ specific due to +// [time.zone.info.sys]/7 +// Effects: Streams out the sys_info object r in an unspecified format. +#ifdef _LIBCPP_VERSION + using namespace std::literals::chrono_literals; + namespace tz = std::chrono; + + std::locale::global(std::locale(LOCALE_fr_FR_UTF_8)); + + // Non localized output + + check(SV("[-10484-10-16 15:30:08, 14423-03-17 15:30:07) 00:00:00 0min \"TZ\""), + SV("{}"), + tz::sys_info{tz::sys_seconds::min(), tz::sys_seconds::max(), 0s, 0min, "TZ"}); + + check(SV("[1970-01-01 00:00:00, 2038-12-31 00:00:00) 12:23:45 -67min \"DMY\""), + SV("{}"), + tz::sys_info{static_cast(tz::year_month_day{1970y, tz::January, 1d}), + static_cast(tz::year_month_day{2038y, tz::December, 31d}), + 12h + 23min + 45s, + -67min, + "DMY"}); + + std::locale::global(std::locale::classic()); +#endif // _LIBCPP_VERSION +} + +template +static void test_valid_values() { + using namespace std::literals::chrono_literals; + + constexpr std::basic_string_view fmt = SV("{:%%z='%z'%t%%Ez='%Ez'%t%%Oz='%Oz'%t%%Z='%Z'%n}"); + constexpr std::basic_string_view lfmt = SV("{:L%%z='%z'%t%%Ez='%Ez'%t%%Oz='%Oz'%t%%Z='%Z'%n}"); + + const std::locale loc(LOCALE_ja_JP_UTF_8); + std::locale::global(std::locale(LOCALE_fr_FR_UTF_8)); + + // Non localized output using C-locale + check(SV("%z='-0200'\t%Ez='-02:00'\t%Oz='-02:00'\t%Z='NEG'\n"), + fmt, + std::chrono::sys_info{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, -2h, 0min, "NEG"}); + + check(SV("%z='+0000'\t%Ez='+00:00'\t%Oz='+00:00'\t%Z='ZERO'\n"), + fmt, + std::chrono::sys_info{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, 0s, 0min, "ZERO"}); + + check(SV("%z='+1115'\t%Ez='+11:15'\t%Oz='+11:15'\t%Z='POS'\n"), + fmt, + std::chrono::sys_info{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, 11h + 15min, 0min, "POS"}); + + // Use the global locale (fr_FR) + check(SV("%z='-0200'\t%Ez='-02:00'\t%Oz='-02:00'\t%Z='NEG'\n"), + lfmt, + std::chrono::sys_info{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, -2h, 0min, "NEG"}); + + check(SV("%z='+0000'\t%Ez='+00:00'\t%Oz='+00:00'\t%Z='ZERO'\n"), + lfmt, + std::chrono::sys_info{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, 0s, 0min, "ZERO"}); + + check(SV("%z='+1115'\t%Ez='+11:15'\t%Oz='+11:15'\t%Z='POS'\n"), + lfmt, + std::chrono::sys_info{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, 11h + 15min, 0min, "POS"}); + + // Use supplied locale (ja_JP). + check(loc, + SV("%z='-0200'\t%Ez='-02:00'\t%Oz='-02:00'\t%Z='NEG'\n"), + lfmt, + std::chrono::sys_info{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, -2h, 0min, "NEG"}); + + check(loc, + SV("%z='+0000'\t%Ez='+00:00'\t%Oz='+00:00'\t%Z='ZERO'\n"), + lfmt, + std::chrono::sys_info{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, 0s, 0min, "ZERO"}); + + check(loc, + SV("%z='+1115'\t%Ez='+11:15'\t%Oz='+11:15'\t%Z='POS'\n"), + lfmt, + std::chrono::sys_info{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, 11h + 15min, 0min, "POS"}); + + std::locale::global(std::locale::classic()); +} + +template +static void test() { + test_no_chrono_specs(); + test_valid_values(); + + check_invalid_types({SV("z"), SV("Z"), SV("Ez"), SV("Oz")}, std::chrono::sys_info{}); +} + +int main(int, char**) { + test(); + +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test(); +#endif + + return 0; +} diff --git a/libcxx/test/std/time/time.zone/time.zone.info/time.zone.info.sys/ostream.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.info/time.zone.info.sys/ostream.pass.cpp new file mode 100644 index 000000000000..82c4844b423c --- /dev/null +++ b/libcxx/test/std/time/time.zone/time.zone.info/time.zone.info.sys/ostream.pass.cpp @@ -0,0 +1,52 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17 +// UNSUPPORTED: no-localization + +// TODO FMT This test should not require std::to_chars(floating-point) +// XFAIL: availability-fp_to_chars-missing + +// XFAIL: libcpp-has-no-incomplete-tzdb + +// + +// template +// basic_ostream& +// operator<<(basic_ostream& os, const sys_info& r); + +// [time.zone.info.sys] +// 7 Effects: Streams out the sys_info object r in an unspecified format. +// 8 Returns: os. +// +// There is a private libc++ test that validates the exact output. + +#include +#include +#include +#include + +#include "test_macros.h" + +template +static void test() { + using namespace std::literals::chrono_literals; + std::chrono::sys_info s{std::chrono::sys_seconds{0s}, std::chrono::sys_seconds{0s}, 0h, 0min, ""}; + std::basic_ostringstream os; + std::basic_ostream& result = std::chrono::operator<<(os, s); + assert(std::addressof(result) == std::addressof(os)); +} + +int main(int, const char**) { + test(); +#ifndef TEST_HAS_NO_WIDE_CHARACTERS + test(); +#endif + + return 0; +} diff --git a/libcxx/test/std/utilities/format/format.formattable/concept.formattable.compile.pass.cpp b/libcxx/test/std/utilities/format/format.formattable/concept.formattable.compile.pass.cpp index fad5277fd6e1..f40784ec446f 100644 --- a/libcxx/test/std/utilities/format/format.formattable/concept.formattable.compile.pass.cpp +++ b/libcxx/test/std/utilities/format/format.formattable/concept.formattable.compile.pass.cpp @@ -176,10 +176,12 @@ void test_P1361() { assert_is_formattable, CharT>(); - //assert_is_formattable(); +# if !defined(TEST_HAS_NO_INCOMPLETE_TZDB) + assert_is_formattable(); //assert_is_formattable(); //assert_is_formattable(); +# endif // !defined(TEST_HAS_NO_INCOMPLETE_TZDB) #endif // TEST_HAS_NO_LOCALIZATION } diff --git a/libcxx/test/support/test_macros.h b/libcxx/test/support/test_macros.h index 7b2dcbb52d0c..fe9207d7e596 100644 --- a/libcxx/test/support/test_macros.h +++ b/libcxx/test/support/test_macros.h @@ -417,6 +417,14 @@ inline Tp const& DoNotOptimize(Tp const& value) { # define TEST_HAS_NO_RANDOM_DEVICE #endif +#if defined(_LIBCPP_HAS_NO_INCOMPLETE_TZDB) +# define TEST_HAS_NO_INCOMPLETE_TZDB +#endif + +#if defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) +# define TEST_HAS_NO_TIME_ZONE_DATABASE +#endif + #if defined(TEST_COMPILER_CLANG) # define TEST_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push") # define TEST_DIAGNOSTIC_POP _Pragma("clang diagnostic pop") -- GitLab From b1dc62f139ef265a36a2a739ce9ba4e1e48a6dbe Mon Sep 17 00:00:00 2001 From: Amirreza Ashouri Date: Wed, 17 Apr 2024 22:28:11 +0330 Subject: [PATCH 057/714] [clang]Treat arguments to builtin type traits as template type arguments (#87132) This change improves error messages for builtins in case of empty parentheses. Fixes llvm#86997 --- clang/lib/Parse/ParseExprCXX.cpp | 12 ++++++------ clang/test/Sema/static-assert.c | 9 ++++----- clang/test/SemaCXX/builtins.cpp | 5 +++++ clang/test/SemaCXX/deprecated-builtins.cpp | 5 +++++ 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp index 43d6105dcf31..0d2ad980696f 100644 --- a/clang/lib/Parse/ParseExprCXX.cpp +++ b/clang/lib/Parse/ParseExprCXX.cpp @@ -3910,10 +3910,10 @@ ExprResult Parser::ParseTypeTrait() { SmallVector Args; do { // Parse the next type. - TypeResult Ty = - ParseTypeName(/*SourceRange=*/nullptr, - getLangOpts().CPlusPlus ? DeclaratorContext::TemplateArg - : DeclaratorContext::TypeName); + TypeResult Ty = ParseTypeName(/*SourceRange=*/nullptr, + getLangOpts().CPlusPlus + ? DeclaratorContext::TemplateTypeArg + : DeclaratorContext::TypeName); if (Ty.isInvalid()) { Parens.skipToEnd(); return ExprError(); @@ -3955,8 +3955,8 @@ ExprResult Parser::ParseArrayTypeTrait() { if (T.expectAndConsume()) return ExprError(); - TypeResult Ty = - ParseTypeName(/*SourceRange=*/nullptr, DeclaratorContext::TemplateArg); + TypeResult Ty = ParseTypeName(/*SourceRange=*/nullptr, + DeclaratorContext::TemplateTypeArg); if (Ty.isInvalid()) { SkipUntil(tok::comma, StopAtSemi); SkipUntil(tok::r_paren, StopAtSemi); diff --git a/clang/test/Sema/static-assert.c b/clang/test/Sema/static-assert.c index ae5e8076e0be..4e9e6b7ee558 100644 --- a/clang/test/Sema/static-assert.c +++ b/clang/test/Sema/static-assert.c @@ -1,6 +1,6 @@ -// RUN: %clang_cc1 -std=c11 -Wgnu-folding-constant -fsyntax-only -verify=expected,c %s -// RUN: %clang_cc1 -fms-compatibility -Wgnu-folding-constant -DMS -fsyntax-only -verify=expected,ms,c %s -// RUN: %clang_cc1 -std=c99 -pedantic -Wgnu-folding-constant -fsyntax-only -verify=expected,ext,c %s +// RUN: %clang_cc1 -std=c11 -Wgnu-folding-constant -fsyntax-only -verify %s +// RUN: %clang_cc1 -fms-compatibility -Wgnu-folding-constant -DMS -fsyntax-only -verify=expected,ms %s +// RUN: %clang_cc1 -std=c99 -pedantic -Wgnu-folding-constant -fsyntax-only -verify=expected,ext %s // RUN: %clang_cc1 -xc++ -std=c++11 -pedantic -fsyntax-only -verify=expected,ext,cxx %s _Static_assert("foo", "string is nonzero"); // ext-warning {{'_Static_assert' is a C11 extension}} @@ -57,8 +57,7 @@ UNION(char[2], short) u2 = { .one = { 'a', 'b' } }; // ext-warning 3 {{'_Static_ typedef UNION(char, short) U3; // expected-error {{static assertion failed due to requirement 'sizeof(char) == sizeof(short)': type size mismatch}} \ // expected-note{{evaluates to '1 == 2'}} \ // ext-warning 3 {{'_Static_assert' is a C11 extension}} -typedef UNION(float, 0.5f) U4; // c-error {{expected a type}} \ - // cxx-error {{type name requires a specifier or qualifier}} \ +typedef UNION(float, 0.5f) U4; // expected-error {{expected a type}} \ // ext-warning 3 {{'_Static_assert' is a C11 extension}} // After defining the assert macro in MS-compatibility mode, we should diff --git a/clang/test/SemaCXX/builtins.cpp b/clang/test/SemaCXX/builtins.cpp index 567094c94c17..080b4476c7ee 100644 --- a/clang/test/SemaCXX/builtins.cpp +++ b/clang/test/SemaCXX/builtins.cpp @@ -76,6 +76,11 @@ using ConstMemFnType = int (Dummy::*)() const; void foo() {} +void test_builtin_empty_parentheses_diags() { + __is_trivially_copyable(); // expected-error {{expected a type}} + __is_trivially_copyable(1); // expected-error {{expected a type}} +} + void test_builtin_launder_diags(void *vp, const void *cvp, FnType *fnp, MemFnType mfp, ConstMemFnType cmfp, int (&Arr)[5]) { __builtin_launder(vp); // expected-error {{void pointer argument to '__builtin_launder' is not allowed}} diff --git a/clang/test/SemaCXX/deprecated-builtins.cpp b/clang/test/SemaCXX/deprecated-builtins.cpp index 849b9b014fff..fafc1da4da13 100644 --- a/clang/test/SemaCXX/deprecated-builtins.cpp +++ b/clang/test/SemaCXX/deprecated-builtins.cpp @@ -17,3 +17,8 @@ void f() { a = __has_trivial_destructor(A); // expected-warning-re {{__has_trivial_destructor {{.*}} use __is_trivially_destructible}} } + +void test_builtin_empty_parentheses_diags(void) { + __has_nothrow_copy(); // expected-error {{expected a type}} + __has_nothrow_copy(1); // expected-error {{expected a type}} +} -- GitLab From 6cea7c491f4c4c68aa0494a9b18f36ff40c22c81 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Wed, 17 Apr 2024 12:04:18 -0700 Subject: [PATCH 058/714] [X86] Always use 64-bit relocations in no-PIC large code model (#89101) This matches other types of relocations, e.g. to constant pool. And makes things more consistent with PIC large code model. Some users of the large code model may not place small data in the lower 2GB of the address space (e.g. https://github.com/ClangBuiltLinux/linux/issues/2016), so just unconditionally use 64-bit relocations in the large code model. So now functions in a section not marked large will use 64-bit relocations to reference everything when using the large code model. This also fixes some lldb tests broken by #88172 (https://lab.llvm.org/buildbot/#/builders/68/builds/72458). --- llvm/lib/Target/X86/X86ISelDAGToDAG.cpp | 7 +++---- llvm/test/CodeGen/X86/code-model-elf.ll | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp index 4e4241efd63d..7dcde2a50894 100644 --- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp +++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp @@ -2927,11 +2927,10 @@ bool X86DAGToDAGISel::selectAddr(SDNode *Parent, SDValue N, SDValue &Base, } bool X86DAGToDAGISel::selectMOV64Imm32(SDValue N, SDValue &Imm) { - // Cannot use 32 bit constants to reference objects in kernel code model. - // Cannot use 32 bit constants to reference objects in large PIC mode since - // GOTOFF is 64 bits. + // Cannot use 32 bit constants to reference objects in kernel/large code + // model. if (TM.getCodeModel() == CodeModel::Kernel || - (TM.getCodeModel() == CodeModel::Large && TM.isPositionIndependent())) + TM.getCodeModel() == CodeModel::Large) return false; // In static codegen with small code model, we can get the address of a label diff --git a/llvm/test/CodeGen/X86/code-model-elf.ll b/llvm/test/CodeGen/X86/code-model-elf.ll index 0da62e3e7a65..f60f75bc2691 100644 --- a/llvm/test/CodeGen/X86/code-model-elf.ll +++ b/llvm/test/CodeGen/X86/code-model-elf.ll @@ -350,7 +350,7 @@ define dso_local ptr @lea_forced_small_data() #0 { ; ; LARGE-STATIC-LABEL: lea_forced_small_data: ; LARGE-STATIC: # %bb.0: -; LARGE-STATIC-NEXT: movl $forced_small_data, %eax +; LARGE-STATIC-NEXT: movabsq $forced_small_data, %rax ; LARGE-STATIC-NEXT: retq ; ; SMALL-PIC-LABEL: lea_forced_small_data: @@ -403,7 +403,7 @@ define dso_local i32 @load_forced_small_data() #0 { ; ; LARGE-STATIC-LABEL: load_forced_small_data: ; LARGE-STATIC: # %bb.0: -; LARGE-STATIC-NEXT: movl $forced_small_data+8, %eax +; LARGE-STATIC-NEXT: movabsq $forced_small_data+8, %rax ; LARGE-STATIC-NEXT: movl (%rax), %eax ; LARGE-STATIC-NEXT: retq ; -- GitLab From 1460b4964c7ada2f7536006722c8585b5bd0a1b5 Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Wed, 17 Apr 2024 19:11:58 +0000 Subject: [PATCH 059/714] [gn build] Manually port d423d80e560d --- llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn index ee44558a4e99..bd08ce044c24 100644 --- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn +++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn @@ -37,9 +37,9 @@ if (current_toolchain == default_toolchain) { "_LIBCPP_INSTRUMENTED_WITH_ASAN=", "_LIBCPP_ABI_DEFINES=", "_LIBCPP_HARDENING_MODE_DEFAULT=_LIBCPP_HARDENING_MODE_NONE", - "_LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH=", - "_LIBCPP_PSTL_CPU_BACKEND_SERIAL=1", - "_LIBCPP_PSTL_CPU_BACKEND_THREAD=", + "_LIBCPP_PSTL_BACKEND_LIBDISPATCH=", + "_LIBCPP_PSTL_BACKEND_SERIAL=1", + "_LIBCPP_PSTL_BACKEND_STD_THREAD=", ] if (libcxx_abi_version != 1) { values += [ "_LIBCPP_ABI_VERSION=$libcxx_abi_version" ] @@ -143,18 +143,12 @@ if (current_toolchain == default_toolchain) { "__algorithm/pop_heap.h", "__algorithm/prev_permutation.h", "__algorithm/pstl_any_all_none_of.h", - "__algorithm/pstl_backend.h", - "__algorithm/pstl_backends/cpu_backend.h", "__algorithm/pstl_backends/cpu_backends/any_of.h", - "__algorithm/pstl_backends/cpu_backends/backend.h", "__algorithm/pstl_backends/cpu_backends/fill.h", "__algorithm/pstl_backends/cpu_backends/find_if.h", "__algorithm/pstl_backends/cpu_backends/for_each.h", - "__algorithm/pstl_backends/cpu_backends/libdispatch.h", "__algorithm/pstl_backends/cpu_backends/merge.h", - "__algorithm/pstl_backends/cpu_backends/serial.h", "__algorithm/pstl_backends/cpu_backends/stable_sort.h", - "__algorithm/pstl_backends/cpu_backends/thread.h", "__algorithm/pstl_backends/cpu_backends/transform.h", "__algorithm/pstl_backends/cpu_backends/transform_reduce.h", "__algorithm/pstl_copy.h", @@ -664,6 +658,11 @@ if (current_toolchain == default_toolchain) { "__numeric/transform_exclusive_scan.h", "__numeric/transform_inclusive_scan.h", "__numeric/transform_reduce.h", + "__pstl/backends/libdispatch.h", + "__pstl/backends/serial.h", + "__pstl/backends/std_thread.h", + "__pstl/configuration.h", + "__pstl/configuration_fwd.h", "__pstl/cpu_algos/cpu_traits.h", "__random/bernoulli_distribution.h", "__random/binomial_distribution.h", -- GitLab From db2f64ee1f743153f242ca92bf627458ba7f8000 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Wed, 17 Apr 2024 21:20:39 +0200 Subject: [PATCH 060/714] AMDGPU: Fix not handling atomicrmw fadd in exotic address spaces correctly We try to interpret unknown address space numbers as aliases of global, but this wasn't applied here. Also improve test coverage for the buffer fat pointer address space. --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 2 +- .../AMDGPU/expand-atomic-rmw-fadd.ll | 414 ++++++++++++++++++ 2 files changed, 415 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 6fde73119f81..0c706d51cb66 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -16062,7 +16062,7 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const { if (HasSystemScope) return AtomicExpansionKind::CmpXChg; - if ((AS == AMDGPUAS::GLOBAL_ADDRESS || + if ((AMDGPU::isExtendedGlobalAddrSpace(AS) || AS == AMDGPUAS::BUFFER_FAT_POINTER) && Ty->isFloatTy()) { // global/buffer atomic fadd f32 no-rtn: gfx908, gfx90a, gfx940, gfx11+. diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll index a5d4c329446f..9010dec31c16 100644 --- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll +++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll @@ -59,6 +59,112 @@ define void @test_atomicrmw_fadd_f32_global_no_use_unsafe(ptr addrspace(1) %ptr, ret void } +define void @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe(ptr addrspace(7) %ptr, float %value) #0 { +; CI-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe( +; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; CI-NEXT: br label [[ATOMICRMW_START:%.*]] +; CI: atomicrmw.start: +; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; CI: atomicrmw.end: +; CI-NEXT: ret void +; +; GFX9-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe( +; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX9: atomicrmw.start: +; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX9: atomicrmw.end: +; GFX9-NEXT: ret void +; +; GFX908-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe( +; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX908-NEXT: ret void +; +; GFX90A-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe( +; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX90A-NEXT: ret void +; +; GFX940-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe( +; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX940-NEXT: ret void +; +; GFX11-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_no_use_unsafe( +; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX11-NEXT: ret void +; + %res = atomicrmw fadd ptr addrspace(7) %ptr, float %value syncscope("wavefront") monotonic + ret void +} + +define void @test_atomicrmw_fadd_f32_as999_no_use_unsafe(ptr addrspace(999) %ptr, float %value) #0 { +; CI-LABEL: @test_atomicrmw_fadd_f32_as999_no_use_unsafe( +; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; CI-NEXT: br label [[ATOMICRMW_START:%.*]] +; CI: atomicrmw.start: +; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; CI: atomicrmw.end: +; CI-NEXT: ret void +; +; GFX9-LABEL: @test_atomicrmw_fadd_f32_as999_no_use_unsafe( +; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX9: atomicrmw.start: +; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX9: atomicrmw.end: +; GFX9-NEXT: ret void +; +; GFX908-LABEL: @test_atomicrmw_fadd_f32_as999_no_use_unsafe( +; GFX908-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX908-NEXT: ret void +; +; GFX90A-LABEL: @test_atomicrmw_fadd_f32_as999_no_use_unsafe( +; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX90A-NEXT: ret void +; +; GFX940-LABEL: @test_atomicrmw_fadd_f32_as999_no_use_unsafe( +; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX940-NEXT: ret void +; +; GFX11-LABEL: @test_atomicrmw_fadd_f32_as999_no_use_unsafe( +; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX11-NEXT: ret void +; + %res = atomicrmw fadd ptr addrspace(999) %ptr, float %value syncscope("wavefront") monotonic + ret void +} + define float @test_atomicrmw_fadd_f32_global_unsafe(ptr addrspace(1) %ptr, float %value) #0 { ; CI-LABEL: @test_atomicrmw_fadd_f32_global_unsafe( ; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 4 @@ -124,6 +230,136 @@ define float @test_atomicrmw_fadd_f32_global_unsafe(ptr addrspace(1) %ptr, float ret float %res } +define float @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe(ptr addrspace(7) %ptr, float %value) #0 { +; CI-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe( +; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; CI-NEXT: br label [[ATOMICRMW_START:%.*]] +; CI: atomicrmw.start: +; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; CI: atomicrmw.end: +; CI-NEXT: ret float [[TMP5]] +; +; GFX9-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe( +; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX9: atomicrmw.start: +; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX9: atomicrmw.end: +; GFX9-NEXT: ret float [[TMP5]] +; +; GFX908-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe( +; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX908: atomicrmw.start: +; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX908: atomicrmw.end: +; GFX908-NEXT: ret float [[TMP5]] +; +; GFX90A-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe( +; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX90A-NEXT: ret float [[RES]] +; +; GFX940-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe( +; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX940-NEXT: ret float [[RES]] +; +; GFX11-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr_unsafe( +; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX11-NEXT: ret float [[RES]] +; + %res = atomicrmw fadd ptr addrspace(7) %ptr, float %value syncscope("wavefront") monotonic + ret float %res +} + +define float @test_atomicrmw_fadd_f32_as999_unsafe(ptr addrspace(999) %ptr, float %value) #0 { +; CI-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe( +; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; CI-NEXT: br label [[ATOMICRMW_START:%.*]] +; CI: atomicrmw.start: +; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; CI: atomicrmw.end: +; CI-NEXT: ret float [[TMP5]] +; +; GFX9-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe( +; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX9: atomicrmw.start: +; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX9: atomicrmw.end: +; GFX9-NEXT: ret float [[TMP5]] +; +; GFX908-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe( +; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX908: atomicrmw.start: +; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] syncscope("wavefront") monotonic monotonic, align 4 +; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX908: atomicrmw.end: +; GFX908-NEXT: ret float [[TMP5]] +; +; GFX90A-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe( +; GFX90A-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX90A-NEXT: ret float [[RES]] +; +; GFX940-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe( +; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX940-NEXT: ret float [[RES]] +; +; GFX11-LABEL: @test_atomicrmw_fadd_f32_as999_unsafe( +; GFX11-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] syncscope("wavefront") monotonic, align 4 +; GFX11-NEXT: ret float [[RES]] +; + %res = atomicrmw fadd ptr addrspace(999) %ptr, float %value syncscope("wavefront") monotonic + ret float %res +} + define double @test_atomicrmw_fadd_f64_global_unsafe(ptr addrspace(1) %ptr, double %value) #0 { ; CI-LABEL: @test_atomicrmw_fadd_f64_global_unsafe( ; CI-NEXT: [[TMP1:%.*]] = load double, ptr addrspace(1) [[PTR:%.*]], align 8 @@ -545,6 +781,184 @@ define float @test_atomicrmw_fadd_f32_global(ptr addrspace(1) %ptr, float %value ret float %res } +define float @test_atomicrmw_fadd_f32_buffer_fat_ptr(ptr addrspace(7) %ptr, float %value) { +; CI-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr( +; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; CI-NEXT: br label [[ATOMICRMW_START:%.*]] +; CI: atomicrmw.start: +; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; CI: atomicrmw.end: +; CI-NEXT: ret float [[TMP5]] +; +; GFX9-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr( +; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX9: atomicrmw.start: +; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX9: atomicrmw.end: +; GFX9-NEXT: ret float [[TMP5]] +; +; GFX908-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr( +; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX908: atomicrmw.start: +; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX908: atomicrmw.end: +; GFX908-NEXT: ret float [[TMP5]] +; +; GFX90A-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr( +; GFX90A-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX90A: atomicrmw.start: +; GFX90A-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX90A-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX90A-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX90A-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX90A: atomicrmw.end: +; GFX90A-NEXT: ret float [[TMP5]] +; +; GFX940-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr( +; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(7) [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 4 +; GFX940-NEXT: ret float [[RES]] +; +; GFX11-LABEL: @test_atomicrmw_fadd_f32_buffer_fat_ptr( +; GFX11-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(7) [[PTR:%.*]], align 4 +; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX11: atomicrmw.start: +; GFX11-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX11-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX11-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX11-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX11: atomicrmw.end: +; GFX11-NEXT: ret float [[TMP5]] +; + %res = atomicrmw fadd ptr addrspace(7) %ptr, float %value seq_cst + ret float %res +} + +define float @test_atomicrmw_fadd_f32_as999(ptr addrspace(999) %ptr, float %value) { +; CI-LABEL: @test_atomicrmw_fadd_f32_as999( +; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; CI-NEXT: br label [[ATOMICRMW_START:%.*]] +; CI: atomicrmw.start: +; CI-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; CI-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; CI-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; CI-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; CI-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; CI-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; CI-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; CI-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; CI-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; CI: atomicrmw.end: +; CI-NEXT: ret float [[TMP5]] +; +; GFX9-LABEL: @test_atomicrmw_fadd_f32_as999( +; GFX9-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; GFX9-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX9: atomicrmw.start: +; GFX9-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX9-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX9-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX9-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX9-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; GFX9-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX9-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX9-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX9-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX9: atomicrmw.end: +; GFX9-NEXT: ret float [[TMP5]] +; +; GFX908-LABEL: @test_atomicrmw_fadd_f32_as999( +; GFX908-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; GFX908-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX908: atomicrmw.start: +; GFX908-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX908-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX908-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX908-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX908-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; GFX908-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX908-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX908-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX908-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX908: atomicrmw.end: +; GFX908-NEXT: ret float [[TMP5]] +; +; GFX90A-LABEL: @test_atomicrmw_fadd_f32_as999( +; GFX90A-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; GFX90A-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX90A: atomicrmw.start: +; GFX90A-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX90A-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX90A-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX90A-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX90A-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; GFX90A-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX90A-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX90A-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX90A-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX90A: atomicrmw.end: +; GFX90A-NEXT: ret float [[TMP5]] +; +; GFX940-LABEL: @test_atomicrmw_fadd_f32_as999( +; GFX940-NEXT: [[RES:%.*]] = atomicrmw fadd ptr addrspace(999) [[PTR:%.*]], float [[VALUE:%.*]] seq_cst, align 4 +; GFX940-NEXT: ret float [[RES]] +; +; GFX11-LABEL: @test_atomicrmw_fadd_f32_as999( +; GFX11-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(999) [[PTR:%.*]], align 4 +; GFX11-NEXT: br label [[ATOMICRMW_START:%.*]] +; GFX11: atomicrmw.start: +; GFX11-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ] +; GFX11-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]] +; GFX11-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32 +; GFX11-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32 +; GFX11-NEXT: [[TMP4:%.*]] = cmpxchg ptr addrspace(999) [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4 +; GFX11-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1 +; GFX11-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0 +; GFX11-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float +; GFX11-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]] +; GFX11: atomicrmw.end: +; GFX11-NEXT: ret float [[TMP5]] +; + %res = atomicrmw fadd ptr addrspace(999) %ptr, float %value seq_cst + ret float %res +} + define void @test_atomicrmw_fadd_f32_global_no_use_ieee(ptr addrspace(1) %ptr, float %value) { ; CI-LABEL: @test_atomicrmw_fadd_f32_global_no_use_ieee( ; CI-NEXT: [[TMP1:%.*]] = load float, ptr addrspace(1) [[PTR:%.*]], align 4 -- GitLab From 652bcf685c72447f3cc46d93d6c9c1948e8499f3 Mon Sep 17 00:00:00 2001 From: Matthias Braun Date: Wed, 17 Apr 2024 12:48:02 -0700 Subject: [PATCH 061/714] CodeGenPrepare: Add support for llvm.threadlocal.address address-mode sinking (#87844) Depending on the TLSMode many thread-local accesses on x86 can be expressed by adding a %fs: segment register to an addressing mode. Even if there are mutliple users of a `llvm.threadlocal.address` intrinsic it is generally not worth sharing the value in a register but instead fold the %fs access into multiple addressing modes. Hence this changes CodeGenPrepare to duplicate the `llvm.threadlocal.address` intrinsic as necessary. Introduces a new `TargetLowering::addressingModeSupportsTLS` callback that allows targets to indicate whether TLS accesses can be part of an addressing mode. This is fixing a performance problem, as this folding of TLS-accesses into multiple addressing modes happened naturally before the introduction of the `llvm.threadlocal.address` intrinsic, but regressed due to `SelectionDAG` keeping things in registers when accessed across basic blocks, so CodeGenPrepare needs to duplicate to mitigate this. We see a ~0.5% recovery in a codebase with heavy TLS usage (HHVM). This fixes most of #87437 --- llvm/include/llvm/CodeGen/TargetLowering.h | 6 + llvm/lib/CodeGen/CodeGenPrepare.cpp | 29 ++- llvm/lib/Target/X86/X86ISelLowering.cpp | 24 +++ llvm/lib/Target/X86/X86ISelLowering.h | 2 + .../X86/codegen-prepare-addrmode-tls.ll | 193 ++++++++++++++++++ .../CodeGenPrepare/X86/sink-addrmode-tls.ll | 86 ++++++++ 6 files changed, 336 insertions(+), 4 deletions(-) create mode 100644 llvm/test/CodeGen/X86/codegen-prepare-addrmode-tls.ll create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-tls.ll diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index e0ade0295902..2dd978c7b584 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -2805,6 +2805,12 @@ public: Type *Ty, unsigned AddrSpace, Instruction *I = nullptr) const; + /// Returns true if the targets addressing mode can target thread local + /// storage (TLS). + virtual bool addressingModeSupportsTLS(const GlobalValue &) const { + return false; + } + /// Return the prefered common base offset. virtual int64_t getPreferredLargeGEPBaseOffset(int64_t MinOffset, int64_t MaxOffset) const { diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index e657872c3828..22a766f8d625 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -5082,6 +5082,15 @@ bool AddressingModeMatcher::matchOperationAddr(User *AddrInst, unsigned Opcode, } return true; } + case Instruction::Call: + if (IntrinsicInst *II = dyn_cast(AddrInst)) { + if (II->getIntrinsicID() == Intrinsic::threadlocal_address) { + GlobalValue &GV = cast(*II->getArgOperand(0)); + if (TLI.addressingModeSupportsTLS(GV)) + return matchAddr(AddrInst->getOperand(0), Depth); + } + } + break; } return false; } @@ -5620,11 +5629,16 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, return Modified; } - if (AddrMode.BaseGV) { + GlobalValue *BaseGV = AddrMode.BaseGV; + if (BaseGV != nullptr) { if (ResultPtr) return Modified; - ResultPtr = AddrMode.BaseGV; + if (BaseGV->isThreadLocal()) { + ResultPtr = Builder.CreateThreadLocalAddress(BaseGV); + } else { + ResultPtr = BaseGV; + } } // If the real base value actually came from an inttoptr, then the matcher @@ -5789,8 +5803,15 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, } // Add in the BaseGV if present. - if (AddrMode.BaseGV) { - Value *V = Builder.CreatePtrToInt(AddrMode.BaseGV, IntPtrTy, "sunkaddr"); + GlobalValue *BaseGV = AddrMode.BaseGV; + if (BaseGV != nullptr) { + Value *BaseGVPtr; + if (BaseGV->isThreadLocal()) { + BaseGVPtr = Builder.CreateThreadLocalAddress(BaseGV); + } else { + BaseGVPtr = BaseGV; + } + Value *V = Builder.CreatePtrToInt(BaseGVPtr, IntPtrTy, "sunkaddr"); if (Result) Result = Builder.CreateAdd(Result, V, "sunkaddr"); else diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 27107f554fcc..bedec0c8974a 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -18920,6 +18920,30 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const { llvm_unreachable("TLS not implemented for this target."); } +bool X86TargetLowering::addressingModeSupportsTLS(const GlobalValue &GV) const { + if (Subtarget.is64Bit() && Subtarget.isTargetELF()) { + const TargetMachine &TM = getTargetMachine(); + TLSModel::Model Model = TM.getTLSModel(&GV); + switch (Model) { + case TLSModel::LocalExec: + case TLSModel::InitialExec: + // We can include the %fs segment register in addressing modes. + return true; + case TLSModel::LocalDynamic: + case TLSModel::GeneralDynamic: + // These models do not result in %fs relative addresses unless + // TLS descriptior are used. + // + // Even in the case of TLS descriptors we currently have no way to model + // the difference between %fs access and the computations needed for the + // offset and returning `true` for TLS-desc currently duplicates both + // which is detrimental :-/ + return false; + } + } + return false; +} + /// Lower SRA_PARTS and friends, which return two i32 values /// and take a 2 x i32 value to shift plus a shift amount. /// TODO: Can this be moved to general expansion code? diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h index 0a1e8ca44273..e348ba6e8ac0 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -1323,6 +1323,8 @@ namespace llvm { Type *Ty, unsigned AS, Instruction *I = nullptr) const override; + bool addressingModeSupportsTLS(const GlobalValue &GV) const override; + /// Return true if the specified immediate is legal /// icmp immediate, that is the target has icmp instructions which can /// compare a register against the immediate without having to materialize diff --git a/llvm/test/CodeGen/X86/codegen-prepare-addrmode-tls.ll b/llvm/test/CodeGen/X86/codegen-prepare-addrmode-tls.ll new file mode 100644 index 000000000000..0ca1da26fa89 --- /dev/null +++ b/llvm/test/CodeGen/X86/codegen-prepare-addrmode-tls.ll @@ -0,0 +1,193 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -o - %s | FileCheck %s --check-prefix=NOPIC +; RUN: llc -o - %s -relocation-model=pic | FileCheck %s --check-prefix=PIC +; RUN: llc -o - %s -relocation-model=pic -enable-tlsdesc | FileCheck %s --check-prefix=TLSDESC + +target triple = "x86_64--linux-gnu" + +declare void @effect() +declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) + +@foo_local = dso_local thread_local(localexec) global i32 0, align 4 + +define i32 @func_local_tls(i32 %arg0, i64 %arg1) nounwind { +; NOPIC-LABEL: func_local_tls: +; NOPIC: # %bb.0: # %entry +; NOPIC-NEXT: pushq %rbp +; NOPIC-NEXT: pushq %rbx +; NOPIC-NEXT: pushq %rax +; NOPIC-NEXT: movl %fs:foo_local@TPOFF, %ebp +; NOPIC-NEXT: testl %edi, %edi +; NOPIC-NEXT: movl %ebp, %eax +; NOPIC-NEXT: jne .LBB0_2 +; NOPIC-NEXT: # %bb.1: # %if.then +; NOPIC-NEXT: movq %rsi, %rbx +; NOPIC-NEXT: callq effect@PLT +; NOPIC-NEXT: movl %fs:foo_local@TPOFF+168(,%rbx,4), %eax +; NOPIC-NEXT: .LBB0_2: # %if.end +; NOPIC-NEXT: addl %ebp, %eax +; NOPIC-NEXT: addq $8, %rsp +; NOPIC-NEXT: popq %rbx +; NOPIC-NEXT: popq %rbp +; NOPIC-NEXT: retq +; +; PIC-LABEL: func_local_tls: +; PIC: # %bb.0: # %entry +; PIC-NEXT: pushq %rbp +; PIC-NEXT: pushq %r14 +; PIC-NEXT: pushq %rbx +; PIC-NEXT: movl %fs:.Lfoo_local$local@TPOFF, %ebp +; PIC-NEXT: testl %edi, %edi +; PIC-NEXT: movl %ebp, %eax +; PIC-NEXT: jne .LBB0_2 +; PIC-NEXT: # %bb.1: # %if.then +; PIC-NEXT: movq %rsi, %rbx +; PIC-NEXT: movq %fs:0, %rax +; PIC-NEXT: leaq .Lfoo_local$local@TPOFF(%rax), %r14 +; PIC-NEXT: callq effect@PLT +; PIC-NEXT: movl 168(%r14,%rbx,4), %eax +; PIC-NEXT: .LBB0_2: # %if.end +; PIC-NEXT: addl %ebp, %eax +; PIC-NEXT: popq %rbx +; PIC-NEXT: popq %r14 +; PIC-NEXT: popq %rbp +; PIC-NEXT: retq +; +; TLSDESC-LABEL: func_local_tls: +; TLSDESC: # %bb.0: # %entry +; TLSDESC-NEXT: pushq %rbp +; TLSDESC-NEXT: pushq %r14 +; TLSDESC-NEXT: pushq %rbx +; TLSDESC-NEXT: movl %fs:.Lfoo_local$local@TPOFF, %ebp +; TLSDESC-NEXT: testl %edi, %edi +; TLSDESC-NEXT: movl %ebp, %eax +; TLSDESC-NEXT: jne .LBB0_2 +; TLSDESC-NEXT: # %bb.1: # %if.then +; TLSDESC-NEXT: movq %rsi, %rbx +; TLSDESC-NEXT: movq %fs:0, %rax +; TLSDESC-NEXT: leaq .Lfoo_local$local@TPOFF(%rax), %r14 +; TLSDESC-NEXT: callq effect@PLT +; TLSDESC-NEXT: movl 168(%r14,%rbx,4), %eax +; TLSDESC-NEXT: .LBB0_2: # %if.end +; TLSDESC-NEXT: addl %ebp, %eax +; TLSDESC-NEXT: popq %rbx +; TLSDESC-NEXT: popq %r14 +; TLSDESC-NEXT: popq %rbp +; TLSDESC-NEXT: retq +entry: + %addr = tail call ptr @llvm.threadlocal.address.p0(ptr @foo_local) + %load0 = load i32, ptr %addr, align 4 + %cond = icmp eq i32 %arg0, 0 + br i1 %cond, label %if.then, label %if.end + +if.then: + tail call void @effect() + %x = add i64 %arg1, 42 + %addr1 = getelementptr inbounds i32, ptr %addr, i64 %x + %load1 = load i32, ptr %addr1, align 4 + br label %if.end + +if.end: + %phi = phi i32 [ %load1, %if.then ], [ %load0, %entry ] + %ret = add i32 %phi, %load0 + ret i32 %ret +} + +@foo_nonlocal = thread_local global i32 0, align 4 + +define i32 @func_nonlocal_tls(i32 %arg0, i64 %arg1) nounwind { +; NOPIC-LABEL: func_nonlocal_tls: +; NOPIC: # %bb.0: # %entry +; NOPIC-NEXT: pushq %rbp +; NOPIC-NEXT: pushq %r14 +; NOPIC-NEXT: pushq %rbx +; NOPIC-NEXT: movq foo_nonlocal@GOTTPOFF(%rip), %r14 +; NOPIC-NEXT: movl %fs:(%r14), %ebp +; NOPIC-NEXT: testl %edi, %edi +; NOPIC-NEXT: movl %ebp, %eax +; NOPIC-NEXT: jne .LBB1_2 +; NOPIC-NEXT: # %bb.1: # %if.then +; NOPIC-NEXT: movq %rsi, %rbx +; NOPIC-NEXT: callq effect@PLT +; NOPIC-NEXT: movl %fs:168(%r14,%rbx,4), %eax +; NOPIC-NEXT: .LBB1_2: # %if.end +; NOPIC-NEXT: addl %ebp, %eax +; NOPIC-NEXT: popq %rbx +; NOPIC-NEXT: popq %r14 +; NOPIC-NEXT: popq %rbp +; NOPIC-NEXT: retq +; +; PIC-LABEL: func_nonlocal_tls: +; PIC: # %bb.0: # %entry +; PIC-NEXT: pushq %rbp +; PIC-NEXT: pushq %r15 +; PIC-NEXT: pushq %r14 +; PIC-NEXT: pushq %rbx +; PIC-NEXT: pushq %rax +; PIC-NEXT: movq %rsi, %rbx +; PIC-NEXT: movl %edi, %ebp +; PIC-NEXT: data16 +; PIC-NEXT: leaq foo_nonlocal@TLSGD(%rip), %rdi +; PIC-NEXT: data16 +; PIC-NEXT: data16 +; PIC-NEXT: rex64 +; PIC-NEXT: callq __tls_get_addr@PLT +; PIC-NEXT: movq %rax, %r14 +; PIC-NEXT: movl (%rax), %r15d +; PIC-NEXT: testl %ebp, %ebp +; PIC-NEXT: movl %r15d, %eax +; PIC-NEXT: jne .LBB1_2 +; PIC-NEXT: # %bb.1: # %if.then +; PIC-NEXT: callq effect@PLT +; PIC-NEXT: movl 168(%r14,%rbx,4), %eax +; PIC-NEXT: .LBB1_2: # %if.end +; PIC-NEXT: addl %r15d, %eax +; PIC-NEXT: addq $8, %rsp +; PIC-NEXT: popq %rbx +; PIC-NEXT: popq %r14 +; PIC-NEXT: popq %r15 +; PIC-NEXT: popq %rbp +; PIC-NEXT: retq +; +; TLSDESC-LABEL: func_nonlocal_tls: +; TLSDESC: # %bb.0: # %entry +; TLSDESC-NEXT: pushq %rbp +; TLSDESC-NEXT: pushq %r14 +; TLSDESC-NEXT: pushq %rbx +; TLSDESC-NEXT: leaq foo_nonlocal@tlsdesc(%rip), %rax +; TLSDESC-NEXT: callq *foo_nonlocal@tlscall(%rax) +; TLSDESC-NEXT: movl %fs:(%rax), %ebp +; TLSDESC-NEXT: testl %edi, %edi +; TLSDESC-NEXT: movl %ebp, %ecx +; TLSDESC-NEXT: jne .LBB1_2 +; TLSDESC-NEXT: # %bb.1: # %if.then +; TLSDESC-NEXT: movq %rsi, %rbx +; TLSDESC-NEXT: addq %fs:0, %rax +; TLSDESC-NEXT: movq %rax, %r14 +; TLSDESC-NEXT: callq effect@PLT +; TLSDESC-NEXT: movl 168(%r14,%rbx,4), %ecx +; TLSDESC-NEXT: .LBB1_2: # %if.end +; TLSDESC-NEXT: addl %ebp, %ecx +; TLSDESC-NEXT: movl %ecx, %eax +; TLSDESC-NEXT: popq %rbx +; TLSDESC-NEXT: popq %r14 +; TLSDESC-NEXT: popq %rbp +; TLSDESC-NEXT: retq +entry: + %addr = tail call ptr @llvm.threadlocal.address.p0(ptr @foo_nonlocal) + %load0 = load i32, ptr %addr, align 4 + %cond = icmp eq i32 %arg0, 0 + br i1 %cond, label %if.then, label %if.end + +if.then: + tail call void @effect() + %x = add i64 %arg1, 42 + %addr1 = getelementptr inbounds i32, ptr %addr, i64 %x + %load1 = load i32, ptr %addr1, align 4 + br label %if.end + +if.end: + %phi = phi i32 [ %load1, %if.then ], [ %load0, %entry ] + %ret = add i32 %phi, %load0 + ret i32 %ret +} diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-tls.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-tls.ll new file mode 100644 index 000000000000..080c807cbad1 --- /dev/null +++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-tls.ll @@ -0,0 +1,86 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -passes='require,function(codegenprepare)' %s | FileCheck %s + +target triple = "x86_64--linux-gnu" + +@foo = dso_local thread_local(localexec) global i32 0, align 4 + +declare void @effect() +declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) + +define i32 @func0(i32 %arg) { +; CHECK-LABEL: define i32 @func0( +; CHECK-SAME: i32 [[ARG:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ADDR:%.*]] = tail call ptr @llvm.threadlocal.address.p0(ptr @foo) +; CHECK-NEXT: [[LOAD0:%.*]] = load i32, ptr [[ADDR]], align 4 +; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[ARG]], 0 +; CHECK-NEXT: br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +; CHECK: if.then: +; CHECK-NEXT: tail call void @effect() +; CHECK-NEXT: [[TMP0:%.*]] = call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @foo) +; CHECK-NEXT: [[LOAD1:%.*]] = load i32, ptr [[TMP0]], align 4 +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[LOAD1]], [[IF_THEN]] ], [ [[LOAD0]], [[ENTRY:%.*]] ] +; CHECK-NEXT: [[RET:%.*]] = add i32 [[PHI]], [[LOAD0]] +; CHECK-NEXT: ret i32 [[RET]] +; +entry: + %addr = tail call ptr @llvm.threadlocal.address.p0(ptr @foo) + %load0 = load i32, ptr %addr, align 4 + %cond = icmp eq i32 %arg, 0 + br i1 %cond, label %if.then, label %if.end + +if.then: + tail call void @effect() + %load1 = load i32, ptr %addr, align 4 + br label %if.end + +if.end: + %phi = phi i32 [ %load1, %if.then ], [ %load0, %entry ] + %ret = add i32 %phi, %load0 + ret i32 %ret +} + +define i32 @func1(i32 %arg0, i32 %arg1) { +; CHECK-LABEL: define i32 @func1( +; CHECK-SAME: i32 [[ARG0:%.*]], i32 [[ARG1:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[ADDR:%.*]] = tail call ptr @llvm.threadlocal.address.p0(ptr @foo) +; CHECK-NEXT: [[LOAD0:%.*]] = load i32, ptr [[ADDR]], align 4 +; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[ARG0]], 0 +; CHECK-NEXT: br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +; CHECK: if.then: +; CHECK-NEXT: tail call void @effect() +; CHECK-NEXT: [[X:%.*]] = add i32 [[ARG1]], 42 +; CHECK-NEXT: [[X64:%.*]] = sext i32 [[X]] to i64 +; CHECK-NEXT: [[TMP0:%.*]] = call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @foo) +; CHECK-NEXT: [[SUNKADDR:%.*]] = mul i64 [[X64]], 4 +; CHECK-NEXT: [[ADDR1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 [[SUNKADDR]] +; CHECK-NEXT: [[LOAD1:%.*]] = load i32, ptr [[ADDR1]], align 4 +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[LOAD1]], [[IF_THEN]] ], [ [[LOAD0]], [[ENTRY:%.*]] ] +; CHECK-NEXT: [[RET:%.*]] = add i32 [[PHI]], [[LOAD0]] +; CHECK-NEXT: ret i32 [[RET]] +; +entry: + %addr = tail call ptr @llvm.threadlocal.address.p0(ptr @foo) + %load0 = load i32, ptr %addr, align 4 + %cond = icmp eq i32 %arg0, 0 + br i1 %cond, label %if.then, label %if.end + +if.then: + tail call void @effect() + %x = add i32 %arg1, 42 + %x64 = sext i32 %x to i64 + %addr1 = getelementptr inbounds i32, ptr %addr, i64 %x64 + %load1 = load i32, ptr %addr1, align 4 + br label %if.end + +if.end: + %phi = phi i32 [ %load1, %if.then ], [ %load0, %entry ] + %ret = add i32 %phi, %load0 + ret i32 %ret +} -- GitLab From 2583b2eea4a509424e5e5f51dffedd9beede76a3 Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Wed, 17 Apr 2024 19:46:43 +0000 Subject: [PATCH 062/714] [flang][NFC] Add missing include for FreeBSD Suggested by dankm here: https://github.com/llvm/llvm-project/pull/88517 --- flang/runtime/extensions.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/flang/runtime/extensions.cpp b/flang/runtime/extensions.cpp index 12498b502ae1..80604d886a70 100644 --- a/flang/runtime/extensions.cpp +++ b/flang/runtime/extensions.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #ifdef _WIN32 inline void CtimeBuffer(char *buffer, size_t bufsize, const time_t cur_time, -- GitLab From d0c51f7d5496015ce410cc758a7caf976ffaaec7 Mon Sep 17 00:00:00 2001 From: Kai Nacke Date: Wed, 17 Apr 2024 16:07:35 -0400 Subject: [PATCH 063/714] [LiveIns] Improve recomputeLiveIns() (#88951) Some small changes to recomputeLiveIns() to improve performance: - Instead of copying the list of old live-ins, and then clearing them, a new method swaps the list for an empty one. - getLiveIns() now returns a constant reference to the list As result, the list-data is never copied. Depending on the implementation details of the vector container, it can also save calls to allocate and deallocate memory. I see a small improvement on CTMark with these changes. --------- Co-authored-by: Nikita Popov --- llvm/include/llvm/CodeGen/LivePhysRegs.h | 9 +++++---- llvm/include/llvm/CodeGen/MachineBasicBlock.h | 6 +++++- llvm/lib/CodeGen/MachineBasicBlock.cpp | 6 ++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/llvm/include/llvm/CodeGen/LivePhysRegs.h b/llvm/include/llvm/CodeGen/LivePhysRegs.h index 9574a6f0c7c0..d315e4ff6f3a 100644 --- a/llvm/include/llvm/CodeGen/LivePhysRegs.h +++ b/llvm/include/llvm/CodeGen/LivePhysRegs.h @@ -199,14 +199,15 @@ void computeAndAddLiveIns(LivePhysRegs &LiveRegs, /// any changes were made. static inline bool recomputeLiveIns(MachineBasicBlock &MBB) { LivePhysRegs LPR; - auto oldLiveIns = MBB.getLiveIns(); + std::vector OldLiveIns; - MBB.clearLiveIns(); + MBB.clearLiveIns(OldLiveIns); computeAndAddLiveIns(LPR, MBB); MBB.sortUniqueLiveIns(); - auto newLiveIns = MBB.getLiveIns(); - return oldLiveIns != newLiveIns; + const std::vector &NewLiveIns = + MBB.getLiveIns(); + return OldLiveIns != NewLiveIns; } /// Convenience function for recomputing live-in's for a set of MBBs until the diff --git a/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/llvm/include/llvm/CodeGen/MachineBasicBlock.h index dc2035fa598c..5b6be3a96b2f 100644 --- a/llvm/include/llvm/CodeGen/MachineBasicBlock.h +++ b/llvm/include/llvm/CodeGen/MachineBasicBlock.h @@ -441,6 +441,10 @@ public: /// Clear live in list. void clearLiveIns(); + /// Clear the live in list, and return the removed live in's in \p OldLiveIns. + /// Requires that the vector \p OldLiveIns is empty. + void clearLiveIns(std::vector &OldLiveIns); + /// Add PhysReg as live in to this block, and ensure that there is a copy of /// PhysReg to a virtual register of class RC. Return the virtual register /// that is a copy of the live in PhysReg. @@ -477,7 +481,7 @@ public: /// Remove entry from the livein set and return iterator to the next. livein_iterator removeLiveIn(livein_iterator I); - std::vector getLiveIns() const { return LiveIns; } + const std::vector &getLiveIns() const { return LiveIns; } class liveout_iterator { public: diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp index b2114c250ac0..0bd5f09564ec 100644 --- a/llvm/lib/CodeGen/MachineBasicBlock.cpp +++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp @@ -1728,6 +1728,12 @@ void MachineBasicBlock::clearLiveIns() { LiveIns.clear(); } +void MachineBasicBlock::clearLiveIns( + std::vector &OldLiveIns) { + assert(OldLiveIns.empty() && "Vector must be empty"); + std::swap(LiveIns, OldLiveIns); +} + MachineBasicBlock::livein_iterator MachineBasicBlock::livein_begin() const { assert(getParent()->getProperties().hasProperty( MachineFunctionProperties::Property::TracksLiveness) && -- GitLab From 60b90b523323f8196a9e4a68b1f33358624c09eb Mon Sep 17 00:00:00 2001 From: Zequan Wu Date: Wed, 17 Apr 2024 16:09:38 -0400 Subject: [PATCH 064/714] [lldb][DynamicLoader] Fix lldb unable to stop at _dl_debug_state if user set it before the process launched. (#88792) If user sets a breakpoint at `_dl_debug_state` before the process launched, the breakpoint is not resolved yet. When lldb loads dynamic loader module, it's created with `Target::GetOrCreateModule` which notifies any pending breakpoint to resolve. However, the module's sections are not loaded at this time. They are loaded after returned from [Target::GetOrCreateModule](https://github.com/llvm/llvm-project/blob/0287a5cc4e2a5ded1ae2e4079f91052e6a6b8d9b/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp#L574-L577). This change fixes it by manually resolving breakpoints after creating dynamic loader module. --- .../POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp | 11 +++++++++-- .../breakpoint_command/TestBreakpointCommand.py | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp b/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp index 9baf86da4dc7..9fa245fc41d4 100644 --- a/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp +++ b/lldb/source/Plugins/DynamicLoader/POSIX-DYLD/DynamicLoaderPOSIXDYLD.cpp @@ -571,10 +571,17 @@ ModuleSP DynamicLoaderPOSIXDYLD::LoadInterpreterModule() { FileSpec file(info.GetName().GetCString()); ModuleSpec module_spec(file, target.GetArchitecture()); - if (ModuleSP module_sp = target.GetOrCreateModule(module_spec, - true /* notify */)) { + // Don't notify that module is added here because its loading section + // addresses are not updated yet. We manually notify it below. + if (ModuleSP module_sp = + target.GetOrCreateModule(module_spec, /*notify=*/false)) { UpdateLoadedSections(module_sp, LLDB_INVALID_ADDRESS, m_interpreter_base, false); + // Manually notify that dynamic linker is loaded after updating load section + // addersses so that breakpoints can be resolved. + ModuleList module_list; + module_list.Append(module_sp); + target.ModulesDidLoad(module_list); m_interpreter_module = module_sp; return module_sp; } diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_command/TestBreakpointCommand.py b/lldb/test/API/functionalities/breakpoint/breakpoint_command/TestBreakpointCommand.py index ea242952e409..850235fdcefa 100644 --- a/lldb/test/API/functionalities/breakpoint/breakpoint_command/TestBreakpointCommand.py +++ b/lldb/test/API/functionalities/breakpoint/breakpoint_command/TestBreakpointCommand.py @@ -671,3 +671,20 @@ class BreakpointCommandTestCase(TestBase): self.assertNotEqual(breakpoints_stats, None) for breakpoint_stats in breakpoints_stats: self.assertIn("hitCount", breakpoint_stats) + + @skipIf(oslist=no_match(["linux"])) + def test_break_at__dl_debug_state(self): + """ + Test lldb is able to stop at _dl_debug_state if it is set before the + process is launched. + """ + self.build() + exe = self.getBuildArtifact("a.out") + self.runCmd("target create %s" % exe) + bpid = lldbutil.run_break_set_by_symbol( + self, "_dl_debug_state", num_expected_locations=0 + ) + self.runCmd("run") + self.assertIsNotNone( + lldbutil.get_one_thread_stopped_at_breakpoint_id(self.process(), bpid) + ) -- GitLab From e15f47f2675a5400464aec00219658882df5e3fa Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 17 Apr 2024 13:16:08 -0700 Subject: [PATCH 065/714] [InstCombine] Don't use dominating conditions to transform sub into xor. (#88566) Other passes are unable to reverse this transform if we use dominating conditions. Fixes #88239. --- llvm/include/llvm/Analysis/SimplifyQuery.h | 6 ++++++ .../Transforms/InstCombine/InstCombineAddSub.cpp | 6 ++++-- llvm/test/Transforms/InstCombine/sub-xor.ll | 4 +++- llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll | 13 ++++++------- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/llvm/include/llvm/Analysis/SimplifyQuery.h b/llvm/include/llvm/Analysis/SimplifyQuery.h index e5e6ae0d3d8e..a10c0dc49fa2 100644 --- a/llvm/include/llvm/Analysis/SimplifyQuery.h +++ b/llvm/include/llvm/Analysis/SimplifyQuery.h @@ -113,6 +113,12 @@ struct SimplifyQuery { using namespace PatternMatch; return match(V, m_Undef()); } + + SimplifyQuery getWithoutDomCondCache() const { + SimplifyQuery Copy(*this); + Copy.DC = nullptr; + return Copy; + } }; } // end namespace llvm diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp index c59b867b10e7..df7f02810ee4 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp @@ -2281,8 +2281,10 @@ Instruction *InstCombinerImpl::visitSub(BinaryOperator &I) { if (match(Op0, m_APInt(Op0C))) { if (Op0C->isMask()) { // Turn this into a xor if LHS is 2^n-1 and the remaining bits are known - // zero. - KnownBits RHSKnown = computeKnownBits(Op1, 0, &I); + // zero. We don't use information from dominating conditions so this + // transform is easier to reverse if necessary. + KnownBits RHSKnown = llvm::computeKnownBits( + Op1, 0, SQ.getWithInstruction(&I).getWithoutDomCondCache()); if ((*Op0C | RHSKnown.Zero).isAllOnes()) return BinaryOperator::CreateXor(Op1, Op0); } diff --git a/llvm/test/Transforms/InstCombine/sub-xor.ll b/llvm/test/Transforms/InstCombine/sub-xor.ll index b4e87d0405fc..2976598e043f 100644 --- a/llvm/test/Transforms/InstCombine/sub-xor.ll +++ b/llvm/test/Transforms/InstCombine/sub-xor.ll @@ -158,13 +158,15 @@ define <2 x i8> @xor_add_splat_undef(<2 x i8> %x) { ret <2 x i8> %add } +; Make sure we don't convert sub to xor using dominating condition. That makes +; it hard for other passe to reverse. define i32 @xor_dominating_cond(i32 %x) { ; CHECK-LABEL: @xor_dominating_cond( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[X:%.*]], 256 ; CHECK-NEXT: br i1 [[COND]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] ; CHECK: if.then: -; CHECK-NEXT: [[A:%.*]] = xor i32 [[X]], 255 +; CHECK-NEXT: [[A:%.*]] = sub nuw nsw i32 255, [[X]] ; CHECK-NEXT: ret i32 [[A]] ; CHECK: if.end: ; CHECK-NEXT: ret i32 [[X]] diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll index 3afa1904fb24..1c16b37144f1 100644 --- a/llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll +++ b/llvm/test/Transforms/PhaseOrdering/X86/pr88239.ll @@ -8,19 +8,18 @@ define void @foo(ptr noalias noundef %0, ptr noalias noundef %1) optsize { ; CHECK-LABEL: define void @foo( ; CHECK-SAME: ptr noalias nocapture noundef readonly [[TMP0:%.*]], ptr noalias nocapture noundef writeonly [[TMP1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: vector.ph: +; CHECK-NEXT: [[INVARIANT_GEP:%.*]] = getelementptr i8, ptr [[TMP0]], i64 -28 ; CHECK-NEXT: br label [[TMP4:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[TMP2:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[TMP4]] ] -; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ , [[TMP2]] ], [ [[VEC_IND_NEXT:%.*]], [[TMP4]] ] -; CHECK-NEXT: [[TMP6:%.*]] = and <8 x i64> [[VEC_IND]], -; CHECK-NEXT: [[TMP3:%.*]] = xor <8 x i64> [[TMP6]], -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], <8 x i64> [[TMP3]] -; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = tail call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> [[TMP7]], i32 4, <8 x i1> , <8 x i32> poison) +; CHECK-NEXT: [[TMP3:%.*]] = sub nuw nsw i64 255, [[INDVARS_IV]] +; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[INVARIANT_GEP]], i64 [[TMP3]] +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = load <8 x i32>, ptr [[GEP]], align 4 ; CHECK-NEXT: [[TMP5:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_GATHER]], +; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <8 x i32> ; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 [[INDVARS_IV]] -; CHECK-NEXT: store <8 x i32> [[TMP5]], ptr [[TMP10]], align 4 +; CHECK-NEXT: store <8 x i32> [[TMP6]], ptr [[TMP10]], align 4 ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw i64 [[INDVARS_IV]], 8 -; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 256 ; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[MIDDLE_BLOCK:%.*]], label [[TMP4]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: -- GitLab From f309c882ea5ad411ac1011a4228c0e49b1025bb4 Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Wed, 17 Apr 2024 20:11:35 +0000 Subject: [PATCH 066/714] Revert "[flang][NFC] Add missing include for FreeBSD" This reverts commit 2583b2eea4a509424e5e5f51dffedd9beede76a3. --- flang/runtime/extensions.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/flang/runtime/extensions.cpp b/flang/runtime/extensions.cpp index 80604d886a70..12498b502ae1 100644 --- a/flang/runtime/extensions.cpp +++ b/flang/runtime/extensions.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #ifdef _WIN32 inline void CtimeBuffer(char *buffer, size_t bufsize, const time_t cur_time, -- GitLab From 0cee89431d77d0bb0809fd9b2c9d21da2a2783aa Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Wed, 17 Apr 2024 19:46:43 +0000 Subject: [PATCH 067/714] [flang][NFC] Add missing include for FreeBSD Suggested by dankm here: https://github.com/llvm/llvm-project/pull/88517 --- flang/runtime/extensions.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flang/runtime/extensions.cpp b/flang/runtime/extensions.cpp index 12498b502ae1..4b110cc10c84 100644 --- a/flang/runtime/extensions.cpp +++ b/flang/runtime/extensions.cpp @@ -43,9 +43,9 @@ inline void CtimeBuffer(char *buffer, size_t bufsize, const time_t cur_time, } #endif -#if _REENTRANT || _POSIX_C_SOURCE >= 199506L -// System is posix-compliant and has getlogin_r -#include +#ifndef _WIN32 +// posix-compliant and has getlogin_r and F_OK +#include #endif extern "C" { -- GitLab From 9435edf628cb2011652897e1f10f7c55313d50be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Wed, 17 Apr 2024 13:45:22 -0700 Subject: [PATCH 068/714] [flang][cuda] Lower DEALLOCATE for device variables (#89091) Replace the runtime call to `AllocatableDeallocate` for CUDA device variable to the newly added `fir.cuda_deallocate` operation. This is similar with #88980 A third patch will handle the case of automatic dealloctaion of device allocatable variables --- flang/lib/Lower/Allocatable.cpp | 33 ++++++++++++++++++++-- flang/test/Lower/CUDA/cuda-allocatable.cuf | 15 ++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/flang/lib/Lower/Allocatable.cpp b/flang/lib/Lower/Allocatable.cpp index 1d434d512d0c..38f61528d7e2 100644 --- a/flang/lib/Lower/Allocatable.cpp +++ b/flang/lib/Lower/Allocatable.cpp @@ -799,6 +799,28 @@ static void postDeallocationAction(Fortran::lower::AbstractConverter &converter, Fortran::lower::attachDeclarePostDeallocAction(converter, builder, sym); } +static mlir::Value genCudaDeallocate(fir::FirOpBuilder &builder, + mlir::Location loc, + const fir::MutableBoxValue &box, + ErrorManager &errorManager, + const Fortran::semantics::Symbol &sym) { + fir::CUDADataAttributeAttr cudaAttr = + Fortran::lower::translateSymbolCUDADataAttribute(builder.getContext(), + sym); + mlir::Value errmsg = + mlir::isa(errorManager.errMsgAddr.getDefiningOp()) + ? nullptr + : errorManager.errMsgAddr; + + // Keep return type the same as a standard AllocatableAllocate call. + mlir::Type retTy = fir::runtime::getModel()(builder.getContext()); + return builder + .create( + loc, retTy, box.getAddr(), errmsg, cudaAttr, + errorManager.hasStatSpec() ? builder.getUnitAttr() : nullptr) + .getResult(); +} + // Generate deallocation of a pointer/allocatable. static mlir::Value genDeallocate(fir::FirOpBuilder &builder, @@ -806,10 +828,11 @@ genDeallocate(fir::FirOpBuilder &builder, const fir::MutableBoxValue &box, ErrorManager &errorManager, mlir::Value declaredTypeDesc = {}, const Fortran::semantics::Symbol *symbol = nullptr) { + bool isCudaSymbol = symbol && Fortran::semantics::HasCUDAAttr(*symbol); // Deallocate intrinsic types inline. if (!box.isDerived() && !box.isPolymorphic() && !box.isUnlimitedPolymorphic() && !errorManager.hasStatSpec() && - !useAllocateRuntime && !box.isPointer()) { + !useAllocateRuntime && !box.isPointer() && !isCudaSymbol) { // Pointers must use PointerDeallocate so that their deallocations // can be validated. mlir::Value ret = fir::factory::genFreemem(builder, loc, box); @@ -820,8 +843,12 @@ genDeallocate(fir::FirOpBuilder &builder, // Use runtime calls to deallocate descriptor cases. Sync MutableBoxValue // with its descriptor before and after calls if needed. errorManager.genStatCheck(builder, loc); - mlir::Value stat = - genRuntimeDeallocate(builder, loc, box, errorManager, declaredTypeDesc); + mlir::Value stat; + if (!isCudaSymbol) + stat = + genRuntimeDeallocate(builder, loc, box, errorManager, declaredTypeDesc); + else + stat = genCudaDeallocate(builder, loc, box, errorManager, *symbol); fir::factory::syncMutableBoxFromIRBox(builder, loc, box); if (symbol) postDeallocationAction(converter, builder, *symbol); diff --git a/flang/test/Lower/CUDA/cuda-allocatable.cuf b/flang/test/Lower/CUDA/cuda-allocatable.cuf index 55223011e8d9..5b10334ecdbc 100644 --- a/flang/test/Lower/CUDA/cuda-allocatable.cuf +++ b/flang/test/Lower/CUDA/cuda-allocatable.cuf @@ -5,6 +5,8 @@ subroutine sub1() real, allocatable, device :: a(:) allocate(a(10)) + + deallocate(a) end subroutine ! CHECK-LABEL: func.func @_QPsub1() @@ -13,10 +15,14 @@ end subroutine ! CHECK: fir.call @_FortranAAllocatableSetBounds ! CHECK: %{{.*}} = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref>>> {cuda_attr = #fir.cuda} -> i32 +! CHECK: %{{.*}} = fir.cuda_deallocate %[[BOX_DECL]]#1 : !fir.ref>>> {cuda_attr = #fir.cuda} -> i32 + subroutine sub2() real, allocatable, managed :: a(:) integer :: istat allocate(a(10), stat=istat) + + deallocate(a, stat=istat) end subroutine ! CHECK-LABEL: func.func @_QPsub2() @@ -28,6 +34,9 @@ end subroutine ! CHECK: %[[STAT:.*]] = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref>>> {cuda_attr = #fir.cuda, hasStat} -> i32 ! CHECK: fir.store %[[STAT]] to %[[ISTAT_DECL]]#1 : !fir.ref +! CHECK: %[[STAT:.*]] = fir.cuda_deallocate %[[BOX_DECL]]#1 : !fir.ref>>> {cuda_attr = #fir.cuda, hasStat} -> i32 +! CHECK: fir.store %[[STAT]] to %[[ISTAT_DECL]]#1 : !fir.ref + subroutine sub3() integer, allocatable, pinned :: a(:,:) logical :: plog @@ -92,6 +101,8 @@ subroutine sub7() integer :: istat character(50) :: err allocate(a(100), stat=istat, errmsg=err) + + deallocate(a, stat=istat, errmsg=err) end subroutine ! CHECK-LABEL: func.func @_QPsub7() @@ -105,3 +116,7 @@ end subroutine ! CHECK: fir.call @_FortranAAllocatableSetBounds ! CHECK: %[[STAT:.*]] = fir.cuda_allocate %[[BOX_DECL]]#1 : !fir.ref>>> errmsg(%[[ERR_BOX]] : !fir.box>) {cuda_attr = #fir.cuda, hasStat} -> i32 ! CHECK: fir.store %[[STAT]] to %[[ISTAT_DECL]]#1 : !fir.ref + +! CHECK: %[[ERR_BOX:.*]] = fir.embox %[[ERR_DECL]]#1 : (!fir.ref>) -> !fir.box> +! CHECK: %[[STAT:.*]] = fir.cuda_deallocate %[[BOX_DECL]]#1 : !fir.ref>>> errmsg(%15 : !fir.box>) {cuda_attr = #fir.cuda, hasStat} -> i32 +! CHECK: fir.store %[[STAT]] to %[[ISTAT_DECL]]#1 : !fir.ref -- GitLab From 1b87418b9960983f54aa67f87018466ae12d0ec6 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Wed, 17 Apr 2024 17:03:39 -0400 Subject: [PATCH 069/714] [bazel] Improve liblldb building (#89095) On Linux using --version-script doesn't force loading of the underlying archives that contain the symbols. By setting alwayslink=True on the API cc_library we virtually get this behavior. This also allows downstream users to use the exports files used by cmake. We could build more configurability into this but there are also a lot of possible variations users might want. --- .../llvm-project-overlay/lldb/BUILD.bazel | 42 +++++++++++-------- .../lldb/source/Plugins/BUILD.bazel | 20 +-------- 2 files changed, 26 insertions(+), 36 deletions(-) diff --git a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel index 1f2b5b476bcc..7958c6024875 100644 --- a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel @@ -16,7 +16,11 @@ package( licenses(["notice"]) -exports_files(["LICENSE.TXT"]) +exports_files([ + "LICENSE.TXT", + "source/API/liblldb-private.exports", + "source/API/liblldb.exports", +]) bool_flag( name = "enable_curses", @@ -203,13 +207,29 @@ cc_library( ":TargetHeaders", ":Utility", ":Version", + "//lldb/source/Plugins:PluginExpressionParserClang", "//lldb/source/Plugins:PluginExpressionParserClangHeaders", "//lldb/source/Plugins:PluginsConfig", + "//llvm:AllTargetsDisassemblers", "//llvm:ExecutionEngine", "//llvm:MCJIT", "//llvm:Support", "//llvm:config", - ], + ] + [ + "//lldb/source/Plugins:Plugin{}".format(x) + for x in DEFAULT_PLUGINS + DEFAULT_SCRIPT_PLUGINS + ] + select({ + "@platforms//os:macos": [ + "//lldb/source/Plugins:PluginProcessMacOSXKernel", + "//lldb/source/Plugins:PluginSymbolLocatorDebugSymbols", + "//lldb/source/Plugins:PluginSymbolVendorMacOSX", + ], + "@platforms//os:linux": [ + "//lldb/source/Plugins:PluginProcessLinux", + ], + "//conditions:default": [], + }), + alwayslink = True, ) cc_library( @@ -690,23 +710,8 @@ cc_library( name = "liblldb.static", deps = [ ":API", - ":Host", ":Interpreter", - "//llvm:AllTargetsDisassemblers", - ] + [ - "//lldb/source/Plugins:Plugin{}".format(x) - for x in DEFAULT_PLUGINS + DEFAULT_SCRIPT_PLUGINS - ] + select({ - "@platforms//os:macos": [ - "//lldb/source/Plugins:PluginProcessMacOSXKernel", - "//lldb/source/Plugins:PluginSymbolLocatorDebugSymbols", - "//lldb/source/Plugins:PluginSymbolVendorMacOSX", - ], - "@platforms//os:linux": [ - "//lldb/source/Plugins:PluginProcessLinux", - ], - "//conditions:default": [], - }), + ], ) cc_shared_library( @@ -889,6 +894,7 @@ cc_binary( deps = [ ":Host", ":Initialization", + ":Interpreter", ":Utility", ":Version", ":lldb_server_opts_gen", diff --git a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel index b5f5bed1698a..e0907a838148 100644 --- a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel @@ -557,7 +557,6 @@ cc_library( "//lldb:Breakpoint", "//lldb:Core", "//lldb:Host", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Target", "//lldb:TargetHeaders", @@ -591,7 +590,6 @@ cc_library( "//lldb:Core", "//lldb:Expression", "//lldb:Host", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Target", "//lldb:TargetHeaders", @@ -664,7 +662,7 @@ cc_library( "//lldb:Breakpoint", "//lldb:Core", "//lldb:Host", - "//lldb:Interpreter", + "//lldb:InterpreterHeaders", "//lldb:SymbolHeaders", "//lldb:Target", "//lldb:TargetHeaders", @@ -1264,7 +1262,6 @@ cc_library( "//lldb:Breakpoint", "//lldb:Core", "//lldb:Host", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1422,7 +1419,6 @@ cc_library( deps = [ ":PluginProcessUtility", "//lldb:Core", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1438,7 +1434,6 @@ cc_library( deps = [ ":PluginProcessUtility", "//lldb:Core", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1475,7 +1470,6 @@ cc_library( ":PluginProcessUtility", "//lldb:Core", "//lldb:Host", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1495,7 +1489,6 @@ cc_library( ":PluginProcessUtility", "//lldb:Core", "//lldb:Host", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1515,7 +1508,6 @@ cc_library( deps = [ ":PluginProcessUtility", "//lldb:Core", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1532,7 +1524,6 @@ cc_library( deps = [ ":PluginProcessUtility", "//lldb:Core", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1570,7 +1561,6 @@ cc_library( "//lldb:Expression", "//lldb:Headers", "//lldb:Host", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1608,7 +1598,7 @@ cc_library( "//lldb:Core", "//lldb:Expression", "//lldb:Headers", - "//lldb:Interpreter", + "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", "//lldb:Target", @@ -1629,7 +1619,6 @@ cc_library( "//lldb:Expression", "//lldb:Headers", "//lldb:Host", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1682,7 +1671,6 @@ cc_library( ":PluginObjectFileMachO", "//lldb:Breakpoint", "//lldb:Core", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -1838,7 +1826,6 @@ cc_library( include_prefix = "Plugins", deps = [ "//lldb:Core", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Utility", "//llvm:Support", @@ -2215,7 +2202,6 @@ cc_library( "//lldb:Core", "//lldb:Headers", "//lldb:Host", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -2262,7 +2248,6 @@ cc_library( "//lldb:DataFormattersHeaders", "//lldb:ExpressionHeaders", "//lldb:Headers", - "//lldb:Interpreter", "//lldb:InterpreterHeaders", "//lldb:Symbol", "//lldb:SymbolHeaders", @@ -2287,7 +2272,6 @@ cc_library( "//lldb:Expression", "//lldb:Headers", "//lldb:Host", - "//lldb:Interpreter", "//lldb:Symbol", "//lldb:Target", "//lldb:TargetHeaders", -- GitLab From 0ab3f160c4bff1c7d57c046b95ab8c5035ae986f Mon Sep 17 00:00:00 2001 From: Philip Reames Date: Wed, 17 Apr 2024 14:23:38 -0700 Subject: [PATCH 070/714] [RISCV] Add coverage of add (mul X, C), Y oppurtunity using shNadd --- llvm/test/CodeGen/RISCV/rv64zba.ll | 73 ++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/llvm/test/CodeGen/RISCV/rv64zba.ll b/llvm/test/CodeGen/RISCV/rv64zba.ll index 6939185947f4..ccb23fc2bbfa 100644 --- a/llvm/test/CodeGen/RISCV/rv64zba.ll +++ b/llvm/test/CodeGen/RISCV/rv64zba.ll @@ -479,6 +479,18 @@ define i64 @addmul20(i64 %a, i64 %b) { ret i64 %d } +define i64 @addmul22(i64 %a, i64 %b) { +; CHECK-LABEL: addmul22: +; CHECK: # %bb.0: +; CHECK-NEXT: li a2, 22 +; CHECK-NEXT: mul a0, a0, a2 +; CHECK-NEXT: add a0, a0, a1 +; CHECK-NEXT: ret + %c = mul i64 %a, 22 + %d = add i64 %c, %b + ret i64 %d +} + define i64 @addmul24(i64 %a, i64 %b) { ; RV64I-LABEL: addmul24: ; RV64I: # %bb.0: @@ -551,6 +563,67 @@ define i64 @addmul72(i64 %a, i64 %b) { ret i64 %d } +define i64 @addmul162(i64 %a, i64 %b) { +; CHECK-LABEL: addmul162: +; CHECK: # %bb.0: +; CHECK-NEXT: li a2, 162 +; CHECK-NEXT: mul a0, a0, a2 +; CHECK-NEXT: add a0, a0, a1 +; CHECK-NEXT: ret + %c = mul i64 %a, 162 + %d = add i64 %c, %b + ret i64 %d +} + +define i64 @addmul180(i64 %a, i64 %b) { +; CHECK-LABEL: addmul180: +; CHECK: # %bb.0: +; CHECK-NEXT: li a2, 180 +; CHECK-NEXT: mul a0, a0, a2 +; CHECK-NEXT: add a0, a0, a1 +; CHECK-NEXT: ret + %c = mul i64 %a, 180 + %d = add i64 %c, %b + ret i64 %d +} + +define i64 @add255mul180(i64 %a) { +; CHECK-LABEL: add255mul180: +; CHECK: # %bb.0: +; CHECK-NEXT: li a1, 180 +; CHECK-NEXT: mul a0, a0, a1 +; CHECK-NEXT: addi a0, a0, 255 +; CHECK-NEXT: ret + %c = mul i64 %a, 180 + %d = add i64 %c, 255 + ret i64 %d +} + + +define i64 @addmul4096(i64 %a, i64 %b) { +; CHECK-LABEL: addmul4096: +; CHECK: # %bb.0: +; CHECK-NEXT: slli a0, a0, 12 +; CHECK-NEXT: add a0, a0, a1 +; CHECK-NEXT: ret + %c = mul i64 %a, 4096 + %d = add i64 %c, %b + ret i64 %d +} + +define i64 @addmul4230(i64 %a, i64 %b) { +; CHECK-LABEL: addmul4230: +; CHECK: # %bb.0: +; CHECK-NEXT: lui a2, 1 +; CHECK-NEXT: addiw a2, a2, 134 +; CHECK-NEXT: mul a0, a0, a2 +; CHECK-NEXT: add a0, a0, a1 +; CHECK-NEXT: ret + %c = mul i64 %a, 4230 + %d = add i64 %c, %b + ret i64 %d +} + define i64 @mul96(i64 %a) { ; RV64I-LABEL: mul96: ; RV64I: # %bb.0: -- GitLab From 678f19f08296fec299438130cf5943714c590b7e Mon Sep 17 00:00:00 2001 From: azhan92 Date: Wed, 17 Apr 2024 17:56:42 -0400 Subject: [PATCH 071/714] [Support] Report EISDIR when opening a directory (#79880) The test `llvm/unittests/Support/CommandLineTest.cpp` that handles errors in expansion of response files was previously disabled for AIX. Originally the code was dependent on `read` returning `EISDIR` which occurs on platforms such as Linux. However, other platforms such as AIX allow use of `read` on file descriptors for directories. This change updates `readNativeFile` to produce `EISDIR` on AIX and z/OS when used on a directory (instead of relying on the call to `read` to do so). --------- Co-authored-by: Alison Zhang Co-authored-by: James Henderson <46713263+jh7370@users.noreply.github.com> --- llvm/lib/Support/Unix/Path.inc | 9 ++++++++ .../tools/llvm-symbolizer/input-file-err.test | 2 -- llvm/unittests/Support/CommandLineTest.cpp | 2 -- llvm/unittests/Support/Path.cpp | 22 +++++++++++++++++++ 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Support/Unix/Path.inc b/llvm/lib/Support/Unix/Path.inc index 968e2c459f3f..6e679f74869f 100644 --- a/llvm/lib/Support/Unix/Path.inc +++ b/llvm/lib/Support/Unix/Path.inc @@ -1191,6 +1191,15 @@ Expected readNativeFile(file_t FD, MutableArrayRef Buf) { ssize_t NumRead = sys::RetryAfterSignal(-1, ::read, FD, Buf.data(), Size); if (ssize_t(NumRead) == -1) return errorCodeToError(errnoAsErrorCode()); +// The underlying operation on these platforms allow opening directories +// for reading in more cases than other platforms. +#if defined(__MVS__) || defined(_AIX) + struct stat Status; + if (fstat(FD, &Status) == -1) + return errorCodeToError(errnoAsErrorCode()); + if (S_ISDIR(Status.st_mode)) + return errorCodeToError(make_error_code(errc::is_a_directory)); +#endif return NumRead; } diff --git a/llvm/test/tools/llvm-symbolizer/input-file-err.test b/llvm/test/tools/llvm-symbolizer/input-file-err.test index df14da2f433c..76115b513470 100644 --- a/llvm/test/tools/llvm-symbolizer/input-file-err.test +++ b/llvm/test/tools/llvm-symbolizer/input-file-err.test @@ -1,5 +1,3 @@ -Failing on AIX due to D153595. The test expects a different error message from the one given on AIX. -XFAIL: target={{.*}}-aix{{.*}} RUN: not llvm-addr2line -e %p/Inputs/nonexistent 0x12 2>&1 | FileCheck %s --check-prefix=CHECK-NONEXISTENT-A2L -DMSG=%errc_ENOENT RUN: not llvm-addr2line -e %p/Inputs/nonexistent 2>&1 | FileCheck %s --check-prefix=CHECK-NONEXISTENT-A2L -DMSG=%errc_ENOENT CHECK-NONEXISTENT-A2L: llvm-addr2line{{.*}}: error: '{{.*}}Inputs/nonexistent': [[MSG]] diff --git a/llvm/unittests/Support/CommandLineTest.cpp b/llvm/unittests/Support/CommandLineTest.cpp index bbeb9d5dc19b..23f6081cd32a 100644 --- a/llvm/unittests/Support/CommandLineTest.cpp +++ b/llvm/unittests/Support/CommandLineTest.cpp @@ -1117,7 +1117,6 @@ TEST(CommandLineTest, BadResponseFile) { ASSERT_STREQ(Argv[0], "clang"); ASSERT_STREQ(Argv[1], AFileExp.c_str()); -#if !defined(_AIX) && !defined(__MVS__) std::string ADirExp = std::string("@") + std::string(ADir.path()); Argv = {"clang", ADirExp.c_str()}; Res = cl::ExpandResponseFiles(Saver, cl::TokenizeGNUCommandLine, Argv); @@ -1125,7 +1124,6 @@ TEST(CommandLineTest, BadResponseFile) { ASSERT_EQ(2U, Argv.size()); ASSERT_STREQ(Argv[0], "clang"); ASSERT_STREQ(Argv[1], ADirExp.c_str()); -#endif } TEST(CommandLineTest, SetDefaultValue) { diff --git a/llvm/unittests/Support/Path.cpp b/llvm/unittests/Support/Path.cpp index 837ca03216f8..bdae7a8ee4b5 100644 --- a/llvm/unittests/Support/Path.cpp +++ b/llvm/unittests/Support/Path.cpp @@ -1757,6 +1757,28 @@ TEST_F(FileSystemTest, OpenFileForRead) { #endif } +TEST_F(FileSystemTest, OpenDirectoryAsFileForRead) { + std::string Buf(5, '?'); + Expected FD = fs::openNativeFileForRead(TestDirectory); +#ifdef _WIN32 + EXPECT_EQ(errorToErrorCode(FD.takeError()), errc::is_a_directory); +#else + ASSERT_THAT_EXPECTED(FD, Succeeded()); + auto Close = make_scope_exit([&] { fs::closeFile(*FD); }); + Expected BytesRead = + fs::readNativeFile(*FD, MutableArrayRef(&*Buf.begin(), Buf.size())); + EXPECT_EQ(errorToErrorCode(BytesRead.takeError()), errc::is_a_directory); +#endif +} + +TEST_F(FileSystemTest, OpenDirectoryAsFileForWrite) { + int FD; + std::error_code EC = fs::openFileForWrite(Twine(TestDirectory), FD); + if (!EC) + ::close(FD); + EXPECT_EQ(EC, errc::is_a_directory); +} + static void createFileWithData(const Twine &Path, bool ShouldExistBefore, fs::CreationDisposition Disp, StringRef Data) { int FD; -- GitLab From 26101e8c50e07ea53869b0c7f343d76bfed61b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Wed, 17 Apr 2024 15:42:09 -0700 Subject: [PATCH 072/714] [flang][cuda] Avoid crash by exiting the check if assignment is not usable (#89149) In the presence of other semantic error, `GetAssignment` would return a nullptr and therefore would make the rest of the check crash when trying to collect symbols. Exiting early when we have a nullptr so the compiler doesn't crash and user can get the meaningful semantic error. --- flang/lib/Semantics/check-cuda.cpp | 4 ++++ flang/test/Semantics/cuf11.cuf | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/flang/lib/Semantics/check-cuda.cpp b/flang/lib/Semantics/check-cuda.cpp index 2cb15437a235..fb1ebadd3785 100644 --- a/flang/lib/Semantics/check-cuda.cpp +++ b/flang/lib/Semantics/check-cuda.cpp @@ -488,6 +488,10 @@ void CUDAChecker::Enter(const parser::AssignmentStmt &x) { } const evaluate::Assignment *assign{semantics::GetAssignment(x)}; + if (!assign) { + return; + } + int nbLhs{evaluate::GetNbOfCUDASymbols(assign->lhs)}; int nbRhs{evaluate::GetNbOfCUDASymbols(assign->rhs)}; diff --git a/flang/test/Semantics/cuf11.cuf b/flang/test/Semantics/cuf11.cuf index b915c7246ed9..de7ff2974324 100644 --- a/flang/test/Semantics/cuf11.cuf +++ b/flang/test/Semantics/cuf11.cuf @@ -22,3 +22,11 @@ subroutine sub1() ahost = adev + adev end subroutine + +logical function compare_h(a,b) +!ERROR: Derived type 'h' not found + type(h) :: a, b +!ERROR: 'a' is not an object of derived type; it is implicitly typed +!ERROR: 'b' is not an object of derived type; it is implicitly typed + compare_h = (a%h .eq. b%h) +end -- GitLab From 823eb1a3252dd773f9c4d92093591f1b39ac27d4 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 17 Apr 2024 16:52:02 -0700 Subject: [PATCH 073/714] [SelectionDAG] Add some validation of (S/U)(ADD/SUB)O_CARRY nodes. (#89133) --- llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index ca0a95750ba8..f3441c9df0f8 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -9915,6 +9915,18 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, } break; } + case ISD::SADDO_CARRY: + case ISD::UADDO_CARRY: + case ISD::SSUBO_CARRY: + case ISD::USUBO_CARRY: + assert(VTList.NumVTs == 2 && Ops.size() == 3 && + "Invalid add/sub overflow op!"); + assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() && + Ops[0].getValueType() == Ops[1].getValueType() && + Ops[0].getValueType() == VTList.VTs[0] && + Ops[2].getValueType().isInteger() && + "Binary operator types must match!"); + break; case ISD::SMUL_LOHI: case ISD::UMUL_LOHI: { assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!"); -- GitLab From 156ab4d4fb06be93b0cfce675e4cf86d330d879c Mon Sep 17 00:00:00 2001 From: Qizhi Hu <836744285@qq.com> Date: Thu, 18 Apr 2024 08:52:25 +0800 Subject: [PATCH 074/714] [Clang][Sema] set declaration invalid earlier to prevent crash in calculating record layout (#87173) Try to fix https://github.com/llvm/llvm-project/issues/75221 This crash caused by calculating record layout which contains a field declaration with dependent type. Make it invalid before it is a complete definition to prevent this crash. Define a new scope type to record this type alias and set the record declaration invalid when it is defined in a type alias template. Co-authored-by: huqizhi <836744285@qq.com> --- clang/docs/ReleaseNotes.rst | 2 ++ clang/include/clang/Sema/Scope.h | 6 ++++++ clang/lib/Parse/ParseDeclCXX.cpp | 5 +++++ clang/lib/Sema/SemaDecl.cpp | 7 +++++++ clang/test/SemaCXX/PR75221.cpp | 6 ++++++ 5 files changed, 26 insertions(+) create mode 100644 clang/test/SemaCXX/PR75221.cpp diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index c19ad9fba58f..6c51c2d1f483 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -536,6 +536,8 @@ Bug Fixes to C++ Support Fixes (#GH70604), (#GH79754), (#GH84163), (#GH84425), (#GH86054), (#GH86398), and (#GH86399). - Fix a crash when deducing ``auto`` from an invalid dereference (#GH88329). - Fix a crash in requires expression with templated base class member function. Fixes (#GH84020). +- Fix a crash caused by defined struct in a type alias template when the structure + has fields with dependent type. Fixes (#GH75221). Bug Fixes to AST Handling ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/include/clang/Sema/Scope.h b/clang/include/clang/Sema/Scope.h index 099c2739e860..1752a25111a7 100644 --- a/clang/include/clang/Sema/Scope.h +++ b/clang/include/clang/Sema/Scope.h @@ -156,6 +156,9 @@ public: /// This is the scope of an OpenACC Compute Construct, which restricts /// jumping into/out of it. OpenACCComputeConstructScope = 0x10000000, + + /// This is a scope of type alias declaration. + TypeAliasScope = 0x20000000, }; private: @@ -580,6 +583,9 @@ public: /// if/switch/while/for statement. bool isControlScope() const { return getFlags() & Scope::ControlScope; } + /// Determine whether this scope is a type alias scope. + bool isTypeAliasScope() const { return getFlags() & Scope::TypeAliasScope; } + /// Returns if rhs has a higher scope depth than this. /// /// The caller is responsible for calling this only if one of the two scopes diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp index 51fd64b2d01a..8e0e86824829 100644 --- a/clang/lib/Parse/ParseDeclCXX.cpp +++ b/clang/lib/Parse/ParseDeclCXX.cpp @@ -799,6 +799,11 @@ Parser::DeclGroupPtrTy Parser::ParseUsingDeclaration( ProhibitAttributes(PrefixAttrs); Decl *DeclFromDeclSpec = nullptr; + Scope *CurScope = getCurScope(); + if (CurScope) + CurScope->setFlags(Scope::ScopeFlags::TypeAliasScope | + CurScope->getFlags()); + Decl *AD = ParseAliasDeclarationAfterDeclarator( TemplateInfo, UsingLoc, D, DeclEnd, AS, Attrs, &DeclFromDeclSpec); return Actions.ConvertDeclToDeclGroup(AD, DeclFromDeclSpec); diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 1bde99d6fce7..356abe09a5ca 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -19529,6 +19529,13 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl, // Okay, we successfully defined 'Record'. if (Record) { bool Completed = false; + if (S) { + Scope *Parent = S->getParent(); + if (Parent && Parent->isTypeAliasScope() && + Parent->isTemplateParamScope()) + Record->setInvalidDecl(); + } + if (CXXRecord) { if (!CXXRecord->isInvalidDecl()) { // Set access bits correctly on the directly-declared conversions. diff --git a/clang/test/SemaCXX/PR75221.cpp b/clang/test/SemaCXX/PR75221.cpp new file mode 100644 index 000000000000..b342e347c560 --- /dev/null +++ b/clang/test/SemaCXX/PR75221.cpp @@ -0,0 +1,6 @@ +// RUN: %clang_cc1 -verify -std=c++11 -fsyntax-only %s + +template using foo = struct foo { // expected-error {{'foo' cannot be defined in a type alias template}} + T size = 0; +}; +foo a; -- GitLab From 888836930b1ed30b2095d45fcffc482560cb0e39 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Thu, 18 Apr 2024 10:15:25 +0900 Subject: [PATCH 075/714] Revert "[SLP]Attempt to vectorize long stores, if short one failed." This reverts commit 6f7160eedb2db02f37d4ffd52fff7b0cf88b3fdc. This still causes large compile-time regressions in some cases. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 106 ++++++------------ .../Transforms/SLPVectorizer/X86/pr46983.ll | 46 ++++++-- 2 files changed, 72 insertions(+), 80 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 806e8085038b..7694627c3b04 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -15164,6 +15164,10 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef Stores, BoUpSLP::ValueSet VectorizedStores; bool Changed = false; + // Stores the pair of stores (first_store, last_store) in a range, that were + // already tried to be vectorized. Allows to skip the store ranges that were + // already tried to be vectorized but the attempts were unsuccessful. + DenseSet> TriedSequences; struct StoreDistCompare { bool operator()(const std::pair &Op1, const std::pair &Op2) const { @@ -15205,10 +15209,8 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef Stores, Type *ValueTy = StoreTy; if (auto *Trunc = dyn_cast(Store->getValueOperand())) ValueTy = Trunc->getSrcTy(); - unsigned MinVF = std::max( - 2, PowerOf2Ceil(TTI->getStoreMinimumVF( - R.getMinVF(DL->getTypeStoreSizeInBits(StoreTy)), StoreTy, - ValueTy))); + unsigned MinVF = PowerOf2Ceil(TTI->getStoreMinimumVF( + R.getMinVF(DL->getTypeStoreSizeInBits(StoreTy)), StoreTy, ValueTy)); if (MaxVF < MinVF) { LLVM_DEBUG(dbgs() << "SLP: Vectorization infeasible as MaxVF (" << MaxVF @@ -15234,74 +15236,40 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef Stores, VF = Size > MaxVF ? NonPowerOf2VF : Size; Size *= 2; }); - unsigned End = Operands.size(); - unsigned Repeat = 0; - constexpr unsigned MaxAttempts = 2; - SmallBitVector Range(Operands.size()); - while (true) { - ++Repeat; - for (unsigned Size : CandidateVFs) { - int StartIdx = Range.find_first_unset(); - while (StartIdx != -1) { - int EndIdx = Range.find_next(StartIdx); - unsigned Sz = EndIdx == -1 ? End : EndIdx; - for (unsigned Cnt = StartIdx; Cnt + Size <= Sz;) { - ArrayRef Slice = ArrayRef(Operands).slice(Cnt, Size); - assert(all_of(Slice, - [&](Value *V) { - return cast(V) - ->getValueOperand() - ->getType() == - cast(Slice.front()) - ->getValueOperand() - ->getType(); - }) && - "Expected all operands of same type."); - if (vectorizeStoreChain(Slice, R, Cnt, MinVF)) { - // Mark the vectorized stores so that we don't vectorize them - // again. - VectorizedStores.insert(Slice.begin(), Slice.end()); - // Mark the vectorized stores so that we don't vectorize them - // again. - Changed = true; - // If we vectorized initial block, no need to try to vectorize - // it again. - Range.set(Cnt, Cnt + Size); - if (Cnt < StartIdx + MinVF) - Range.set(StartIdx, Cnt); - if (Cnt > EndIdx - Size - MinVF) { - Range.set(Cnt + Size, EndIdx); - End = Cnt; - } - Cnt += Size; - continue; - } - ++Cnt; - } - if (Sz >= End) - break; - StartIdx = Range.find_next_unset(EndIdx); + unsigned StartIdx = 0; + for (unsigned Size : CandidateVFs) { + for (unsigned Cnt = StartIdx, E = Operands.size(); Cnt + Size <= E;) { + ArrayRef Slice = ArrayRef(Operands).slice(Cnt, Size); + assert( + all_of( + Slice, + [&](Value *V) { + return cast(V)->getValueOperand()->getType() == + cast(Slice.front()) + ->getValueOperand() + ->getType(); + }) && + "Expected all operands of same type."); + if (!VectorizedStores.count(Slice.front()) && + !VectorizedStores.count(Slice.back()) && + TriedSequences.insert(std::make_pair(Slice.front(), Slice.back())) + .second && + vectorizeStoreChain(Slice, R, Cnt, MinVF)) { + // Mark the vectorized stores so that we don't vectorize them again. + VectorizedStores.insert(Slice.begin(), Slice.end()); + Changed = true; + // If we vectorized initial block, no need to try to vectorize it + // again. + if (Cnt == StartIdx) + StartIdx += Size; + Cnt += Size; + continue; } + ++Cnt; } - // All values vectorize - exit. - if (Range.all()) - break; - // Check if tried all attempts or no need for the last attempts at all. - if (Repeat >= MaxAttempts) - break; - constexpr unsigned MaxVFScale = 4; - constexpr unsigned StoresLimit = 16; - const unsigned MaxTotalNum = std::min( - std::max(StoresLimit, MaxVFScale * MaxVF), - bit_floor(static_cast(Range.find_last_unset() - - Range.find_first_unset() + 1))); - if (MaxVF >= MaxTotalNum) + // Check if the whole array was vectorized already - exit. + if (StartIdx >= Operands.size()) break; - // Last attempt to vectorize max number of elements, if all previous - // attempts were unsuccessful because of the cost issues. - CandidateVFs.clear(); - for (unsigned Size = MaxTotalNum; Size > MaxVF; Size /= 2) - CandidateVFs.push_back(Size); } } }; diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll index 3deab0975ce7..75505f632a43 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll @@ -100,17 +100,41 @@ define void @store_i8(ptr nocapture %0, i32 %1, i32 %2) { define void @store_i64(ptr nocapture %0, i32 %1, i32 %2) { ; SSE-LABEL: @store_i64( ; SSE-NEXT: [[TMP4:%.*]] = zext i32 [[TMP1:%.*]] to i64 -; SSE-NEXT: [[TMP5:%.*]] = load <4 x i64>, ptr [[TMP0:%.*]], align 8, !tbaa [[TBAA5:![0-9]+]] -; SSE-NEXT: [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0 -; SSE-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> zeroinitializer -; SSE-NEXT: [[TMP8:%.*]] = mul <4 x i64> [[TMP5]], [[TMP7]] -; SSE-NEXT: [[TMP9:%.*]] = lshr <4 x i64> [[TMP8]], -; SSE-NEXT: [[TMP10:%.*]] = trunc <4 x i64> [[TMP9]] to <4 x i32> -; SSE-NEXT: [[TMP11:%.*]] = icmp ult <4 x i32> [[TMP10]], -; SSE-NEXT: [[TMP12:%.*]] = trunc <4 x i64> [[TMP9]] to <4 x i32> -; SSE-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP11]], <4 x i32> [[TMP12]], <4 x i32> -; SSE-NEXT: [[TMP14:%.*]] = zext <4 x i32> [[TMP13]] to <4 x i64> -; SSE-NEXT: store <4 x i64> [[TMP14]], ptr [[TMP0]], align 8, !tbaa [[TBAA5]] +; SSE-NEXT: [[TMP5:%.*]] = load i64, ptr [[TMP0:%.*]], align 8, !tbaa [[TBAA5:![0-9]+]] +; SSE-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], [[TMP4]] +; SSE-NEXT: [[TMP7:%.*]] = lshr i64 [[TMP6]], 15 +; SSE-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32 +; SSE-NEXT: [[TMP9:%.*]] = icmp ult i32 [[TMP8]], 255 +; SSE-NEXT: [[TMP10:%.*]] = and i64 [[TMP7]], 4294967295 +; SSE-NEXT: [[TMP11:%.*]] = select i1 [[TMP9]], i64 [[TMP10]], i64 255 +; SSE-NEXT: store i64 [[TMP11]], ptr [[TMP0]], align 8, !tbaa [[TBAA5]] +; SSE-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8 +; SSE-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP12]], align 8, !tbaa [[TBAA5]] +; SSE-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], [[TMP4]] +; SSE-NEXT: [[TMP15:%.*]] = lshr i64 [[TMP14]], 15 +; SSE-NEXT: [[TMP16:%.*]] = trunc i64 [[TMP15]] to i32 +; SSE-NEXT: [[TMP17:%.*]] = icmp ult i32 [[TMP16]], 255 +; SSE-NEXT: [[TMP18:%.*]] = and i64 [[TMP15]], 4294967295 +; SSE-NEXT: [[TMP19:%.*]] = select i1 [[TMP17]], i64 [[TMP18]], i64 255 +; SSE-NEXT: store i64 [[TMP19]], ptr [[TMP12]], align 8, !tbaa [[TBAA5]] +; SSE-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16 +; SSE-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP20]], align 8, !tbaa [[TBAA5]] +; SSE-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], [[TMP4]] +; SSE-NEXT: [[TMP23:%.*]] = lshr i64 [[TMP22]], 15 +; SSE-NEXT: [[TMP24:%.*]] = trunc i64 [[TMP23]] to i32 +; SSE-NEXT: [[TMP25:%.*]] = icmp ult i32 [[TMP24]], 255 +; SSE-NEXT: [[TMP26:%.*]] = and i64 [[TMP23]], 4294967295 +; SSE-NEXT: [[TMP27:%.*]] = select i1 [[TMP25]], i64 [[TMP26]], i64 255 +; SSE-NEXT: store i64 [[TMP27]], ptr [[TMP20]], align 8, !tbaa [[TBAA5]] +; SSE-NEXT: [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 24 +; SSE-NEXT: [[TMP29:%.*]] = load i64, ptr [[TMP28]], align 8, !tbaa [[TBAA5]] +; SSE-NEXT: [[TMP30:%.*]] = mul i64 [[TMP29]], [[TMP4]] +; SSE-NEXT: [[TMP31:%.*]] = lshr i64 [[TMP30]], 15 +; SSE-NEXT: [[TMP32:%.*]] = trunc i64 [[TMP31]] to i32 +; SSE-NEXT: [[TMP33:%.*]] = icmp ult i32 [[TMP32]], 255 +; SSE-NEXT: [[TMP34:%.*]] = and i64 [[TMP31]], 4294967295 +; SSE-NEXT: [[TMP35:%.*]] = select i1 [[TMP33]], i64 [[TMP34]], i64 255 +; SSE-NEXT: store i64 [[TMP35]], ptr [[TMP28]], align 8, !tbaa [[TBAA5]] ; SSE-NEXT: ret void ; ; AVX-LABEL: @store_i64( -- GitLab From 748ef7eccc8b997ee5553cc5aadf04d7c7d0556f Mon Sep 17 00:00:00 2001 From: "Yaxun (Sam) Liu" Date: Wed, 17 Apr 2024 21:44:12 -0400 Subject: [PATCH 076/714] [CUDA][HIP] Fix record layout on Windows (#87651) On windows, record layout should be consistent with host side, otherwise host code is not able to access fields of the record correctly. Fixes: https://github.com/llvm/llvm-project/issues/51031 Fixes: SWDEV-446010 --- clang/lib/AST/RecordLayoutBuilder.cpp | 5 + clang/test/CodeGenCUDA/record-layout.cu | 200 ++++++++++++++++++++++++ 2 files changed, 205 insertions(+) create mode 100644 clang/test/CodeGenCUDA/record-layout.cu diff --git a/clang/lib/AST/RecordLayoutBuilder.cpp b/clang/lib/AST/RecordLayoutBuilder.cpp index a3b7431f7ffd..d9bf62c2bbb0 100644 --- a/clang/lib/AST/RecordLayoutBuilder.cpp +++ b/clang/lib/AST/RecordLayoutBuilder.cpp @@ -2458,6 +2458,11 @@ static bool mustSkipTailPadding(TargetCXXABI ABI, const CXXRecordDecl *RD) { } static bool isMsLayout(const ASTContext &Context) { + // Check if it's CUDA device compilation; ensure layout consistency with host. + if (Context.getLangOpts().CUDA && Context.getLangOpts().CUDAIsDevice && + Context.getAuxTargetInfo()) + return Context.getAuxTargetInfo()->getCXXABI().isMicrosoft(); + return Context.getTargetInfo().getCXXABI().isMicrosoft(); } diff --git a/clang/test/CodeGenCUDA/record-layout.cu b/clang/test/CodeGenCUDA/record-layout.cu new file mode 100644 index 000000000000..dd34121ccb9d --- /dev/null +++ b/clang/test/CodeGenCUDA/record-layout.cu @@ -0,0 +1,200 @@ +// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -fdump-record-layouts \ +// RUN: -emit-llvm -o %t -xhip %s 2>&1 | FileCheck %s --check-prefix=AST +// RUN: cat %t | FileCheck --check-prefixes=CHECK,HOST %s +// RUN: %clang_cc1 -fcuda-is-device -triple amdgcn-amd-amdhsa -target-cpu gfx1100 \ +// RUN: -emit-llvm -fdump-record-layouts -aux-triple x86_64-pc-windows-msvc \ +// RUN: -o %t -xhip %s | FileCheck %s --check-prefix=AST +// RUN: cat %t | FileCheck --check-prefixes=CHECK,DEV %s + +#include "Inputs/cuda.h" + +// AST: *** Dumping AST Record Layout +// AST-LABEL: 0 | struct C +// AST-NEXT: 0 | struct A (base) (empty) +// AST-NEXT: 1 | struct B (base) (empty) +// AST-NEXT: 4 | int i +// AST-NEXT: | [sizeof=8, align=4, +// AST-NEXT: | nvsize=8, nvalign=4] + +// CHECK: %struct.C = type { [4 x i8], i32 } + +struct A {}; +struct B {}; +struct C : A, B { + int i; +}; + +// AST: *** Dumping AST Record Layout +// AST-LABEL: 0 | struct I +// AST-NEXT: 0 | (I vftable pointer) +// AST-NEXT: 8 | int i +// AST-NEXT: | [sizeof=16, align=8, +// AST-NEXT: | nvsize=16, nvalign=8] + +// AST: *** Dumping AST Record Layout +// AST-LABEL: 0 | struct J +// AST-NEXT: 0 | struct I (primary base) +// AST-NEXT: 0 | (I vftable pointer) +// AST-NEXT: 8 | int i +// AST-NEXT: 16 | int j +// AST-NEXT: | [sizeof=24, align=8, +// AST-NEXT: | nvsize=24, nvalign=8] + +// CHECK: %struct.I = type { ptr, i32 } +// CHECK: %struct.J = type { %struct.I, i32 } + +// HOST: @0 = private unnamed_addr constant { [4 x ptr] } { [4 x ptr] [ptr @"??_R4J@@6B@", ptr @"?f@J@@UEAAXXZ", ptr null, ptr @"?h@J@@UEAAXXZ"] }, comdat($"??_7J@@6B@") +// HOST: @1 = private unnamed_addr constant { [4 x ptr] } { [4 x ptr] [ptr @"??_R4I@@6B@", ptr @_purecall, ptr null, ptr @_purecall] }, comdat($"??_7I@@6B@") +// HOST: @"??_7J@@6B@" = unnamed_addr alias ptr, getelementptr inbounds ({ [4 x ptr] }, ptr @0, i32 0, i32 0, i32 1) +// HOST: @"??_7I@@6B@" = unnamed_addr alias ptr, getelementptr inbounds ({ [4 x ptr] }, ptr @1, i32 0, i32 0, i32 1) + +// DEV: @_ZTV1J = linkonce_odr unnamed_addr addrspace(1) constant { [5 x ptr addrspace(1)] } { [5 x ptr addrspace(1)] [ptr addrspace(1) null, ptr addrspace(1) null, ptr addrspace(1) null, ptr addrspace(1) addrspacecast (ptr @_ZN1J1gEv to ptr addrspace(1)), ptr addrspace(1) addrspacecast (ptr @_ZN1J1hEv to ptr addrspace(1))] }, comdat, align 8 +// DEV: @_ZTV1I = linkonce_odr unnamed_addr addrspace(1) constant { [5 x ptr addrspace(1)] } { [5 x ptr addrspace(1)] [ptr addrspace(1) null, ptr addrspace(1) null, ptr addrspace(1) null, ptr addrspace(1) addrspacecast (ptr @__cxa_pure_virtual to ptr addrspace(1)), ptr addrspace(1) addrspacecast (ptr @__cxa_pure_virtual to ptr addrspace(1))] }, comdat, align 8 +struct I { + virtual void f() = 0; + __device__ virtual void g() = 0; + __device__ __host__ virtual void h() = 0; + int i; +}; + +struct J : I { + void f() override {} + __device__ void g() override {} + __device__ __host__ void h() override {} + int j; +}; + +// DEV: define dso_local amdgpu_kernel void @_Z8C_kernel1C(ptr addrspace(4) noundef byref(%struct.C) align 4 %0) +// DEV: %coerce = alloca %struct.C, align 4, addrspace(5) +// DEV: %c = addrspacecast ptr addrspace(5) %coerce to ptr +// DEV: call void @llvm.memcpy.p0.p4.i64(ptr align 4 %c, ptr addrspace(4) align 4 %0, i64 8, i1 false) +// DEV: %i = getelementptr inbounds %struct.C, ptr %c, i32 0, i32 1 +// DEV: store i32 1, ptr %i, align 4 + +__global__ void C_kernel(C c) +{ + c.i = 1; +} + +// HOST-LABEL: define dso_local void @"?test_C@@YAXXZ"() +// HOST: %c = alloca %struct.C, align 4 +// HOST: %i = getelementptr inbounds %struct.C, ptr %c, i32 0, i32 1 +// HOST: store i32 11, ptr %i, align 4 + +void test_C() { + C c; + c.i = 11; + C_kernel<<<1, 1>>>(c); +} + +// DEV: define dso_local void @_Z5J_devP1J(ptr noundef %j) +// DEV: %j.addr = alloca ptr, align 8, addrspace(5) +// DEV: %j.addr.ascast = addrspacecast ptr addrspace(5) %j.addr to ptr +// DEV: store ptr %j, ptr %j.addr.ascast, align 8 +// DEV: %0 = load ptr, ptr %j.addr.ascast, align 8 +// DEV: %i = getelementptr inbounds %struct.I, ptr %0, i32 0, i32 1 +// DEV: store i32 2, ptr %i, align 8 +// DEV: %1 = load ptr, ptr %j.addr.ascast, align 8 +// DEV: %j1 = getelementptr inbounds %struct.J, ptr %1, i32 0, i32 1 +// DEV: store i32 3, ptr %j1, align 8 +// DEV: %2 = load ptr, ptr %j.addr.ascast, align 8 +// DEV: %vtable = load ptr addrspace(1), ptr %2, align 8 +// DEV: %vfn = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) %vtable, i64 1 +// DEV: %3 = load ptr addrspace(1), ptr addrspace(1) %vfn, align 8 +// DEV: call addrspace(1) void %3(ptr noundef nonnull align 8 dereferenceable(24) %2) +// DEV: %4 = load ptr, ptr %j.addr.ascast, align 8 +// DEV: %vtable2 = load ptr addrspace(1), ptr %4, align 8 +// DEV: %vfn3 = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) %vtable2, i64 2 +// DEV: %5 = load ptr addrspace(1), ptr addrspace(1) %vfn3, align 8 +// DEV: call addrspace(1) void %5(ptr noundef nonnull align 8 dereferenceable(24) %4) + +__device__ void J_dev(J *j) { + j->i = 2; + j->j = 3; + j->g(); + j->h(); +} + +// DEV: define dso_local amdgpu_kernel void @_Z8J_kernelv() +// DEV: %j = alloca %struct.J, align 8, addrspace(5) +// DEV: %j.ascast = addrspacecast ptr addrspace(5) %j to ptr +// DEV: call void @_ZN1JC1Ev(ptr noundef nonnull align 8 dereferenceable(24) %j.ascast) +// DEV: call void @_Z5J_devP1J(ptr noundef %j.ascast) + +__global__ void J_kernel() { + J j; + J_dev(&j); +} + +// HOST-LABEL: define dso_local void @"?J_host@@YAXPEAUJ@@@Z"(ptr noundef %j) +// HOST: %0 = load ptr, ptr %j.addr, align 8 +// HOST: %i = getelementptr inbounds %struct.I, ptr %0, i32 0, i32 1 +// HOST: store i32 12, ptr %i, align 8 +// HOST: %1 = load ptr, ptr %j.addr, align 8 +// HOST: %j1 = getelementptr inbounds %struct.J, ptr %1, i32 0, i32 1 +// HOST: store i32 13, ptr %j1, align 8 +// HOST: %2 = load ptr, ptr %j.addr, align 8 +// HOST: %vtable = load ptr, ptr %2, align 8 +// HOST: %vfn = getelementptr inbounds ptr, ptr %vtable, i64 0 +// HOST: %3 = load ptr, ptr %vfn, align 8 +// HOST: call void %3(ptr noundef nonnull align 8 dereferenceable(24) %2) +// HOST: %4 = load ptr, ptr %j.addr, align 8 +// HOST: %vtable2 = load ptr, ptr %4, align 8 +// HOST: %vfn3 = getelementptr inbounds ptr, ptr %vtable2, i64 2 +// HOST: %5 = load ptr, ptr %vfn3, align 8 +// HOST: call void %5(ptr noundef nonnull align 8 dereferenceable(24) %4) + +void J_host(J *j) { + j->i = 12; + j->j = 13; + j->f(); + j->h(); +} + +// HOST: define dso_local void @"?test_J@@YAXXZ"() +// HOST: %j = alloca %struct.J, align 8 +// HOST: %call = call noundef ptr @"??0J@@QEAA@XZ"(ptr noundef nonnull align 8 dereferenceable(24) %j) +// HOST: call void @"?J_host@@YAXPEAUJ@@@Z"(ptr noundef %j) + +void test_J() { + J j; + J_host(&j); + J_kernel<<<1, 1>>>(); +} + +// HOST: define linkonce_odr dso_local noundef ptr @"??0J@@QEAA@XZ"(ptr noundef nonnull returned align 8 dereferenceable(24) %this) +// HOST: %this.addr = alloca ptr, align 8 +// HOST: store ptr %this, ptr %this.addr, align 8 +// HOST: %this1 = load ptr, ptr %this.addr, align 8 +// HOST: %call = call noundef ptr @"??0I@@QEAA@XZ"(ptr noundef nonnull align 8 dereferenceable(16) %this1) #5 +// HOST: store ptr @"??_7J@@6B@", ptr %this1, align 8 +// HOST: ret ptr %this1 + +// HOST: define linkonce_odr dso_local noundef ptr @"??0I@@QEAA@XZ"(ptr noundef nonnull returned align 8 dereferenceable(16) %this) +// HOST: %this.addr = alloca ptr, align 8 +// HOST: store ptr %this, ptr %this.addr, align 8 +// HOST: %this1 = load ptr, ptr %this.addr, align 8 +// HOST: store ptr @"??_7I@@6B@", ptr %this1, align 8 +// HOST: ret ptr %this1 + +// DEV: define linkonce_odr void @_ZN1JC1Ev(ptr noundef nonnull align 8 dereferenceable(24) %this) +// DEV: %this.addr = alloca ptr, align 8, addrspace(5) +// DEV: %this.addr.ascast = addrspacecast ptr addrspace(5) %this.addr to ptr +// DEV: store ptr %this, ptr %this.addr.ascast, align 8 +// DEV: %this1 = load ptr, ptr %this.addr.ascast, align 8 +// DEV: call void @_ZN1JC2Ev(ptr noundef nonnull align 8 dereferenceable(24) %this1) + +// DEV: define linkonce_odr void @_ZN1JC2Ev(ptr noundef nonnull align 8 dereferenceable(24) %this) +// DEV: %this.addr = alloca ptr, align 8, addrspace(5) +// DEV: %this.addr.ascast = addrspacecast ptr addrspace(5) %this.addr to ptr +// DEV: store ptr %this, ptr %this.addr.ascast, align 8 +// DEV: %this1 = load ptr, ptr %this.addr.ascast, align 8 +// DEV: call void @_ZN1IC2Ev(ptr noundef nonnull align 8 dereferenceable(16) %this1) +// DEV: store ptr addrspace(1) getelementptr inbounds inrange(-16, 24) ({ [5 x ptr addrspace(1)] }, ptr addrspace(1) @_ZTV1J, i32 0, i32 0, i32 2), ptr %this1, align 8 + +// DEV: define linkonce_odr void @_ZN1IC2Ev(ptr noundef nonnull align 8 dereferenceable(16) %this) +// DEV: %this.addr = alloca ptr, align 8, addrspace(5) +// DEV: %this.addr.ascast = addrspacecast ptr addrspace(5) %this.addr to ptr +// DEV: store ptr %this, ptr %this.addr.ascast, align 8 +// DEV: %this1 = load ptr, ptr %this.addr.ascast, align 8 +// DEV: store ptr addrspace(1) getelementptr inbounds inrange(-16, 24) ({ [5 x ptr addrspace(1)] }, ptr addrspace(1) @_ZTV1I, i32 0, i32 0, i32 2), ptr %this1, align 8 -- GitLab From 097b68ff067565af744b5b0327732d7c199710ab Mon Sep 17 00:00:00 2001 From: Shih-Po Hung Date: Thu, 18 Apr 2024 09:44:31 +0800 Subject: [PATCH 077/714] [RISCV][TTI] Refine the cost of FCmp (#88833) This patch introduces following changes - Support all fp predicates - Use the Val type to estimate the latency/throughput cost - Assign a cost of 1 for mask operations as LMULCost for mask types cannot be correctly estimated. --- .../Target/RISCV/RISCVTargetTransformInfo.cpp | 48 +- llvm/test/Analysis/CostModel/RISCV/rvv-cmp.ll | 582 +++++++++++------- .../Analysis/CostModel/RISCV/rvv-select.ll | 8 +- 3 files changed, 396 insertions(+), 242 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 56f5bd8794ae..ce26e61880fd 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -91,6 +91,10 @@ RISCVTTIImpl::getRISCVInstructionCost(ArrayRef OpCodes, MVT VT, case RISCV::VMV_S_X: case RISCV::VFMV_F_S: case RISCV::VFMV_S_F: + case RISCV::VMOR_MM: + case RISCV::VMXOR_MM: + case RISCV::VMAND_MM: + case RISCV::VMANDN_MM: case RISCV::VMNAND_MM: case RISCV::VCPOP_M: Cost += 1; @@ -1383,7 +1387,13 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, getRISCVInstructionCost(RISCV::VMSLT_VV, LT.second, CostKind); } - if ((Opcode == Instruction::FCmp) && ValTy->isVectorTy()) { + if ((Opcode == Instruction::FCmp) && ValTy->isVectorTy() && + CmpInst::isFPPredicate(VecPred)) { + + // Use VMXOR_MM and VMXNOR_MM to generate all true/false mask + if ((VecPred == CmpInst::FCMP_FALSE) || (VecPred == CmpInst::FCMP_TRUE)) + return getRISCVInstructionCost(RISCV::VMXOR_MM, LT.second, CostKind); + // If we do not support the input floating point vector type, use the base // one which will calculate as: // ScalarizeCost + Num * Cost for fixed vector, @@ -1393,16 +1403,34 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, (ValTy->getScalarSizeInBits() == 64 && !ST->hasVInstructionsF64())) return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I); + + // Assuming vector fp compare and mask instructions are all the same cost + // until a need arises to differentiate them. switch (VecPred) { - // Support natively. - case CmpInst::FCMP_OEQ: - case CmpInst::FCMP_OGT: - case CmpInst::FCMP_OGE: - case CmpInst::FCMP_OLT: - case CmpInst::FCMP_OLE: - case CmpInst::FCMP_UNE: - return LT.first * 1; - // TODO: Other comparisons? + case CmpInst::FCMP_ONE: // vmflt.vv + vmflt.vv + vmor.mm + case CmpInst::FCMP_ORD: // vmfeq.vv + vmfeq.vv + vmand.mm + case CmpInst::FCMP_UNO: // vmfne.vv + vmfne.vv + vmor.mm + case CmpInst::FCMP_UEQ: // vmflt.vv + vmflt.vv + vmnor.mm + return LT.first * getRISCVInstructionCost( + {RISCV::VMFLT_VV, RISCV::VMFLT_VV, RISCV::VMOR_MM}, + LT.second, CostKind); + + case CmpInst::FCMP_UGT: // vmfle.vv + vmnot.m + case CmpInst::FCMP_UGE: // vmflt.vv + vmnot.m + case CmpInst::FCMP_ULT: // vmfle.vv + vmnot.m + case CmpInst::FCMP_ULE: // vmflt.vv + vmnot.m + return LT.first * + getRISCVInstructionCost({RISCV::VMFLT_VV, RISCV::VMNAND_MM}, + LT.second, CostKind); + + case CmpInst::FCMP_OEQ: // vmfeq.vv + case CmpInst::FCMP_OGT: // vmflt.vv + case CmpInst::FCMP_OGE: // vmfle.vv + case CmpInst::FCMP_OLT: // vmflt.vv + case CmpInst::FCMP_OLE: // vmfle.vv + case CmpInst::FCMP_UNE: // vmfne.vv + return LT.first * + getRISCVInstructionCost(RISCV::VMFLT_VV, LT.second, CostKind); default: break; } diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-cmp.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-cmp.ll index caa6d6f483a2..8aa78ec95e8a 100644 --- a/llvm/test/Analysis/CostModel/RISCV/rvv-cmp.ll +++ b/llvm/test/Analysis/CostModel/RISCV/rvv-cmp.ll @@ -878,28 +878,28 @@ define void @fcmp_oeq() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp oeq <2 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp oeq <4 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp oeq <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp oeq <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16f16 = fcmp oeq <16 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp oeq undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp oeq undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp oeq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp oeq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp oeq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8f16 = fcmp oeq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv16f16 = fcmp oeq undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp oeq <2 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp oeq <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp oeq <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp oeq <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f32 = fcmp oeq <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16f32 = fcmp oeq <16 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp oeq undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp oeq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp oeq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp oeq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp oeq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32 = fcmp oeq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv8f32 = fcmp oeq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16f32 = fcmp oeq undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp oeq <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp oeq <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp oeq <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f64 = fcmp oeq <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8f64 = fcmp oeq <8 x double> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp oeq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp oeq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp oeq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp oeq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64 = fcmp oeq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64 = fcmp oeq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv8f64 = fcmp oeq undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp oeq <2 x half> undef, undef @@ -938,31 +938,31 @@ define void @fcmp_oeq() { define void @fcmp_one() { ; CHECK-LABEL: 'fcmp_one' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp one <2 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp one <4 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp one <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp one <16 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp one <2 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp one <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp one <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp one <16 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp one <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp one <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp one <8 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp one undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2f16 = fcmp one <2 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f16 = fcmp one <4 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8f16 = fcmp one <8 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v16f16 = fcmp one <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv1f16 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv2f16 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv4f16 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv8f16 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16f16 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2f32 = fcmp one <2 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f32 = fcmp one <4 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v8f32 = fcmp one <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v16f32 = fcmp one <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv1f32 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv2f32 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv4f32 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv8f32 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %nxv16f32 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2f64 = fcmp one <2 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v4f64 = fcmp one <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v8f64 = fcmp one <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv1f64 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv2f64 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv4f64 = fcmp one undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %nxv8f64 = fcmp one undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp one <2 x half> undef, undef @@ -1004,28 +1004,28 @@ define void @fcmp_olt() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp olt <2 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp olt <4 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp olt <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp olt <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16f16 = fcmp olt <16 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp olt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp olt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp olt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp olt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp olt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8f16 = fcmp olt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv16f16 = fcmp olt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp olt <2 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp olt <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp olt <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp olt <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f32 = fcmp olt <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16f32 = fcmp olt <16 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp olt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp olt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp olt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp olt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp olt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32 = fcmp olt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv8f32 = fcmp olt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16f32 = fcmp olt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp olt <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp olt <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp olt <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f64 = fcmp olt <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8f64 = fcmp olt <8 x double> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp olt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp olt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp olt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp olt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64 = fcmp olt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64 = fcmp olt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv8f64 = fcmp olt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp olt <2 x half> undef, undef @@ -1067,28 +1067,28 @@ define void @fcmp_ole() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp ole <2 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp ole <4 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp ole <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp ole <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16f16 = fcmp ole <16 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp ole undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp ole undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp ole undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp ole undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp ole undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8f16 = fcmp ole undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv16f16 = fcmp ole undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp ole <2 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp ole <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp ole <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp ole <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f32 = fcmp ole <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16f32 = fcmp ole <16 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp ole undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp ole undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp ole undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp ole undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp ole undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32 = fcmp ole undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv8f32 = fcmp ole undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16f32 = fcmp ole undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp ole <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp ole <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp ole <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f64 = fcmp ole <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8f64 = fcmp ole <8 x double> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp ole undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp ole undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp ole undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp ole undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64 = fcmp ole undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64 = fcmp ole undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv8f64 = fcmp ole undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp ole <2 x half> undef, undef @@ -1130,28 +1130,28 @@ define void @fcmp_ogt() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp ogt <2 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp ogt <4 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp ogt <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp ogt <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16f16 = fcmp ogt <16 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp ogt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp ogt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp ogt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp ogt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp ogt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8f16 = fcmp ogt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv16f16 = fcmp ogt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp ogt <2 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp ogt <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp ogt <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp ogt <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f32 = fcmp ogt <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16f32 = fcmp ogt <16 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp ogt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp ogt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp ogt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp ogt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp ogt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32 = fcmp ogt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv8f32 = fcmp ogt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16f32 = fcmp ogt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp ogt <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp ogt <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp ogt <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f64 = fcmp ogt <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8f64 = fcmp ogt <8 x double> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp ogt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp ogt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp ogt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp ogt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64 = fcmp ogt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64 = fcmp ogt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv8f64 = fcmp ogt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp ogt <2 x half> undef, undef @@ -1193,28 +1193,28 @@ define void @fcmp_oge() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp oge <2 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp oge <4 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp oge <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp oge <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16f16 = fcmp oge <16 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp oge undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp oge undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp oge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp oge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp oge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8f16 = fcmp oge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv16f16 = fcmp oge undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp oge <2 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp oge <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp oge <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp oge <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f32 = fcmp oge <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16f32 = fcmp oge <16 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp oge undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp oge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp oge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp oge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp oge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32 = fcmp oge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv8f32 = fcmp oge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16f32 = fcmp oge undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp oge <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp oge <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp oge <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f64 = fcmp oge <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8f64 = fcmp oge <8 x double> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp oge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp oge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp oge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp oge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64 = fcmp oge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64 = fcmp oge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv8f64 = fcmp oge undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp oge <2 x half> undef, undef @@ -1253,31 +1253,31 @@ define void @fcmp_oge() { define void @fcmp_ueq() { ; CHECK-LABEL: 'fcmp_ueq' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp ueq <2 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp ueq <4 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp ueq <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp ueq <16 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp ueq <2 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp ueq <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp ueq <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp ueq <16 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp ueq <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp ueq <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp ueq <8 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp ueq undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2f16 = fcmp ueq <2 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f16 = fcmp ueq <4 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8f16 = fcmp ueq <8 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v16f16 = fcmp ueq <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv1f16 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv2f16 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv4f16 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv8f16 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16f16 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2f32 = fcmp ueq <2 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f32 = fcmp ueq <4 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v8f32 = fcmp ueq <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v16f32 = fcmp ueq <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv1f32 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv2f32 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv4f32 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv8f32 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %nxv16f32 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v2f64 = fcmp ueq <2 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v4f64 = fcmp ueq <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v8f64 = fcmp ueq <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv1f64 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv2f64 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv4f64 = fcmp ueq undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %nxv8f64 = fcmp ueq undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp ueq <2 x half> undef, undef @@ -1319,28 +1319,28 @@ define void @fcmp_une() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp une <2 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp une <4 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp une <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp une <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v16f16 = fcmp une <16 x half> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp une undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp une undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp une undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp une undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp une undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv8f16 = fcmp une undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv16f16 = fcmp une undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp une <2 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp une <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp une <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp une <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f32 = fcmp une <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v16f32 = fcmp une <16 x float> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp une undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp une undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp une undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp une undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp une undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32 = fcmp une undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv8f32 = fcmp une undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv16f32 = fcmp une undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp une <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp une <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp une <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f64 = fcmp une <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8f64 = fcmp une <8 x double> undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp une undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp une undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp une undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp une undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64 = fcmp une undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64 = fcmp une undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %nxv8f64 = fcmp une undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp une <2 x half> undef, undef @@ -1379,31 +1379,31 @@ define void @fcmp_une() { define void @fcmp_ult() { ; CHECK-LABEL: 'fcmp_ult' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp ult <2 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp ult <4 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp ult <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp ult <16 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp ult <2 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp ult <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp ult <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp ult <16 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp ult <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp ult <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp ult <8 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp ult undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp ult <2 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f16 = fcmp ult <4 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f16 = fcmp ult <8 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v16f16 = fcmp ult <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f16 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f16 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f16 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv8f16 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv16f16 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32 = fcmp ult <2 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f32 = fcmp ult <4 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8f32 = fcmp ult <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v16f32 = fcmp ult <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f32 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv4f32 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv8f32 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16f32 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f64 = fcmp ult <2 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f64 = fcmp ult <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v8f64 = fcmp ult <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f64 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv2f64 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv4f64 = fcmp ult undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv8f64 = fcmp ult undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp ult <2 x half> undef, undef @@ -1442,31 +1442,31 @@ define void @fcmp_ult() { define void @fcmp_ule() { ; CHECK-LABEL: 'fcmp_ule' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp ule <2 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp ule <4 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp ule <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp ule <16 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp ule <2 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp ule <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp ule <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp ule <16 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp ule <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp ule <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp ule <8 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp ule undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp ule <2 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f16 = fcmp ule <4 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f16 = fcmp ule <8 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v16f16 = fcmp ule <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f16 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f16 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f16 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv8f16 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv16f16 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32 = fcmp ule <2 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f32 = fcmp ule <4 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8f32 = fcmp ule <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v16f32 = fcmp ule <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f32 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv4f32 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv8f32 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16f32 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f64 = fcmp ule <2 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f64 = fcmp ule <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v8f64 = fcmp ule <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f64 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv2f64 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv4f64 = fcmp ule undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv8f64 = fcmp ule undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp ule <2 x half> undef, undef @@ -1505,31 +1505,31 @@ define void @fcmp_ule() { define void @fcmp_ugt() { ; CHECK-LABEL: 'fcmp_ugt' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp ugt <2 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp ugt <4 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp ugt <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp ugt <16 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp ugt <2 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp ugt <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp ugt <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp ugt <16 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp ugt <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp ugt <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp ugt <8 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp ugt undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp ugt <2 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f16 = fcmp ugt <4 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f16 = fcmp ugt <8 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v16f16 = fcmp ugt <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f16 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f16 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f16 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv8f16 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv16f16 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32 = fcmp ugt <2 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f32 = fcmp ugt <4 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8f32 = fcmp ugt <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v16f32 = fcmp ugt <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f32 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv4f32 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv8f32 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16f32 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f64 = fcmp ugt <2 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f64 = fcmp ugt <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v8f64 = fcmp ugt <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f64 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv2f64 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv4f64 = fcmp ugt undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv8f64 = fcmp ugt undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp ugt <2 x half> undef, undef @@ -1568,31 +1568,31 @@ define void @fcmp_ugt() { define void @fcmp_uge() { ; CHECK-LABEL: 'fcmp_uge' -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp uge <2 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp uge <4 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp uge <8 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp uge <16 x half> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp uge <2 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp uge <4 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp uge <8 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp uge <16 x float> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp uge <2 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp uge <4 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp uge <8 x double> undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp uge undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f16 = fcmp uge <2 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f16 = fcmp uge <4 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v8f16 = fcmp uge <8 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v16f16 = fcmp uge <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f16 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f16 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv4f16 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv8f16 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv16f16 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f32 = fcmp uge <2 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4f32 = fcmp uge <4 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v8f32 = fcmp uge <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v16f32 = fcmp uge <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f32 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv4f32 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv8f32 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv16f32 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v2f64 = fcmp uge <2 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4f64 = fcmp uge <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %v8f64 = fcmp uge <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %nxv1f64 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %nxv2f64 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %nxv4f64 = fcmp uge undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %nxv8f64 = fcmp uge undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %v2f16 = fcmp uge <2 x half> undef, undef @@ -1628,3 +1628,129 @@ define void @fcmp_uge() { ret void } + +define void @fcmp_true() { +; CHECK-LABEL: 'fcmp_true' +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp true <2 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp true <4 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp true <8 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp true <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp true <2 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp true <4 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp true <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp true <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp true <2 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp true <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp true <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp true undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + %v2f16 = fcmp true <2 x half> undef, undef + %v4f16 = fcmp true <4 x half> undef, undef + %v8f16 = fcmp true <8 x half> undef, undef + %v16f16 = fcmp true <16 x half> undef, undef + + %nxv1f16 = fcmp true undef, undef + %nxv2f16 = fcmp true undef, undef + %nxv4f16 = fcmp true undef, undef + %nxv8f16 = fcmp true undef, undef + %nxv16f16 = fcmp true undef, undef + + %v2f32 = fcmp true <2 x float> undef, undef + %v4f32 = fcmp true <4 x float> undef, undef + %v8f32 = fcmp true <8 x float> undef, undef + %v16f32 = fcmp true <16 x float> undef, undef + + %nxv1f32 = fcmp true undef, undef + %nxv2f32 = fcmp true undef, undef + %nxv4f32 = fcmp true undef, undef + %nxv8f32 = fcmp true undef, undef + %nxv16f32 = fcmp true undef, undef + + %v2f64 = fcmp true <2 x double> undef, undef + %v4f64 = fcmp true <4 x double> undef, undef + %v8f64 = fcmp true <8 x double> undef, undef + + %nxv1f64 = fcmp true undef, undef + %nxv2f64 = fcmp true undef, undef + %nxv4f64 = fcmp true undef, undef + %nxv8f64 = fcmp true undef, undef + + ret void +} + +define void @fcmp_false() { +; CHECK-LABEL: 'fcmp_false' +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f16 = fcmp false <2 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f16 = fcmp false <4 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f16 = fcmp false <8 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f16 = fcmp false <16 x half> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f16 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f16 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f16 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f32 = fcmp false <2 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f32 = fcmp false <4 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f32 = fcmp false <8 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16f32 = fcmp false <16 x float> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f32 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv16f32 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2f64 = fcmp false <2 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4f64 = fcmp false <4 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8f64 = fcmp false <8 x double> undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv4f64 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %nxv8f64 = fcmp false undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + %v2f16 = fcmp false <2 x half> undef, undef + %v4f16 = fcmp false <4 x half> undef, undef + %v8f16 = fcmp false <8 x half> undef, undef + %v16f16 = fcmp false <16 x half> undef, undef + + %nxv1f16 = fcmp false undef, undef + %nxv2f16 = fcmp false undef, undef + %nxv4f16 = fcmp false undef, undef + %nxv8f16 = fcmp false undef, undef + %nxv16f16 = fcmp false undef, undef + + %v2f32 = fcmp false <2 x float> undef, undef + %v4f32 = fcmp false <4 x float> undef, undef + %v8f32 = fcmp false <8 x float> undef, undef + %v16f32 = fcmp false <16 x float> undef, undef + + %nxv1f32 = fcmp false undef, undef + %nxv2f32 = fcmp false undef, undef + %nxv4f32 = fcmp false undef, undef + %nxv8f32 = fcmp false undef, undef + %nxv16f32 = fcmp false undef, undef + + %v2f64 = fcmp false <2 x double> undef, undef + %v4f64 = fcmp false <4 x double> undef, undef + %v8f64 = fcmp false <8 x double> undef, undef + + %nxv1f64 = fcmp false undef, undef + %nxv2f64 = fcmp false undef, undef + %nxv4f64 = fcmp false undef, undef + %nxv8f64 = fcmp false undef, undef + + ret void +} diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-select.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-select.ll index 6dcbc73674aa..b80096b057a6 100644 --- a/llvm/test/Analysis/CostModel/RISCV/rvv-select.ll +++ b/llvm/test/Analysis/CostModel/RISCV/rvv-select.ll @@ -22,14 +22,14 @@ define void @select() { ; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %15 = select i1 undef, undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %16 = select i1 undef, undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %17 = select i1 undef, undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %18 = select i1 undef, undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %19 = select i1 undef, undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %18 = select i1 undef, undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %19 = select i1 undef, undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %20 = select undef, undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %21 = select undef, undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %22 = select undef, undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %23 = select undef, undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %24 = select undef, undef, undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %25 = select undef, undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %24 = select undef, undef, undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %25 = select undef, undef, undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %26 = select i1 undef, i8 undef, i8 undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %27 = select i1 undef, <1 x i8> undef, <1 x i8> undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %28 = select i1 undef, <2 x i8> undef, <2 x i8> undef -- GitLab From 525d00e5edc4b83105c6ad518926b174f76c3a1c Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Thu, 18 Apr 2024 10:56:20 +0900 Subject: [PATCH 078/714] [InstCombine] Fix poison propagation in round up alignment fold We can't directly use the high bits value if it is more poisonous due to poison elements in the masks. This fixes the issue reported in https://github.com/llvm/llvm-project/pull/88217#issuecomment-2061034941. --- .../InstCombine/InstCombineSelect.cpp | 3 ++- .../integer-round-up-pow2-alignment.ll | 20 +++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp index 2d78fcee1152..41c3ab85940e 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -2965,7 +2965,8 @@ foldRoundUpIntegerWithPow2Alignment(SelectInst &SI, return nullptr; if (!XBiasedHighBits->hasOneUse()) { - if (*BiasCst == *LowBitMaskCst) + // We can't directly return XBiasedHighBits if it is more poisonous. + if (*BiasCst == *LowBitMaskCst && impliesPoison(XBiasedHighBits, X)) return XBiasedHighBits; return nullptr; } diff --git a/llvm/test/Transforms/InstCombine/integer-round-up-pow2-alignment.ll b/llvm/test/Transforms/InstCombine/integer-round-up-pow2-alignment.ll index c7e0553992b9..afd56abf40a5 100644 --- a/llvm/test/Transforms/InstCombine/integer-round-up-pow2-alignment.ll +++ b/llvm/test/Transforms/InstCombine/integer-round-up-pow2-alignment.ll @@ -437,14 +437,17 @@ define i8 @t17_oneuse(i8 %x) { ret i8 %x.roundedup } -; Bias is equal to the alignment-1 (as opposed to alignment), -; so we can just replace %x.roundedup with %x.biased.highbits +; Negative test: We can't replace with %x.biased.highbits because it is +; more poisonous. define <2 x i4> @t18_replacement_0b0001(<2 x i4> %x) { ; CHECK-LABEL: @t18_replacement_0b0001( -; CHECK-NEXT: [[X_BIASED:%.*]] = add <2 x i4> [[X:%.*]], +; CHECK-NEXT: [[X_LOWBITS:%.*]] = and <2 x i4> [[X:%.*]], +; CHECK-NEXT: [[X_LOWBITS_ARE_ZERO:%.*]] = icmp eq <2 x i4> [[X_LOWBITS]], zeroinitializer +; CHECK-NEXT: [[X_BIASED:%.*]] = add <2 x i4> [[X]], ; CHECK-NEXT: [[X_BIASED_HIGHBITS:%.*]] = and <2 x i4> [[X_BIASED]], ; CHECK-NEXT: call void @use.v2i4(<2 x i4> [[X_BIASED_HIGHBITS]]) -; CHECK-NEXT: ret <2 x i4> [[X_BIASED_HIGHBITS]] +; CHECK-NEXT: [[X_ROUNDEDUP:%.*]] = select <2 x i1> [[X_LOWBITS_ARE_ZERO]], <2 x i4> [[X]], <2 x i4> [[X_BIASED_HIGHBITS]] +; CHECK-NEXT: ret <2 x i4> [[X_ROUNDEDUP]] ; %x.lowbits = and <2 x i4> %x, %x.lowbits.are.zero = icmp eq <2 x i4> %x.lowbits, @@ -454,12 +457,17 @@ define <2 x i4> @t18_replacement_0b0001(<2 x i4> %x) { %x.roundedup = select <2 x i1> %x.lowbits.are.zero, <2 x i4> %x, <2 x i4> %x.biased.highbits ret <2 x i4> %x.roundedup } +; Negative test: We can't replace with %x.biased.highbits because it is +; more poisonous. define <2 x i4> @t18_replacement_0b0010(<2 x i4> %x) { ; CHECK-LABEL: @t18_replacement_0b0010( -; CHECK-NEXT: [[X_BIASED:%.*]] = add <2 x i4> [[X:%.*]], +; CHECK-NEXT: [[X_LOWBITS:%.*]] = and <2 x i4> [[X:%.*]], +; CHECK-NEXT: [[X_LOWBITS_ARE_ZERO:%.*]] = icmp eq <2 x i4> [[X_LOWBITS]], zeroinitializer +; CHECK-NEXT: [[X_BIASED:%.*]] = add <2 x i4> [[X]], ; CHECK-NEXT: [[X_BIASED_HIGHBITS:%.*]] = and <2 x i4> [[X_BIASED]], ; CHECK-NEXT: call void @use.v2i4(<2 x i4> [[X_BIASED_HIGHBITS]]) -; CHECK-NEXT: ret <2 x i4> [[X_BIASED_HIGHBITS]] +; CHECK-NEXT: [[X_ROUNDEDUP:%.*]] = select <2 x i1> [[X_LOWBITS_ARE_ZERO]], <2 x i4> [[X]], <2 x i4> [[X_BIASED_HIGHBITS]] +; CHECK-NEXT: ret <2 x i4> [[X_ROUNDEDUP]] ; %x.lowbits = and <2 x i4> %x, %x.lowbits.are.zero = icmp eq <2 x i4> %x.lowbits, -- GitLab From 0ee260ec37a039ffc276f2f827c842f71513a3e9 Mon Sep 17 00:00:00 2001 From: Kai Luo Date: Thu, 18 Apr 2024 10:00:56 +0800 Subject: [PATCH 079/714] [PowerPC] `ANDI_rec_1_*` should define CR0 (#89034) These pseudo instructions finally copy the result to CR0 so they should define `CR0` in their definitions. --- llvm/lib/Target/PowerPC/PPCInstrInfo.td | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td index 43e3902cf407..261b9a3d1dff 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td @@ -4191,6 +4191,7 @@ def : Pat<(v4i32 (selectcc i1:$lhs, i1:$rhs, v4i32:$tval, v4i32:$fval, SETUGT)), def : Pat<(v4i32 (selectcc i1:$lhs, i1:$rhs, v4i32:$tval, v4i32:$fval, SETNE)), (SELECT_VRRC (CRXOR $lhs, $rhs), $tval, $fval)>; +let Defs = [CR0] in { def ANDI_rec_1_EQ_BIT : PPCCustomInserterPseudo<(outs crbitrc:$dst), (ins gprc:$in), "#ANDI_rec_1_EQ_BIT", [(set i1:$dst, (trunc (not i32:$in)))]>; @@ -4204,6 +4205,7 @@ def ANDI_rec_1_EQ_BIT8 : PPCCustomInserterPseudo<(outs crbitrc:$dst), (ins g8rc: def ANDI_rec_1_GT_BIT8 : PPCCustomInserterPseudo<(outs crbitrc:$dst), (ins g8rc:$in), "#ANDI_rec_1_GT_BIT8", [(set i1:$dst, (trunc i64:$in))]>; +} def : Pat<(i1 (not (trunc i32:$in))), (ANDI_rec_1_EQ_BIT $in)>; -- GitLab From 29ecd6d50f1400e3101f27567b535eb8af905f58 Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Wed, 17 Apr 2024 19:51:46 -0700 Subject: [PATCH 080/714] [clang-format] Revert breaking stream operators to previous default (#89016) Reverts commit d68826dfbd98, which changes the previous default behavior of always breaking before a stream insertion operator `<<` if both operands are string literals. Also reverts the related commits 27f547968cce and bf05be5b87fc. See the discussion in #88483. --- clang/lib/Format/TokenAnnotator.cpp | 8 ++------ clang/unittests/Format/FormatTest.cpp | 7 +------ clang/unittests/Format/TokenAnnotatorTest.cpp | 9 --------- polly/lib/Analysis/DependenceInfo.cpp | 4 ++-- polly/lib/Analysis/ScopBuilder.cpp | 7 ++++--- 5 files changed, 9 insertions(+), 26 deletions(-) diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index 628f70417866..80e5605fb577 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -5595,12 +5595,8 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line, return true; if (Left.IsUnterminatedLiteral) return true; - // FIXME: Breaking after newlines seems useful in general. Turn this into an - // option and recognize more cases like endl etc, and break independent of - // what comes after operator lessless. - if (Right.is(tok::lessless) && Right.Next && - Right.Next->is(tok::string_literal) && Left.is(tok::string_literal) && - Left.TokenText.ends_with("\\n\"")) { + if (Right.is(tok::lessless) && Right.Next && Left.is(tok::string_literal) && + Right.Next->is(tok::string_literal)) { return true; } if (Right.is(TT_RequiresClause)) { diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp index 4906b3350b5b..bc61b9c089e9 100644 --- a/clang/unittests/Format/FormatTest.cpp +++ b/clang/unittests/Format/FormatTest.cpp @@ -27339,12 +27339,6 @@ TEST_F(FormatTest, PPDirectivesAndCommentsInBracedInit) { getLLVMStyleWithColumns(30)); } -TEST_F(FormatTest, StreamOutputOperator) { - verifyFormat("std::cout << \"foo\" << \"bar\" << baz;"); - verifyFormat("std::cout << \"foo\\n\"\n" - " << \"bar\";"); -} - TEST_F(FormatTest, BreakAdjacentStringLiterals) { constexpr StringRef Code{ "return \"Code\" \"\\0\\52\\26\\55\\55\\0\" \"x013\" \"\\02\\xBA\";"}; @@ -27359,6 +27353,7 @@ TEST_F(FormatTest, BreakAdjacentStringLiterals) { Style.BreakAdjacentStringLiterals = false; verifyFormat(Code, Style); } + } // namespace } // namespace test } // namespace format diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp index da02ced8c7a9..a71b68064769 100644 --- a/clang/unittests/Format/TokenAnnotatorTest.cpp +++ b/clang/unittests/Format/TokenAnnotatorTest.cpp @@ -2841,15 +2841,6 @@ TEST_F(TokenAnnotatorTest, BraceKind) { EXPECT_BRACE_KIND(Tokens[16], BK_BracedInit); } -TEST_F(TokenAnnotatorTest, StreamOperator) { - auto Tokens = annotate("\"foo\\n\" << aux << \"foo\\n\" << \"foo\";"); - ASSERT_EQ(Tokens.size(), 9u) << Tokens; - EXPECT_FALSE(Tokens[1]->MustBreakBefore); - EXPECT_FALSE(Tokens[3]->MustBreakBefore); - // Only break between string literals if the former ends with \n. - EXPECT_TRUE(Tokens[5]->MustBreakBefore); -} - TEST_F(TokenAnnotatorTest, UnderstandsElaboratedTypeSpecifier) { auto Tokens = annotate("auto foo() -> enum En {}"); ASSERT_EQ(Tokens.size(), 10u) << Tokens; diff --git a/polly/lib/Analysis/DependenceInfo.cpp b/polly/lib/Analysis/DependenceInfo.cpp index 9ee004fbac9e..a530fa7b58fa 100644 --- a/polly/lib/Analysis/DependenceInfo.cpp +++ b/polly/lib/Analysis/DependenceInfo.cpp @@ -951,8 +951,8 @@ public: bool runOnScop(Scop &S) override { DependenceInfo &P = getAnalysis(); - OS << "Printing analysis '" << P.getPassName() << "' for " << "region: '" - << S.getRegion().getNameStr() << "' in function '" + OS << "Printing analysis '" << P.getPassName() << "' for " + << "region: '" << S.getRegion().getNameStr() << "' in function '" << S.getFunction().getName() << "':\n"; P.printScop(OS, S); diff --git a/polly/lib/Analysis/ScopBuilder.cpp b/polly/lib/Analysis/ScopBuilder.cpp index 214e4d360d6b..a34f29315272 100644 --- a/polly/lib/Analysis/ScopBuilder.cpp +++ b/polly/lib/Analysis/ScopBuilder.cpp @@ -2715,9 +2715,10 @@ void ScopBuilder::addUserContext() { if (NameContext != NameUserContext) { std::string SpaceStr = stringFromIslObj(Space, "null"); errs() << "Error: the name of dimension " << i - << " provided in -polly-context " << "is '" << NameUserContext - << "', but the name in the computed " << "context is '" - << NameContext << "'. Due to this name mismatch, " + << " provided in -polly-context " + << "is '" << NameUserContext << "', but the name in the computed " + << "context is '" << NameContext + << "'. Due to this name mismatch, " << "the -polly-context option is ignored. Please provide " << "the context in the parameter space: " << SpaceStr << ".\n"; return; -- GitLab From b6cc667190e3bee7485a225d3dadd8a57c0a22b6 Mon Sep 17 00:00:00 2001 From: Owen Pan Date: Wed, 17 Apr 2024 19:53:24 -0700 Subject: [PATCH 081/714] [clang-format] Annotate ampamp after new/delete as BinaryOperator (#89033) Fixes #78789. --- clang/lib/Format/TokenAnnotator.cpp | 2 ++ clang/unittests/Format/TokenAnnotatorTest.cpp | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index 80e5605fb577..a679683077ac 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -2912,6 +2912,8 @@ private: return TT_UnaryOperator; if (PrevToken->is(TT_TypeName)) return TT_PointerOrReference; + if (PrevToken->isOneOf(tok::kw_new, tok::kw_delete) && Tok.is(tok::ampamp)) + return TT_BinaryOperator; const FormatToken *NextToken = Tok.getNextNonComment(); diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp index a71b68064769..4f445c64ab30 100644 --- a/clang/unittests/Format/TokenAnnotatorTest.cpp +++ b/clang/unittests/Format/TokenAnnotatorTest.cpp @@ -309,6 +309,16 @@ TEST_F(TokenAnnotatorTest, UnderstandsUsesOfStarAndAmp) { EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_OverloadedOperatorLParen); EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace); EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference); + + Tokens = annotate("if (new && num) {\n" + " new = 1;\n" + "}\n" + "if (!delete && num) {\n" + " delete = 1;\n" + "}"); + ASSERT_EQ(Tokens.size(), 26u) << Tokens; + EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_BinaryOperator); + EXPECT_TOKEN(Tokens[16], tok::ampamp, TT_BinaryOperator); } TEST_F(TokenAnnotatorTest, UnderstandsUsesOfPlusAndMinus) { -- GitLab From f2695a1c2f561da42b973187a254b6eeb7da76a9 Mon Sep 17 00:00:00 2001 From: Chuanqi Xu Date: Wed, 17 Apr 2024 16:40:00 +0800 Subject: [PATCH 082/714] [C++20] [Modules] Avoid writing untouched DeclUpdates from GMF in Reduced BMI Mitigate https://github.com/llvm/llvm-project/issues/61447 The root cause of the above problem is that when we write a declaration, we need to lookup all the redeclarations in the imported modules. Then it will be pretty slow if there are too many redeclarations in different modules. This patch doesn't solve the porblem. What the patchs mitigated is, when we writing a named module, we shouldn't write the declarations from GMF if it is unreferenced **in current module unit**. The difference here is that, if the declaration is used in the imported modules, we used to emit it as an update. But we definitely want to avoid that after this patch. For that reproducer in https://github.com/llvm/llvm-project/issues/61447, it used to take 2.5s to compile and now it only takes 0.49s to compile, which is a big win. --- clang/include/clang/AST/ASTMutationListener.h | 27 ++++++ clang/include/clang/AST/Decl.h | 2 +- clang/include/clang/Serialization/ASTWriter.h | 10 ++ clang/lib/AST/ASTContext.cpp | 18 +++- clang/lib/AST/Decl.cpp | 7 ++ clang/lib/Frontend/MultiplexConsumer.cpp | 29 ++++++ clang/lib/Sema/SemaModule.cpp | 4 + clang/lib/Serialization/ASTWriter.cpp | 64 +++++++++---- .../reduced-bmi-empty-module-purview.cppm | 96 +++++++++++++++++++ 9 files changed, 231 insertions(+), 26 deletions(-) create mode 100644 clang/test/Modules/reduced-bmi-empty-module-purview.cppm diff --git a/clang/include/clang/AST/ASTMutationListener.h b/clang/include/clang/AST/ASTMutationListener.h index 8879f9f3229f..2c4ec2ce67f3 100644 --- a/clang/include/clang/AST/ASTMutationListener.h +++ b/clang/include/clang/AST/ASTMutationListener.h @@ -27,6 +27,7 @@ namespace clang { class FunctionTemplateDecl; class Module; class NamedDecl; + class NamespaceDecl; class ObjCCategoryDecl; class ObjCContainerDecl; class ObjCInterfaceDecl; @@ -35,6 +36,7 @@ namespace clang { class QualType; class RecordDecl; class TagDecl; + class TranslationUnitDecl; class ValueDecl; class VarDecl; class VarTemplateDecl; @@ -147,6 +149,31 @@ public: virtual void AddedAttributeToRecord(const Attr *Attr, const RecordDecl *Record) {} + /// The parser find the named module declaration. + virtual void EnteringModulePurview() {} + + /// An mangling number was added to a Decl + /// + /// \param D The decl that got a mangling number + /// + /// \param Number The mangling number that was added to the Decl + virtual void AddedManglingNumber(const Decl *D, unsigned Number) {} + + /// An static local number was added to a Decl + /// + /// \param D The decl that got a static local number + /// + /// \param Number The static local number that was added to the Decl + virtual void AddedStaticLocalNumbers(const Decl *D, unsigned Number) {} + + /// An anonymous namespace was added the translation unit decl + /// + /// \param TU The translation unit decl that got a new anonymous namespace + /// + /// \param AnonNamespace The anonymous namespace that was added + virtual void AddedAnonymousNamespace(const TranslationUnitDecl *TU, + NamespaceDecl *AnonNamespace) {} + // NOTE: If new methods are added they should also be added to // MultiplexASTMutationListener. }; diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h index 01af50ca694f..91449ebcceec 100644 --- a/clang/include/clang/AST/Decl.h +++ b/clang/include/clang/AST/Decl.h @@ -120,7 +120,7 @@ public: ASTContext &getASTContext() const { return Ctx; } NamespaceDecl *getAnonymousNamespace() const { return AnonymousNamespace; } - void setAnonymousNamespace(NamespaceDecl *D) { AnonymousNamespace = D; } + void setAnonymousNamespace(NamespaceDecl *D); static TranslationUnitDecl *Create(ASTContext &C); diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h index 443f77031047..892d7363e06d 100644 --- a/clang/include/clang/Serialization/ASTWriter.h +++ b/clang/include/clang/Serialization/ASTWriter.h @@ -399,6 +399,11 @@ private: /// record containing modifications to them. DeclUpdateMap DeclUpdates; + /// DeclUpdates added during parsing the GMF. We split these from + /// DeclUpdates since we want to add these updates in GMF on need. + /// Only meaningful for reduced BMI. + DeclUpdateMap DeclUpdatesFromGMF; + using FirstLatestDeclMap = llvm::DenseMap; /// Map of first declarations from a chained PCH that point to the @@ -866,6 +871,11 @@ private: void RedefinedHiddenDefinition(const NamedDecl *D, Module *M) override; void AddedAttributeToRecord(const Attr *Attr, const RecordDecl *Record) override; + void EnteringModulePurview() override; + void AddedManglingNumber(const Decl *D, unsigned) override; + void AddedStaticLocalNumbers(const Decl *D, unsigned) override; + void AddedAnonymousNamespace(const TranslationUnitDecl *, + NamespaceDecl *AnonNamespace) override; }; /// AST and semantic-analysis consumer that generates a diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp index 6ce233704a58..bcae6d705b78 100644 --- a/clang/lib/AST/ASTContext.cpp +++ b/clang/lib/AST/ASTContext.cpp @@ -12245,8 +12245,13 @@ QualType ASTContext::getRealTypeForBitwidth(unsigned DestWidth, } void ASTContext::setManglingNumber(const NamedDecl *ND, unsigned Number) { - if (Number > 1) - MangleNumbers[ND] = Number; + if (Number <= 1) + return; + + MangleNumbers[ND] = Number; + + if (Listener) + Listener->AddedManglingNumber(ND, Number); } unsigned ASTContext::getManglingNumber(const NamedDecl *ND, @@ -12265,8 +12270,13 @@ unsigned ASTContext::getManglingNumber(const NamedDecl *ND, } void ASTContext::setStaticLocalNumber(const VarDecl *VD, unsigned Number) { - if (Number > 1) - StaticLocalNumbers[VD] = Number; + if (Number <= 1) + return; + + StaticLocalNumbers[VD] = Number; + + if (Listener) + Listener->AddedStaticLocalNumbers(VD, Number); } unsigned ASTContext::getStaticLocalNumber(const VarDecl *VD) const { diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp index 33b6f8611f21..63388d79bbf2 100644 --- a/clang/lib/AST/Decl.cpp +++ b/clang/lib/AST/Decl.cpp @@ -5274,6 +5274,13 @@ TranslationUnitDecl *TranslationUnitDecl::Create(ASTContext &C) { return new (C, (DeclContext *)nullptr) TranslationUnitDecl(C); } +void TranslationUnitDecl::setAnonymousNamespace(NamespaceDecl *D) { + AnonymousNamespace = D; + + if (ASTMutationListener *Listener = Ctx.getASTMutationListener()) + Listener->AddedAnonymousNamespace(this, D); +} + void PragmaCommentDecl::anchor() {} PragmaCommentDecl *PragmaCommentDecl::Create(const ASTContext &C, diff --git a/clang/lib/Frontend/MultiplexConsumer.cpp b/clang/lib/Frontend/MultiplexConsumer.cpp index 737877329c9c..744ea70cc24d 100644 --- a/clang/lib/Frontend/MultiplexConsumer.cpp +++ b/clang/lib/Frontend/MultiplexConsumer.cpp @@ -20,6 +20,9 @@ using namespace clang; namespace clang { +class NamespaceDecl; +class TranslationUnitDecl; + MultiplexASTDeserializationListener::MultiplexASTDeserializationListener( const std::vector& L) : Listeners(L) { @@ -115,6 +118,11 @@ public: void RedefinedHiddenDefinition(const NamedDecl *D, Module *M) override; void AddedAttributeToRecord(const Attr *Attr, const RecordDecl *Record) override; + void EnteringModulePurview() override; + void AddedManglingNumber(const Decl *D, unsigned) override; + void AddedStaticLocalNumbers(const Decl *D, unsigned) override; + void AddedAnonymousNamespace(const TranslationUnitDecl *, + NamespaceDecl *AnonNamespace) override; private: std::vector Listeners; @@ -238,6 +246,27 @@ void MultiplexASTMutationListener::AddedAttributeToRecord( L->AddedAttributeToRecord(Attr, Record); } +void MultiplexASTMutationListener::EnteringModulePurview() { + for (auto *L : Listeners) + L->EnteringModulePurview(); +} + +void MultiplexASTMutationListener::AddedManglingNumber(const Decl *D, + unsigned Number) { + for (auto *L : Listeners) + L->AddedManglingNumber(D, Number); +} +void MultiplexASTMutationListener::AddedStaticLocalNumbers(const Decl *D, + unsigned Number) { + for (auto *L : Listeners) + L->AddedStaticLocalNumbers(D, Number); +} +void MultiplexASTMutationListener::AddedAnonymousNamespace( + const TranslationUnitDecl *TU, NamespaceDecl *AnonNamespace) { + for (auto *L : Listeners) + L->AddedAnonymousNamespace(TU, AnonNamespace); +} + } // end namespace clang MultiplexConsumer::MultiplexConsumer( diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp index 2ddf9d70263a..67658c93ed3b 100644 --- a/clang/lib/Sema/SemaModule.cpp +++ b/clang/lib/Sema/SemaModule.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "clang/AST/ASTConsumer.h" +#include "clang/AST/ASTMutationListener.h" #include "clang/Lex/HeaderSearch.h" #include "clang/Lex/Preprocessor.h" #include "clang/Sema/SemaInternal.h" @@ -475,6 +476,9 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc, getASTContext().setCurrentNamedModule(Mod); + if (auto *Listener = getASTMutationListener()) + Listener->EnteringModulePurview(); + // We already potentially made an implicit import (in the case of a module // implementation unit importing its interface). Make this module visible // and return the import decl to be added to the current TU. diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index b2a078b6d80f..c3cf242391b7 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -5026,27 +5026,6 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, GetDeclRef(D); } - // If the translation unit has an anonymous namespace, and we don't already - // have an update block for it, write it as an update block. - // FIXME: Why do we not do this if there's already an update block? - if (NamespaceDecl *NS = TU->getAnonymousNamespace()) { - ASTWriter::UpdateRecord &Record = DeclUpdates[TU]; - if (Record.empty()) - Record.push_back(DeclUpdate(UPD_CXX_ADDED_ANONYMOUS_NAMESPACE, NS)); - } - - // Add update records for all mangling numbers and static local numbers. - // These aren't really update records, but this is a convenient way of - // tagging this rare extra data onto the declarations. - for (const auto &Number : Context.MangleNumbers) - if (!Number.first->isFromASTFile()) - DeclUpdates[Number.first].push_back(DeclUpdate(UPD_MANGLING_NUMBER, - Number.second)); - for (const auto &Number : Context.StaticLocalNumbers) - if (!Number.first->isFromASTFile()) - DeclUpdates[Number.first].push_back(DeclUpdate(UPD_STATIC_LOCAL_NUMBER, - Number.second)); - // Make sure visible decls, added to DeclContexts previously loaded from // an AST file, are registered for serialization. Likewise for template // specializations added to imported templates. @@ -5315,6 +5294,41 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, return backpatchSignature(); } +void ASTWriter::EnteringModulePurview() { + // In C++20 named modules, all entities before entering the module purview + // lives in the GMF. + if (GeneratingReducedBMI) + DeclUpdatesFromGMF.swap(DeclUpdates); +} + +// Add update records for all mangling numbers and static local numbers. +// These aren't really update records, but this is a convenient way of +// tagging this rare extra data onto the declarations. +void ASTWriter::AddedManglingNumber(const Decl *D, unsigned Number) { + if (D->isFromASTFile()) + return; + + DeclUpdates[D].push_back(DeclUpdate(UPD_MANGLING_NUMBER, Number)); +} +void ASTWriter::AddedStaticLocalNumbers(const Decl *D, unsigned Number) { + if (D->isFromASTFile()) + return; + + DeclUpdates[D].push_back(DeclUpdate(UPD_STATIC_LOCAL_NUMBER, Number)); +} + +void ASTWriter::AddedAnonymousNamespace(const TranslationUnitDecl *TU, + NamespaceDecl *AnonNamespace) { + // If the translation unit has an anonymous namespace, and we don't already + // have an update block for it, write it as an update block. + // FIXME: Why do we not do this if there's already an update block? + if (NamespaceDecl *NS = TU->getAnonymousNamespace()) { + ASTWriter::UpdateRecord &Record = DeclUpdates[TU]; + if (Record.empty()) + Record.push_back(DeclUpdate(UPD_CXX_ADDED_ANONYMOUS_NAMESPACE, NS)); + } +} + void ASTWriter::WriteDeclAndTypes(ASTContext &Context) { // Keep writing types, declarations, and declaration update records // until we've emitted all of them. @@ -5860,6 +5874,14 @@ DeclID ASTWriter::GetDeclRef(const Decl *D) { return 0; } + // If the DeclUpdate from the GMF gets touched, emit it. + if (auto *Iter = DeclUpdatesFromGMF.find(D); + Iter != DeclUpdatesFromGMF.end()) { + for (DeclUpdate &Update : Iter->second) + DeclUpdates[D].push_back(Update); + DeclUpdatesFromGMF.erase(Iter); + } + // If D comes from an AST file, its declaration ID is already known and // fixed. if (D->isFromASTFile()) diff --git a/clang/test/Modules/reduced-bmi-empty-module-purview.cppm b/clang/test/Modules/reduced-bmi-empty-module-purview.cppm new file mode 100644 index 000000000000..54a1e9b77e6e --- /dev/null +++ b/clang/test/Modules/reduced-bmi-empty-module-purview.cppm @@ -0,0 +1,96 @@ +// Test that we won't write additional information into the Reduced BMI if the +// module purview is empty. +// +// RUN: rm -rf %t +// RUN: mkdir -p %t +// RUN: split-file %s %t +// +// RUN: %clang_cc1 -std=c++20 %t/M.cppm -emit-reduced-module-interface -o %t/M.pcm +// RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-reduced-module-interface -o %t/A.pcm \ +// RUN: -fmodule-file=M=%t/M.pcm +// RUN: llvm-bcanalyzer --dump --disable-histogram --show-binary-blobs %t/A.pcm > %t/A.dump +// RUN: cat %t/A.dump | FileCheck %t/A.cppm +// +// RUN: %clang_cc1 -std=c++20 %t/A1.cppm -emit-reduced-module-interface -o %t/A1.pcm \ +// RUN: -fmodule-file=M=%t/M.pcm +// RUN: llvm-bcanalyzer --dump --disable-histogram --show-binary-blobs %t/A1.pcm > %t/A1.dump +// RUN: cat %t/A1.dump | FileCheck %t/A1.cppm + +//--- foo.h +namespace ns { +template +class A { + +}; + +extern template class A; + +inline A a() { return A(); } +template +A _av_ = A(); + +auto _av_1 = _av_; +auto _av_2 = _av_; + +template <> +class A { + +}; + +void func(A, ...) { + +} + +} + +struct S { + union { + unsigned int V; + struct { + int v1; + int v2; + ns::A a1; + } WESQ; + }; + + union { + double d; + struct { + int v1; + unsigned v2; + ns::A a1; + } Another; + }; +}; + +//--- M.cppm +module; +#include "foo.h" +export module M; +export namespace nv { + using ns::A; + using ns::a; + using ns::_av_; + + using ns::func; +} +using ::S; + +//--- A.cppm +module; +#include "foo.h" +export module A; +import M; + +// CHECK-NOT: Date: Thu, 18 Apr 2024 11:04:06 +0800 Subject: [PATCH 083/714] [RISCV] Check that VLMAX is the same when demanding exact VL (#89080) --- llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp index fa37d1ccccd7..331253e39c0a 100644 --- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp +++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp @@ -616,7 +616,7 @@ public: if (SEWLMULRatioOnly) return false; - if (Used.VLAny && !hasSameAVL(Require)) + if (Used.VLAny && !(hasSameAVL(Require) && hasSameVLMAX(Require))) return false; if (Used.VLZeroness && !hasEquallyZeroAVL(Require, MRI)) -- GitLab From 3e2aad412ada8f1db88c9e0f984a302deed74f4b Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 17 Apr 2024 21:03:35 -0700 Subject: [PATCH 084/714] [RISCV] Speed up RISCVRegisterInfo::needsFrameBaseReg when frame pointer isn't used. NFC (#89163) The callee saved size is only used if there is a frame pointer. Sink the code onto the frame pointer only path. --- llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp | 22 +++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp index 84af6eec40ee..367a62e830cb 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp @@ -607,20 +607,22 @@ bool RISCVRegisterInfo::needsFrameBaseReg(MachineInstr *MI, const MachineFrameInfo &MFI = MF.getFrameInfo(); const RISCVFrameLowering *TFI = getFrameLowering(MF); const MachineRegisterInfo &MRI = MF.getRegInfo(); - unsigned CalleeSavedSize = 0; Offset += getFrameIndexInstrOffset(MI, FIOperandNum); - // Estimate the stack size used to store callee saved registers( - // excludes reserved registers). - BitVector ReservedRegs = getReservedRegs(MF); - for (const MCPhysReg *R = MRI.getCalleeSavedRegs(); MCPhysReg Reg = *R; ++R) { - if (!ReservedRegs.test(Reg)) - CalleeSavedSize += getSpillSize(*getMinimalPhysRegClass(Reg)); - } + if (TFI->hasFP(MF) && !shouldRealignStack(MF)) { + // Estimate the stack size used to store callee saved registers( + // excludes reserved registers). + unsigned CalleeSavedSize = 0; + BitVector ReservedRegs = getReservedRegs(MF); + for (const MCPhysReg *R = MRI.getCalleeSavedRegs(); MCPhysReg Reg = *R; + ++R) { + if (!ReservedRegs.test(Reg)) + CalleeSavedSize += getSpillSize(*getMinimalPhysRegClass(Reg)); + } - int64_t MaxFPOffset = Offset - CalleeSavedSize; - if (TFI->hasFP(MF) && !shouldRealignStack(MF)) + int64_t MaxFPOffset = Offset - CalleeSavedSize; return !isFrameOffsetLegal(MI, RISCV::X8, MaxFPOffset); + } // Assume 128 bytes spill slots size to estimate the maximum possible // offset relative to the stack pointer. -- GitLab From 8b37ec1f7bda40c240f7bfda6737df5043860103 Mon Sep 17 00:00:00 2001 From: Will Hawkins Date: Thu, 18 Apr 2024 00:29:57 -0400 Subject: [PATCH 085/714] [libc++][NFC] Add additional tests for begin/end of std::ranges::take_view (#79085) Add additional tests for `begin`/`end` of `std::ranges::take_view`. In partial fulfillment of #72406. --- .../range.adaptors/range.take/begin.pass.cpp | 100 +++++++++++++----- .../range.adaptors/range.take/end.pass.cpp | 14 +++ .../ranges/range.adaptors/range.take/types.h | 50 +++++++++ 3 files changed, 139 insertions(+), 25 deletions(-) diff --git a/libcxx/test/std/ranges/range.adaptors/range.take/begin.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.take/begin.pass.cpp index 1873481d7322..9f11a991535e 100644 --- a/libcxx/test/std/ranges/range.adaptors/range.take/begin.pass.cpp +++ b/libcxx/test/std/ranges/range.adaptors/range.take/begin.pass.cpp @@ -11,8 +11,9 @@ // constexpr auto begin() requires (!simple-view); // constexpr auto begin() const requires range; -#include #include +#include +#include #include "test_macros.h" #include "test_iterators.h" @@ -27,55 +28,104 @@ struct NonCommonSimpleView : std::ranges::view_base { static_assert(std::ranges::sized_range); static_assert(!std::ranges::sized_range); +using CommonInputIterPtrConstInt = common_input_iterator; +using CountedCommonInputIterPtrConstInt = std::counted_iterator; + constexpr bool test() { int buffer[8] = {1, 2, 3, 4, 5, 6, 7, 8}; - // sized_range && random_access_iterator + // simple-view && sized_range && random_access_range { - std::ranges::take_view tv(SizedRandomAccessView(buffer), 4); - assert(tv.begin() == SizedRandomAccessView(buffer).begin()); - ASSERT_SAME_TYPE(decltype(tv.begin()), RandomAccessIter); - } + using ViewTested = SizedRandomAccessView; + static_assert(simple_view); + static_assert(std::ranges::sized_range); + static_assert(std::ranges::random_access_range); - { - const std::ranges::take_view tv(SizedRandomAccessView(buffer), 4); - assert(tv.begin() == SizedRandomAccessView(buffer).begin()); + std::ranges::take_view tv(ViewTested(buffer), 4); + assert(tv.begin() == ViewTested(buffer).begin()); ASSERT_SAME_TYPE(decltype(tv.begin()), RandomAccessIter); - } - // sized_range && !random_access_iterator - { - std::ranges::take_view tv(SizedForwardView{buffer}, 4); - assert(tv.begin() == std::counted_iterator(ForwardIter(buffer), 4)); - ASSERT_SAME_TYPE(decltype(tv.begin()), std::counted_iterator); + const std::ranges::take_view ctv(ViewTested(buffer), 4); + assert(ctv.begin() == ViewTested(buffer).begin()); + ASSERT_SAME_TYPE(decltype(ctv.begin()), RandomAccessIter); } + // simple-view && sized_range && !random_access_range { - const std::ranges::take_view tv(SizedForwardView{buffer}, 4); - assert(tv.begin() == std::counted_iterator(ForwardIter(buffer), 4)); + using ViewTested = SizedForwardView; + static_assert(simple_view); + static_assert(std::ranges::sized_range); + static_assert(!std::ranges::random_access_range); + + std::ranges::take_view tv(ViewTested{buffer}, 16); // underlying size is 8 + assert(tv.begin() == std::counted_iterator(ForwardIter(buffer), 8)); // expect min(8, 16) ASSERT_SAME_TYPE(decltype(tv.begin()), std::counted_iterator); + + const std::ranges::take_view ctv(ViewTested{buffer}, 4); + assert(ctv.begin() == std::counted_iterator(ForwardIter(buffer), 4)); + ASSERT_SAME_TYPE(decltype(ctv.begin()), std::counted_iterator); } - // !sized_range + // simple-view && !sized_range { - std::ranges::take_view tv(MoveOnlyView{buffer}, 4); + using ViewTested = MoveOnlyView; + static_assert(simple_view); + std::ranges::take_view tv(ViewTested{buffer}, 4); assert(tv.begin() == std::counted_iterator(buffer, 4)); ASSERT_SAME_TYPE(decltype(tv.begin()), std::counted_iterator); + + const std::ranges::take_view ctv(ViewTested{buffer}, 4); + assert(ctv.begin() == std::counted_iterator(buffer, 4)); + ASSERT_SAME_TYPE(decltype(ctv.begin()), std::counted_iterator); } + // simple-view && sized_range && !sized_range { - const std::ranges::take_view tv(MoveOnlyView{buffer}, 4); - assert(tv.begin() == std::counted_iterator(buffer, 4)); + using ViewTested = NonCommonSimpleView; + static_assert(simple_view); + static_assert(std::ranges::sized_range); + static_assert(!std::ranges::sized_range); + + std::ranges::take_view tv{}; ASSERT_SAME_TYPE(decltype(tv.begin()), std::counted_iterator); + ASSERT_SAME_TYPE(decltype(std::as_const(tv).begin()), std::counted_iterator); } - // simple-view && sized_range && !size_range + // !simple-view && !sized_range { - std::ranges::take_view tv{}; - ASSERT_SAME_TYPE(decltype(tv.begin()), std::counted_iterator); - ASSERT_SAME_TYPE(decltype(std::as_const(tv).begin()), std::counted_iterator); + using ViewTested = NonSimpleNonSizedView; + static_assert(!simple_view); + static_assert(!std::ranges::sized_range); + + std::ranges::take_view tv{ViewTested{buffer, buffer + 2}, 4}; + // The count for the counted iterator is the count of the take_view (i.e., 4) + assert(tv.begin() == CountedCommonInputIterPtrConstInt(CommonInputIterPtrConstInt(buffer), 4)); + ASSERT_SAME_TYPE(decltype(tv.begin()), CountedCommonInputIterPtrConstInt); } + // !simple-view && sized_range + { + using ViewTested = NonSimpleSizedView; + static_assert(!simple_view); + static_assert(std::ranges::sized_range); + + std::ranges::take_view tv{ViewTested{buffer, buffer + 2}, 4}; + // The count for the counted iterator is the min(2, 4) (i.e., 2). + assert(tv.begin() == CountedCommonInputIterPtrConstInt(CommonInputIterPtrConstInt(buffer), 2)); + ASSERT_SAME_TYPE(decltype(tv.begin()), CountedCommonInputIterPtrConstInt); + } + + // !simple-view && sized_range && random_access_range + { + using ViewTested = NonSimpleSizedRandomView; + static_assert(!simple_view); + static_assert(std::ranges::sized_range); + static_assert(std::ranges::random_access_range); + + std::ranges::take_view tv{ViewTested{buffer, buffer + 2}, 4}; + assert(tv.begin() == random_access_iterator(buffer)); + ASSERT_SAME_TYPE(decltype(tv.begin()), random_access_iterator); + } return true; } diff --git a/libcxx/test/std/ranges/range.adaptors/range.take/end.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.take/end.pass.cpp index 6cab05daa9e0..eddb39af2712 100644 --- a/libcxx/test/std/ranges/range.adaptors/range.take/end.pass.cpp +++ b/libcxx/test/std/ranges/range.adaptors/range.take/end.pass.cpp @@ -69,6 +69,20 @@ constexpr bool test() { assert(tv.end() == std::ranges::next(tv.begin(), 8)); } + { + // __iterator has base with type std::ranges::sentinel_t; adding a const qualifier + // would change the equality. + std::ranges::take_view tvns(NonSimpleNonSizedView{buffer, buffer + 8}, 0); + static_assert(!std::is_same_v>); + } + + { + // __iterator has base with type std::ranges::sentinel_t; adding a const qualifier + // would not change the equality. + std::ranges::take_view tvs(SimpleViewNonSized{buffer, buffer + 8}, 0); + static_assert(std::is_same_v>); + } + return true; } diff --git a/libcxx/test/std/ranges/range.adaptors/range.take/types.h b/libcxx/test/std/ranges/range.adaptors/range.take/types.h index db80e68bb21a..7590ce33bffc 100644 --- a/libcxx/test/std/ranges/range.adaptors/range.take/types.h +++ b/libcxx/test/std/ranges/range.adaptors/range.take/types.h @@ -65,4 +65,54 @@ private: int* end_; }; +template