aboutsummaryrefslogtreecommitdiff
path: root/libc
diff options
context:
space:
mode:
authorGuillaume Chatelet <gchatelet@google.com>2023-07-17 21:35:03 +0000
committerGuillaume Chatelet <gchatelet@google.com>2023-07-18 12:20:23 +0000
commit23dcdbfba790eb24141d0d2523aab34a2a4d1ef5 (patch)
tree517dfee0a74e8afecd8619b8134f91bf77cbcdbf /libc
parent30d11e5ef98c9d481daac5d3bd784416e71d22a2 (diff)
downloadllvm-23dcdbfba790eb24141d0d2523aab34a2a4d1ef5.zip
llvm-23dcdbfba790eb24141d0d2523aab34a2a4d1ef5.tar.gz
llvm-23dcdbfba790eb24141d0d2523aab34a2a4d1ef5.tar.bz2
[libc][NFC] Split memmove implementations per platform
This is a follow up on D154800 and D154770 to make the code structure more principled and avoid too many nested #ifdef/#endif. Reviewed By: courbet Differential Revision: https://reviews.llvm.org/D155515
Diffstat (limited to 'libc')
-rw-r--r--libc/src/string/memory_utils/CMakeLists.txt3
-rw-r--r--libc/src/string/memory_utils/aarch64/memmove_implementations.h53
-rw-r--r--libc/src/string/memory_utils/memmove_implementations.h74
-rw-r--r--libc/src/string/memory_utils/riscv/memmove_implementations.h27
-rw-r--r--libc/src/string/memory_utils/x86_64/memmove_implementations.h66
5 files changed, 163 insertions, 60 deletions
diff --git a/libc/src/string/memory_utils/CMakeLists.txt b/libc/src/string/memory_utils/CMakeLists.txt
index 7580b08..f9b3c86 100644
--- a/libc/src/string/memory_utils/CMakeLists.txt
+++ b/libc/src/string/memory_utils/CMakeLists.txt
@@ -5,6 +5,7 @@ add_header_library(
aarch64/bcmp_implementations.h
aarch64/memcmp_implementations.h
aarch64/memcpy_implementations.h
+ aarch64/memmove_implementations.h
aarch64/memset_implementations.h
bcmp_implementations.h
bzero_implementations.h
@@ -21,11 +22,13 @@ add_header_library(
riscv/bcmp_implementations.h
riscv/memcmp_implementations.h
riscv/memcpy_implementations.h
+ riscv/memmove_implementations.h
riscv/memset_implementations.h
utils.h
x86_64/bcmp_implementations.h
x86_64/memcmp_implementations.h
x86_64/memcpy_implementations.h
+ x86_64/memmove_implementations.h
x86_64/memset_implementations.h
DEPS
libc.src.__support.common
diff --git a/libc/src/string/memory_utils/aarch64/memmove_implementations.h b/libc/src/string/memory_utils/aarch64/memmove_implementations.h
new file mode 100644
index 0000000..37e5656
--- /dev/null
+++ b/libc/src/string/memory_utils/aarch64/memmove_implementations.h
@@ -0,0 +1,53 @@
+//===-- Memmove implementation for aarch64 ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMMOVE_IMPLEMENTATIONS_H
+#define LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMMOVE_IMPLEMENTATIONS_H
+
+#include "src/__support/macros/config.h" // LIBC_INLINE
+#include "src/string/memory_utils/op_aarch64.h" // aarch64::kNeon
+#include "src/string/memory_utils/op_builtin.h"
+#include "src/string/memory_utils/op_generic.h"
+#include "src/string/memory_utils/utils.h"
+
+#include <stddef.h> // size_t
+
+namespace __llvm_libc {
+
+LIBC_INLINE void inline_memmove_aarch64(Ptr dst, CPtr src, size_t count) {
+ static_assert(aarch64::kNeon, "aarch64 supports vector types");
+ using uint128_t = generic_v128;
+ using uint256_t = generic_v256;
+ using uint512_t = generic_v512;
+ if (count == 0)
+ return;
+ if (count == 1)
+ return generic::Memmove<uint8_t>::block(dst, src);
+ if (count <= 4)
+ return generic::Memmove<uint16_t>::head_tail(dst, src, count);
+ if (count <= 8)
+ return generic::Memmove<uint32_t>::head_tail(dst, src, count);
+ if (count <= 16)
+ return generic::Memmove<uint64_t>::head_tail(dst, src, count);
+ if (count <= 32)
+ return generic::Memmove<uint128_t>::head_tail(dst, src, count);
+ if (count <= 64)
+ return generic::Memmove<uint256_t>::head_tail(dst, src, count);
+ if (count <= 128)
+ return generic::Memmove<uint512_t>::head_tail(dst, src, count);
+ if (dst < src) {
+ generic::Memmove<uint256_t>::align_forward<Arg::Src>(dst, src, count);
+ return generic::Memmove<uint512_t>::loop_and_tail_forward(dst, src, count);
+ } else {
+ generic::Memmove<uint256_t>::align_backward<Arg::Src>(dst, src, count);
+ return generic::Memmove<uint512_t>::loop_and_tail_backward(dst, src, count);
+ }
+}
+
+} // namespace __llvm_libc
+
+#endif // LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMMOVE_IMPLEMENTATIONS_H
diff --git a/libc/src/string/memory_utils/memmove_implementations.h b/libc/src/string/memory_utils/memmove_implementations.h
index f8b0b78..78ae7a8 100644
--- a/libc/src/string/memory_utils/memmove_implementations.h
+++ b/libc/src/string/memory_utils/memmove_implementations.h
@@ -9,74 +9,28 @@
#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H
#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE_IMPLEMENTATIONS_H
-#include "src/__support/common.h"
-#include "src/__support/macros/optimization.h"
-#include "src/string/memory_utils/generic/byte_per_byte.h"
-#include "src/string/memory_utils/op_aarch64.h"
-#include "src/string/memory_utils/op_builtin.h"
-#include "src/string/memory_utils/op_generic.h"
-#include "src/string/memory_utils/op_x86.h"
#include <stddef.h> // size_t, ptrdiff_t
-namespace __llvm_libc {
-
-LIBC_INLINE void inline_memmove(Ptr dst, CPtr src, size_t count) {
-#if defined(LIBC_TARGET_ARCH_IS_X86) || defined(LIBC_TARGET_ARCH_IS_AARCH64)
#if defined(LIBC_TARGET_ARCH_IS_X86)
-#if defined(__AVX512F__)
- using uint128_t = generic_v128;
- using uint256_t = generic_v256;
- using uint512_t = generic_v512;
-#elif defined(__AVX__)
- using uint128_t = generic_v128;
- using uint256_t = generic_v256;
- using uint512_t = cpp::array<generic_v256, 2>;
-#elif defined(__SSE2__)
- using uint128_t = generic_v128;
- using uint256_t = cpp::array<generic_v128, 2>;
- using uint512_t = cpp::array<generic_v128, 4>;
-#else
- using uint128_t = cpp::array<uint64_t, 2>;
- using uint256_t = cpp::array<uint64_t, 4>;
- using uint512_t = cpp::array<uint64_t, 8>;
-#endif
+#include "src/string/memory_utils/x86_64/memmove_implementations.h"
+#define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE inline_memmove_x86
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
- static_assert(aarch64::kNeon, "aarch64 supports vector types");
- using uint128_t = generic_v128;
- using uint256_t = generic_v256;
- using uint512_t = generic_v512;
-#endif
- if (count == 0)
- return;
- if (count == 1)
- return generic::Memmove<uint8_t>::block(dst, src);
- if (count <= 4)
- return generic::Memmove<uint16_t>::head_tail(dst, src, count);
- if (count <= 8)
- return generic::Memmove<uint32_t>::head_tail(dst, src, count);
- if (count <= 16)
- return generic::Memmove<uint64_t>::head_tail(dst, src, count);
- if (count <= 32)
- return generic::Memmove<uint128_t>::head_tail(dst, src, count);
- if (count <= 64)
- return generic::Memmove<uint256_t>::head_tail(dst, src, count);
- if (count <= 128)
- return generic::Memmove<uint512_t>::head_tail(dst, src, count);
- if (dst < src) {
- generic::Memmove<uint256_t>::align_forward<Arg::Src>(dst, src, count);
- return generic::Memmove<uint512_t>::loop_and_tail_forward(dst, src, count);
- } else {
- generic::Memmove<uint256_t>::align_backward<Arg::Src>(dst, src, count);
- return generic::Memmove<uint512_t>::loop_and_tail_backward(dst, src, count);
- }
+#include "src/string/memory_utils/aarch64/memmove_implementations.h"
+#define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE inline_memmove_aarch64
+#elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV)
+#include "src/string/memory_utils/riscv/memmove_implementations.h"
+#define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE inline_memmove_riscv
#else
- return inline_memmove_byte_per_byte(dst, src, count);
+// We may want to error instead of defaulting to suboptimal implementation.
+#include "src/string/memory_utils/generic/byte_per_byte.h"
+#define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE inline_memmove_byte_per_byte
#endif
-}
+
+namespace __llvm_libc {
LIBC_INLINE void inline_memmove(void *dst, const void *src, size_t count) {
- inline_memmove(reinterpret_cast<Ptr>(dst), reinterpret_cast<CPtr>(src),
- count);
+ LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE(reinterpret_cast<Ptr>(dst),
+ reinterpret_cast<CPtr>(src), count);
}
} // namespace __llvm_libc
diff --git a/libc/src/string/memory_utils/riscv/memmove_implementations.h b/libc/src/string/memory_utils/riscv/memmove_implementations.h
new file mode 100644
index 0000000..19fc956
--- /dev/null
+++ b/libc/src/string/memory_utils/riscv/memmove_implementations.h
@@ -0,0 +1,27 @@
+//===-- Memmove implementation for riscv ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LIBC_SRC_STRING_MEMORY_UTILS_RISCV_MEMMOVE_IMPLEMENTATIONS_H
+#define LIBC_SRC_STRING_MEMORY_UTILS_RISCV_MEMMOVE_IMPLEMENTATIONS_H
+
+#include "src/__support/macros/attributes.h" // LIBC_INLINE
+#include "src/__support/macros/properties/architectures.h" // LIBC_TARGET_ARCH_IS_RISCV64
+#include "src/string/memory_utils/generic/byte_per_byte.h"
+#include "src/string/memory_utils/utils.h" // Ptr, CPtr
+
+#include <stddef.h> // size_t
+
+namespace __llvm_libc {
+
+[[maybe_unused]] LIBC_INLINE void
+inline_memmove_riscv(Ptr __restrict dst, CPtr __restrict src, size_t count) {
+ return inline_memmove_byte_per_byte(dst, src, count);
+}
+
+} // namespace __llvm_libc
+
+#endif // LIBC_SRC_STRING_MEMORY_UTILS_RISCV_MEMMOVE_IMPLEMENTATIONS_H
diff --git a/libc/src/string/memory_utils/x86_64/memmove_implementations.h b/libc/src/string/memory_utils/x86_64/memmove_implementations.h
new file mode 100644
index 0000000..218f543
--- /dev/null
+++ b/libc/src/string/memory_utils/x86_64/memmove_implementations.h
@@ -0,0 +1,66 @@
+//===-- Memmove implementation for x86_64 -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMMOVE_IMPLEMENTATIONS_H
+#define LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMMOVE_IMPLEMENTATIONS_H
+
+#include "src/__support/macros/config.h" // LIBC_INLINE
+#include "src/string/memory_utils/op_builtin.h"
+#include "src/string/memory_utils/op_generic.h"
+#include "src/string/memory_utils/op_x86.h"
+#include "src/string/memory_utils/utils.h"
+
+#include <stddef.h> // size_t
+
+namespace __llvm_libc {
+
+LIBC_INLINE void inline_memmove_x86(Ptr dst, CPtr src, size_t count) {
+#if defined(__AVX512F__)
+ using uint128_t = generic_v128;
+ using uint256_t = generic_v256;
+ using uint512_t = generic_v512;
+#elif defined(__AVX__)
+ using uint128_t = generic_v128;
+ using uint256_t = generic_v256;
+ using uint512_t = cpp::array<generic_v256, 2>;
+#elif defined(__SSE2__)
+ using uint128_t = generic_v128;
+ using uint256_t = cpp::array<generic_v128, 2>;
+ using uint512_t = cpp::array<generic_v128, 4>;
+#else
+ using uint128_t = cpp::array<uint64_t, 2>;
+ using uint256_t = cpp::array<uint64_t, 4>;
+ using uint512_t = cpp::array<uint64_t, 8>;
+#endif
+ if (count == 0)
+ return;
+ if (count == 1)
+ return generic::Memmove<uint8_t>::block(dst, src);
+ if (count <= 4)
+ return generic::Memmove<uint16_t>::head_tail(dst, src, count);
+ if (count <= 8)
+ return generic::Memmove<uint32_t>::head_tail(dst, src, count);
+ if (count <= 16)
+ return generic::Memmove<uint64_t>::head_tail(dst, src, count);
+ if (count <= 32)
+ return generic::Memmove<uint128_t>::head_tail(dst, src, count);
+ if (count <= 64)
+ return generic::Memmove<uint256_t>::head_tail(dst, src, count);
+ if (count <= 128)
+ return generic::Memmove<uint512_t>::head_tail(dst, src, count);
+ if (dst < src) {
+ generic::Memmove<uint256_t>::align_forward<Arg::Src>(dst, src, count);
+ return generic::Memmove<uint512_t>::loop_and_tail_forward(dst, src, count);
+ } else {
+ generic::Memmove<uint256_t>::align_backward<Arg::Src>(dst, src, count);
+ return generic::Memmove<uint512_t>::loop_and_tail_backward(dst, src, count);
+ }
+}
+
+} // namespace __llvm_libc
+
+#endif // LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMMOVE_IMPLEMENTATIONS_H