diff options
Diffstat (limited to 'compiler-rt/lib')
11 files changed, 119 insertions, 30 deletions
diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64.c b/compiler-rt/lib/builtins/cpu_model/aarch64.c index 8af736d..06bc7c4a 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64.c +++ b/compiler-rt/lib/builtins/cpu_model/aarch64.c @@ -19,34 +19,34 @@ #error This file is intended only for aarch64-based targets #endif -#if __has_include(<sys/ifunc.h>) -#include <sys/ifunc.h> -#else typedef struct __ifunc_arg_t { unsigned long _size; unsigned long _hwcap; unsigned long _hwcap2; + unsigned long _hwcap3; + unsigned long _hwcap4; } __ifunc_arg_t; -#endif // __has_include(<sys/ifunc.h>) // LSE support detection for out-of-line atomics // using HWCAP and Auxiliary vector _Bool __aarch64_have_lse_atomics __attribute__((visibility("hidden"), nocommon)) = false; +// The formatter wants to re-order these includes, but doing so is incorrect: +// clang-format off #if defined(__FreeBSD__) || defined(__OpenBSD__) -// clang-format off: should not reorder sys/auxv.h alphabetically #include <sys/auxv.h> -// clang-format on #include "aarch64/hwcap.inc" #include "aarch64/lse_atomics/elf_aux_info.inc" #elif defined(__Fuchsia__) #include "aarch64/hwcap.inc" #include "aarch64/lse_atomics/fuchsia.inc" #elif defined(__ANDROID__) +#include <sys/auxv.h> #include "aarch64/hwcap.inc" #include "aarch64/lse_atomics/android.inc" -#elif defined(__linux__) && __has_include(<sys/auxv.h>) +#elif defined(__linux__) +#include <sys/auxv.h> #include "aarch64/hwcap.inc" #include "aarch64/lse_atomics/getauxval.inc" #elif defined(_WIN32) @@ -54,6 +54,7 @@ _Bool __aarch64_have_lse_atomics #else // When unimplemented, we leave __aarch64_have_lse_atomics initialized to false. #endif +// clang-format on #if !defined(DISABLE_AARCH64_FMV) @@ -76,7 +77,7 @@ struct { #elif defined(__ANDROID__) #include "aarch64/fmv/hwcap.inc" #include "aarch64/fmv/android.inc" -#elif defined(__linux__) && __has_include(<sys/auxv.h>) +#elif defined(__linux__) #include "aarch64/fmv/hwcap.inc" #include "aarch64/fmv/getauxval.inc" #elif defined(_WIN32) diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/android.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/android.inc index a9e3594..d19beca 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/android.inc +++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/android.inc @@ -27,10 +27,14 @@ void CONSTRUCTOR_ATTRIBUTE __init_cpu_features(void) { unsigned long hwcap = getauxval(AT_HWCAP); unsigned long hwcap2 = getauxval(AT_HWCAP2); + unsigned long hwcap3 = getauxval(AT_HWCAP3); + unsigned long hwcap4 = getauxval(AT_HWCAP4); __ifunc_arg_t arg; arg._size = sizeof(__ifunc_arg_t); arg._hwcap = hwcap; arg._hwcap2 = hwcap2; + arg._hwcap3 = hwcap3; + arg._hwcap4 = hwcap4; __init_cpu_features_constructor(hwcap | _IFUNC_ARG_HWCAP, &arg); } diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/elf_aux_info.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/elf_aux_info.inc index aa975dc..1ada7b6 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/elf_aux_info.inc +++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/elf_aux_info.inc @@ -7,21 +7,21 @@ void __init_cpu_features_resolver(unsigned long hwcap, } void CONSTRUCTOR_ATTRIBUTE __init_cpu_features(void) { - unsigned long hwcap = 0; - unsigned long hwcap2 = 0; + unsigned long hwcap, hwcap2, hwcap3, hwcap4 = 0; // CPU features already initialized. if (__atomic_load_n(&__aarch64_cpu_features.features, __ATOMIC_RELAXED)) return; - int res = 0; - res = elf_aux_info(AT_HWCAP, &hwcap, sizeof hwcap); - res |= elf_aux_info(AT_HWCAP2, &hwcap2, sizeof hwcap2); - if (res) - return; + elf_aux_info(AT_HWCAP, &hwcap, sizeof hwcap); + elf_aux_info(AT_HWCAP2, &hwcap2, sizeof hwcap2); + elf_aux_info(AT_HWCAP3, &hwcap3, sizeof hwcap3); + elf_aux_info(AT_HWCAP4, &hwcap4, sizeof hwcap4); __ifunc_arg_t arg; arg._size = sizeof(__ifunc_arg_t); arg._hwcap = hwcap; arg._hwcap2 = hwcap2; + arg._hwcap3 = hwcap3; + arg._hwcap4 = hwcap4; __init_cpu_features_constructor(hwcap | _IFUNC_ARG_HWCAP, &arg); } diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/getauxval.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/getauxval.inc index 486f77a..6c52c53 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/getauxval.inc +++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/getauxval.inc @@ -12,10 +12,14 @@ void CONSTRUCTOR_ATTRIBUTE __init_cpu_features(void) { unsigned long hwcap = getauxval(AT_HWCAP); unsigned long hwcap2 = getauxval(AT_HWCAP2); + unsigned long hwcap3 = getauxval(AT_HWCAP3); + unsigned long hwcap4 = getauxval(AT_HWCAP4); __ifunc_arg_t arg; arg._size = sizeof(__ifunc_arg_t); arg._hwcap = hwcap; arg._hwcap2 = hwcap2; + arg._hwcap3 = hwcap3; + arg._hwcap4 = hwcap4; __init_cpu_features_constructor(hwcap | _IFUNC_ARG_HWCAP, &arg); } diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/hwcap.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/hwcap.inc index 2f44e9e..1c53191 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64/hwcap.inc +++ b/compiler-rt/lib/builtins/cpu_model/aarch64/hwcap.inc @@ -1,13 +1,8 @@ -#if __has_include(<sys/hwcap.h>) -#include <sys/hwcap.h> -#define HAVE_SYS_HWCAP_H -#endif - #ifndef _IFUNC_ARG_HWCAP #define _IFUNC_ARG_HWCAP (1ULL << 62) #endif #ifndef AT_HWCAP -#define AT_HWCAP 16 +#define AT_HWCAP 16 // Linux value #endif #ifndef HWCAP_CPUID #define HWCAP_CPUID (1 << 11) @@ -95,7 +90,7 @@ #endif #ifndef AT_HWCAP2 -#define AT_HWCAP2 26 +#define AT_HWCAP2 26 // Linux value #endif #ifndef HWCAP2_DCPODP #define HWCAP2_DCPODP (1 << 0) @@ -190,3 +185,19 @@ #ifndef HWCAP2_CSSC #define HWCAP2_CSSC (1UL << 34) #endif + +#ifndef AT_HWCAP3 +#ifdef __linux__ +#define AT_HWCAP3 29 // Linux value +#else +#define AT_HWCAP3 38 // BSD value +#endif +#endif + +#ifndef AT_HWCAP4 +#ifdef __linux__ +#define AT_HWCAP4 30 // Linux value +#else +#define AT_HWCAP4 39 // BSD value +#endif +#endif diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/lse_atomics/android.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/lse_atomics/android.inc index 94bf64a..8d8a913 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64/lse_atomics/android.inc +++ b/compiler-rt/lib/builtins/cpu_model/aarch64/lse_atomics/android.inc @@ -1,5 +1,4 @@ #include <string.h> -#include <sys/auxv.h> #include <sys/system_properties.h> static bool __isExynos9810(void) { diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/lse_atomics/getauxval.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/lse_atomics/getauxval.inc index 6642c1f..670aba5 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64/lse_atomics/getauxval.inc +++ b/compiler-rt/lib/builtins/cpu_model/aarch64/lse_atomics/getauxval.inc @@ -1,5 +1,3 @@ -#include <sys/auxv.h> - static void CONSTRUCTOR_ATTRIBUTE init_have_lse_atomics(void) { unsigned long hwcap = getauxval(AT_HWCAP); __aarch64_have_lse_atomics = (hwcap & HWCAP_ATOMICS) != 0; diff --git a/compiler-rt/lib/orc/coff_platform.cpp b/compiler-rt/lib/orc/coff_platform.cpp index 78144ec..4cce20e 100644 --- a/compiler-rt/lib/orc/coff_platform.cpp +++ b/compiler-rt/lib/orc/coff_platform.cpp @@ -110,6 +110,7 @@ public: const char *dlerror(); void *dlopen(std::string_view Name, int Mode); + int dlupdate(void *DSOHandle); int dlclose(void *Header); void *dlsym(void *Header, std::string_view Symbol); @@ -141,6 +142,10 @@ private: Error dlopenFull(JITDylibState &JDS); Error dlopenInitialize(JITDylibState &JDS, COFFJITDylibDepInfoMap &DepInfo); + Error dlupdateImpl(void *DSOHandle); + Error dlupdateFull(JITDylibState &JDS); + Error dlupdateInitialize(JITDylibState &JDS); + Error dlcloseImpl(void *DSOHandle); Error dlcloseDeinitialize(JITDylibState &JDS); @@ -265,6 +270,20 @@ void *COFFPlatformRuntimeState::dlopen(std::string_view Path, int Mode) { } } +int COFFPlatformRuntimeState::dlupdate(void *DSOHandle) { + ORC_RT_DEBUG({ + std::string S; + printdbg("COFFPlatform::dlupdate(%p) (%s)\n", DSOHandle, S.c_str()); + }); + std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); + if (auto Err = dlupdateImpl(DSOHandle)) { + // FIXME: Make dlerror thread safe. + DLFcnError = toString(std::move(Err)); + return -1; + } + return 0; +} + int COFFPlatformRuntimeState::dlclose(void *DSOHandle) { ORC_RT_DEBUG({ auto *JDS = getJITDylibStateByHeader(DSOHandle); @@ -390,6 +409,55 @@ Error COFFPlatformRuntimeState::dlopenInitialize( return Error::success(); } +Error COFFPlatformRuntimeState::dlupdateImpl(void *DSOHandle) { + // Try to find JITDylib state by header. + auto *JDS = getJITDylibStateByHeader(DSOHandle); + + if (!JDS) { + std::ostringstream ErrStream; + ErrStream << "No registered JITDylib for " << DSOHandle; + return make_error<StringError>(ErrStream.str()); + } + + if (!JDS->referenced()) + return make_error<StringError>("dlupdate failed, JITDylib must be open."); + + if (auto Err = dlupdateFull(*JDS)) + return Err; + + return Error::success(); +} + +Error COFFPlatformRuntimeState::dlupdateFull(JITDylibState &JDS) { + // Call back to the JIT to push the initializers. + Expected<COFFJITDylibDepInfoMap> DepInfoMap((COFFJITDylibDepInfoMap())); + if (auto Err = WrapperFunction<SPSExpected<SPSCOFFJITDylibDepInfoMap>( + SPSExecutorAddr)>:: + call(JITDispatch(&__orc_rt_coff_push_initializers_tag), DepInfoMap, + ExecutorAddr::fromPtr(JDS.Header))) + return Err; + if (!DepInfoMap) + return DepInfoMap.takeError(); + + if (auto Err = dlupdateInitialize(JDS)) + return Err; + + return Error::success(); +} + +Error COFFPlatformRuntimeState::dlupdateInitialize(JITDylibState &JDS) { + ORC_RT_DEBUG({ + printdbg("COFFPlatformRuntimeState::dlupdateInitialize(\"%s\")\n", + JDS.Name.c_str()); + }); + + // Run static initializers. + JDS.CInitSection.RunAllNewAndFlush(); + JDS.CXXInitSection.RunAllNewAndFlush(); + + return Error::success(); +} + Error COFFPlatformRuntimeState::dlcloseImpl(void *DSOHandle) { // Try to find JITDylib state by header. auto *JDS = getJITDylibStateByHeader(DSOHandle); @@ -667,6 +735,10 @@ void *__orc_rt_coff_jit_dlopen(const char *path, int mode) { return COFFPlatformRuntimeState::get().dlopen(path, mode); } +int __orc_rt_coff_jit_dlupdate(void *dso_handle) { + return COFFPlatformRuntimeState::get().dlupdate(dso_handle); +} + int __orc_rt_coff_jit_dlclose(void *header) { return COFFPlatformRuntimeState::get().dlclose(header); } diff --git a/compiler-rt/lib/orc/coff_platform.h b/compiler-rt/lib/orc/coff_platform.h index aae57bc..e9891f0b 100644 --- a/compiler-rt/lib/orc/coff_platform.h +++ b/compiler-rt/lib/orc/coff_platform.h @@ -19,6 +19,7 @@ // dlfcn functions. ORC_RT_INTERFACE const char *__orc_rt_coff_jit_dlerror(); ORC_RT_INTERFACE void *__orc_rt_coff_jit_dlopen(const char *path, int mode); +ORC_RT_INTERFACE int __orc_rt_coff_jit_dlupdate(void *dso_handle); ORC_RT_INTERFACE int __orc_rt_coff_jit_dlclose(void *header); ORC_RT_INTERFACE void *__orc_rt_coff_jit_dlsym(void *header, const char *symbol); diff --git a/compiler-rt/lib/orc/dlfcn_wrapper.cpp b/compiler-rt/lib/orc/dlfcn_wrapper.cpp index 02e3796..cb92a1f 100644 --- a/compiler-rt/lib/orc/dlfcn_wrapper.cpp +++ b/compiler-rt/lib/orc/dlfcn_wrapper.cpp @@ -42,7 +42,6 @@ __orc_rt_jit_dlopen_wrapper(const char *ArgData, size_t ArgSize) { .release(); } -#ifndef _WIN32 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult __orc_rt_jit_dlupdate_wrapper(const char *ArgData, size_t ArgSize) { return WrapperFunction<int32_t(SPSExecutorAddr)>::handle( @@ -52,7 +51,6 @@ __orc_rt_jit_dlupdate_wrapper(const char *ArgData, size_t ArgSize) { }) .release(); } -#endif ORC_RT_INTERFACE orc_rt_WrapperFunctionResult __orc_rt_jit_dlclose_wrapper(const char *ArgData, size_t ArgSize) { diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp b/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp index 6b65387..c974f54 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp @@ -415,7 +415,7 @@ void InitializePlatform() { // is not compiled with -pie. #if !SANITIZER_GO { -# if SANITIZER_LINUX && (defined(__aarch64__) || defined(__loongarch_lp64)) +# if INIT_LONGJMP_XOR_KEY // Initialize the xor key used in {sig}{set,long}jump. InitializeLongjmpXorKey(); # endif @@ -486,7 +486,7 @@ int ExtractRecvmsgFDs(void *msgp, int *fds, int nfd) { // Reverse operation of libc stack pointer mangling static uptr UnmangleLongJmpSp(uptr mangled_sp) { -# if SANITIZER_ANDROID +# if SANITIZER_ANDROID && INIT_LONGJMP_XOR_KEY if (longjmp_xor_key == 0) { // bionic libc initialization process: __libc_init_globals -> // __libc_init_vdso (calls strcmp) -> __libc_init_setjmp_cookie. strcmp is @@ -680,7 +680,8 @@ ThreadState *cur_thread() { // significant bit of TLS_SLOT_SANITIZER to 1. Scudo allocator uses this bit // as a flag to disable memory initialization. This is a workaround to get the // correct ThreadState pointer. - reinterpret_cast<ThreadState*>(addr & ~1ULL); + uptr addr = reinterpret_cast<uptr>(thr); + return reinterpret_cast<ThreadState*>(addr & ~1ULL); } void set_cur_thread(ThreadState *thr) { |