diff options
Diffstat (limited to 'libatomic/config/x86/host-config.h')
| -rw-r--r-- | libatomic/config/x86/host-config.h | 43 |
1 files changed, 36 insertions, 7 deletions
diff --git a/libatomic/config/x86/host-config.h b/libatomic/config/x86/host-config.h index f20ce09..007b7e1 100644 --- a/libatomic/config/x86/host-config.h +++ b/libatomic/config/x86/host-config.h @@ -55,31 +55,37 @@ load_feat1 (void) } #ifdef __x86_64__ -# define IFUNC_COND_1 (load_feat1 () & bit_CMPXCHG16B) +# define IFUNC_COND_1 ((load_feat1 () & (bit_AVX | bit_CMPXCHG16B)) \ + == (bit_AVX | bit_CMPXCHG16B)) +# define IFUNC_COND_2 (load_feat1 () & bit_CMPXCHG16B) #else # define IFUNC_COND_1 (load_feat1 () & bit_CMPXCHG8B) #endif #ifdef __x86_64__ -# define IFUNC_NCOND(N) (N == 16) +# define IFUNC_NCOND(N) (2 * (N == 16)) #else # define IFUNC_NCOND(N) (N == 8) #endif #ifdef __x86_64__ # undef MAYBE_HAVE_ATOMIC_CAS_16 -# define MAYBE_HAVE_ATOMIC_CAS_16 IFUNC_COND_1 +# define MAYBE_HAVE_ATOMIC_CAS_16 IFUNC_COND_2 # undef MAYBE_HAVE_ATOMIC_EXCHANGE_16 -# define MAYBE_HAVE_ATOMIC_EXCHANGE_16 IFUNC_COND_1 +# define MAYBE_HAVE_ATOMIC_EXCHANGE_16 IFUNC_COND_2 # undef MAYBE_HAVE_ATOMIC_LDST_16 -# define MAYBE_HAVE_ATOMIC_LDST_16 IFUNC_COND_1 +# define MAYBE_HAVE_ATOMIC_LDST_16 IFUNC_COND_2 /* Since load and store are implemented with CAS, they are not fast. */ # undef FAST_ATOMIC_LDST_16 # define FAST_ATOMIC_LDST_16 0 -# if IFUNC_ALT == 1 +# if IFUNC_ALT != 0 # undef HAVE_ATOMIC_CAS_16 # define HAVE_ATOMIC_CAS_16 1 # endif +# if IFUNC_ALT == 1 +# undef HAVE_ATOMIC_LDST_16 +# define HAVE_ATOMIC_LDST_16 1 +# endif #else # undef MAYBE_HAVE_ATOMIC_CAS_8 # define MAYBE_HAVE_ATOMIC_CAS_8 IFUNC_COND_1 @@ -93,7 +99,7 @@ load_feat1 (void) # endif #endif -#if defined(__x86_64__) && N == 16 && IFUNC_ALT == 1 +#if defined(__x86_64__) && N == 16 && IFUNC_ALT != 0 static inline bool atomic_compare_exchange_n (UTYPE *mptr, UTYPE *eptr, UTYPE newval, bool weak_p UNUSED, int sm UNUSED, int fm UNUSED) @@ -108,6 +114,29 @@ atomic_compare_exchange_n (UTYPE *mptr, UTYPE *eptr, UTYPE newval, # define atomic_compare_exchange_n atomic_compare_exchange_n #endif /* Have CAS 16 */ +#if defined(__x86_64__) && N == 16 && IFUNC_ALT == 1 +#define __atomic_load_n(ptr, model) \ + (sizeof (*ptr) == 16 ? atomic_load_n (ptr, model) \ + : (__atomic_load_n) (ptr, model)) +#define __atomic_store_n(ptr, val, model) \ + (sizeof (*ptr) == 16 ? atomic_store_n (ptr, val, model) \ + : (__atomic_store_n) (ptr, val, model)) + +static inline UTYPE +atomic_load_n (UTYPE *ptr, int model UNUSED) +{ + UTYPE ret; + __asm__ ("vmovdqa\t{%1, %0|%0, %1}" : "=x" (ret) : "m" (*ptr)); + return ret; +} + +static inline void +atomic_store_n (UTYPE *ptr, UTYPE val, int model UNUSED) +{ + __asm__ ("vmovdqa\t{%1, %0|%0, %1}\n\tmfence" : "=m" (*ptr) : "x" (val)); +} +#endif + #endif /* HAVE_IFUNC */ #include_next <host-config.h> |