/*
 * Copyright 2013-2014 MongoDB, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "bson-prelude.h"


#ifndef BSON_ATOMIC_H
#define BSON_ATOMIC_H


#include "bson-config.h"
#include "bson-compat.h"
#include "bson-macros.h"

#ifdef _MSC_VER
#include <intrin.h>
#endif


BSON_BEGIN_DECLS

enum bson_memory_order {
   bson_memory_order_seq_cst,
   bson_memory_order_acquire,
   bson_memory_order_release,
   bson_memory_order_relaxed,
   bson_memory_order_acq_rel,
   bson_memory_order_consume,
};

#if defined(_M_ARM) /* MSVC memorder atomics are only avail on ARM */
#define MSVC_MEMORDER_SUFFIX(X) X
#else
#define MSVC_MEMORDER_SUFFIX(X)
#endif

#if defined(USE_LEGACY_GCC_ATOMICS) || (!defined(__clang__) && __GNUC__ == 4) || defined(__xlC__)
#define BSON_USE_LEGACY_GCC_ATOMICS
#else
#undef BSON_USE_LEGACY_GCC_ATOMICS
#endif

/* Not all GCC-like compilers support the current __atomic built-ins.  Older
 * GCC (pre-5) used different built-ins named with the __sync prefix.  When
 * compiling with such older GCC versions, it is necessary to use the applicable
 * functions, which requires redefining BSON_IF_GNU_LIKE and defining the
 * additional BSON_IF_GNU_LEGACY_ATOMICS macro here. */
#ifdef BSON_USE_LEGACY_GCC_ATOMICS
#undef BSON_IF_GNU_LIKE
#define BSON_IF_GNU_LIKE(...)
#define BSON_IF_MSVC(...)
#define BSON_IF_GNU_LEGACY_ATOMICS(...) __VA_ARGS__
#else
#define BSON_IF_GNU_LEGACY_ATOMICS(...)
#endif

/* CDRIVER-4229 zSeries with gcc 4.8.4 produces illegal instructions for int and
 * int32 atomic intrinsics. */
#if defined(__s390__) || defined(__s390x__) || defined(__zarch__)
#define BSON_EMULATE_INT32
#define BSON_EMULATE_INT
#endif

/* CDRIVER-4264 Contrary to documentation, VS 2013 targeting x86 does not
 * correctly/consistently provide _InterlockedPointerExchange. */
#if defined(_MSC_VER) && _MSC_VER < 1900 && defined(_M_IX86)
#define BSON_EMULATE_PTR
#endif

#define DEF_ATOMIC_OP(                                                       \
   MSVC_Intrinsic, GNU_Intrinsic, GNU_Legacy_Intrinsic, Order, ...)          \
   do {                                                                      \
      switch (Order) {                                                       \
      case bson_memory_order_acq_rel:                                        \
         BSON_IF_MSVC (return MSVC_Intrinsic (__VA_ARGS__);)                 \
         BSON_IF_GNU_LIKE (                                                  \
            return GNU_Intrinsic (__VA_ARGS__, __ATOMIC_ACQ_REL);)           \
         BSON_IF_GNU_LEGACY_ATOMICS (                                        \
            return GNU_Legacy_Intrinsic (__VA_ARGS__);)                      \
      case bson_memory_order_seq_cst:                                        \
         BSON_IF_MSVC (return MSVC_Intrinsic (__VA_ARGS__);)                 \
         BSON_IF_GNU_LIKE (                                                  \
            return GNU_Intrinsic (__VA_ARGS__, __ATOMIC_SEQ_CST);)           \
         BSON_IF_GNU_LEGACY_ATOMICS (                                        \
            return GNU_Legacy_Intrinsic (__VA_ARGS__);)                      \
      case bson_memory_order_acquire:                                        \
         BSON_IF_MSVC (                                                      \
            return BSON_CONCAT (MSVC_Intrinsic,                              \
                                MSVC_MEMORDER_SUFFIX (_acq)) (__VA_ARGS__);) \
         BSON_IF_GNU_LIKE (                                                  \
            return GNU_Intrinsic (__VA_ARGS__, __ATOMIC_ACQUIRE);)           \
         BSON_IF_GNU_LEGACY_ATOMICS (                                        \
            return GNU_Legacy_Intrinsic (__VA_ARGS__);)                      \
      case bson_memory_order_consume:                                        \
         BSON_IF_MSVC (                                                      \
            return BSON_CONCAT (MSVC_Intrinsic,                              \
                                MSVC_MEMORDER_SUFFIX (_acq)) (__VA_ARGS__);) \
         BSON_IF_GNU_LIKE (                                                  \
            return GNU_Intrinsic (__VA_ARGS__, __ATOMIC_CONSUME);)           \
         BSON_IF_GNU_LEGACY_ATOMICS (                                        \
            return GNU_Legacy_Intrinsic (__VA_ARGS__);)                      \
      case bson_memory_order_release:                                        \
         BSON_IF_MSVC (                                                      \
            return BSON_CONCAT (MSVC_Intrinsic,                              \
                                MSVC_MEMORDER_SUFFIX (_rel)) (__VA_ARGS__);) \
         BSON_IF_GNU_LIKE (                                                  \
            return GNU_Intrinsic (__VA_ARGS__, __ATOMIC_RELEASE);)           \
         BSON_IF_GNU_LEGACY_ATOMICS (                                        \
            return GNU_Legacy_Intrinsic (__VA_ARGS__);)                      \
      case bson_memory_order_relaxed:                                        \
         BSON_IF_MSVC (                                                      \
            return BSON_CONCAT (MSVC_Intrinsic,                              \
                                MSVC_MEMORDER_SUFFIX (_nf)) (__VA_ARGS__);)  \
         BSON_IF_GNU_LIKE (                                                  \
            return GNU_Intrinsic (__VA_ARGS__, __ATOMIC_RELAXED);)           \
         BSON_IF_GNU_LEGACY_ATOMICS (                                        \
            return GNU_Legacy_Intrinsic (__VA_ARGS__);)                      \
      default:                                                               \
         BSON_UNREACHABLE ("Invalid bson_memory_order value");               \
      }                                                                      \
   } while (0)


#define DEF_ATOMIC_CMPEXCH_STRONG(                                            \
   VCSuffix1, VCSuffix2, GNU_MemOrder, Ptr, ExpectActualVar, NewValue)        \
   do {                                                                       \
      BSON_IF_MSVC (ExpectActualVar = BSON_CONCAT3 (                          \
                       _InterlockedCompareExchange, VCSuffix1, VCSuffix2) (   \
                       Ptr, NewValue, ExpectActualVar);)                      \
      BSON_IF_GNU_LIKE (                                                      \
         (void) __atomic_compare_exchange_n (Ptr,                             \
                                             &ExpectActualVar,                \
                                             NewValue,                        \
                                             false, /* Not weak */            \
                                             GNU_MemOrder,                    \
                                             GNU_MemOrder);)                  \
      BSON_IF_GNU_LEGACY_ATOMICS (                                            \
         __typeof__ (ExpectActualVar) _val;                                   \
         _val = __sync_val_compare_and_swap (Ptr, ExpectActualVar, NewValue); \
         ExpectActualVar = _val;)                                             \
   } while (0)


#define DEF_ATOMIC_CMPEXCH_WEAK(                                              \
   VCSuffix1, VCSuffix2, GNU_MemOrder, Ptr, ExpectActualVar, NewValue)        \
   do {                                                                       \
      BSON_IF_MSVC (ExpectActualVar = BSON_CONCAT3 (                          \
                       _InterlockedCompareExchange, VCSuffix1, VCSuffix2) (   \
                       Ptr, NewValue, ExpectActualVar);)                      \
      BSON_IF_GNU_LIKE (                                                      \
         (void) __atomic_compare_exchange_n (Ptr,                             \
                                             &ExpectActualVar,                \
                                             NewValue,                        \
                                             true, /* Yes weak */             \
                                             GNU_MemOrder,                    \
                                             GNU_MemOrder);)                  \
      BSON_IF_GNU_LEGACY_ATOMICS (                                            \
         __typeof__ (ExpectActualVar) _val;                                   \
         _val = __sync_val_compare_and_swap (Ptr, ExpectActualVar, NewValue); \
         ExpectActualVar = _val;)                                             \
   } while (0)


#define DECL_ATOMIC_INTEGRAL(NamePart, Type, VCIntrinSuffix)                  \
   static BSON_INLINE Type bson_atomic_##NamePart##_fetch_add (               \
      Type volatile *a, Type addend, enum bson_memory_order ord)              \
   {                                                                          \
      DEF_ATOMIC_OP (BSON_CONCAT (_InterlockedExchangeAdd, VCIntrinSuffix),   \
                     __atomic_fetch_add,                                      \
                     __sync_fetch_and_add,                                    \
                     ord,                                                     \
                     a,                                                       \
                     addend);                                                 \
   }                                                                          \
                                                                              \
   static BSON_INLINE Type bson_atomic_##NamePart##_fetch_sub (               \
      Type volatile *a, Type subtrahend, enum bson_memory_order ord)          \
   {                                                                          \
      /* MSVC doesn't have a subtract intrinsic, so just reuse addition    */ \
      BSON_IF_MSVC (                                                          \
         return bson_atomic_##NamePart##_fetch_add (a, -subtrahend, ord);)    \
      BSON_IF_GNU_LIKE (                                                      \
         DEF_ATOMIC_OP (~, __atomic_fetch_sub, ~, ord, a, subtrahend);)       \
      BSON_IF_GNU_LEGACY_ATOMICS (                                            \
         DEF_ATOMIC_OP (~, ~, __sync_fetch_and_sub, ord, a, subtrahend);)     \
   }                                                                          \
                                                                              \
   static BSON_INLINE Type bson_atomic_##NamePart##_fetch (                   \
      Type volatile const *a, enum bson_memory_order order)                   \
   {                                                                          \
      /* MSVC doesn't have a load intrinsic, so just add zero */              \
      BSON_IF_MSVC (return bson_atomic_##NamePart##_fetch_add (               \
                              (Type volatile *) a, 0, order);)                \
      /* GNU doesn't want RELEASE order for the fetch operation, so we can't  \
       * just use DEF_ATOMIC_OP. */                                           \
      BSON_IF_GNU_LIKE (switch (order) {                                      \
         case bson_memory_order_release: /* Fall back to seqcst */            \
         case bson_memory_order_acq_rel: /* Fall back to seqcst */            \
         case bson_memory_order_seq_cst:                                      \
            return __atomic_load_n (a, __ATOMIC_SEQ_CST);                     \
         case bson_memory_order_acquire:                                      \
            return __atomic_load_n (a, __ATOMIC_ACQUIRE);                     \
         case bson_memory_order_consume:                                      \
            return __atomic_load_n (a, __ATOMIC_CONSUME);                     \
         case bson_memory_order_relaxed:                                      \
            return __atomic_load_n (a, __ATOMIC_RELAXED);                     \
         default:                                                             \
            BSON_UNREACHABLE ("Invalid bson_memory_order value");             \
      })                                                                      \
      BSON_IF_GNU_LEGACY_ATOMICS ({                                           \
         __sync_synchronize ();                                               \
         return *a;                                                           \
      })                                                                      \
   }                                                                          \
                                                                              \
   static BSON_INLINE Type bson_atomic_##NamePart##_exchange (                \
      Type volatile *a, Type value, enum bson_memory_order ord)               \
   {                                                                          \
      BSON_IF_MSVC (                                                          \
         DEF_ATOMIC_OP (BSON_CONCAT (_InterlockedExchange, VCIntrinSuffix),   \
                        ~,                                                    \
                        ~,                                                    \
                        ord,                                                  \
                        a,                                                    \
                        value);)                                              \
      /* GNU doesn't want CONSUME order for the exchange operation, so we     \
       * cannot use DEF_ATOMIC_OP. */                                         \
      BSON_IF_GNU_LIKE (switch (ord) {                                        \
         case bson_memory_order_acq_rel:                                      \
            return __atomic_exchange_n (a, value, __ATOMIC_ACQ_REL);          \
         case bson_memory_order_release:                                      \
            return __atomic_exchange_n (a, value, __ATOMIC_RELEASE);          \
         case bson_memory_order_seq_cst:                                      \
            return __atomic_exchange_n (a, value, __ATOMIC_SEQ_CST);          \
         case bson_memory_order_consume: /* Fall back to acquire */           \
         case bson_memory_order_acquire:                                      \
            return __atomic_exchange_n (a, value, __ATOMIC_ACQUIRE);          \
         case bson_memory_order_relaxed:                                      \
            return __atomic_exchange_n (a, value, __ATOMIC_RELAXED);          \
         default:                                                             \
            BSON_UNREACHABLE ("Invalid bson_memory_order value");             \
      })                                                                      \
      BSON_IF_GNU_LEGACY_ATOMICS (                                            \
         return __sync_val_compare_and_swap (a, *a, value);)                  \
   }                                                                          \
                                                                              \
   static BSON_INLINE Type bson_atomic_##NamePart##_compare_exchange_strong ( \
      Type volatile *a,                                                       \
      Type expect,                                                            \
      Type new_value,                                                         \
      enum bson_memory_order ord)                                             \
   {                                                                          \
      Type actual = expect;                                                   \
      switch (ord) {                                                          \
      case bson_memory_order_release:                                         \
      case bson_memory_order_acq_rel:                                         \
      case bson_memory_order_seq_cst:                                         \
         DEF_ATOMIC_CMPEXCH_STRONG (                                          \
            VCIntrinSuffix, , __ATOMIC_SEQ_CST, a, actual, new_value);        \
         break;                                                               \
      case bson_memory_order_acquire:                                         \
         DEF_ATOMIC_CMPEXCH_STRONG (VCIntrinSuffix,                           \
                                    MSVC_MEMORDER_SUFFIX (_acq),              \
                                    __ATOMIC_ACQUIRE,                         \
                                    a,                                        \
                                    actual,                                   \
                                    new_value);                               \
         break;                                                               \
      case bson_memory_order_consume:                                         \
         DEF_ATOMIC_CMPEXCH_STRONG (VCIntrinSuffix,                           \
                                    MSVC_MEMORDER_SUFFIX (_acq),              \
                                    __ATOMIC_CONSUME,                         \
                                    a,                                        \
                                    actual,                                   \
                                    new_value);                               \
         break;                                                               \
      case bson_memory_order_relaxed:                                         \
         DEF_ATOMIC_CMPEXCH_STRONG (VCIntrinSuffix,                           \
                                    MSVC_MEMORDER_SUFFIX (_nf),               \
                                    __ATOMIC_RELAXED,                         \
                                    a,                                        \
                                    actual,                                   \
                                    new_value);                               \
         break;                                                               \
      default:                                                                \
         BSON_UNREACHABLE ("Invalid bson_memory_order value");                \
      }                                                                       \
      return actual;                                                          \
   }                                                                          \
                                                                              \
   static BSON_INLINE Type bson_atomic_##NamePart##_compare_exchange_weak (   \
      Type volatile *a,                                                       \
      Type expect,                                                            \
      Type new_value,                                                         \
      enum bson_memory_order ord)                                             \
   {                                                                          \
      Type actual = expect;                                                   \
      switch (ord) {                                                          \
      case bson_memory_order_release:                                         \
      case bson_memory_order_acq_rel:                                         \
      case bson_memory_order_seq_cst:                                         \
         DEF_ATOMIC_CMPEXCH_WEAK (                                            \
            VCIntrinSuffix, , __ATOMIC_SEQ_CST, a, actual, new_value);        \
         break;                                                               \
      case bson_memory_order_acquire:                                         \
         DEF_ATOMIC_CMPEXCH_WEAK (VCIntrinSuffix,                             \
                                  MSVC_MEMORDER_SUFFIX (_acq),                \
                                  __ATOMIC_ACQUIRE,                           \
                                  a,                                          \
                                  actual,                                     \
                                  new_value);                                 \
         break;                                                               \
      case bson_memory_order_consume:                                         \
         DEF_ATOMIC_CMPEXCH_WEAK (VCIntrinSuffix,                             \
                                  MSVC_MEMORDER_SUFFIX (_acq),                \
                                  __ATOMIC_CONSUME,                           \
                                  a,                                          \
                                  actual,                                     \
                                  new_value);                                 \
         break;                                                               \
      case bson_memory_order_relaxed:                                         \
         DEF_ATOMIC_CMPEXCH_WEAK (VCIntrinSuffix,                             \
                                  MSVC_MEMORDER_SUFFIX (_nf),                 \
                                  __ATOMIC_RELAXED,                           \
                                  a,                                          \
                                  actual,                                     \
                                  new_value);                                 \
         break;                                                               \
      default:                                                                \
         BSON_UNREACHABLE ("Invalid bson_memory_order value");                \
      }                                                                       \
      return actual;                                                          \
   }

#define DECL_ATOMIC_STDINT(Name, VCSuffix) \
   DECL_ATOMIC_INTEGRAL (Name, Name##_t, VCSuffix)

#if defined(_MSC_VER) || defined(BSON_USE_LEGACY_GCC_ATOMICS)
/* MSVC and GCC require built-in types (not typedefs) for their atomic
 * intrinsics. */
#if defined(_MSC_VER)
#define DECL_ATOMIC_INTEGRAL_INT8 char
#define DECL_ATOMIC_INTEGRAL_INT32 long
#define DECL_ATOMIC_INTEGRAL_INT long
#else
#define DECL_ATOMIC_INTEGRAL_INT8 signed char
#define DECL_ATOMIC_INTEGRAL_INT32 int
#define DECL_ATOMIC_INTEGRAL_INT int
#endif
DECL_ATOMIC_INTEGRAL (int8, DECL_ATOMIC_INTEGRAL_INT8, 8)
DECL_ATOMIC_INTEGRAL (int16, short, 16)
#if !defined(BSON_EMULATE_INT32)
DECL_ATOMIC_INTEGRAL (int32, DECL_ATOMIC_INTEGRAL_INT32, )
#endif
#if !defined(BSON_EMULATE_INT)
DECL_ATOMIC_INTEGRAL (int, DECL_ATOMIC_INTEGRAL_INT, )
#endif
#else
/* Other compilers that we support provide generic intrinsics */
DECL_ATOMIC_STDINT (int8, 8)
DECL_ATOMIC_STDINT (int16, 16)
#if !defined(BSON_EMULATE_INT32)
DECL_ATOMIC_STDINT (int32, )
#endif
#if !defined(BSON_EMULATE_INT)
DECL_ATOMIC_INTEGRAL (int, int, )
#endif
#endif

BSON_EXPORT (int64_t)
_bson_emul_atomic_int64_fetch_add (int64_t volatile *val,
                                   int64_t v,
                                   enum bson_memory_order);
BSON_EXPORT (int64_t)
_bson_emul_atomic_int64_exchange (int64_t volatile *val,
                                  int64_t v,
                                  enum bson_memory_order);
BSON_EXPORT (int64_t)
_bson_emul_atomic_int64_compare_exchange_strong (int64_t volatile *val,
                                                 int64_t expect_value,
                                                 int64_t new_value,
                                                 enum bson_memory_order);

BSON_EXPORT (int64_t)
_bson_emul_atomic_int64_compare_exchange_weak (int64_t volatile *val,
                                               int64_t expect_value,
                                               int64_t new_value,
                                               enum bson_memory_order);

BSON_EXPORT (int32_t)
_bson_emul_atomic_int32_fetch_add (int32_t volatile *val,
                                   int32_t v,
                                   enum bson_memory_order);
BSON_EXPORT (int32_t)
_bson_emul_atomic_int32_exchange (int32_t volatile *val,
                                  int32_t v,
                                  enum bson_memory_order);
BSON_EXPORT (int32_t)
_bson_emul_atomic_int32_compare_exchange_strong (int32_t volatile *val,
                                                 int32_t expect_value,
                                                 int32_t new_value,
                                                 enum bson_memory_order);

BSON_EXPORT (int32_t)
_bson_emul_atomic_int32_compare_exchange_weak (int32_t volatile *val,
                                               int32_t expect_value,
                                               int32_t new_value,
                                               enum bson_memory_order);

BSON_EXPORT (int)
_bson_emul_atomic_int_fetch_add (int volatile *val,
                                 int v,
                                 enum bson_memory_order);
BSON_EXPORT (int)
_bson_emul_atomic_int_exchange (int volatile *val,
                                int v,
                                enum bson_memory_order);
BSON_EXPORT (int)
_bson_emul_atomic_int_compare_exchange_strong (int volatile *val,
                                               int expect_value,
                                               int new_value,
                                               enum bson_memory_order);

BSON_EXPORT (int)
_bson_emul_atomic_int_compare_exchange_weak (int volatile *val,
                                             int expect_value,
                                             int new_value,
                                             enum bson_memory_order);

BSON_EXPORT (void *)
_bson_emul_atomic_ptr_exchange (void *volatile *val,
                                void *v,
                                enum bson_memory_order);

BSON_EXPORT (void)
bson_thrd_yield (void);

#if (defined(_MSC_VER) && !defined(_M_IX86)) || (defined(__LP64__) && __LP64__)
/* (64-bit intrinsics are only available in x64) */
#ifdef _MSC_VER
DECL_ATOMIC_INTEGRAL (int64, __int64, 64)
#else
DECL_ATOMIC_STDINT (int64, 64)
#endif
#else
static BSON_INLINE int64_t
bson_atomic_int64_fetch (const int64_t volatile *val,
                         enum bson_memory_order order)
{
   return _bson_emul_atomic_int64_fetch_add (
      (int64_t volatile *) val, 0, order);
}

static BSON_INLINE int64_t
bson_atomic_int64_fetch_add (int64_t volatile *val,
                             int64_t v,
                             enum bson_memory_order order)
{
   return _bson_emul_atomic_int64_fetch_add (val, v, order);
}

static BSON_INLINE int64_t
bson_atomic_int64_fetch_sub (int64_t volatile *val,
                             int64_t v,
                             enum bson_memory_order order)
{
   return _bson_emul_atomic_int64_fetch_add (val, -v, order);
}

static BSON_INLINE int64_t
bson_atomic_int64_exchange (int64_t volatile *val,
                            int64_t v,
                            enum bson_memory_order order)
{
   return _bson_emul_atomic_int64_exchange (val, v, order);
}

static BSON_INLINE int64_t
bson_atomic_int64_compare_exchange_strong (int64_t volatile *val,
                                           int64_t expect_value,
                                           int64_t new_value,
                                           enum bson_memory_order order)
{
   return _bson_emul_atomic_int64_compare_exchange_strong (
      val, expect_value, new_value, order);
}

static BSON_INLINE int64_t
bson_atomic_int64_compare_exchange_weak (int64_t volatile *val,
                                         int64_t expect_value,
                                         int64_t new_value,
                                         enum bson_memory_order order)
{
   return _bson_emul_atomic_int64_compare_exchange_weak (
      val, expect_value, new_value, order);
}
#endif

#if defined(BSON_EMULATE_INT32)
static BSON_INLINE int32_t
bson_atomic_int32_fetch (const int32_t volatile *val,
                         enum bson_memory_order order)
{
   return _bson_emul_atomic_int32_fetch_add (
      (int32_t volatile *) val, 0, order);
}

static BSON_INLINE int32_t
bson_atomic_int32_fetch_add (int32_t volatile *val,
                             int32_t v,
                             enum bson_memory_order order)
{
   return _bson_emul_atomic_int32_fetch_add (val, v, order);
}

static BSON_INLINE int32_t
bson_atomic_int32_fetch_sub (int32_t volatile *val,
                             int32_t v,
                             enum bson_memory_order order)
{
   return _bson_emul_atomic_int32_fetch_add (val, -v, order);
}

static BSON_INLINE int32_t
bson_atomic_int32_exchange (int32_t volatile *val,
                            int32_t v,
                            enum bson_memory_order order)
{
   return _bson_emul_atomic_int32_exchange (val, v, order);
}

static BSON_INLINE int32_t
bson_atomic_int32_compare_exchange_strong (int32_t volatile *val,
                                           int32_t expect_value,
                                           int32_t new_value,
                                           enum bson_memory_order order)
{
   return _bson_emul_atomic_int32_compare_exchange_strong (
      val, expect_value, new_value, order);
}

static BSON_INLINE int32_t
bson_atomic_int32_compare_exchange_weak (int32_t volatile *val,
                                         int32_t expect_value,
                                         int32_t new_value,
                                         enum bson_memory_order order)
{
   return _bson_emul_atomic_int32_compare_exchange_weak (
      val, expect_value, new_value, order);
}
#endif /* BSON_EMULATE_INT32 */

#if defined(BSON_EMULATE_INT)
static BSON_INLINE int
bson_atomic_int_fetch (const int volatile *val, enum bson_memory_order order)
{
   return _bson_emul_atomic_int_fetch_add ((int volatile *) val, 0, order);
}

static BSON_INLINE int
bson_atomic_int_fetch_add (int volatile *val,
                           int v,
                           enum bson_memory_order order)
{
   return _bson_emul_atomic_int_fetch_add (val, v, order);
}

static BSON_INLINE int
bson_atomic_int_fetch_sub (int volatile *val,
                           int v,
                           enum bson_memory_order order)
{
   return _bson_emul_atomic_int_fetch_add (val, -v, order);
}

static BSON_INLINE int
bson_atomic_int_exchange (int volatile *val,
                          int v,
                          enum bson_memory_order order)
{
   return _bson_emul_atomic_int_exchange (val, v, order);
}

static BSON_INLINE int
bson_atomic_int_compare_exchange_strong (int volatile *val,
                                         int expect_value,
                                         int new_value,
                                         enum bson_memory_order order)
{
   return _bson_emul_atomic_int_compare_exchange_strong (
      val, expect_value, new_value, order);
}

static BSON_INLINE int
bson_atomic_int_compare_exchange_weak (int volatile *val,
                                       int expect_value,
                                       int new_value,
                                       enum bson_memory_order order)
{
   return _bson_emul_atomic_int_compare_exchange_weak (
      val, expect_value, new_value, order);
}
#endif /* BSON_EMULATE_INT */

static BSON_INLINE void *
bson_atomic_ptr_exchange (void *volatile *ptr,
                          void *new_value,
                          enum bson_memory_order ord)
{
#if defined(BSON_EMULATE_PTR)
   return _bson_emul_atomic_ptr_exchange (ptr, new_value, ord);
#elif defined(BSON_USE_LEGACY_GCC_ATOMICS)
   /* The older __sync_val_compare_and_swap also takes oldval */
   DEF_ATOMIC_OP (_InterlockedExchangePointer,
                  ,
                  __sync_val_compare_and_swap,
                  ord,
                  ptr,
                  *ptr,
                  new_value);
#else
   DEF_ATOMIC_OP (
      _InterlockedExchangePointer, __atomic_exchange_n, , ord, ptr, new_value);
#endif
}

static BSON_INLINE void *
bson_atomic_ptr_compare_exchange_strong (void *volatile *ptr,
                                         void *expect,
                                         void *new_value,
                                         enum bson_memory_order ord)
{
   switch (ord) {
   case bson_memory_order_release:
   case bson_memory_order_acq_rel:
   case bson_memory_order_seq_cst:
      DEF_ATOMIC_CMPEXCH_STRONG (
         Pointer, , __ATOMIC_SEQ_CST, ptr, expect, new_value);
      return expect;
   case bson_memory_order_relaxed:
      DEF_ATOMIC_CMPEXCH_STRONG (Pointer,
                                 MSVC_MEMORDER_SUFFIX (_nf),
                                 __ATOMIC_RELAXED,
                                 ptr,
                                 expect,
                                 new_value);
      return expect;
   case bson_memory_order_consume:
      DEF_ATOMIC_CMPEXCH_STRONG (Pointer,
                                 MSVC_MEMORDER_SUFFIX (_acq),
                                 __ATOMIC_CONSUME,
                                 ptr,
                                 expect,
                                 new_value);
      return expect;
   case bson_memory_order_acquire:
      DEF_ATOMIC_CMPEXCH_STRONG (Pointer,
                                 MSVC_MEMORDER_SUFFIX (_acq),
                                 __ATOMIC_ACQUIRE,
                                 ptr,
                                 expect,
                                 new_value);
      return expect;
   default:
      BSON_UNREACHABLE ("Invalid bson_memory_order value");
   }
}


static BSON_INLINE void *
bson_atomic_ptr_compare_exchange_weak (void *volatile *ptr,
                                       void *expect,
                                       void *new_value,
                                       enum bson_memory_order ord)
{
   switch (ord) {
   case bson_memory_order_release:
   case bson_memory_order_acq_rel:
   case bson_memory_order_seq_cst:
      DEF_ATOMIC_CMPEXCH_WEAK (
         Pointer, , __ATOMIC_SEQ_CST, ptr, expect, new_value);
      return expect;
   case bson_memory_order_relaxed:
      DEF_ATOMIC_CMPEXCH_WEAK (Pointer,
                               MSVC_MEMORDER_SUFFIX (_nf),
                               __ATOMIC_RELAXED,
                               ptr,
                               expect,
                               new_value);
      return expect;
   case bson_memory_order_consume:
      DEF_ATOMIC_CMPEXCH_WEAK (Pointer,
                               MSVC_MEMORDER_SUFFIX (_acq),
                               __ATOMIC_CONSUME,
                               ptr,
                               expect,
                               new_value);
      return expect;
   case bson_memory_order_acquire:
      DEF_ATOMIC_CMPEXCH_WEAK (Pointer,
                               MSVC_MEMORDER_SUFFIX (_acq),
                               __ATOMIC_ACQUIRE,
                               ptr,
                               expect,
                               new_value);
      return expect;
   default:
      BSON_UNREACHABLE ("Invalid bson_memory_order value");
   }
}


static BSON_INLINE void *
bson_atomic_ptr_fetch (void *volatile const *ptr, enum bson_memory_order ord)
{
   return bson_atomic_ptr_compare_exchange_strong (
      (void *volatile *) ptr, NULL, NULL, ord);
}

#undef DECL_ATOMIC_STDINT
#undef DECL_ATOMIC_INTEGRAL
#undef DEF_ATOMIC_OP
#undef DEF_ATOMIC_CMPEXCH_STRONG
#undef DEF_ATOMIC_CMPEXCH_WEAK
#undef MSVC_MEMORDER_SUFFIX

/**
 * @brief Generate a full-fence memory barrier at the call site.
 */
static BSON_INLINE void
bson_atomic_thread_fence (void)
{
   BSON_IF_MSVC (MemoryBarrier ();)
   BSON_IF_GNU_LIKE (__sync_synchronize ();)
   BSON_IF_GNU_LEGACY_ATOMICS (__sync_synchronize ();)
}

#ifdef BSON_USE_LEGACY_GCC_ATOMICS
#undef BSON_IF_GNU_LIKE
#define BSON_IF_GNU_LIKE(...) __VA_ARGS__
#endif
#undef BSON_IF_GNU_LEGACY_ATOMICS
#undef BSON_USE_LEGACY_GCC_ATOMICS

BSON_GNUC_DEPRECATED_FOR ("bson_atomic_thread_fence")
BSON_EXPORT (void) bson_memory_barrier (void);

BSON_GNUC_DEPRECATED_FOR ("bson_atomic_int_fetch_add")
BSON_EXPORT (int32_t) bson_atomic_int_add (volatile int32_t *p, int32_t n);

BSON_GNUC_DEPRECATED_FOR ("bson_atomic_int64_fetch_add")
BSON_EXPORT (int64_t) bson_atomic_int64_add (volatile int64_t *p, int64_t n);


#undef BSON_EMULATE_PTR
#undef BSON_EMULATE_INT32
#undef BSON_EMULATE_INT

BSON_END_DECLS


#endif /* BSON_ATOMIC_H */
