diff options
| author | Wenju He <wenju.he@intel.com> | 2025-07-18 08:09:14 +0800 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2025-07-18 08:09:14 +0800 |
| commit | 9c26f37ce34de3be3550cbd67fcea534721724fd (patch) | |
| tree | f3e49a7f0cf17098676c3bb226bb0e98a4554642 | |
| parent | 64205adc3bfdaab0f35f7909de59dadd20ef6e6a (diff) | |
| download | llvm-9c26f37ce34de3be3550cbd67fcea534721724fd.zip llvm-9c26f37ce34de3be3550cbd67fcea534721724fd.tar.gz llvm-9c26f37ce34de3be3550cbd67fcea534721724fd.tar.bz2 | |
[libclc] Add generic implementation of some atomic functions in OpenCL spec section 6.15.12.7 (#146814)
Add corresponding clc functions, which are implemented with clang
__scoped_atomic builtins. OpenCL functions are implemented as a wrapper
over clc functions.
Also change legacy atomic_inc and atomic_dec to re-use the newly added
clc_atomic_inc/dec implementations. llvm-diff only no change to
atomic_inc and atomic_dec in bitcode.
Notes:
* Generic OpenCL built-ins functions uses __ATOMIC_SEQ_CST and
__MEMORY_SCOPE_DEVICE for memory order and memory scope parameters.
* OpenCL atomic_*_explicit, atomic_flag* built-ins are not implemented
yet.
* OpenCL built-ins of atomic_intptr_t, atomic_uintptr_t, atomic_size_t
and atomic_ptrdiff_t types are not implemented yet.
* llvm-diff shows no change to nvptx64--nvidiacl.bc and
amdgcn--amdhsa.bc since __opencl_c_atomic_order_seq_cst and
__opencl_c_atomic_scope_device are not defined in these two targets.
69 files changed, 1511 insertions, 43 deletions
diff --git a/libclc/clc/include/clc/atomic/atomic_decl.inc b/libclc/clc/include/clc/atomic/atomic_decl.inc new file mode 100644 index 0000000..b790a94 --- /dev/null +++ b/libclc/clc/include/clc/atomic/atomic_decl.inc @@ -0,0 +1,47 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// MemoryOrder is memory order supported by Clang __scoped_atomic* builtins. +// MemoryScope is memory scope supported by Clang __scoped_atomic* builtins. + +#ifdef __CLC_SCALAR +#if defined(__CLC_FPSIZE) || (__CLC_GENSIZE >= 32) + +#ifdef __CLC_NO_VALUE_ARG +#define __CLC_DECLARE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION( \ + volatile ADDRSPACE __CLC_GENTYPE *Ptr, int MemoryOrder, \ + int MemoryScope); +#elif defined(__CLC_RETURN_VOID) +#define __CLC_DECLARE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL void FUNCTION(volatile ADDRSPACE __CLC_GENTYPE *Ptr, \ + __CLC_GENTYPE Value, int MemoryOrder, \ + int MemoryScope); +#elif defined(__CLC_COMPARE_EXCHANGE) +#define __CLC_DECLARE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION( \ + volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Comparator, \ + __CLC_GENTYPE Value, int MemoryOrderEqual, int MemoryOrderUnequal, \ + int MemoryScope); +#else +#define __CLC_DECLARE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION( \ + volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Value, \ + int MemoryOrder, int MemoryScope); +#endif + +__CLC_DECLARE_ATOMIC(global) +__CLC_DECLARE_ATOMIC(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DECLARE_ATOMIC() +#endif + +#undef __CLC_DECLARE_ATOMIC + +#endif // defined(__CLC_FPSIZE) || (__CLC_GENSIZE >= 32) +#endif // __CLC_SCALAR diff --git a/libclc/clc/include/clc/atomic/clc_atomic_compare_exchange.h b/libclc/clc/include/clc/atomic/clc_atomic_compare_exchange.h new file mode 100644 index 0000000..ae7918a --- /dev/null +++ b/libclc/clc/include/clc/atomic/clc_atomic_compare_exchange.h @@ -0,0 +1,26 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_ATOMIC_CLC_ATOMIC_COMPARE_EXCHANGE_H__ +#define __CLC_ATOMIC_CLC_ATOMIC_COMPARE_EXCHANGE_H__ + +#include <clc/internal/clc.h> + +#define FUNCTION __clc_atomic_compare_exchange +#define __CLC_COMPARE_EXCHANGE + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/math/gentype.inc> + +#undef __CLC_COMPARE_EXCHANGE +#undef FUNCTION + +#endif // __CLC_ATOMIC_CLC_ATOMIC_COMPARE_EXCHANGE_H__ diff --git a/libclc/clc/include/clc/atomic/clc_atomic_dec.h b/libclc/clc/include/clc/atomic/clc_atomic_dec.h new file mode 100644 index 0000000..ada36ba --- /dev/null +++ b/libclc/clc/include/clc/atomic/clc_atomic_dec.h @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_ATOMIC_CLC_ATOMIC_DEC_H__ +#define __CLC_ATOMIC_CLC_ATOMIC_DEC_H__ + +#include <clc/internal/clc.h> + +#define FUNCTION __clc_atomic_dec +#define __CLC_NO_VALUE_ARG + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#undef __CLC_NO_VALUE_ARG +#undef FUNCTION + +#endif // __CLC_ATOMIC_CLC_ATOMIC_DEC_H__ diff --git a/libclc/clc/include/clc/atomic/clc_atomic_exchange.h b/libclc/clc/include/clc/atomic/clc_atomic_exchange.h new file mode 100644 index 0000000..7e626d4 --- /dev/null +++ b/libclc/clc/include/clc/atomic/clc_atomic_exchange.h @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_ATOMIC_CLC_ATOMIC_EXCHANGE_H__ +#define __CLC_ATOMIC_CLC_ATOMIC_EXCHANGE_H__ + +#include <clc/internal/clc.h> + +#define FUNCTION __clc_atomic_exchange + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/math/gentype.inc> + +#undef FUNCTION + +#endif // __CLC_ATOMIC_CLC_ATOMIC_EXCHANGE_H__ diff --git a/libclc/clc/include/clc/atomic/clc_atomic_fetch_add.h b/libclc/clc/include/clc/atomic/clc_atomic_fetch_add.h new file mode 100644 index 0000000..ad0c2eb4 --- /dev/null +++ b/libclc/clc/include/clc/atomic/clc_atomic_fetch_add.h @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_ATOMIC_CLC_ATOMIC_FETCH_ADD_H__ +#define __CLC_ATOMIC_CLC_ATOMIC_FETCH_ADD_H__ + +#include <clc/internal/clc.h> + +#define FUNCTION __clc_atomic_fetch_add + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/math/gentype.inc> + +#undef FUNCTION + +#endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_ADD_H__ diff --git a/libclc/clc/include/clc/atomic/clc_atomic_fetch_and.h b/libclc/clc/include/clc/atomic/clc_atomic_fetch_and.h new file mode 100644 index 0000000..80810c3 --- /dev/null +++ b/libclc/clc/include/clc/atomic/clc_atomic_fetch_and.h @@ -0,0 +1,21 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_ATOMIC_CLC_ATOMIC_FETCH_AND_H__ +#define __CLC_ATOMIC_CLC_ATOMIC_FETCH_AND_H__ + +#include <clc/internal/clc.h> + +#define FUNCTION __clc_atomic_fetch_and + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#undef FUNCTION + +#endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_AND_H__ diff --git a/libclc/clc/include/clc/atomic/clc_atomic_fetch_max.h b/libclc/clc/include/clc/atomic/clc_atomic_fetch_max.h new file mode 100644 index 0000000..56f5119 --- /dev/null +++ b/libclc/clc/include/clc/atomic/clc_atomic_fetch_max.h @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_ATOMIC_CLC_ATOMIC_FETCH_MAX_H__ +#define __CLC_ATOMIC_CLC_ATOMIC_FETCH_MAX_H__ + +#include <clc/internal/clc.h> + +#define FUNCTION __clc_atomic_fetch_max + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/math/gentype.inc> + +#undef FUNCTION + +#endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_MAX_H__ diff --git a/libclc/clc/include/clc/atomic/clc_atomic_fetch_min.h b/libclc/clc/include/clc/atomic/clc_atomic_fetch_min.h new file mode 100644 index 0000000..f17408d --- /dev/null +++ b/libclc/clc/include/clc/atomic/clc_atomic_fetch_min.h @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_ATOMIC_CLC_ATOMIC_FETCH_MIN_H__ +#define __CLC_ATOMIC_CLC_ATOMIC_FETCH_MIN_H__ + +#include <clc/internal/clc.h> + +#define FUNCTION __clc_atomic_fetch_min + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/math/gentype.inc> + +#undef FUNCTION + +#endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_MIN_H__ diff --git a/libclc/clc/include/clc/atomic/clc_atomic_fetch_or.h b/libclc/clc/include/clc/atomic/clc_atomic_fetch_or.h new file mode 100644 index 0000000..b82069e --- /dev/null +++ b/libclc/clc/include/clc/atomic/clc_atomic_fetch_or.h @@ -0,0 +1,21 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_ATOMIC_CLC_ATOMIC_FETCH_OR_H__ +#define __CLC_ATOMIC_CLC_ATOMIC_FETCH_OR_H__ + +#include <clc/internal/clc.h> + +#define FUNCTION __clc_atomic_fetch_or + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#undef FUNCTION + +#endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_OR_H__ diff --git a/libclc/clc/include/clc/atomic/clc_atomic_fetch_sub.h b/libclc/clc/include/clc/atomic/clc_atomic_fetch_sub.h new file mode 100644 index 0000000..6cfd224 --- /dev/null +++ b/libclc/clc/include/clc/atomic/clc_atomic_fetch_sub.h @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_ATOMIC_CLC_ATOMIC_FETCH_SUB_H__ +#define __CLC_ATOMIC_CLC_ATOMIC_FETCH_SUB_H__ + +#include <clc/internal/clc.h> + +#define FUNCTION __clc_atomic_fetch_sub + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/math/gentype.inc> + +#undef FUNCTION + +#endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_SUB_H__ diff --git a/libclc/clc/include/clc/atomic/clc_atomic_fetch_xor.h b/libclc/clc/include/clc/atomic/clc_atomic_fetch_xor.h new file mode 100644 index 0000000..b007b47 --- /dev/null +++ b/libclc/clc/include/clc/atomic/clc_atomic_fetch_xor.h @@ -0,0 +1,21 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_ATOMIC_CLC_ATOMIC_FETCH_XOR_H__ +#define __CLC_ATOMIC_CLC_ATOMIC_FETCH_XOR_H__ + +#include <clc/internal/clc.h> + +#define FUNCTION __clc_atomic_fetch_xor + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#undef FUNCTION + +#endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_XOR_H__ diff --git a/libclc/clc/include/clc/atomic/clc_atomic_inc.h b/libclc/clc/include/clc/atomic/clc_atomic_inc.h new file mode 100644 index 0000000..3ddef4a --- /dev/null +++ b/libclc/clc/include/clc/atomic/clc_atomic_inc.h @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_ATOMIC_CLC_ATOMIC_INC_H__ +#define __CLC_ATOMIC_CLC_ATOMIC_INC_H__ + +#include <clc/internal/clc.h> + +#define FUNCTION __clc_atomic_inc +#define __CLC_NO_VALUE_ARG + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#undef __CLC_NO_VALUE_ARG +#undef FUNCTION + +#endif // __CLC_ATOMIC_CLC_ATOMIC_INC_H__ diff --git a/libclc/clc/include/clc/atomic/clc_atomic_load.h b/libclc/clc/include/clc/atomic/clc_atomic_load.h new file mode 100644 index 0000000..a4899b3 --- /dev/null +++ b/libclc/clc/include/clc/atomic/clc_atomic_load.h @@ -0,0 +1,26 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_ATOMIC_CLC_ATOMIC_LOAD_H__ +#define __CLC_ATOMIC_CLC_ATOMIC_LOAD_H__ + +#include <clc/internal/clc.h> + +#define FUNCTION __clc_atomic_load +#define __CLC_NO_VALUE_ARG + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/math/gentype.inc> + +#undef __CLC_NO_VALUE_ARG +#undef FUNCTION + +#endif // __CLC_ATOMIC_CLC_ATOMIC_LOAD_H__ diff --git a/libclc/clc/include/clc/atomic/clc_atomic_store.h b/libclc/clc/include/clc/atomic/clc_atomic_store.h new file mode 100644 index 0000000..6baf0eb --- /dev/null +++ b/libclc/clc/include/clc/atomic/clc_atomic_store.h @@ -0,0 +1,26 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_ATOMIC_CLC_ATOMIC_STORE_H__ +#define __CLC_ATOMIC_CLC_ATOMIC_STORE_H__ + +#include <clc/internal/clc.h> + +#define FUNCTION __clc_atomic_store +#define __CLC_RETURN_VOID + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc/atomic/atomic_decl.inc> +#include <clc/math/gentype.inc> + +#undef __CLC_RETURN_VOID +#undef FUNCTION + +#endif // __CLC_ATOMIC_CLC_ATOMIC_STORE_H__ diff --git a/libclc/clc/lib/generic/SOURCES b/libclc/clc/lib/generic/SOURCES index 9d792c4..ee4f771 100644 --- a/libclc/clc/lib/generic/SOURCES +++ b/libclc/clc/lib/generic/SOURCES @@ -1,4 +1,17 @@ async/clc_prefetch.cl +atomic/clc_atomic_compare_exchange.cl +atomic/clc_atomic_dec.cl +atomic/clc_atomic_exchange.cl +atomic/clc_atomic_fetch_add.cl +atomic/clc_atomic_fetch_and.cl +atomic/clc_atomic_fetch_max.cl +atomic/clc_atomic_fetch_min.cl +atomic/clc_atomic_fetch_or.cl +atomic/clc_atomic_fetch_sub.cl +atomic/clc_atomic_fetch_xor.cl +atomic/clc_atomic_inc.cl +atomic/clc_atomic_load.cl +atomic/clc_atomic_store.cl common/clc_degrees.cl common/clc_radians.cl common/clc_sign.cl diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_compare_exchange.cl b/libclc/clc/lib/generic/atomic/clc_atomic_compare_exchange.cl new file mode 100644 index 0000000..796dedc --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_compare_exchange.cl @@ -0,0 +1,15 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/atomic/clc_atomic_compare_exchange.h> + +#define __CLC_BODY <clc_atomic_compare_exchange.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc_atomic_compare_exchange.inc> +#include <clc/math/gentype.inc> diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_compare_exchange.inc b/libclc/clc/lib/generic/atomic/clc_atomic_compare_exchange.inc new file mode 100644 index 0000000..32ff9b4 --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_compare_exchange.inc @@ -0,0 +1,64 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifdef __CLC_SCALAR + +#if defined(__SPIR32__) || defined(CLC_NVPTX) +#if (defined(__CLC_FPSIZE) && __CLC_FPSIZE <= 32) || \ + (defined(__CLC_GENSIZE) && (__CLC_GENSIZE == 32)) +#define __CLC_HAS_ATOMIC +#endif +#else // defined(__SPIR32__) || defined(CLC_NVPTX) +#if defined(__CLC_FPSIZE) || (__CLC_GENSIZE >= 32) +#define __CLC_HAS_ATOMIC +#endif +#endif // defined(__SPIR32__) || defined(CLC_NVPTX) + +#ifdef __CLC_HAS_ATOMIC + +#ifdef __CLC_FPSIZE + +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __clc_atomic_compare_exchange( \ + volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Comparator, \ + __CLC_GENTYPE Value, int MemoryOrderEqual, int MemoryOrderUnequal, \ + int MemoryScope) { \ + __CLC_U_GENTYPE Comp = __CLC_AS_U_GENTYPE(Comparator); \ + __scoped_atomic_compare_exchange_n( \ + (ADDRSPACE __CLC_U_GENTYPE *)Ptr, &Comp, __CLC_AS_U_GENTYPE(Value), \ + false, MemoryOrderEqual, MemoryOrderUnequal, MemoryScope); \ + return __CLC_AS_GENTYPE(Comp); \ + } + +#else + +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __clc_atomic_compare_exchange( \ + volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Comparator, \ + __CLC_GENTYPE Value, int MemoryOrderEqual, int MemoryOrderUnequal, \ + int MemoryScope) { \ + __scoped_atomic_compare_exchange_n(Ptr, &Comparator, Value, false, \ + MemoryOrderEqual, MemoryOrderUnequal, \ + MemoryScope); \ + return Comparator; \ + } + +#endif // __CLC_FPSIZE + +__CLC_DEFINE_ATOMIC(global) +__CLC_DEFINE_ATOMIC(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DEFINE_ATOMIC() +#endif + +#undef __CLC_DEFINE_ATOMIC + +#endif // __CLC_HAS_ATOMIC +#undef __CLC_HAS_ATOMIC + +#endif // __CLC_SCALAR diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_dec.cl b/libclc/clc/lib/generic/atomic/clc_atomic_dec.cl new file mode 100644 index 0000000..f35a962 --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_dec.cl @@ -0,0 +1,16 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/atomic/clc_atomic_dec.h> + +#define FUNCTION __clc_atomic_dec +#define __IMPL_FUNCTION __scoped_atomic_fetch_add +#define __CLC_INC_DEC + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/integer/gentype.inc> diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_def.inc b/libclc/clc/lib/generic/atomic/clc_atomic_def.inc new file mode 100644 index 0000000..2c45f49 --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_def.inc @@ -0,0 +1,79 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifdef __CLC_SCALAR + +#if defined(__SPIR32__) || defined(CLC_NVPTX) +#if (defined(__CLC_FPSIZE) && __CLC_FPSIZE <= 32) || \ + (defined(__CLC_GENSIZE) && (__CLC_GENSIZE == 32)) +#define __CLC_HAS_ATOMIC +#endif +#else // defined(__SPIR32__) || defined(CLC_NVPTX) +#if defined(__CLC_FPSIZE) || (__CLC_GENSIZE >= 32) +#define __CLC_HAS_ATOMIC +#endif +#endif // defined(__SPIR32__) || defined(CLC_NVPTX) + +#ifdef __CLC_HAS_ATOMIC + +#ifndef __CLC_PTR_CASTTYPE +#define __CLC_PTR_CASTTYPE __CLC_GENTYPE +#endif + +#ifndef __CLC_AS_RETTYPE +#define __CLC_AS_RETTYPE(x) x +#endif + +#ifdef __CLC_NO_VALUE_ARG +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION( \ + volatile ADDRSPACE __CLC_GENTYPE *Ptr, int MemoryOrder, \ + int MemoryScope) { \ + return __CLC_AS_RETTYPE(__IMPL_FUNCTION( \ + (ADDRSPACE __CLC_PTR_CASTTYPE *)Ptr, MemoryOrder, MemoryScope)); \ + } +#elif defined(__CLC_INC_DEC) +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION( \ + volatile ADDRSPACE __CLC_GENTYPE *Ptr, int MemoryOrder, \ + int MemoryScope) { \ + return __CLC_AS_RETTYPE( \ + __IMPL_FUNCTION((ADDRSPACE __CLC_PTR_CASTTYPE *)Ptr, (__CLC_GENTYPE)1, \ + MemoryOrder, MemoryScope)); \ + } +#elif defined(__CLC_RETURN_VOID) +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL void FUNCTION(volatile ADDRSPACE __CLC_GENTYPE *Ptr, \ + __CLC_GENTYPE Value, int MemoryOrder, \ + int MemoryScope) { \ + __IMPL_FUNCTION((ADDRSPACE __CLC_PTR_CASTTYPE *)Ptr, Value, MemoryOrder, \ + MemoryScope); \ + } +#else +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION( \ + volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Value, \ + int MemoryOrder, int MemoryScope) { \ + return __CLC_AS_RETTYPE( \ + __IMPL_FUNCTION((ADDRSPACE __CLC_PTR_CASTTYPE *)Ptr, Value, \ + MemoryOrder, MemoryScope)); \ + } +#endif + +__CLC_DEFINE_ATOMIC(global) +__CLC_DEFINE_ATOMIC(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DEFINE_ATOMIC() +#endif + +#undef __CLC_DEFINE_ATOMIC + +#endif // __CLC_HAS_ATOMIC +#undef __CLC_HAS_ATOMIC + +#endif // __CLC_SCALAR diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_exchange.cl b/libclc/clc/lib/generic/atomic/clc_atomic_exchange.cl new file mode 100644 index 0000000..52fd11af --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_exchange.cl @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/atomic/clc_atomic_exchange.h> + +#define FUNCTION __clc_atomic_exchange +#define __IMPL_FUNCTION __scoped_atomic_exchange_n + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/integer/gentype.inc> + +#undef __CLC_PTR_CASTTYPE +#undef __CLC_AS_RETTYPE +#define __CLC_PTR_CASTTYPE __CLC_BIT_INTN +#define __CLC_AS_RETTYPE(x) __CLC_AS_GENTYPE(x) + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/math/gentype.inc> diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_add.cl b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_add.cl new file mode 100644 index 0000000..0dc4491 --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_add.cl @@ -0,0 +1,18 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/atomic/clc_atomic_fetch_add.h> + +#define FUNCTION __clc_atomic_fetch_add +#define __IMPL_FUNCTION __scoped_atomic_fetch_add + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/math/gentype.inc> diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_and.cl b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_and.cl new file mode 100644 index 0000000..ec89738 --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_and.cl @@ -0,0 +1,15 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/atomic/clc_atomic_fetch_and.h> + +#define FUNCTION __clc_atomic_fetch_and +#define __IMPL_FUNCTION __scoped_atomic_fetch_and + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/integer/gentype.inc> diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_max.cl b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_max.cl new file mode 100644 index 0000000..0acac71 --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_max.cl @@ -0,0 +1,18 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/atomic/clc_atomic_fetch_max.h> + +#define FUNCTION __clc_atomic_fetch_max +#define __IMPL_FUNCTION __scoped_atomic_fetch_max + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/math/gentype.inc> diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_min.cl b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_min.cl new file mode 100644 index 0000000..7a09858 --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_min.cl @@ -0,0 +1,18 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/atomic/clc_atomic_fetch_min.h> + +#define FUNCTION __clc_atomic_fetch_min +#define __IMPL_FUNCTION __scoped_atomic_fetch_min + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/math/gentype.inc> diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_or.cl b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_or.cl new file mode 100644 index 0000000..e0f48fa --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_or.cl @@ -0,0 +1,15 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/atomic/clc_atomic_fetch_or.h> + +#define FUNCTION __clc_atomic_fetch_or +#define __IMPL_FUNCTION __scoped_atomic_fetch_or + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/integer/gentype.inc> diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_sub.cl b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_sub.cl new file mode 100644 index 0000000..a4c2c1da --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_sub.cl @@ -0,0 +1,18 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/atomic/clc_atomic_fetch_sub.h> + +#define FUNCTION __clc_atomic_fetch_sub +#define __IMPL_FUNCTION __scoped_atomic_fetch_sub + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/math/gentype.inc> diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_xor.cl b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_xor.cl new file mode 100644 index 0000000..4424a29 --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_xor.cl @@ -0,0 +1,15 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/atomic/clc_atomic_fetch_xor.h> + +#define FUNCTION __clc_atomic_fetch_xor +#define __IMPL_FUNCTION __scoped_atomic_fetch_xor + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/integer/gentype.inc> diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_inc.cl b/libclc/clc/lib/generic/atomic/clc_atomic_inc.cl new file mode 100644 index 0000000..019aa8d --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_inc.cl @@ -0,0 +1,16 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/atomic/clc_atomic_inc.h> + +#define FUNCTION __clc_atomic_inc +#define __IMPL_FUNCTION __scoped_atomic_fetch_sub +#define __CLC_INC_DEC + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/integer/gentype.inc> diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_load.cl b/libclc/clc/lib/generic/atomic/clc_atomic_load.cl new file mode 100644 index 0000000..1f08307 --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_load.cl @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/atomic/clc_atomic_load.h> + +#define FUNCTION __clc_atomic_load +#define __IMPL_FUNCTION __scoped_atomic_load_n +#define __CLC_NO_VALUE_ARG + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/integer/gentype.inc> + +#undef __CLC_PTR_CASTTYPE +#undef __CLC_AS_RETTYPE +#define __CLC_PTR_CASTTYPE __CLC_BIT_INTN +#define __CLC_AS_RETTYPE(x) __CLC_AS_GENTYPE(x) + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/math/gentype.inc> diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_store.cl b/libclc/clc/lib/generic/atomic/clc_atomic_store.cl new file mode 100644 index 0000000..8fd165b --- /dev/null +++ b/libclc/clc/lib/generic/atomic/clc_atomic_store.cl @@ -0,0 +1,22 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include <clc/atomic/clc_atomic_store.h> + +#define FUNCTION __clc_atomic_store +#define __IMPL_FUNCTION __scoped_atomic_store_n +#define __CLC_RETURN_VOID + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/integer/gentype.inc> + +#undef __CLC_PTR_CASTTYPE +#define __CLC_PTR_CASTTYPE __CLC_BIT_INTN + +#define __CLC_BODY <clc_atomic_def.inc> +#include <clc/math/gentype.inc> diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_add.h b/libclc/opencl/include/clc/opencl/atomic/atomic_add.h index 821ae7a..50fb99d1 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_add.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_add.h @@ -12,6 +12,6 @@ #include <clc/opencl/opencl-base.h> #define FUNCTION atomic_add -#include <clc/opencl/atomic/atomic_decl.inc> +#include <clc/opencl/atomic/atomic_decl_legacy.inc> #endif // __CLC_OPENCL_ATOMIC_ATOMIC_ADD_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_and.h b/libclc/opencl/include/clc/opencl/atomic/atomic_and.h index d10cfed..8ce328c 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_and.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_and.h @@ -12,6 +12,6 @@ #include <clc/opencl/opencl-base.h> #define FUNCTION atomic_and -#include <clc/opencl/atomic/atomic_decl.inc> +#include <clc/opencl/atomic/atomic_decl_legacy.inc> #endif // __CLC_OPENCL_ATOMIC_ATOMIC_AND_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_strong.h b/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_strong.h new file mode 100644 index 0000000..76eeda7 --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_strong.h @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_STRONG_H__ +#define __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_STRONG_H__ + +#define FUNCTION atomic_compare_exchange_strong +#define __CLC_COMPARE_EXCHANGE + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/math/gentype.inc> + +#undef __CLC_COMPARE_EXCHANGE +#undef FUNCTION + +#endif // __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_STRONG_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_weak.h b/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_weak.h new file mode 100644 index 0000000..12788ad --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_weak.h @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_WEAK_H__ +#define __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_WEAK_H__ + +#define FUNCTION atomic_compare_exchange_weak +#define __CLC_COMPARE_EXCHANGE + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/math/gentype.inc> + +#undef __CLC_COMPARE_EXCHANGE +#undef FUNCTION + +#endif // __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_WEAK_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc b/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc index e060e3a..1b2bf17 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc @@ -6,17 +6,55 @@ // //===----------------------------------------------------------------------===// -#define __CLC_DECLARE_ATOMIC(ADDRSPACE, TYPE) \ - _CLC_OVERLOAD _CLC_DECL TYPE FUNCTION(volatile ADDRSPACE TYPE *, TYPE); +#ifdef __CLC_SCALAR -#define __CLC_DECLARE_ATOMIC_ADDRSPACE(TYPE) \ - __CLC_DECLARE_ATOMIC(global, TYPE) \ - __CLC_DECLARE_ATOMIC(local, TYPE) +#if defined(__opencl_c_fp64) && (defined(cl_khr_int64_base_atomics) && \ + defined(cl_khr_int64_extended_atomics)) +#define HAVE_64_ATOMIC +#endif +#if defined(__CLC_FPSIZE) && (__CLC_FPSIZE < 64 || defined(HAVE_64_ATOMIC)) +#define HAVE_FP_ATOMIC +#endif +#if defined(__CLC_GENSIZE) && \ + ((__CLC_GENSIZE == 32) || \ + (__CLC_GENSIZE == 64 && defined(HAVE_64_ATOMIC))) +#define HAVE_INT_ATOMIC +#endif +#if defined(HAVE_FP_ATOMIC) || defined(HAVE_INT_ATOMIC) -__CLC_DECLARE_ATOMIC_ADDRSPACE(int) -__CLC_DECLARE_ATOMIC_ADDRSPACE(uint) +#define __CLC_ATOMIC_GENTYPE __CLC_XCONCAT(atomic_, __CLC_GENTYPE) -#undef __CLC_DECLARE_ATOMIC_ADDRSPACE -#undef __CLC_DECLARE_ATOMIC +#ifdef __CLC_NO_VALUE_ARG +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr); +#elif defined(__CLC_RETURN_VOID) +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL void FUNCTION( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value); +#elif defined(__CLC_COMPARE_EXCHANGE) +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, \ + ADDRSPACE __CLC_GENTYPE *Expected, __CLC_GENTYPE Desired); +#else +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value); +#endif -#undef FUNCTION +__CLC_DEFINE_ATOMIC(global) +__CLC_DEFINE_ATOMIC(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DEFINE_ATOMIC() +#endif + +#undef __CLC_DEFINE_ATOMIC + +#endif // HAVE_FP_ATOMIC || HAVE_INT_ATOMIC + +#undef HAVE_INT_ATOMIC +#undef HAVE_FP_ATOMIC +#undef HAVE_64_ATOMIC + +#endif // __CLC_SCALAR diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_decl_legacy.inc b/libclc/opencl/include/clc/opencl/atomic/atomic_decl_legacy.inc new file mode 100644 index 0000000..e060e3a --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_decl_legacy.inc @@ -0,0 +1,22 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#define __CLC_DECLARE_ATOMIC(ADDRSPACE, TYPE) \ + _CLC_OVERLOAD _CLC_DECL TYPE FUNCTION(volatile ADDRSPACE TYPE *, TYPE); + +#define __CLC_DECLARE_ATOMIC_ADDRSPACE(TYPE) \ + __CLC_DECLARE_ATOMIC(global, TYPE) \ + __CLC_DECLARE_ATOMIC(local, TYPE) + +__CLC_DECLARE_ATOMIC_ADDRSPACE(int) +__CLC_DECLARE_ATOMIC_ADDRSPACE(uint) + +#undef __CLC_DECLARE_ATOMIC_ADDRSPACE +#undef __CLC_DECLARE_ATOMIC + +#undef FUNCTION diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_exchange.h b/libclc/opencl/include/clc/opencl/atomic/atomic_exchange.h new file mode 100644 index 0000000..3949bc1 --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_exchange.h @@ -0,0 +1,22 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_EXCHANGE_H__ +#define __CLC_OPENCL_ATOMIC_ATOMIC_EXCHANGE_H__ + +#define FUNCTION atomic_exchange + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/math/gentype.inc> + +#undef FUNCTION + +#endif // __CLC_OPENCL_ATOMIC_ATOMIC_EXCHANGE_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_add.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_add.h new file mode 100644 index 0000000..972c1fa --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_add.h @@ -0,0 +1,22 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_ADD_H__ +#define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_ADD_H__ + +#define FUNCTION atomic_fetch_add + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/math/gentype.inc> + +#undef FUNCTION + +#endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_ADD_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_and.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_and.h new file mode 100644 index 0000000..fdac049 --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_and.h @@ -0,0 +1,19 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_AND_H__ +#define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_AND_H__ + +#define FUNCTION atomic_fetch_and + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#undef FUNCTION + +#endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_AND_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_max.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_max.h new file mode 100644 index 0000000..513b60f --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_max.h @@ -0,0 +1,22 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MAX_H__ +#define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MAX_H__ + +#define FUNCTION atomic_fetch_max + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/math/gentype.inc> + +#undef FUNCTION + +#endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MAX_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_min.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_min.h new file mode 100644 index 0000000..c961c4a --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_min.h @@ -0,0 +1,22 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MIN_H__ +#define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MIN_H__ + +#define FUNCTION atomic_fetch_min + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/math/gentype.inc> + +#undef FUNCTION + +#endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MIN_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_or.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_or.h new file mode 100644 index 0000000..25923e3 --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_or.h @@ -0,0 +1,19 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_OR_H__ +#define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_OR_H__ + +#define FUNCTION atomic_fetch_or + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#undef FUNCTION + +#endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_OR_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_sub.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_sub.h new file mode 100644 index 0000000..b307c30 --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_sub.h @@ -0,0 +1,22 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_SUB_H__ +#define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_SUB_H__ + +#define FUNCTION atomic_fetch_sub + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/math/gentype.inc> + +#undef FUNCTION + +#endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_SUB_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_xor.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_xor.h new file mode 100644 index 0000000..52510d0 --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_xor.h @@ -0,0 +1,19 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_XOR_H__ +#define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_XOR_H__ + +#define FUNCTION atomic_fetch_xor + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#undef FUNCTION + +#endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_XOR_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_load.h b/libclc/opencl/include/clc/opencl/atomic/atomic_load.h new file mode 100644 index 0000000..3998a4d --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_load.h @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_LOAD_H__ +#define __CLC_OPENCL_ATOMIC_ATOMIC_LOAD_H__ + +#define FUNCTION atomic_load +#define __CLC_NO_VALUE_ARG + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/math/gentype.inc> + +#undef __CLC_NO_VALUE_ARG +#undef FUNCTION + +#endif // __CLC_OPENCL_ATOMIC_ATOMIC_LOAD_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_max.h b/libclc/opencl/include/clc/opencl/atomic/atomic_max.h index 667fa36..6b95ad7 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_max.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_max.h @@ -12,6 +12,6 @@ #include <clc/opencl/opencl-base.h> #define FUNCTION atomic_max -#include <clc/opencl/atomic/atomic_decl.inc> +#include <clc/opencl/atomic/atomic_decl_legacy.inc> #endif // __CLC_OPENCL_ATOMIC_ATOMIC_MAX_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_min.h b/libclc/opencl/include/clc/opencl/atomic/atomic_min.h index 91bb636..c1dfacb 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_min.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_min.h @@ -12,6 +12,6 @@ #include <clc/opencl/opencl-base.h> #define FUNCTION atomic_min -#include <clc/opencl/atomic/atomic_decl.inc> +#include <clc/opencl/atomic/atomic_decl_legacy.inc> #endif // __CLC_OPENCL_ATOMIC_ATOMIC_MIN_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_or.h b/libclc/opencl/include/clc/opencl/atomic/atomic_or.h index 5c03fd1..30c32fe 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_or.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_or.h @@ -12,6 +12,6 @@ #include <clc/opencl/opencl-base.h> #define FUNCTION atomic_or -#include <clc/opencl/atomic/atomic_decl.inc> +#include <clc/opencl/atomic/atomic_decl_legacy.inc> #endif // __CLC_OPENCL_ATOMIC_ATOMIC_OR_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_store.h b/libclc/opencl/include/clc/opencl/atomic/atomic_store.h new file mode 100644 index 0000000..4893a5b --- /dev/null +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_store.h @@ -0,0 +1,24 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef __CLC_OPENCL_ATOMIC_ATOMIC_STORE_H__ +#define __CLC_OPENCL_ATOMIC_ATOMIC_STORE_H__ + +#define FUNCTION atomic_store +#define __CLC_RETURN_VOID + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc> +#include <clc/math/gentype.inc> + +#undef __CLC_RETURN_VOID +#undef FUNCTION + +#endif // __CLC_OPENCL_ATOMIC_ATOMIC_STORE_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_sub.h b/libclc/opencl/include/clc/opencl/atomic/atomic_sub.h index 25ffe9f..1e7ac55 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_sub.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_sub.h @@ -12,6 +12,6 @@ #include <clc/opencl/opencl-base.h> #define FUNCTION atomic_sub -#include <clc/opencl/atomic/atomic_decl.inc> +#include <clc/opencl/atomic/atomic_decl_legacy.inc> #endif // __CLC_OPENCL_ATOMIC_ATOMIC_SUB_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_xchg.h b/libclc/opencl/include/clc/opencl/atomic/atomic_xchg.h index 6b4206d..043d782 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_xchg.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_xchg.h @@ -15,6 +15,6 @@ _CLC_OVERLOAD _CLC_DECL float FUNCTION(volatile local float *, float); _CLC_OVERLOAD _CLC_DECL float FUNCTION(volatile global float *, float); -#include <clc/opencl/atomic/atomic_decl.inc> +#include <clc/opencl/atomic/atomic_decl_legacy.inc> #endif // __CLC_OPENCL_ATOMIC_ATOMIC_XCHG_H__ diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_xor.h b/libclc/opencl/include/clc/opencl/atomic/atomic_xor.h index e94560c..a9bee00 100644 --- a/libclc/opencl/include/clc/opencl/atomic/atomic_xor.h +++ b/libclc/opencl/include/clc/opencl/atomic/atomic_xor.h @@ -12,6 +12,6 @@ #include <clc/opencl/opencl-base.h> #define FUNCTION atomic_xor -#include <clc/opencl/atomic/atomic_decl.inc> +#include <clc/opencl/atomic/atomic_decl_legacy.inc> #endif // __CLC_OPENCL_ATOMIC_ATOMIC_XOR_H__ diff --git a/libclc/opencl/lib/generic/SOURCES b/libclc/opencl/lib/generic/SOURCES index a59a82e..61757ef 100644 --- a/libclc/opencl/lib/generic/SOURCES +++ b/libclc/opencl/lib/generic/SOURCES @@ -8,24 +8,36 @@ atomic/atom_add.cl atomic/atom_and.cl atomic/atom_cmpxchg.cl atomic/atom_dec.cl -atomic/atom_inc.cl -atomic/atom_max.cl -atomic/atom_min.cl -atomic/atom_or.cl -atomic/atom_sub.cl -atomic/atom_xchg.cl -atomic/atom_xor.cl atomic/atomic_add.cl atomic/atomic_and.cl atomic/atomic_cmpxchg.cl +atomic/atomic_compare_exchange_strong.cl +atomic/atomic_compare_exchange_weak.cl atomic/atomic_dec.cl +atomic/atomic_exchange.cl +atomic/atomic_fetch_add.cl +atomic/atomic_fetch_and.cl +atomic/atomic_fetch_max.cl +atomic/atomic_fetch_min.cl +atomic/atomic_fetch_or.cl +atomic/atomic_fetch_sub.cl +atomic/atomic_fetch_xor.cl atomic/atomic_inc.cl +atomic/atomic_load.cl atomic/atomic_max.cl atomic/atomic_min.cl atomic/atomic_or.cl +atomic/atomic_store.cl atomic/atomic_sub.cl atomic/atomic_xchg.cl atomic/atomic_xor.cl +atomic/atom_inc.cl +atomic/atom_max.cl +atomic/atom_min.cl +atomic/atom_or.cl +atomic/atom_sub.cl +atomic/atom_xchg.cl +atomic/atom_xor.cl common/degrees.cl common/mix.cl common/radians.cl diff --git a/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_strong.cl b/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_strong.cl new file mode 100644 index 0000000..422c03f --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_strong.cl @@ -0,0 +1,25 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) + +#include <clc/atomic/clc_atomic_compare_exchange.h> +#include <clc/opencl/atomic/atomic_compare_exchange_strong.h> + +#define FUNCTION atomic_compare_exchange_strong +#define __CLC_COMPARE_EXCHANGE + +#define __CLC_BODY <atomic_def.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <atomic_def.inc> +#include <clc/math/gentype.inc> + +#endif // defined(__opencl_c_atomic_order_seq_cst) && + // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_weak.cl b/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_weak.cl new file mode 100644 index 0000000..8a6b3c4 --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_weak.cl @@ -0,0 +1,25 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) + +#include <clc/atomic/clc_atomic_compare_exchange.h> +#include <clc/opencl/atomic/atomic_compare_exchange_weak.h> + +#define FUNCTION atomic_compare_exchange_weak +#define __CLC_COMPARE_EXCHANGE + +#define __CLC_BODY <atomic_def.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <atomic_def.inc> +#include <clc/math/gentype.inc> + +#endif // defined(__opencl_c_atomic_order_seq_cst) && + // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_dec.cl b/libclc/opencl/lib/generic/atomic/atomic_dec.cl index 6f18cdf..6de55bc 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_dec.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_dec.cl @@ -6,15 +6,11 @@ // //===----------------------------------------------------------------------===// +#include <clc/atomic/clc_atomic_dec.h> #include <clc/opencl/atomic/atomic_dec.h> -#define IMPL(TYPE, AS) \ - _CLC_OVERLOAD _CLC_DEF TYPE atomic_dec(volatile AS TYPE *p) { \ - return __sync_fetch_and_sub(p, (TYPE)1); \ - } +#define FUNCTION atomic_dec +#define __IMPL_FUNCTION __clc_atomic_dec -IMPL(int, global) -IMPL(unsigned int, global) -IMPL(int, local) -IMPL(unsigned int, local) -#undef IMPL +#define __CLC_BODY <atomic_inc_dec.inc> +#include <clc/integer/gentype.inc> diff --git a/libclc/opencl/lib/generic/atomic/atomic_def.inc b/libclc/opencl/lib/generic/atomic/atomic_def.inc new file mode 100644 index 0000000..ce192bf --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_def.inc @@ -0,0 +1,79 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifdef __CLC_SCALAR + +#if defined(__opencl_c_fp64) && (defined(cl_khr_int64_base_atomics) && \ + defined(cl_khr_int64_extended_atomics)) +#define HAVE_64_ATOMIC +#endif +#if defined(__CLC_FPSIZE) && (__CLC_FPSIZE < 64 || defined(HAVE_64_ATOMIC) +#define HAVE_FP_ATOMIC +#endif +#if defined(__CLC_GENSIZE) && \ + ((__CLC_GENSIZE == 32) || \ + (__CLC_GENSIZE == 64 && defined(HAVE_64_ATOMIC))) +#define HAVE_INT_ATOMIC +#endif +#if defined(HAVE_FP_ATOMIC) || defined(HAVE_INT_ATOMIC) + +#define __CLC_ATOMIC_GENTYPE __CLC_XCONCAT(atomic_, __CLC_GENTYPE) + +#ifdef __CLC_NO_VALUE_ARG +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr) { \ + return __IMPL_FUNCTION((volatile ADDRSPACE __CLC_GENTYPE *)Ptr, \ + __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE); \ + } +#elif defined(__CLC_RETURN_VOID) +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF void FUNCTION( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value) { \ + __IMPL_FUNCTION((volatile ADDRSPACE __CLC_GENTYPE *)Ptr, Value, \ + __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE); \ + } +#elif defined(__CLC_COMPARE_EXCHANGE) +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, \ + ADDRSPACE __CLC_GENTYPE *Expected, __CLC_GENTYPE Desired) { \ + __CLC_GENTYPE Comparator = *Expected; \ + __CLC_GENTYPE RetValue = __clc_atomic_compare_exchange( \ + (volatile ADDRSPACE __CLC_GENTYPE *)Ptr, Comparator, Desired, \ + __ATOMIC_SEQ_CST, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); \ + if (Comparator != RetValue) { \ + *Expected = RetValue; \ + return true; \ + } \ + return false; \ + } +#else +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION( \ + volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value) { \ + return __IMPL_FUNCTION((volatile ADDRSPACE __CLC_GENTYPE *)Ptr, Value, \ + __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE); \ + } +#endif + +__CLC_DEFINE_ATOMIC(global) +__CLC_DEFINE_ATOMIC(local) +#if _CLC_GENERIC_AS_SUPPORTED +__CLC_DEFINE_ATOMIC() +#endif + +#undef __CLC_DEFINE_ATOMIC + +#endif // HAVE_FP_ATOMIC || HAVE_INT_ATOMIC + +#undef HAVE_INT_ATOMIC +#undef HAVE_FP_ATOMIC +#undef HAVE_64_ATOMIC + +#endif // __CLC_SCALAR diff --git a/libclc/opencl/lib/generic/atomic/atomic_exchange.cl b/libclc/opencl/lib/generic/atomic/atomic_exchange.cl new file mode 100644 index 0000000..6dae6c0 --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_exchange.cl @@ -0,0 +1,25 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) + +#include <clc/atomic/clc_atomic_exchange.h> +#include <clc/opencl/atomic/atomic_exchange.h> + +#define FUNCTION atomic_exchange +#define __IMPL_FUNCTION __clc_atomic_exchange + +#define __CLC_BODY <atomic_def.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <atomic_def.inc> +#include <clc/math/gentype.inc> + +#endif // defined(__opencl_c_atomic_order_seq_cst) && + // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_add.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_add.cl new file mode 100644 index 0000000..bbaa1c2 --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_add.cl @@ -0,0 +1,25 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) + +#include <clc/atomic/clc_atomic_fetch_add.h> +#include <clc/opencl/atomic/atomic_fetch_add.h> + +#define FUNCTION atomic_fetch_add +#define __IMPL_FUNCTION __clc_atomic_fetch_add + +#define __CLC_BODY <atomic_def.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <atomic_def.inc> +#include <clc/math/gentype.inc> + +#endif // defined(__opencl_c_atomic_order_seq_cst) && + // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_and.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_and.cl new file mode 100644 index 0000000..7392584 --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_and.cl @@ -0,0 +1,22 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) + +#include <clc/atomic/clc_atomic_fetch_and.h> +#include <clc/opencl/atomic/atomic_fetch_and.h> + +#define FUNCTION atomic_fetch_and +#define __IMPL_FUNCTION __clc_atomic_fetch_and + +#define __CLC_BODY <atomic_def.inc> +#include <clc/integer/gentype.inc> + +#endif // defined(__opencl_c_atomic_order_seq_cst) && + // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_max.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_max.cl new file mode 100644 index 0000000..8c8ce11 --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_max.cl @@ -0,0 +1,25 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) + +#include <clc/atomic/clc_atomic_fetch_max.h> +#include <clc/opencl/atomic/atomic_fetch_max.h> + +#define FUNCTION atomic_fetch_max +#define __IMPL_FUNCTION __clc_atomic_fetch_max + +#define __CLC_BODY <atomic_def.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <atomic_def.inc> +#include <clc/math/gentype.inc> + +#endif // defined(__opencl_c_atomic_order_seq_cst) && + // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_min.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_min.cl new file mode 100644 index 0000000..550459c --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_min.cl @@ -0,0 +1,25 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) + +#include <clc/atomic/clc_atomic_fetch_min.h> +#include <clc/opencl/atomic/atomic_fetch_min.h> + +#define FUNCTION atomic_fetch_min +#define __IMPL_FUNCTION __clc_atomic_fetch_min + +#define __CLC_BODY <atomic_def.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <atomic_def.inc> +#include <clc/math/gentype.inc> + +#endif // defined(__opencl_c_atomic_order_seq_cst) && + // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_or.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_or.cl new file mode 100644 index 0000000..2606ff3 --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_or.cl @@ -0,0 +1,22 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) + +#include <clc/atomic/clc_atomic_fetch_or.h> +#include <clc/opencl/atomic/atomic_fetch_or.h> + +#define FUNCTION atomic_fetch_or +#define __IMPL_FUNCTION __clc_atomic_fetch_or + +#define __CLC_BODY <atomic_def.inc> +#include <clc/integer/gentype.inc> + +#endif // defined(__opencl_c_atomic_order_seq_cst) && + // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_sub.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_sub.cl new file mode 100644 index 0000000..3377223 --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_sub.cl @@ -0,0 +1,25 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) + +#include <clc/atomic/clc_atomic_fetch_sub.h> +#include <clc/opencl/atomic/atomic_fetch_sub.h> + +#define FUNCTION atomic_fetch_sub +#define __IMPL_FUNCTION __clc_atomic_fetch_sub + +#define __CLC_BODY <atomic_def.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <atomic_def.inc> +#include <clc/math/gentype.inc> + +#endif // defined(__opencl_c_atomic_order_seq_cst) && + // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_xor.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_xor.cl new file mode 100644 index 0000000..6f6503e --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_xor.cl @@ -0,0 +1,22 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) + +#include <clc/atomic/clc_atomic_fetch_xor.h> +#include <clc/opencl/atomic/atomic_fetch_xor.h> + +#define FUNCTION atomic_fetch_xor +#define __IMPL_FUNCTION __clc_atomic_fetch_xor + +#define __CLC_BODY <atomic_def.inc> +#include <clc/integer/gentype.inc> + +#endif // defined(__opencl_c_atomic_order_seq_cst) && + // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_inc.cl b/libclc/opencl/lib/generic/atomic/atomic_inc.cl index 13349e5..a160b2e 100644 --- a/libclc/opencl/lib/generic/atomic/atomic_inc.cl +++ b/libclc/opencl/lib/generic/atomic/atomic_inc.cl @@ -6,15 +6,11 @@ // //===----------------------------------------------------------------------===// +#include <clc/atomic/clc_atomic_inc.h> #include <clc/opencl/atomic/atomic_inc.h> -#define IMPL(TYPE, AS) \ - _CLC_OVERLOAD _CLC_DEF TYPE atomic_inc(volatile AS TYPE *p) { \ - return __sync_fetch_and_add(p, (TYPE)1); \ - } +#define FUNCTION atomic_inc +#define __IMPL_FUNCTION __clc_atomic_inc -IMPL(int, global) -IMPL(unsigned int, global) -IMPL(int, local) -IMPL(unsigned int, local) -#undef IMPL +#define __CLC_BODY <atomic_inc_dec.inc> +#include <clc/integer/gentype.inc> diff --git a/libclc/opencl/lib/generic/atomic/atomic_inc_dec.inc b/libclc/opencl/lib/generic/atomic/atomic_inc_dec.inc new file mode 100644 index 0000000..0bcf300 --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_inc_dec.inc @@ -0,0 +1,26 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifdef __CLC_SCALAR + +#if __CLC_GENSIZE == 32 + +#define __CLC_DEFINE_ATOMIC(ADDRSPACE) \ + _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION( \ + volatile ADDRSPACE __CLC_GENTYPE *Ptr) { \ + return __IMPL_FUNCTION(Ptr, __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE); \ + } + +__CLC_DEFINE_ATOMIC(global) +__CLC_DEFINE_ATOMIC(local) + +#undef __CLC_DEFINE_ATOMIC + +#endif // __CLC_GENSIZE == 32 + +#endif // __CLC_SCALAR diff --git a/libclc/opencl/lib/generic/atomic/atomic_load.cl b/libclc/opencl/lib/generic/atomic/atomic_load.cl new file mode 100644 index 0000000..4592654 --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_load.cl @@ -0,0 +1,26 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) + +#include <clc/atomic/clc_atomic_load.h> +#include <clc/opencl/atomic/atomic_load.h> + +#define FUNCTION atomic_load +#define __IMPL_FUNCTION __clc_atomic_load +#define __CLC_NO_VALUE_ARG + +#define __CLC_BODY <atomic_def.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <atomic_def.inc> +#include <clc/math/gentype.inc> + +#endif // defined(__opencl_c_atomic_order_seq_cst) && + // defined(__opencl_c_atomic_scope_device) diff --git a/libclc/opencl/lib/generic/atomic/atomic_store.cl b/libclc/opencl/lib/generic/atomic/atomic_store.cl new file mode 100644 index 0000000..67f2c84 --- /dev/null +++ b/libclc/opencl/lib/generic/atomic/atomic_store.cl @@ -0,0 +1,26 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if defined(__opencl_c_atomic_order_seq_cst) && \ + defined(__opencl_c_atomic_scope_device) + +#include <clc/atomic/clc_atomic_store.h> +#include <clc/opencl/atomic/atomic_store.h> + +#define FUNCTION atomic_store +#define __IMPL_FUNCTION __clc_atomic_store +#define __CLC_RETURN_VOID + +#define __CLC_BODY <atomic_def.inc> +#include <clc/integer/gentype.inc> + +#define __CLC_BODY <atomic_def.inc> +#include <clc/math/gentype.inc> + +#endif // defined(__opencl_c_atomic_order_seq_cst) && + // defined(__opencl_c_atomic_scope_device) |