diff options
author | Michael Halkenhaeuser <MichaelGerald.Halkenhauser@amd.com> | 2023-07-25 08:14:59 -0400 |
---|---|---|
committer | Tobias Hieta <tobias@hieta.se> | 2023-07-26 19:27:46 +0200 |
commit | 7e68c9e5c27182b867c1602a1095395089d3039c (patch) | |
tree | 89d44be47ae0adba0d263f0d918d600250be8141 | |
parent | ce53e91738036d4ed3ff4b060c64e712610cf0a4 (diff) | |
download | llvm-7e68c9e5c27182b867c1602a1095395089d3039c.zip llvm-7e68c9e5c27182b867c1602a1095395089d3039c.tar.gz llvm-7e68c9e5c27182b867c1602a1095395089d3039c.tar.bz2 |
[OpenMP] [OMPT] [7/8] Invoke tool-supplied callbacks before and after target launch and data transfer operations
Implemented RAII objects, initialized at target entry points, that
invoke tool-supplied callbacks. Updated status of target callbacks as
implemented.
Depends on D127365
Patch from John Mellor-Crummey <johnmc@rice.edu>
With contributions from:
Dhruva Chakrabarti <Dhruva.Chakrabarti@amd.com>
Jan-Patrick Lehr <janpatrick.lehr@amd.com>
Reviewed By: jdoerfert, dhruvachak, jplehr
Differential Revision: https://reviews.llvm.org/D127367
(cherry picked from commit 1dec417ac4a533e40f637cd1a7f0628803d9e634)
-rw-r--r-- | openmp/libomptarget/include/OmptCallback.h | 9 | ||||
-rw-r--r-- | openmp/libomptarget/src/OmptCallback.cpp | 99 | ||||
-rw-r--r-- | openmp/libomptarget/src/OmptInterface.h | 120 | ||||
-rw-r--r-- | openmp/libomptarget/src/device.cpp | 32 | ||||
-rw-r--r-- | openmp/libomptarget/src/interface.cpp | 30 | ||||
-rw-r--r-- | openmp/libomptarget/src/omptarget.cpp | 16 | ||||
-rw-r--r-- | openmp/libomptarget/test/ompt/veccopy.c | 27 | ||||
-rw-r--r-- | openmp/libomptarget/test/ompt/veccopy_disallow_both.c | 42 | ||||
-rw-r--r-- | openmp/libomptarget/test/ompt/veccopy_emi.c | 44 | ||||
-rw-r--r-- | openmp/libomptarget/test/ompt/veccopy_emi_map.c | 46 | ||||
-rw-r--r-- | openmp/libomptarget/test/ompt/veccopy_map.c | 27 | ||||
-rw-r--r-- | openmp/libomptarget/test/ompt/veccopy_no_device_init.c | 27 | ||||
-rw-r--r-- | openmp/libomptarget/test/ompt/veccopy_wrong_return.c | 29 | ||||
-rw-r--r-- | openmp/runtime/src/ompt-event-specific.h | 13 |
14 files changed, 473 insertions, 88 deletions
diff --git a/openmp/libomptarget/include/OmptCallback.h b/openmp/libomptarget/include/OmptCallback.h index 885029f..79d6bf8 100644 --- a/openmp/libomptarget/include/OmptCallback.h +++ b/openmp/libomptarget/include/OmptCallback.h @@ -27,6 +27,13 @@ FOREACH_OMPT_NOEMI_EVENT(macro) \ FOREACH_OMPT_EMI_EVENT(macro) +#define performIfOmptInitialized(stmt) \ + do { \ + if (llvm::omp::target::ompt::Initialized) { \ + stmt; \ + } \ + } while (0) + #define performOmptCallback(CallbackName, ...) \ do { \ if (ompt_callback_##CallbackName##_fn) \ @@ -89,6 +96,8 @@ extern bool Initialized; } // namespace omp } // namespace llvm +#else +#define performIfOmptInitialized(stmt) #endif // OMPT_SUPPORT #pragma pop_macro("DEBUG_PREFIX") diff --git a/openmp/libomptarget/src/OmptCallback.cpp b/openmp/libomptarget/src/OmptCallback.cpp index 2d9959e0..cd44d09 100644 --- a/openmp/libomptarget/src/OmptCallback.cpp +++ b/openmp/libomptarget/src/OmptCallback.cpp @@ -35,12 +35,20 @@ FOREACH_OMPT_NOEMI_EVENT(defineOmptCallback) FOREACH_OMPT_EMI_EVENT(defineOmptCallback) #undef defineOmptCallback -/// Thread local state for target region and associated metadata -thread_local llvm::omp::target::ompt::Interface OmptInterface; +/// Forward declaration +class LibomptargetRtlFinalizer; -/// Define function pointers -ompt_get_task_data_t ompt_get_task_data_fn = nullptr; +/// Object that will maintain the RTL finalizer from the plugin +LibomptargetRtlFinalizer *LibraryFinalizer = nullptr; + +thread_local Interface llvm::omp::target::ompt::RegionInterface; + +bool llvm::omp::target::ompt::Initialized = false; + +ompt_get_callback_t llvm::omp::target::ompt::lookupCallbackByCode = nullptr; +ompt_function_lookup_t llvm::omp::target::ompt::lookupCallbackByName = nullptr; ompt_get_target_task_data_t ompt_get_target_task_data_fn = nullptr; +ompt_get_task_data_t ompt_get_task_data_fn = nullptr; /// Unique correlation id static std::atomic<uint64_t> IdCounter(1); @@ -51,14 +59,14 @@ static uint64_t createId() { return IdCounter.fetch_add(1); } /// Create a new correlation id and update the operations id static uint64_t createOpId() { uint64_t NewId = createId(); - OmptInterface.setHostOpId(NewId); + RegionInterface.setHostOpId(NewId); return NewId; } /// Create a new correlation id and update the target region id static uint64_t createRegionId() { uint64_t NewId = createId(); - OmptInterface.setTargetDataValue(NewId); + RegionInterface.setTargetDataValue(NewId); return NewId; } @@ -68,18 +76,19 @@ void Interface::beginTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin, if (ompt_callback_target_data_op_emi_fn) { // HostOpId will be set by the tool. Invoke the tool supplied data op EMI // callback - ompt_callback_target_data_op_emi_fn(ompt_scope_begin, TargetTaskData, - &TargetData, &TargetRegionOpId, - ompt_target_data_alloc, HstPtrBegin, - DeviceId, /* TgtPtrBegin */ nullptr, - /* TgtDeviceNum */ 0, Size, Code); + ompt_callback_target_data_op_emi_fn( + ompt_scope_begin, TargetTaskData, &TargetData, &TargetRegionOpId, + ompt_target_data_alloc, HstPtrBegin, + /* SrcDeviceNum */ omp_get_initial_device(), /* TgtPtrBegin */ nullptr, + /* TgtDeviceNum */ DeviceId, Size, Code); } else if (ompt_callback_target_data_op_fn) { // HostOpId is set by the runtime HostOpId = createOpId(); // Invoke the tool supplied data op callback ompt_callback_target_data_op_fn( TargetData.value, HostOpId, ompt_target_data_alloc, HstPtrBegin, - DeviceId, /* TgtPtrBegin */ nullptr, /* TgtDeviceNum */ 0, Size, Code); + /* SrcDeviceNum */ omp_get_initial_device(), /* TgtPtrBegin */ nullptr, + /* TgtDeviceNum */ DeviceId, Size, Code); } } @@ -89,11 +98,11 @@ void Interface::endTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin, if (ompt_callback_target_data_op_emi_fn) { // HostOpId will be set by the tool. Invoke the tool supplied data op EMI // callback - ompt_callback_target_data_op_emi_fn(ompt_scope_end, TargetTaskData, - &TargetData, &TargetRegionOpId, - ompt_target_data_alloc, HstPtrBegin, - DeviceId, /* TgtPtrBegin */ nullptr, - /* TgtDeviceNum */ 0, Size, Code); + ompt_callback_target_data_op_emi_fn( + ompt_scope_end, TargetTaskData, &TargetData, &TargetRegionOpId, + ompt_target_data_alloc, HstPtrBegin, + /* SrcDeviceNum */ omp_get_initial_device(), /* TgtPtrBegin */ nullptr, + /* TgtDeviceNum */ DeviceId, Size, Code); } endTargetDataOperation(); } @@ -108,14 +117,16 @@ void Interface::beginTargetDataSubmit(int64_t DeviceId, void *TgtPtrBegin, ompt_callback_target_data_op_emi_fn( ompt_scope_begin, TargetTaskData, &TargetData, &TargetRegionOpId, ompt_target_data_transfer_to_device, HstPtrBegin, - /* SrcDeviceNum */ 0, TgtPtrBegin, DeviceId, Size, Code); + /* SrcDeviceNum */ omp_get_initial_device(), TgtPtrBegin, DeviceId, + Size, Code); } else if (ompt_callback_target_data_op_fn) { // HostOpId is set by the runtime HostOpId = createOpId(); // Invoke the tool supplied data op callback ompt_callback_target_data_op_fn( TargetData.value, HostOpId, ompt_target_data_transfer_to_device, - HstPtrBegin, /* SrcDeviceNum */ 0, TgtPtrBegin, DeviceId, Size, Code); + HstPtrBegin, /* SrcDeviceNum */ omp_get_initial_device(), TgtPtrBegin, + DeviceId, Size, Code); } } @@ -129,7 +140,8 @@ void Interface::endTargetDataSubmit(int64_t DeviceId, void *TgtPtrBegin, ompt_callback_target_data_op_emi_fn( ompt_scope_end, TargetTaskData, &TargetData, &TargetRegionOpId, ompt_target_data_transfer_to_device, HstPtrBegin, - /* SrcDeviceNum */ 0, TgtPtrBegin, DeviceId, Size, Code); + /* SrcDeviceNum */ omp_get_initial_device(), TgtPtrBegin, DeviceId, + Size, Code); } endTargetDataOperation(); } @@ -143,7 +155,7 @@ void Interface::beginTargetDataDelete(int64_t DeviceId, void *TgtPtrBegin, ompt_callback_target_data_op_emi_fn( ompt_scope_begin, TargetTaskData, &TargetData, &TargetRegionOpId, ompt_target_data_delete, TgtPtrBegin, DeviceId, - /* TgtPtrBegin */ nullptr, /* TgtDeviceNum */ 0, /* Bytes */ 0, Code); + /* TgtPtrBegin */ nullptr, /* TgtDeviceNum */ -1, /* Bytes */ 0, Code); } else if (ompt_callback_target_data_op_fn) { // HostOpId is set by the runtime HostOpId = createOpId(); @@ -151,7 +163,7 @@ void Interface::beginTargetDataDelete(int64_t DeviceId, void *TgtPtrBegin, ompt_callback_target_data_op_fn(TargetData.value, HostOpId, ompt_target_data_delete, TgtPtrBegin, DeviceId, /* TgtPtrBegin */ nullptr, - /* TgtDeviceNum */ 0, /* Bytes */ 0, Code); + /* TgtDeviceNum */ -1, /* Bytes */ 0, Code); } } @@ -164,7 +176,7 @@ void Interface::endTargetDataDelete(int64_t DeviceId, void *TgtPtrBegin, ompt_callback_target_data_op_emi_fn( ompt_scope_end, TargetTaskData, &TargetData, &TargetRegionOpId, ompt_target_data_delete, TgtPtrBegin, DeviceId, - /* TgtPtrBegin */ nullptr, /* TgtDeviceNum */ 0, /* Bytes */ 0, Code); + /* TgtPtrBegin */ nullptr, /* TgtDeviceNum */ -1, /* Bytes */ 0, Code); } endTargetDataOperation(); } @@ -176,19 +188,19 @@ void Interface::beginTargetDataRetrieve(int64_t DeviceId, void *HstPtrBegin, if (ompt_callback_target_data_op_emi_fn) { // HostOpId will be set by the tool. Invoke the tool supplied data op EMI // callback - ompt_callback_target_data_op_emi_fn(ompt_scope_begin, TargetTaskData, - &TargetData, &TargetRegionOpId, - ompt_target_data_transfer_from_device, - TgtPtrBegin, DeviceId, HstPtrBegin, - /* TgtDeviceNum */ 0, Size, Code); + ompt_callback_target_data_op_emi_fn( + ompt_scope_begin, TargetTaskData, &TargetData, &TargetRegionOpId, + ompt_target_data_transfer_from_device, TgtPtrBegin, DeviceId, + HstPtrBegin, + /* TgtDeviceNum */ omp_get_initial_device(), Size, Code); } else if (ompt_callback_target_data_op_fn) { // HostOpId is set by the runtime HostOpId = createOpId(); // Invoke the tool supplied data op callback - ompt_callback_target_data_op_fn(TargetData.value, HostOpId, - ompt_target_data_transfer_from_device, - TgtPtrBegin, DeviceId, HstPtrBegin, - /* TgtDeviceNum */ 0, Size, Code); + ompt_callback_target_data_op_fn( + TargetData.value, HostOpId, ompt_target_data_transfer_from_device, + TgtPtrBegin, DeviceId, HstPtrBegin, + /* TgtDeviceNum */ omp_get_initial_device(), Size, Code); } } @@ -199,11 +211,11 @@ void Interface::endTargetDataRetrieve(int64_t DeviceId, void *HstPtrBegin, if (ompt_callback_target_data_op_emi_fn) { // HostOpId will be set by the tool. Invoke the tool supplied data op EMI // callback - ompt_callback_target_data_op_emi_fn(ompt_scope_end, TargetTaskData, - &TargetData, &TargetRegionOpId, - ompt_target_data_transfer_from_device, - TgtPtrBegin, DeviceId, HstPtrBegin, - /* TgtDeviceNum */ 0, Size, Code); + ompt_callback_target_data_op_emi_fn( + ompt_scope_end, TargetTaskData, &TargetData, &TargetRegionOpId, + ompt_target_data_transfer_from_device, TgtPtrBegin, DeviceId, + HstPtrBegin, + /* TgtDeviceNum */ omp_get_initial_device(), Size, Code); } endTargetDataOperation(); } @@ -230,6 +242,7 @@ void Interface::endTargetSubmit(unsigned int numTeams) { numTeams); } } + void Interface::beginTargetDataEnter(int64_t DeviceId, void *Code) { beginTargetRegion(); if (ompt_callback_target_emi_fn) { @@ -391,14 +404,6 @@ private: llvm::SmallVector<ompt_finalize_t> RtlFinalizationFunctions; }; -/// Object that will maintain the RTL finalizer from the plugin -LibomptargetRtlFinalizer *LibraryFinalizer = nullptr; - -bool llvm::omp::target::ompt::Initialized = false; - -ompt_get_callback_t llvm::omp::target::ompt::lookupCallbackByCode = nullptr; -ompt_function_lookup_t llvm::omp::target::ompt::lookupCallbackByName = nullptr; - int llvm::omp::target::ompt::initializeLibrary(ompt_function_lookup_t lookup, int initial_device_num, ompt_data_t *tool_data) { @@ -418,6 +423,9 @@ int llvm::omp::target::ompt::initializeLibrary(ompt_function_lookup_t lookup, assert(lookupCallbackByCode && "lookupCallbackByCode should be non-null"); assert(lookupCallbackByName && "lookupCallbackByName should be non-null"); + assert(ompt_get_task_data_fn && "ompt_get_task_data_fn should be non-null"); + assert(ompt_get_target_task_data_fn && + "ompt_get_target_task_data_fn should be non-null"); assert(LibraryFinalizer == nullptr && "LibraryFinalizer should not be initialized yet"); @@ -434,6 +442,7 @@ void llvm::omp::target::ompt::finalizeLibrary(ompt_data_t *data) { // with this library LibraryFinalizer->finalize(); delete LibraryFinalizer; + Initialized = false; } void llvm::omp::target::ompt::connectLibrary() { diff --git a/openmp/libomptarget/src/OmptInterface.h b/openmp/libomptarget/src/OmptInterface.h index e2553c1..c3a5296 100644 --- a/openmp/libomptarget/src/OmptInterface.h +++ b/openmp/libomptarget/src/OmptInterface.h @@ -13,14 +13,18 @@ #ifndef _OMPTARGET_OMPTINTERFACE_H #define _OMPTARGET_OMPTINTERFACE_H +// Only provide functionality if target OMPT support is enabled +#ifdef OMPT_SUPPORT +#include <functional> +#include <tuple> + +#include "OmptCallback.h" #include "omp-tools.h" -// If target OMPT support is compiled in -#ifdef OMPT_SUPPORT +#include "llvm/Support/ErrorHandling.h" + #define OMPT_IF_BUILT(stmt) stmt -#else -#define OMPT_IF_BUILT(stmt) -#endif +#define OMPT_GET_RETURN_ADDRESS(level) __builtin_return_address(level) /// Callbacks for target regions require task_data representing the /// encountering task. @@ -108,6 +112,66 @@ public: /// Top-level function for invoking callback after target construct void endTarget(int64_t DeviceId, void *Code); + // Callback getter: Target data operations + template <ompt_target_data_op_t OpType> auto getCallbacks() { + if constexpr (OpType == ompt_target_data_alloc || + OpType == ompt_target_data_alloc_async) + return std::make_pair(std::mem_fn(&Interface::beginTargetDataAlloc), + std::mem_fn(&Interface::endTargetDataAlloc)); + + if constexpr (OpType == ompt_target_data_delete || + OpType == ompt_target_data_delete_async) + return std::make_pair(std::mem_fn(&Interface::beginTargetDataDelete), + std::mem_fn(&Interface::endTargetDataDelete)); + + if constexpr (OpType == ompt_target_data_transfer_to_device || + OpType == ompt_target_data_transfer_to_device_async) + return std::make_pair(std::mem_fn(&Interface::beginTargetDataSubmit), + std::mem_fn(&Interface::endTargetDataSubmit)); + + if constexpr (OpType == ompt_target_data_transfer_from_device || + OpType == ompt_target_data_transfer_from_device_async) + return std::make_pair(std::mem_fn(&Interface::beginTargetDataRetrieve), + std::mem_fn(&Interface::endTargetDataRetrieve)); + + llvm_unreachable("Unhandled target data operation type!"); + } + + // Callback getter: Target region operations + template <ompt_target_t OpType> auto getCallbacks() { + if constexpr (OpType == ompt_target_enter_data || + OpType == ompt_target_enter_data_nowait) + return std::make_pair(std::mem_fn(&Interface::beginTargetDataEnter), + std::mem_fn(&Interface::endTargetDataEnter)); + + if constexpr (OpType == ompt_target_exit_data || + OpType == ompt_target_exit_data_nowait) + return std::make_pair(std::mem_fn(&Interface::beginTargetDataExit), + std::mem_fn(&Interface::endTargetDataExit)); + + if constexpr (OpType == ompt_target_update || + OpType == ompt_target_update_nowait) + return std::make_pair(std::mem_fn(&Interface::beginTargetUpdate), + std::mem_fn(&Interface::endTargetUpdate)); + + if constexpr (OpType == ompt_target || OpType == ompt_target_nowait) + return std::make_pair(std::mem_fn(&Interface::beginTarget), + std::mem_fn(&Interface::endTarget)); + + llvm_unreachable("Unknown target region operation type!"); + } + + // Callback getter: Kernel launch operation + template <ompt_callbacks_t OpType> auto getCallbacks() { + // We use 'ompt_callbacks_t', because no other enum is currently available + // to model a kernel launch / target submit operation. + if constexpr (OpType == ompt_callback_target_submit) + return std::make_pair(std::mem_fn(&Interface::beginTargetSubmit), + std::mem_fn(&Interface::endTargetSubmit)); + + llvm_unreachable("Unhandled target operation!"); + } + /// Setters for target region and target operation correlation ids void setTargetDataValue(uint64_t DataValue) { TargetData.value = DataValue; } void setTargetDataPtr(void *DataPtr) { TargetData.ptr = DataPtr; } @@ -147,11 +211,53 @@ private: void endTargetRegion(); }; +/// Thread local state for target region and associated metadata +extern thread_local Interface RegionInterface; + +template <typename FuncTy, typename ArgsTy, size_t... IndexSeq> +void InvokeInterfaceFunction(FuncTy Func, ArgsTy Args, + std::index_sequence<IndexSeq...>) { + std::invoke(Func, RegionInterface, std::get<IndexSeq>(Args)...); +} + +template <typename CallbackPairTy, typename... ArgsTy> class InterfaceRAII { +public: + InterfaceRAII(CallbackPairTy Callbacks, ArgsTy... Args) + : Arguments(Args...), beginFunction(std::get<0>(Callbacks)), + endFunction(std::get<1>(Callbacks)) { + performIfOmptInitialized(begin()); + } + ~InterfaceRAII() { performIfOmptInitialized(end()); } + +private: + void begin() { + auto IndexSequence = + std::make_index_sequence<std::tuple_size_v<decltype(Arguments)>>{}; + InvokeInterfaceFunction(beginFunction, Arguments, IndexSequence); + } + + void end() { + auto IndexSequence = + std::make_index_sequence<std::tuple_size_v<decltype(Arguments)>>{}; + InvokeInterfaceFunction(endFunction, Arguments, IndexSequence); + } + + std::tuple<ArgsTy...> Arguments; + typename CallbackPairTy::first_type beginFunction; + typename CallbackPairTy::second_type endFunction; +}; + +// InterfaceRAII's class template argument deduction guide +template <typename CallbackPairTy, typename... ArgsTy> +InterfaceRAII(CallbackPairTy Callbacks, ArgsTy... Args) + -> InterfaceRAII<CallbackPairTy, ArgsTy...>; + } // namespace ompt } // namespace target } // namespace omp } // namespace llvm - -extern thread_local llvm::omp::target::ompt::Interface OmptInterface; +#else +#define OMPT_IF_BUILT(stmt) +#endif #endif // _OMPTARGET_OMPTINTERFACE_H diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp index a5409e2..276b7c9 100644 --- a/openmp/libomptarget/src/device.cpp +++ b/openmp/libomptarget/src/device.cpp @@ -11,6 +11,8 @@ //===----------------------------------------------------------------------===// #include "device.h" +#include "OmptCallback.h" +#include "OmptInterface.h" #include "omptarget.h" #include "private.h" #include "rtl.h" @@ -23,6 +25,10 @@ #include <string> #include <thread> +#ifdef OMPT_SUPPORT +using namespace llvm::omp::target::ompt; +#endif + int HostDataToTargetTy::addEventIfNecessary(DeviceTy &Device, AsyncInfoTy &AsyncInfo) const { // First, check if the user disabled atomic map transfer/malloc/dealloc. @@ -554,10 +560,22 @@ __tgt_target_table *DeviceTy::loadBinary(void *Img) { } void *DeviceTy::allocData(int64_t Size, void *HstPtr, int32_t Kind) { + /// RAII to establish tool anchors before and after data allocation + OMPT_IF_BUILT(InterfaceRAII TargetDataAllocRAII( + RegionInterface.getCallbacks<ompt_target_data_alloc>(), + RTLDeviceID, HstPtr, Size, + /* CodePtr */ OMPT_GET_RETURN_ADDRESS(0));) + return RTL->data_alloc(RTLDeviceID, Size, HstPtr, Kind); } int32_t DeviceTy::deleteData(void *TgtAllocBegin, int32_t Kind) { + /// RAII to establish tool anchors before and after data deletion + OMPT_IF_BUILT(InterfaceRAII TargetDataDeleteRAII( + RegionInterface.getCallbacks<ompt_target_data_delete>(), + RTLDeviceID, TgtAllocBegin, + /* CodePtr */ OMPT_GET_RETURN_ADDRESS(0));) + return RTL->data_delete(RTLDeviceID, TgtAllocBegin, Kind); } @@ -589,6 +607,13 @@ int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size, Entry); } + /// RAII to establish tool anchors before and after data submit + OMPT_IF_BUILT( + InterfaceRAII TargetDataSubmitRAII( + RegionInterface.getCallbacks<ompt_target_data_transfer_to_device>(), + RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size, + /* CodePtr */ OMPT_GET_RETURN_ADDRESS(0));) + if (!AsyncInfo || !RTL->data_submit_async || !RTL->synchronize) return RTL->data_submit(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size); return RTL->data_submit_async(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size, @@ -610,6 +635,13 @@ int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin, Entry); } + /// RAII to establish tool anchors before and after data retrieval + OMPT_IF_BUILT( + InterfaceRAII TargetDataRetrieveRAII( + RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(), + RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size, + /* CodePtr */ OMPT_GET_RETURN_ADDRESS(0));) + if (!RTL->data_retrieve_async || !RTL->synchronize) return RTL->data_retrieve(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size); return RTL->data_retrieve_async(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size, diff --git a/openmp/libomptarget/src/interface.cpp b/openmp/libomptarget/src/interface.cpp index 6f9c63a..7fb72e16 100644 --- a/openmp/libomptarget/src/interface.cpp +++ b/openmp/libomptarget/src/interface.cpp @@ -11,6 +11,8 @@ // //===----------------------------------------------------------------------===// +#include "OmptCallback.h" +#include "OmptInterface.h" #include "device.h" #include "omptarget.h" #include "private.h" @@ -24,6 +26,10 @@ #include <mutex> #include <type_traits> +#ifdef OMPT_SUPPORT +using namespace llvm::omp::target::ompt; +#endif + //////////////////////////////////////////////////////////////////////////////// /// adds requires flags EXTERN void __tgt_register_requires(int64_t Flags) { @@ -123,6 +129,12 @@ EXTERN void __tgt_target_data_begin_mapper(ident_t *Loc, int64_t DeviceId, map_var_info_t *ArgNames, void **ArgMappers) { TIMESCOPE_WITH_IDENT(Loc); + /// RAII to establish tool anchors before and after data begin + OMPT_IF_BUILT(InterfaceRAII TargetDataEnterRAII( + RegionInterface.getCallbacks<ompt_target_enter_data>(), + DeviceId, + /* CodePtr */ OMPT_GET_RETURN_ADDRESS(0));) + targetDataMapper<AsyncInfoTy>(Loc, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, ArgTypes, ArgNames, ArgMappers, targetDataBegin, "Entering OpenMP data region", "begin"); @@ -149,6 +161,12 @@ EXTERN void __tgt_target_data_end_mapper(ident_t *Loc, int64_t DeviceId, map_var_info_t *ArgNames, void **ArgMappers) { TIMESCOPE_WITH_IDENT(Loc); + /// RAII to establish tool anchors before and after data end + OMPT_IF_BUILT(InterfaceRAII TargetDataExitRAII( + RegionInterface.getCallbacks<ompt_target_exit_data>(), + DeviceId, + /* CodePtr */ OMPT_GET_RETURN_ADDRESS(0));) + targetDataMapper<AsyncInfoTy>(Loc, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, ArgTypes, ArgNames, ArgMappers, targetDataEnd, "Exiting OpenMP data region", "end"); @@ -172,6 +190,12 @@ EXTERN void __tgt_target_data_update_mapper(ident_t *Loc, int64_t DeviceId, map_var_info_t *ArgNames, void **ArgMappers) { TIMESCOPE_WITH_IDENT(Loc); + /// RAII to establish tool anchors before and after data update + OMPT_IF_BUILT(InterfaceRAII TargetDataUpdateRAII( + RegionInterface.getCallbacks<ompt_target_update>(), + DeviceId, + /* CodePtr */ OMPT_GET_RETURN_ADDRESS(0));) + targetDataMapper<AsyncInfoTy>( Loc, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, ArgTypes, ArgNames, ArgMappers, targetDataUpdate, "Updating OpenMP data", "update"); @@ -271,6 +295,9 @@ static inline int targetKernel(ident_t *Loc, int64_t DeviceId, int32_t NumTeams, DeviceTy &Device = *PM->Devices[DeviceId]; TargetAsyncInfoTy TargetAsyncInfo(Device); AsyncInfoTy &AsyncInfo = TargetAsyncInfo; + OMPT_IF_BUILT(InterfaceRAII TargetDataAllocRAII( + RegionInterface.getCallbacks<ompt_target>(), DeviceId, + /* CodePtr */ OMPT_GET_RETURN_ADDRESS(0));) int Rc = OFFLOAD_SUCCESS; Rc = target(Loc, Device, HostPtr, *KernelArgs, AsyncInfo); @@ -336,6 +363,9 @@ EXTERN int __tgt_target_kernel_replay(ident_t *Loc, int64_t DeviceId, return OMP_TGT_FAIL; } DeviceTy &Device = *PM->Devices[DeviceId]; + OMPT_IF_BUILT(InterfaceRAII TargetDataAllocRAII( + RegionInterface.getCallbacks<ompt_target>(), DeviceId, + /* CodePtr */ OMPT_GET_RETURN_ADDRESS(0));) AsyncInfoTy AsyncInfo(Device); int Rc = target_replay(Loc, Device, HostPtr, DeviceMemory, DeviceMemorySize, diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp index 45576c6..0719f1b 100644 --- a/openmp/libomptarget/src/omptarget.cpp +++ b/openmp/libomptarget/src/omptarget.cpp @@ -12,6 +12,8 @@ //===----------------------------------------------------------------------===// #include "omptarget.h" +#include "OmptCallback.h" +#include "OmptInterface.h" #include "device.h" #include "private.h" #include "rtl.h" @@ -24,6 +26,9 @@ #include <vector> using llvm::SmallVector; +#ifdef OMPT_SUPPORT +using namespace llvm::omp::target::ompt; +#endif int AsyncInfoTy::synchronize() { int Result = OFFLOAD_SUCCESS; @@ -1670,6 +1675,17 @@ int target(ident_t *Loc, DeviceTy &Device, void *HostPtr, { assert(KernelArgs.NumArgs == TgtArgs.size() && "Argument count mismatch!"); TIMESCOPE_WITH_NAME_AND_IDENT("Initiate Kernel Launch", Loc); + +#ifdef OMPT_SUPPORT + assert(KernelArgs.NumTeams[1] == 0 && KernelArgs.NumTeams[2] == 0 && + "Multi dimensional launch not supported yet."); + /// RAII to establish tool anchors before and after kernel launch + int32_t NumTeams = KernelArgs.NumTeams[0]; + // No need to guard this with OMPT_IF_BUILT + InterfaceRAII TargetSubmitRAII( + RegionInterface.getCallbacks<ompt_callback_target_submit>(), NumTeams); +#endif + Ret = Device.launchKernel(TgtEntryPtr, TgtArgs.data(), TgtOffsets.data(), KernelArgs, AsyncInfo); } diff --git a/openmp/libomptarget/test/ompt/veccopy.c b/openmp/libomptarget/test/ompt/veccopy.c index 2293e46..eb14f10 100644 --- a/openmp/libomptarget/test/ompt/veccopy.c +++ b/openmp/libomptarget/test/ompt/veccopy.c @@ -55,10 +55,29 @@ int main() { return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op' -/// CHECK: Could not register callback 'ompt_callback_target' -/// CHECK: Could not register callback 'ompt_callback_target_submit' - /// CHECK: Callback Init: /// CHECK: Callback Load: +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 + +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 /// CHECK: Callback Fini: diff --git a/openmp/libomptarget/test/ompt/veccopy_disallow_both.c b/openmp/libomptarget/test/ompt/veccopy_disallow_both.c index ba16cda..6fdcfdb 100644 --- a/openmp/libomptarget/test/ompt/veccopy_disallow_both.c +++ b/openmp/libomptarget/test/ompt/veccopy_disallow_both.c @@ -58,10 +58,44 @@ int main() { return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op' -/// CHECK: Could not register callback 'ompt_callback_target' -/// CHECK: Could not register callback 'ompt_callback_target_submit' - /// CHECK: Callback Init: /// CHECK: Callback Load: +/// CHECK: Callback Target EMI: kind=1 endpoint=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback Target EMI: kind=1 endpoint=2 +/// CHECK: Callback Target EMI: kind=1 endpoint=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback Target EMI: kind=1 endpoint=2 /// CHECK: Callback Fini: diff --git a/openmp/libomptarget/test/ompt/veccopy_emi.c b/openmp/libomptarget/test/ompt/veccopy_emi.c index 6f22ed2..f15dfb1 100644 --- a/openmp/libomptarget/test/ompt/veccopy_emi.c +++ b/openmp/libomptarget/test/ompt/veccopy_emi.c @@ -56,10 +56,46 @@ int main() { return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op_emi' -/// CHECK: Could not register callback 'ompt_callback_target_emi' -/// CHECK: Could not register callback 'ompt_callback_target_submit_emi' - /// CHECK: Callback Init: /// CHECK: Callback Load: +/// CHECK: Callback Target EMI: kind=1 endpoint=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback Submit EMI: endpoint=1 req_num_teams=1 +/// CHECK: Callback Submit EMI: endpoint=2 req_num_teams=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback Target EMI: kind=1 endpoint=2 +/// CHECK: Callback Target EMI: kind=1 endpoint=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback Submit EMI: endpoint=1 req_num_teams=0 +/// CHECK: Callback Submit EMI: endpoint=2 req_num_teams=0 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback Target EMI: kind=1 endpoint=2 /// CHECK: Callback Fini: diff --git a/openmp/libomptarget/test/ompt/veccopy_emi_map.c b/openmp/libomptarget/test/ompt/veccopy_emi_map.c index 25e0b78..af0743f 100644 --- a/openmp/libomptarget/test/ompt/veccopy_emi_map.c +++ b/openmp/libomptarget/test/ompt/veccopy_emi_map.c @@ -56,11 +56,47 @@ int main() { return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op_emi' -/// CHECK: Could not register callback 'ompt_callback_target_emi' -/// CHECK: Could not register callback 'ompt_callback_target_submit_emi' -/// CHECK: Could not register callback 'ompt_callback_target_map_emi' - +/// CHECK: 0: Could not register callback 'ompt_callback_target_map_emi' /// CHECK: Callback Init: /// CHECK: Callback Load: +/// CHECK: Callback Target EMI: kind=1 endpoint=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback Submit EMI: endpoint=1 req_num_teams=1 +/// CHECK: Callback Submit EMI: endpoint=2 req_num_teams=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback Target EMI: kind=1 endpoint=2 +/// CHECK: Callback Target EMI: kind=1 endpoint=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=1 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=2 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=2 +/// CHECK: Callback Submit EMI: endpoint=1 req_num_teams=0 +/// CHECK: Callback Submit EMI: endpoint=2 req_num_teams=0 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=3 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=1 optype=4 +/// CHECK: Callback DataOp EMI: endpoint=2 optype=4 +/// CHECK: Callback Target EMI: kind=1 endpoint=2 /// CHECK: Callback Fini: diff --git a/openmp/libomptarget/test/ompt/veccopy_map.c b/openmp/libomptarget/test/ompt/veccopy_map.c index 9929e50..6b3f408 100644 --- a/openmp/libomptarget/test/ompt/veccopy_map.c +++ b/openmp/libomptarget/test/ompt/veccopy_map.c @@ -55,10 +55,31 @@ int main() { return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op' -/// CHECK: Could not register callback 'ompt_callback_target' -/// CHECK: Could not register callback 'ompt_callback_target_submit' +/// CHECK: 0: Could not register callback 'ompt_callback_target_map' /// CHECK: Callback Init: /// CHECK: Callback Load: +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 + +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 /// CHECK: Callback Fini: diff --git a/openmp/libomptarget/test/ompt/veccopy_no_device_init.c b/openmp/libomptarget/test/ompt/veccopy_no_device_init.c index 0bb9c46..0b4f012 100644 --- a/openmp/libomptarget/test/ompt/veccopy_no_device_init.c +++ b/openmp/libomptarget/test/ompt/veccopy_no_device_init.c @@ -54,10 +54,29 @@ int main() { return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op' -/// CHECK: Could not register callback 'ompt_callback_target' -/// CHECK: Could not register callback 'ompt_callback_target_submit' - /// CHECK-NOT: Callback Init: /// CHECK-NOT: Callback Load: +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 + +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 /// CHECK-NOT: Callback Fini: diff --git a/openmp/libomptarget/test/ompt/veccopy_wrong_return.c b/openmp/libomptarget/test/ompt/veccopy_wrong_return.c index eef2094..a51a6dc 100644 --- a/openmp/libomptarget/test/ompt/veccopy_wrong_return.c +++ b/openmp/libomptarget/test/ompt/veccopy_wrong_return.c @@ -54,10 +54,29 @@ int main() { return rc; } -/// CHECK: Could not register callback 'ompt_callback_target_data_op' -/// CHECK: Could not register callback 'ompt_callback_target' -/// CHECK: Could not register callback 'ompt_callback_target_submit' - /// CHECK-NOT: Callback Init: /// CHECK-NOT: Callback Load: -/// CHECK-NOT: Callback Fini: +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 + +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2 +/// CHECK-NOT: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4 +/// CHECK-NOT: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2 +/// CHECK-NOT: Callback Fini diff --git a/openmp/runtime/src/ompt-event-specific.h b/openmp/runtime/src/ompt-event-specific.h index 5ac7f6d..7736ba8 100644 --- a/openmp/runtime/src/ompt-event-specific.h +++ b/openmp/runtime/src/ompt-event-specific.h @@ -55,13 +55,12 @@ #define ompt_callback_implicit_task_implemented ompt_event_MAY_ALWAYS -#define ompt_callback_target_implemented ompt_event_UNIMPLEMENTED -#define ompt_callback_target_emi_implemented ompt_event_UNIMPLEMENTED -#define ompt_callback_target_data_op_implemented ompt_event_UNIMPLEMENTED -#define ompt_callback_target_data_op_emi_implemented ompt_event_UNIMPLEMENTED -#define ompt_callback_target_submit_implemented ompt_event_UNIMPLEMENTED -#define ompt_callback_target_submit_emi_implemented ompt_event_UNIMPLEMENTED - +#define ompt_callback_target_implemented ompt_event_MAY_ALWAYS +#define ompt_callback_target_emi_implemented ompt_event_MAY_ALWAYS +#define ompt_callback_target_data_op_implemented ompt_event_MAY_ALWAYS +#define ompt_callback_target_data_op_emi_implemented ompt_event_MAY_ALWAYS +#define ompt_callback_target_submit_implemented ompt_event_MAY_ALWAYS +#define ompt_callback_target_submit_emi_implemented ompt_event_MAY_ALWAYS #define ompt_callback_control_tool_implemented ompt_event_MAY_ALWAYS #define ompt_callback_device_initialize_implemented ompt_event_MAY_ALWAYS |