Revert " [OpenMP][NFC] Remove `DelayedBinDesc`" (#74679)

Reverts llvm/llvm-project#74360

As I wrote in the analysis of #74360:

Since
https://github.com/llvm/llvm-project/commit/bc4e0c048aa3cd940b0cea787014c7e8680e5040
we will not add PluginAdaptors into the container of all plugin adaptors
before the plugin is not ready. The error is thereby gone. When and old
HSA loads other libraries they can call register_image but that will
simply not register the image with the plugin we are currently
initializing. That seems like reasonable behavior, thought it is good to
keep in mind if we ever want a kernel library (@jhuber6 @mjklemm). We
can still have a standalone kernel library though or load it late after
all plugins are setup (which seems reasonable).

I did not expect one our tests actually doing exactly what this will not
allow anymore, at least when you use rocm <5.5.0. Need to figure out if
we want this behavior (for rocm <5.5.0).

5 files changed, 25 insertions, 7 deletions

diff --git a/openmp/libomptarget/include/PluginManager.h b/openmp/libomptarget/include/PluginManager.h
index 6af99ef..0b09747 100644
--- a/openmp/libomptarget/include/PluginManager.h
+++ b/openmp/libomptarget/include/PluginManager.h
@@ -150,6 +150,24 @@ struct PluginManager {
   HostPtrToTableMapTy HostPtrToTableMap;
   std::mutex TblMapMtx; ///< For HostPtrToTableMap
 
+  // Work around for plugins that call dlopen on shared libraries that call
+  // tgt_register_lib during their initialisation. Stash the pointers in a
+  // vector until the plugins are all initialised and then register them.
+  bool delayRegisterLib(__tgt_bin_desc *Desc) {
+    if (RTLsLoaded)
+      return false;
+    DelayedBinDesc.push_back(Desc);
+    return true;
+  }
+
+  void registerDelayedLibraries() {
+    // Only called by libomptarget constructor
+    RTLsLoaded = true;
+    for (auto *Desc : DelayedBinDesc)
+      __tgt_register_lib(Desc);
+    DelayedBinDesc.clear();
+  }
+
   /// Return the number of usable devices.
   int getNumDevices() { return getExclusiveDevicesAccessor()->size(); }
 
@@ -178,6 +196,9 @@ struct PluginManager {
   void addRequirements(int64_t Flags) { Requirements.addRequirements(Flags); }
 
 private:
+  bool RTLsLoaded = false;
+  llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc;
+
   // List of all plugin adaptors, in use or not.
   llvm::SmallVector<std::unique_ptr<PluginAdaptorTy>> PluginAdaptors;
 
diff --git a/openmp/libomptarget/src/interface.cpp b/openmp/libomptarget/src/interface.cpp
index 777dc07..d92f40c 100644
--- a/openmp/libomptarget/src/interface.cpp
+++ b/openmp/libomptarget/src/interface.cpp
@@ -46,6 +46,9 @@ EXTERN void __tgt_register_requires(int64_t Flags) {
 /// adds a target shared library to the target execution image
 EXTERN void __tgt_register_lib(__tgt_bin_desc *Desc) {
   TIMESCOPE();
+  if (PM->delayRegisterLib(Desc))
+    return;
+
   PM->registerLib(Desc);
 }
 
diff --git a/openmp/libomptarget/src/rtl.cpp b/openmp/libomptarget/src/rtl.cpp
index 27db703..5eb1c55 100644
--- a/openmp/libomptarget/src/rtl.cpp
+++ b/openmp/libomptarget/src/rtl.cpp
@@ -51,6 +51,7 @@ __attribute__((constructor(101))) void init() {
   PM->init();
 
   Profiler::get();
+  PM->registerDelayedLibraries();
 }
 
 __attribute__((destructor(101))) void deinit() {
diff --git a/openmp/libomptarget/test/Inputs/empty.c b/openmp/libomptarget/test/Inputs/empty.c
deleted file mode 100644
index 8b13789..0000000
--- a/openmp/libomptarget/test/Inputs/empty.c
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/openmp/libomptarget/test/offloading/bug60119.c b/openmp/libomptarget/test/offloading/bug60119.c
deleted file mode 100644
index e32f1cc..0000000
--- a/openmp/libomptarget/test/offloading/bug60119.c
+++ /dev/null
@@ -1,6 +0,0 @@
-// RUN: %clang-generic -fPIC -shared %S/../Inputs/empty.c -o %T/liba.so
-// RUN: %clang-generic -fPIC -shared %S/../Inputs/empty.c -o %T/libb.so
-// RUN: %clang-generic -rpath %T -L %T -l a -l b %s -o %t
-// RUN: %t
-
-int main() {}