[OpenMP][MLIR][Flang][bbc][Driver] Add OpenMP RTL Flags to Flang and generate omp.FlagsAttr from them

This patch ports OpenMP RTL flags from the shared Clang compiler
options to Flang. As well as adding a limited subset to bbc.

This patch enables the flags below (and any equals or inverse variants)
for Flang that exist in Clang:

-fopenmp-target-debug
-fopenmp-assume-threads-oversubscription
-fopenmp-assume-teams-oversubscription
-fopenmp-assume-no-nested-parallelism
-fopenmp-assume-no-thread-state

For the bbc tool it only utilises the primary variants to minimize
additional complexity in the tool.

The patch also provides FlagAttr generation from these flags. Which
will be lowered to LLVM-IR in a subsequent patch.

Reviewers: kiranchandramohan, awarzynski

Differential Revision: https://reviews.llvm.org/D147324

8 files changed, 201 insertions, 6 deletions

diff --git a/flang/include/flang/Frontend/LangOptions.def b/flang/include/flang/Frontend/LangOptions.def
index 3648e09..67ce2a9 100644
--- a/flang/include/flang/Frontend/LangOptions.def
+++ b/flang/include/flang/Frontend/LangOptions.def
@@ -36,6 +36,18 @@ LANGOPT(AssociativeMath, 1, false)
 LANGOPT(ReciprocalMath, 1, false)
 /// Generate code only for OpenMP target device
 LANGOPT(OpenMPIsDevice, 1, false)
+/// Enable debugging in the OpenMP offloading device RTL
+LANGOPT(OpenMPTargetDebug, 32, 0)
+/// Assume work-shared loops do not have more iterations than participating
+/// threads.
+LANGOPT(OpenMPThreadSubscription, 1, 0)
+/// Assume distributed loops do not have more iterations than participating
+/// teams.
+LANGOPT(OpenMPTeamSubscription, 1, 0)
+/// Assume that no thread in a parallel region will modify an ICV.
+LANGOPT(OpenMPNoThreadState, 1, 0)
+/// Assume that no thread in a parallel region will encounter a parallel region
+LANGOPT(OpenMPNoNestedParallelism, 1, 0)
 
 #undef LANGOPT
 #undef ENUM_LANGOPT
diff --git a/flang/include/flang/Tools/CrossToolHelpers.h b/flang/include/flang/Tools/CrossToolHelpers.h
index 63b6c99..19d31df 100644
--- a/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/flang/include/flang/Tools/CrossToolHelpers.h
@@ -13,17 +13,53 @@
 #ifndef FORTRAN_TOOLS_CROSS_TOOL_HELPERS_H
 #define FORTRAN_TOOLS_CROSS_TOOL_HELPERS_H
 
+#include "flang/Frontend/LangOptions.h"
+#include <cstdint>
+
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/IR/BuiltinOps.h"
 
+struct OffloadModuleOpts {
+  OffloadModuleOpts() {}
+  OffloadModuleOpts(uint32_t OpenMPTargetDebug, bool OpenMPTeamSubscription,
+      bool OpenMPThreadSubscription, bool OpenMPNoThreadState,
+      bool OpenMPNoNestedParallelism, bool OpenMPIsDevice)
+      : OpenMPTargetDebug(OpenMPTargetDebug),
+        OpenMPTeamSubscription(OpenMPTeamSubscription),
+        OpenMPThreadSubscription(OpenMPThreadSubscription),
+        OpenMPNoThreadState(OpenMPNoThreadState),
+        OpenMPNoNestedParallelism(OpenMPNoNestedParallelism),
+        OpenMPIsDevice(OpenMPIsDevice) {}
+
+  OffloadModuleOpts(Fortran::frontend::LangOptions &Opts)
+      : OpenMPTargetDebug(Opts.OpenMPTargetDebug),
+        OpenMPTeamSubscription(Opts.OpenMPTeamSubscription),
+        OpenMPThreadSubscription(Opts.OpenMPThreadSubscription),
+        OpenMPNoThreadState(Opts.OpenMPNoThreadState),
+        OpenMPNoNestedParallelism(Opts.OpenMPNoNestedParallelism),
+        OpenMPIsDevice(Opts.OpenMPIsDevice) {}
+
+  uint32_t OpenMPTargetDebug = 0;
+  bool OpenMPTeamSubscription = false;
+  bool OpenMPThreadSubscription = false;
+  bool OpenMPNoThreadState = false;
+  bool OpenMPNoNestedParallelism = false;
+  bool OpenMPIsDevice = false;
+};
+
 //  Shares assinging of the OpenMP OffloadModuleInterface and its assorted
 //  attributes accross Flang tools (bbc/flang)
 void setOffloadModuleInterfaceAttributes(
-    mlir::ModuleOp &module, bool isDevice) {
+    mlir::ModuleOp &module, OffloadModuleOpts Opts) {
   // Should be registered by the OpenMPDialect
   if (auto offloadMod = llvm::dyn_cast<mlir::omp::OffloadModuleInterface>(
           module.getOperation())) {
-    offloadMod.setIsDevice(isDevice);
+    offloadMod.setIsDevice(Opts.OpenMPIsDevice);
+    if (Opts.OpenMPIsDevice) {
+      offloadMod.setFlags(Opts.OpenMPTargetDebug, Opts.OpenMPTeamSubscription,
+          Opts.OpenMPThreadSubscription, Opts.OpenMPNoThreadState,
+          Opts.OpenMPNoNestedParallelism);
+    }
   }
 }
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 9fbf5bb..e05ce43 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -721,6 +721,39 @@ static bool parseDialectArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
 
     if (args.hasArg(clang::driver::options::OPT_fopenmp_is_device)) {
       res.getLangOpts().OpenMPIsDevice = 1;
+
+      if (args.hasFlag(
+              clang::driver::options::OPT_fopenmp_assume_teams_oversubscription,
+              clang::driver::options::
+                  OPT_fno_openmp_assume_teams_oversubscription,
+              /*Default=*/false))
+        res.getLangOpts().OpenMPTeamSubscription = true;
+
+      if (args.hasArg(
+              clang::driver::options::OPT_fopenmp_assume_no_thread_state))
+        res.getLangOpts().OpenMPNoThreadState = 1;
+
+      if (args.hasArg(
+              clang::driver::options::OPT_fopenmp_assume_no_nested_parallelism))
+        res.getLangOpts().OpenMPNoNestedParallelism = 1;
+
+      if (args.hasFlag(clang::driver::options::
+                           OPT_fopenmp_assume_threads_oversubscription,
+                       clang::driver::options::
+                           OPT_fno_openmp_assume_threads_oversubscription,
+                       /*Default=*/false))
+        res.getLangOpts().OpenMPThreadSubscription = true;
+
+      if ((args.hasArg(clang::driver::options::OPT_fopenmp_target_debug) ||
+           args.hasArg(clang::driver::options::OPT_fopenmp_target_debug_EQ))) {
+        res.getLangOpts().OpenMPTargetDebug = getLastArgIntValue(
+            args, clang::driver::options::OPT_fopenmp_target_debug_EQ,
+            res.getLangOpts().OpenMPTargetDebug, diags);
+
+        if (!res.getLangOpts().OpenMPTargetDebug &&
+            args.hasArg(clang::driver::options::OPT_fopenmp_target_debug))
+          res.getLangOpts().OpenMPTargetDebug = 1;
+      }
     }
   }
 
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index b2f25d1..af6d52d 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -281,8 +281,8 @@ bool CodeGenAction::beginSourceFileAction() {
 
   if (ci.getInvocation().getFrontendOpts().features.IsEnabled(
           Fortran::common::LanguageFeature::OpenMP)) {
-    setOffloadModuleInterfaceAttributes(
-        *mlirModule, ci.getInvocation().getLangOpts().OpenMPIsDevice);
+    setOffloadModuleInterfaceAttributes(*mlirModule,
+                                        ci.getInvocation().getLangOpts());
     setOffloadModuleInterfaceTargetAttribute(*mlirModule, tm->getTargetCPU(),
                                              tm->getTargetFeatureString());
   }
diff --git a/flang/test/Driver/driver-help.f90 b/flang/test/Driver/driver-help.f90
index f50b14e..8470779 100644
--- a/flang/test/Driver/driver-help.f90
+++ b/flang/test/Driver/driver-help.f90
@@ -145,6 +145,7 @@
 ! HELP-FC1-NEXT: -fno-stack-arrays      Allocate array temporaries on the heap (default)
 ! HELP-FC1-NEXT: -fopenacc              Enable OpenACC
 ! HELP-FC1-NEXT: -fopenmp-is-device     Generate code only for an OpenMP target device.
+! HELP-FC1-NEXT: -fopenmp-target-debug  Enable debugging in the OpenMP offloading device RTL
 ! HELP-FC1-NEXT: -fopenmp               Parse OpenMP pragmas and generate parallel code.
 ! HELP-FC1-NEXT: -fpass-plugin=<dsopath> Load pass plugin from a dynamic shared object file (only with new pass manager).
 ! HELP-FC1-NEXT: -freciprocal-math      Allow division operations to be reassociated
diff --git a/flang/test/Driver/omp-frontend-forwarding.f90 b/flang/test/Driver/omp-frontend-forwarding.f90
index d10fd9f..ef4875c 100644
--- a/flang/test/Driver/omp-frontend-forwarding.f90
+++ b/flang/test/Driver/omp-frontend-forwarding.f90
@@ -20,3 +20,50 @@
 ! CHECK-OPENMP-EMBED-NEXT: "{{[^"]*}}flang-new" "-fc1" "-triple" "amdgcn-amd-amdhsa" {{.*}} "-fopenmp" {{.*}} "-fopenmp-is-device" {{.*}}.f90"
 ! CHECK-OPENMP-EMBED: "{{[^"]*}}clang-offload-packager{{.*}} "--image=file={{.*}}.bc,triple=amdgcn-amd-amdhsa,arch=gfx90a,kind=openmp"
 ! CHECK-OPENMP-EMBED-NEXT: "{{[^"]*}}flang-new" "-fc1" "-triple" "aarch64-unknown-linux-gnu" {{.*}} "-fopenmp" {{.*}} "-fembed-offload-object={{.*}}.out" {{.*}}.bc"
+
+! Test -fopenmp with offload for RTL Flag Options
+! RUN: %flang -### %s -o %t 2>&1 \
+! RUN: -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa \
+! RUN: -fopenmp-assume-threads-oversubscription \
+! RUN: | FileCheck %s --check-prefixes=CHECK-THREADS-OVS
+! CHECK-THREADS-OVS: "{{[^"]*}}flang-new" "-fc1" {{.*}} "-fopenmp" {{.*}} "-fopenmp-is-device" "-fopenmp-assume-threads-oversubscription" {{.*}}.f90"
+
+! RUN: %flang -### %s -o %t 2>&1 \
+! RUN: -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa \
+! RUN: -fopenmp-assume-teams-oversubscription  \
+! RUN: | FileCheck %s --check-prefixes=CHECK-TEAMS-OVS
+! CHECK-TEAMS-OVS: "{{[^"]*}}flang-new" "-fc1" {{.*}} "-fopenmp" {{.*}} "-fopenmp-is-device" "-fopenmp-assume-teams-oversubscription" {{.*}}.f90"
+
+! RUN: %flang -### %s -o %t 2>&1 \
+! RUN: -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa \
+! RUN: -fopenmp-assume-no-nested-parallelism  \
+! RUN: | FileCheck %s --check-prefixes=CHECK-NEST-PAR
+! CHECK-NEST-PAR: "{{[^"]*}}flang-new" "-fc1" {{.*}} "-fopenmp" {{.*}} "-fopenmp-is-device" "-fopenmp-assume-no-nested-parallelism" {{.*}}.f90"
+
+! RUN: %flang -### %s -o %t 2>&1 \
+! RUN: -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa \
+! RUN: -fopenmp-assume-no-thread-state \
+! RUN: | FileCheck %s --check-prefixes=CHECK-THREAD-STATE
+! CHECK-THREAD-STATE: "{{[^"]*}}flang-new" "-fc1" {{.*}} "-fopenmp" {{.*}} "-fopenmp-is-device" "-fopenmp-assume-no-thread-state" {{.*}}.f90"
+
+! RUN: %flang -### %s -o %t 2>&1 \
+! RUN: -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa \
+! RUN: -fopenmp-target-debug \
+! RUN: | FileCheck %s --check-prefixes=CHECK-TARGET-DEBUG
+! CHECK-TARGET-DEBUG: "{{[^"]*}}flang-new" "-fc1" {{.*}} "-fopenmp" {{.*}} "-fopenmp-is-device" "-fopenmp-target-debug" {{.*}}.f90"
+
+! RUN: %flang -### %s -o %t 2>&1 \
+! RUN: -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa \
+! RUN: -fopenmp-target-debug \
+! RUN: | FileCheck %s --check-prefixes=CHECK-TARGET-DEBUG
+! CHECK-TARGET-DEBUG-EQ: "{{[^"]*}}flang-new" "-fc1" {{.*}} "-fopenmp" {{.*}} "-fopenmp-is-device" "-fopenmp-target-debug=111" {{.*}}.f90"
+
+! RUN: %flang -S -### %s -o %t 2>&1 \
+! RUN: -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa \
+! RUN: -fopenmp-target-debug -fopenmp-assume-threads-oversubscription \
+! RUN: -fopenmp-assume-teams-oversubscription -fopenmp-assume-no-nested-parallelism \
+! RUN: -fopenmp-assume-no-thread-state \
+! RUN: | FileCheck %s --check-prefixes=CHECK-RTL-ALL
+! CHECK-RTL-ALL: "{{[^"]*}}flang-new" "-fc1" {{.*}} "-fopenmp" {{.*}} "-fopenmp-is-device" "-fopenmp-target-debug" "-fopenmp-assume-teams-oversubscription"
+! CHECK-RTL-ALL: "-fopenmp-assume-threads-oversubscription" "-fopenmp-assume-no-thread-state" "-fopenmp-assume-no-nested-parallelism"
+! CHECK-RTL-ALL: {{.*}}.f90"
diff --git a/flang/test/Lower/OpenMP/rtl-flags.f90 b/flang/test/Lower/OpenMP/rtl-flags.f90
new file mode 100644
index 0000000..4b9a0c9
--- /dev/null
+++ b/flang/test/Lower/OpenMP/rtl-flags.f90
@@ -0,0 +1,29 @@
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-is-device %s -o - | FileCheck %s --check-prefix=DEFAULT-DEVICE-FIR
+!RUN: %flang_fc1 -emit-fir -fopenmp %s -o - | FileCheck %s --check-prefix=DEFAULT-HOST-FIR
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-target-debug -fopenmp-is-device %s -o - | FileCheck %s --check-prefix=DBG-DEVICE-FIR
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-target-debug=111 -fopenmp-is-device %s -o - | FileCheck %s --check-prefix=DBG-EQ-DEVICE-FIR
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-assume-teams-oversubscription -fopenmp-is-device %s -o - | FileCheck %s --check-prefix=TEAMS-OSUB-DEVICE-FIR
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-assume-threads-oversubscription -fopenmp-is-device %s -o - | FileCheck %s --check-prefix=THREAD-OSUB-DEVICE-FIR
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-assume-no-thread-state -fopenmp-is-device %s -o - | FileCheck %s --check-prefix=THREAD-STATE-DEVICE-FIR
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-assume-no-nested-parallelism -fopenmp-is-device %s -o - | FileCheck %s --check-prefix=NEST-PAR-DEVICE-FIR
+!RUN: %flang_fc1 -emit-fir -fopenmp -fopenmp-target-debug -fopenmp-assume-teams-oversubscription -fopenmp-assume-no-nested-parallelism -fopenmp-assume-threads-oversubscription -fopenmp-assume-no-thread-state -fopenmp-is-device %s -o - | FileCheck %s --check-prefix=ALL-DEVICE-FIR
+!RUN: bbc -emit-fir -fopenmp -fopenmp-is-device -o - %s | FileCheck %s --check-prefix=DEFAULT-DEVICE-FIR
+!RUN: bbc -emit-fir -fopenmp -o - %s | FileCheck %s --check-prefix=DEFAULT-HOST-FIR
+!RUN: bbc -emit-fir -fopenmp -fopenmp-target-debug=111 -fopenmp-is-device -o - %s | FileCheck %s --check-prefix=DBG-EQ-DEVICE-FIR
+!RUN: bbc -emit-fir -fopenmp -fopenmp-assume-teams-oversubscription -fopenmp-is-device -o - %s | FileCheck %s --check-prefix=TEAMS-OSUB-DEVICE-FIR
+!RUN: bbc -emit-fir -fopenmp -fopenmp-assume-threads-oversubscription -fopenmp-is-device -o - %s | FileCheck %s --check-prefix=THREAD-OSUB-DEVICE-FIR
+!RUN: bbc -emit-fir -fopenmp -fopenmp-assume-no-thread-state -fopenmp-is-device -o - %s | FileCheck %s --check-prefix=THREAD-STATE-DEVICE-FIR
+!RUN: bbc -emit-fir -fopenmp -fopenmp-assume-no-nested-parallelism -fopenmp-is-device -o - %s | FileCheck %s --check-prefix=NEST-PAR-DEVICE-FIR
+!RUN: bbc -emit-fir -fopenmp -fopenmp-target-debug=1 -fopenmp-assume-teams-oversubscription -fopenmp-assume-no-nested-parallelism -fopenmp-assume-threads-oversubscription -fopenmp-assume-no-thread-state -fopenmp-is-device -o - %s | FileCheck %s --check-prefix=ALL-DEVICE-FIR
+
+!DEFAULT-DEVICE-FIR: module attributes {{{.*}}, omp.flags = #omp.flags<>, omp.is_device = #omp.isdevice<is_device = true>{{.*}}}
+!DEFAULT-HOST-FIR: module attributes {{{.*}},  omp.is_device = #omp.isdevice<is_device = false>{{.*}}}
+!DBG-DEVICE-FIR: module attributes {{{.*}}, omp.flags = #omp.flags<debug_kind = 1>{{.*}}}
+!DBG-EQ-DEVICE-FIR: module attributes {{{.*}}, omp.flags = #omp.flags<debug_kind = 111>{{.*}}}
+!TEAMS-OSUB-DEVICE-FIR: module attributes {{{.*}}, omp.flags = #omp.flags<assume_teams_oversubscription = true>{{.*}}}
+!THREAD-OSUB-DEVICE-FIR: module attributes {{{.*}}, omp.flags = #omp.flags<assume_threads_oversubscription = true>{{.*}}}
+!THREAD-STATE-DEVICE-FIR: module attributes {{{.*}}, omp.flags = #omp.flags<assume_no_thread_state = true>{{.*}}}
+!NEST-PAR-DEVICE-FIR: module attributes {{{.*}}, omp.flags = #omp.flags<assume_no_nested_parallelism = true>{{.*}}}
+!ALL-DEVICE-FIR: module attributes {{{.*}}, omp.flags = #omp.flags<debug_kind = 1, assume_teams_oversubscription = true, assume_threads_oversubscription = true, assume_no_thread_state = true, assume_no_nested_parallelism = true>{{.*}}}
+subroutine omp_subroutine()
+end subroutine omp_subroutine
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index d021a96..e9cf208 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -129,6 +129,38 @@ static llvm::cl::opt<bool>
                        llvm::cl::desc("enable openmp device compilation"),
                        llvm::cl::init(false));
 
+// A simplified subset of the OpenMP RTL Flags from Flang, only the primary
+// positive options are available, no negative options e.g. fopen_assume* vs
+// fno_open_assume*
+static llvm::cl::opt<uint32_t> setOpenMPTargetDebug(
+    "fopenmp-target-debug",
+    llvm::cl::desc("Enable debugging in the OpenMP offloading device RTL"),
+    llvm::cl::init(0));
+
+static llvm::cl::opt<bool> setOpenMPThreadSubscription(
+    "fopenmp-assume-threads-oversubscription",
+    llvm::cl::desc("Assume work-shared loops do not have more "
+                   "iterations than participating threads."),
+    llvm::cl::init(false));
+
+static llvm::cl::opt<bool> setOpenMPTeamSubscription(
+    "fopenmp-assume-teams-oversubscription",
+    llvm::cl::desc("Assume distributed loops do not have more iterations than "
+                   "participating teams."),
+    llvm::cl::init(false));
+
+static llvm::cl::opt<bool> setOpenMPNoThreadState(
+    "fopenmp-assume-no-thread-state",
+    llvm::cl::desc(
+        "Assume that no thread in a parallel region will modify an ICV."),
+    llvm::cl::init(false));
+
+static llvm::cl::opt<bool> setOpenMPNoNestedParallelism(
+    "fopenmp-assume-no-nested-parallelism",
+    llvm::cl::desc("Assume that no thread in a parallel region will encounter "
+                   "a parallel region."),
+    llvm::cl::init(false));
+
 static llvm::cl::opt<bool> enableOpenACC("fopenacc",
                                          llvm::cl::desc("enable openacc"),
                                          llvm::cl::init(false));
@@ -244,8 +276,13 @@ static mlir::LogicalResult convertFortranSourceToMLIR(
       kindMap, loweringOptions, {});
   burnside.lower(parseTree, semanticsContext);
   mlir::ModuleOp mlirModule = burnside.getModule();
-  if (enableOpenMP)
-    setOffloadModuleInterfaceAttributes(mlirModule, enableOpenMPDevice);
+  if (enableOpenMP) {
+    auto offloadModuleOpts =
+        OffloadModuleOpts(setOpenMPTargetDebug, setOpenMPTeamSubscription,
+                          setOpenMPThreadSubscription, setOpenMPNoThreadState,
+                          setOpenMPNoNestedParallelism, enableOpenMPDevice);
+    setOffloadModuleInterfaceAttributes(mlirModule, offloadModuleOpts);
+  }
   std::error_code ec;
   std::string outputName = outputFilename;
   if (!outputName.size())