[flang] Added lowering and runtime for COMPLEX(16) intrinsics. (#83874)

For `LDBL_MANT_DIG == 113` targets the FortranFloat128Math library
is just an interface library that provides sources and compilation
options to be used for building FortranRuntime - there are not extra
dependencies on other libraries, so it can be a part of FortranRuntime,
which helps to avoid extra linking steps in the compiler driver.
Targets with __float128 support in libc will also use this path.
Other targets, where the math support comes from
FLANG_RUNTIME_F128_MATH_LIB,
FortranFloat128Math is built as a standalone static library,
and the compiler driver needs to conduct the linking.

Flang APIs for COMPLEX(16) are just thin C wrappers around
the C math functions. Flang uses C _Complex ABI for passing/returning
COMPLEX values, so the runtime is aligned to this.

34 files changed, 444 insertions, 233 deletions

diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index fb9b58e..25598ed 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -689,22 +689,22 @@ prettyPrintIntrinsicName(fir::FirOpBuilder &builder, mlir::Location loc,
 }
 
 // Generate a call to the Fortran runtime library providing
-// support for 128-bit float math via a third-party library.
-// If the compiler is built without FLANG_RUNTIME_F128_MATH_LIB,
-// this function will report an error.
+// support for 128-bit float math.
+// On 'LDBL_MANT_DIG == 113' targets the implementation
+// is provided by FortranRuntime, otherwise, it is done via
+// FortranFloat128Math library. In the latter case the compiler
+// has to be built with FLANG_RUNTIME_F128_MATH_LIB to guarantee
+// proper linking actions in the driver.
 static mlir::Value genLibF128Call(fir::FirOpBuilder &builder,
                                   mlir::Location loc,
                                   const MathOperation &mathOp,
                                   mlir::FunctionType libFuncType,
                                   llvm::ArrayRef<mlir::Value> args) {
-#ifndef FLANG_RUNTIME_F128_MATH_LIB
-  std::string message = prettyPrintIntrinsicName(
-      builder, loc, "compiler is built without support for '", mathOp.key, "'",
-      libFuncType);
-  fir::emitFatalError(loc, message, /*genCrashDiag=*/false);
-#else  // FLANG_RUNTIME_F128_MATH_LIB
+  // TODO: if we knew that the C 'long double' does not have 113-bit mantissa
+  // on the target, we could have asserted that FLANG_RUNTIME_F128_MATH_LIB
+  // must be specified. For now just always generate the call even
+  // if it will be unresolved.
   return genLibCall(builder, loc, mathOp, libFuncType, args);
-#endif // FLANG_RUNTIME_F128_MATH_LIB
 }
 
 mlir::Value genLibCall(fir::FirOpBuilder &builder, mlir::Location loc,
@@ -926,6 +926,10 @@ constexpr auto FuncTypeInteger8Real16 =
     genFuncType<Ty::Integer<8>, Ty::Real<16>>;
 constexpr auto FuncTypeReal16Complex16 =
     genFuncType<Ty::Real<16>, Ty::Complex<16>>;
+constexpr auto FuncTypeComplex16Complex16 =
+    genFuncType<Ty::Complex<16>, Ty::Complex<16>>;
+constexpr auto FuncTypeComplex16Complex16Complex16 =
+    genFuncType<Ty::Complex<16>, Ty::Complex<16>, Ty::Complex<16>>;
 
 static constexpr MathOperation mathOperations[] = {
     {"abs", "fabsf", genFuncType<Ty::Real<4>, Ty::Real<4>>,
@@ -944,6 +948,8 @@ static constexpr MathOperation mathOperations[] = {
     {"acos", RTNAME_STRING(AcosF128), FuncTypeReal16Real16, genLibF128Call},
     {"acos", "cacosf", genFuncType<Ty::Complex<4>, Ty::Complex<4>>, genLibCall},
     {"acos", "cacos", genFuncType<Ty::Complex<8>, Ty::Complex<8>>, genLibCall},
+    {"acos", RTNAME_STRING(CAcosF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"acosh", "acoshf", genFuncType<Ty::Real<4>, Ty::Real<4>>, genLibCall},
     {"acosh", "acosh", genFuncType<Ty::Real<8>, Ty::Real<8>>, genLibCall},
     {"acosh", RTNAME_STRING(AcoshF128), FuncTypeReal16Real16, genLibF128Call},
@@ -951,6 +957,8 @@ static constexpr MathOperation mathOperations[] = {
      genLibCall},
     {"acosh", "cacosh", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genLibCall},
+    {"acosh", RTNAME_STRING(CAcoshF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     // llvm.trunc behaves the same way as libm's trunc.
     {"aint", "llvm.trunc.f32", genFuncType<Ty::Real<4>, Ty::Real<4>>,
      genLibCall},
@@ -972,6 +980,8 @@ static constexpr MathOperation mathOperations[] = {
     {"asin", RTNAME_STRING(AsinF128), FuncTypeReal16Real16, genLibF128Call},
     {"asin", "casinf", genFuncType<Ty::Complex<4>, Ty::Complex<4>>, genLibCall},
     {"asin", "casin", genFuncType<Ty::Complex<8>, Ty::Complex<8>>, genLibCall},
+    {"asin", RTNAME_STRING(CAsinF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"asinh", "asinhf", genFuncType<Ty::Real<4>, Ty::Real<4>>, genLibCall},
     {"asinh", "asinh", genFuncType<Ty::Real<8>, Ty::Real<8>>, genLibCall},
     {"asinh", RTNAME_STRING(AsinhF128), FuncTypeReal16Real16, genLibF128Call},
@@ -979,6 +989,8 @@ static constexpr MathOperation mathOperations[] = {
      genLibCall},
     {"asinh", "casinh", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genLibCall},
+    {"asinh", RTNAME_STRING(CAsinhF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"atan", "atanf", genFuncType<Ty::Real<4>, Ty::Real<4>>,
      genMathOp<mlir::math::AtanOp>},
     {"atan", "atan", genFuncType<Ty::Real<8>, Ty::Real<8>>,
@@ -986,6 +998,8 @@ static constexpr MathOperation mathOperations[] = {
     {"atan", RTNAME_STRING(AtanF128), FuncTypeReal16Real16, genLibF128Call},
     {"atan", "catanf", genFuncType<Ty::Complex<4>, Ty::Complex<4>>, genLibCall},
     {"atan", "catan", genFuncType<Ty::Complex<8>, Ty::Complex<8>>, genLibCall},
+    {"atan", RTNAME_STRING(CAtanF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"atan2", "atan2f", genFuncType<Ty::Real<4>, Ty::Real<4>, Ty::Real<4>>,
      genMathOp<mlir::math::Atan2Op>},
     {"atan2", "atan2", genFuncType<Ty::Real<8>, Ty::Real<8>, Ty::Real<8>>,
@@ -999,6 +1013,8 @@ static constexpr MathOperation mathOperations[] = {
      genLibCall},
     {"atanh", "catanh", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genLibCall},
+    {"atanh", RTNAME_STRING(CAtanhF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"bessel_j0", "j0f", genFuncType<Ty::Real<4>, Ty::Real<4>>, genLibCall},
     {"bessel_j0", "j0", genFuncType<Ty::Real<8>, Ty::Real<8>>, genLibCall},
     {"bessel_j0", RTNAME_STRING(J0F128), FuncTypeReal16Real16, genLibF128Call},
@@ -1038,11 +1054,15 @@ static constexpr MathOperation mathOperations[] = {
      genComplexMathOp<mlir::complex::CosOp>},
     {"cos", "ccos", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genComplexMathOp<mlir::complex::CosOp>},
+    {"cos", RTNAME_STRING(CCosF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"cosh", "coshf", genFuncType<Ty::Real<4>, Ty::Real<4>>, genLibCall},
     {"cosh", "cosh", genFuncType<Ty::Real<8>, Ty::Real<8>>, genLibCall},
     {"cosh", RTNAME_STRING(CoshF128), FuncTypeReal16Real16, genLibF128Call},
     {"cosh", "ccoshf", genFuncType<Ty::Complex<4>, Ty::Complex<4>>, genLibCall},
     {"cosh", "ccosh", genFuncType<Ty::Complex<8>, Ty::Complex<8>>, genLibCall},
+    {"cosh", RTNAME_STRING(CCoshF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"divc",
      {},
      genFuncType<Ty::Complex<2>, Ty::Complex<2>, Ty::Complex<2>>,
@@ -1080,6 +1100,8 @@ static constexpr MathOperation mathOperations[] = {
      genComplexMathOp<mlir::complex::ExpOp>},
     {"exp", "cexp", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genComplexMathOp<mlir::complex::ExpOp>},
+    {"exp", RTNAME_STRING(CExpF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"feclearexcept", "feclearexcept",
      genFuncType<Ty::Integer<4>, Ty::Integer<4>>, genLibCall},
     {"fedisableexcept", "fedisableexcept",
@@ -1131,6 +1153,8 @@ static constexpr MathOperation mathOperations[] = {
      genComplexMathOp<mlir::complex::LogOp>},
     {"log", "clog", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genComplexMathOp<mlir::complex::LogOp>},
+    {"log", RTNAME_STRING(CLogF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"log10", "log10f", genFuncType<Ty::Real<4>, Ty::Real<4>>,
      genMathOp<mlir::math::Log10Op>},
     {"log10", "log10", genFuncType<Ty::Real<8>, Ty::Real<8>>,
@@ -1178,6 +1202,8 @@ static constexpr MathOperation mathOperations[] = {
      genComplexMathOp<mlir::complex::PowOp>},
     {"pow", "cpow", genFuncType<Ty::Complex<8>, Ty::Complex<8>, Ty::Complex<8>>,
      genComplexMathOp<mlir::complex::PowOp>},
+    {"pow", RTNAME_STRING(CPowF128), FuncTypeComplex16Complex16Complex16,
+     genLibF128Call},
     {"pow", RTNAME_STRING(FPow4i),
      genFuncType<Ty::Real<4>, Ty::Real<4>, Ty::Integer<4>>,
      genMathOp<mlir::math::FPowIOp>},
@@ -1222,11 +1248,15 @@ static constexpr MathOperation mathOperations[] = {
      genComplexMathOp<mlir::complex::SinOp>},
     {"sin", "csin", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genComplexMathOp<mlir::complex::SinOp>},
+    {"sin", RTNAME_STRING(CSinF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"sinh", "sinhf", genFuncType<Ty::Real<4>, Ty::Real<4>>, genLibCall},
     {"sinh", "sinh", genFuncType<Ty::Real<8>, Ty::Real<8>>, genLibCall},
     {"sinh", RTNAME_STRING(SinhF128), FuncTypeReal16Real16, genLibF128Call},
     {"sinh", "csinhf", genFuncType<Ty::Complex<4>, Ty::Complex<4>>, genLibCall},
     {"sinh", "csinh", genFuncType<Ty::Complex<8>, Ty::Complex<8>>, genLibCall},
+    {"sinh", RTNAME_STRING(CSinhF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"sqrt", "sqrtf", genFuncType<Ty::Real<4>, Ty::Real<4>>,
      genMathOp<mlir::math::SqrtOp>},
     {"sqrt", "sqrt", genFuncType<Ty::Real<8>, Ty::Real<8>>,
@@ -1236,6 +1266,8 @@ static constexpr MathOperation mathOperations[] = {
      genComplexMathOp<mlir::complex::SqrtOp>},
     {"sqrt", "csqrt", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genComplexMathOp<mlir::complex::SqrtOp>},
+    {"sqrt", RTNAME_STRING(CSqrtF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"tan", "tanf", genFuncType<Ty::Real<4>, Ty::Real<4>>,
      genMathOp<mlir::math::TanOp>},
     {"tan", "tan", genFuncType<Ty::Real<8>, Ty::Real<8>>,
@@ -1245,6 +1277,8 @@ static constexpr MathOperation mathOperations[] = {
      genComplexMathOp<mlir::complex::TanOp>},
     {"tan", "ctan", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genComplexMathOp<mlir::complex::TanOp>},
+    {"tan", RTNAME_STRING(CTanF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
     {"tanh", "tanhf", genFuncType<Ty::Real<4>, Ty::Real<4>>,
      genMathOp<mlir::math::TanhOp>},
     {"tanh", "tanh", genFuncType<Ty::Real<8>, Ty::Real<8>>,
@@ -1254,6 +1288,8 @@ static constexpr MathOperation mathOperations[] = {
      genComplexMathOp<mlir::complex::TanhOp>},
     {"tanh", "ctanh", genFuncType<Ty::Complex<8>, Ty::Complex<8>>,
      genComplexMathOp<mlir::complex::TanhOp>},
+    {"tanh", RTNAME_STRING(CTanhF128), FuncTypeComplex16Complex16,
+     genLibF128Call},
 };
 
 // This helper class computes a "distance" between two function types.
diff --git a/flang/runtime/CMakeLists.txt b/flang/runtime/CMakeLists.txt
index ac89184..7dd60b5 100644
--- a/flang/runtime/CMakeLists.txt
+++ b/flang/runtime/CMakeLists.txt
@@ -57,12 +57,6 @@ if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
     REAL(16) is mapped to __float128, or libm for targets where REAL(16) \
     is mapped to long double, etc."
     )
-
-  if (NOT FLANG_RUNTIME_F128_MATH_LIB STREQUAL "")
-    add_compile_definitions(
-      -DFLANG_RUNTIME_F128_MATH_LIB="${FLANG_RUNTIME_F128_MATH_LIB}"
-      )
-  endif()
 endif()
 
 include(CheckCXXSymbolExists)
@@ -78,6 +72,16 @@ check_cxx_source_compiles(
   "
   HAVE_DECL_STRERROR_S)
 
+# Check if 128-bit float computations can be done via long double.
+check_cxx_source_compiles(
+  "#include <cfloat>
+   #if LDBL_MANT_DIG != 113
+   #error LDBL_MANT_DIG != 113
+   #endif
+   int main() { return 0; }
+  "
+  HAVE_LDBL_MANT_DIG_113)
+
 check_cxx_compiler_flag(-fno-lto FLANG_RUNTIME_HAS_FNO_LTO_FLAG)
 if (FLANG_RUNTIME_HAS_FNO_LTO_FLAG)
   set(NO_LTO_FLAGS "-fno-lto")
@@ -100,9 +104,7 @@ add_definitions(-U_GLIBCXX_ASSERTIONS)
 add_definitions(-U_LIBCPP_ENABLE_ASSERTIONS)
 
 add_subdirectory(FortranMain)
-if (NOT ${FLANG_RUNTIME_F128_MATH_LIB} STREQUAL "")
-  add_subdirectory(Float128Math)
-endif()
+add_subdirectory(Float128Math)
 
 set(sources
   ISO_Fortran_binding.cpp
@@ -319,6 +321,29 @@ if (NOT FLANG_EXPERIMENTAL_OMP_OFFLOAD_BUILD STREQUAL "off")
   endif()
 endif()
 
+if (NOT TARGET FortranFloat128Math)
+  # If FortranFloat128Math is not defined, then we are not building
+  # standalone FortranFloat128Math library. Instead, include
+  # the relevant sources into FortranRuntime itself.
+  # The information is provided via FortranFloat128MathILib
+  # interface library.
+  get_target_property(f128_sources
+    FortranFloat128MathILib INTERFACE_SOURCES
+    )
+  if (f128_sources)
+    # The interface may define special macros for Float128Math files,
+    # so we need to propagate them.
+    get_target_property(f128_defs
+      FortranFloat128MathILib INTERFACE_COMPILE_DEFINITIONS
+      )
+    set_property(SOURCE ${f128_sources}
+      APPEND PROPERTY COMPILE_DEFINITIONS
+      ${f128_defs}
+      )
+    list(APPEND sources ${f128_sources})
+  endif()
+endif()
+
 if (NOT DEFINED MSVC)
   add_flang_library(FortranRuntime
     ${sources}
diff --git a/flang/runtime/Float128Math/CMakeLists.txt b/flang/runtime/Float128Math/CMakeLists.txt
index 60d44c7..9803561 100644
--- a/flang/runtime/Float128Math/CMakeLists.txt
+++ b/flang/runtime/Float128Math/CMakeLists.txt
@@ -16,34 +16,6 @@
 
 include(CheckLibraryExists)
 
-if (${FLANG_RUNTIME_F128_MATH_LIB} STREQUAL "libquadmath")
-  check_include_file(quadmath.h FOUND_QUADMATH_HEADER)
-  if(FOUND_QUADMATH_HEADER)
-    add_compile_definitions(HAS_QUADMATHLIB)
-  else()
-    message(FATAL_ERROR
-      "FLANG_RUNTIME_F128_MATH_LIB setting requires quadmath.h "
-      "to be available: ${FLANG_RUNTIME_F128_MATH_LIB}"
-      )
-  endif()
-elseif (${FLANG_RUNTIME_F128_MATH_LIB} STREQUAL "libm")
-  check_library_exists(m sinl "" FOUND_LIBM)
-  check_library_exists(m sinf128 "" FOUND_LIBMF128)
-  if (FOUND_LIBM)
-    add_compile_definitions(HAS_LIBM)
-  endif()
-  if (FOUND_LIBMF128)
-    add_compile_definitions(HAS_LIBMF128)
-  endif()
-endif()
-
-if (NOT FOUND_QUADMATH_HEADER AND NOT FOUND_LIBM)
-  message(FATAL_ERROR
-    "Unsupported third-party library for Fortran F128 math runtime: "
-    "${FLANG_RUNTIME_F128_MATH_LIB}"
-    )
-endif()
-
 set(sources
   acos.cpp
   acosh.cpp
@@ -52,8 +24,8 @@ set(sources
   atan.cpp
   atan2.cpp
   atanh.cpp
-  cabs.cpp
   ceil.cpp
+  complex-math.c
   cos.cpp
   cosh.cpp
   erf.cpp
@@ -94,18 +66,61 @@ set(sources
   )
 
 include_directories(AFTER "${CMAKE_CURRENT_SOURCE_DIR}/..")
-add_flang_library(FortranFloat128Math STATIC INSTALL_WITH_TOOLCHAIN ${sources})
+add_library(FortranFloat128MathILib INTERFACE)
 
-if (DEFINED MSVC)
-  set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreaded)
-  add_flang_library(FortranFloat128Math.static STATIC INSTALL_WITH_TOOLCHAIN
-    ${sources}
-    )
-  set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreadedDebug)
-  add_flang_library(FortranFloat128Math.static_dbg STATIC INSTALL_WITH_TOOLCHAIN
-    ${sources}
-    )
-  add_dependencies(FortranFloat128Math FortranFloat128Math.static
-    FortranFloat128Math.static_dbg
-    )
+if (FLANG_RUNTIME_F128_MATH_LIB)
+  if (${FLANG_RUNTIME_F128_MATH_LIB} STREQUAL "libquadmath")
+    check_include_file(quadmath.h FOUND_QUADMATH_HEADER)
+    if(FOUND_QUADMATH_HEADER)
+      add_compile_definitions(HAS_QUADMATHLIB)
+    else()
+      message(FATAL_ERROR
+        "FLANG_RUNTIME_F128_MATH_LIB setting requires quadmath.h "
+        "to be available: ${FLANG_RUNTIME_F128_MATH_LIB}"
+        )
+    endif()
+  else()
+    message(FATAL_ERROR
+      "Unsupported third-party library for Fortran F128 math runtime: "
+      "${FLANG_RUNTIME_F128_MATH_LIB}"
+      )
+  endif()
+
+  add_flang_library(FortranFloat128Math STATIC INSTALL_WITH_TOOLCHAIN
+    ${sources})
+
+  if (DEFINED MSVC)
+    set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreaded)
+    add_flang_library(FortranFloat128Math.static STATIC INSTALL_WITH_TOOLCHAIN
+      ${sources}
+      )
+    set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreadedDebug)
+    add_flang_library(FortranFloat128Math.static_dbg STATIC INSTALL_WITH_TOOLCHAIN
+      ${sources}
+      )
+    add_dependencies(FortranFloat128Math FortranFloat128Math.static
+      FortranFloat128Math.static_dbg
+      )
+  endif()
+elseif (HAVE_LDBL_MANT_DIG_113)
+  # We can use 'long double' versions from libc.
+  check_library_exists(m sinl "" FOUND_LIBM)
+  if (FOUND_LIBM)
+    target_compile_definitions(FortranFloat128MathILib INTERFACE
+      HAS_LIBM
+      )
+    target_sources(FortranFloat128MathILib INTERFACE ${sources})
+  else()
+    message(FATAL_ERROR "FortranRuntime cannot build without libm")
+  endif()
+else()
+  # We can use '__float128' version from libc, if it has them.
+  check_library_exists(m sinf128 "" FOUND_LIBMF128)
+  if (FOUND_LIBMF128)
+    target_compile_definitions(FortranFloat128MathILib INTERFACE
+      HAS_LIBMF128
+      )
+    # Enable this, when math-entries.h and complex-math.h is ready.
+    # target_sources(FortranFloat128MathILib INTERFACE ${sources})
+  endif()
 endif()
diff --git a/flang/runtime/Float128Math/cabs.cpp b/flang/runtime/Float128Math/cabs.cpp
deleted file mode 100644
index 3b8c9d1..0000000
--- a/flang/runtime/Float128Math/cabs.cpp
+++ /dev/null
@@ -1,24 +0,0 @@
-//===-- runtime/Float128Math/cabs.cpp -------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "math-entries.h"
-
-namespace Fortran::runtime {
-extern "C" {
-#if 0
-// FIXME: temporarily disabled. Need to add pure C entry point
-// using C _Complex ABI.
-#if LDBL_MANT_DIG == 113 || HAS_FLOAT128
-// NOTE: Flang calls the runtime APIs using C _Complex ABI
-CppTypeFor<TypeCategory::Real, 16> RTDEF(CAbsF128)(CFloat128ComplexType x) {
-  return CAbs<true>::invoke(x);
-}
-#endif
-#endif
-} // extern "C"
-} // namespace Fortran::runtime
diff --git a/flang/runtime/Float128Math/complex-math.c b/flang/runtime/Float128Math/complex-math.c
new file mode 100644
index 0000000..d0180c6
--- /dev/null
+++ b/flang/runtime/Float128Math/complex-math.c
@@ -0,0 +1,55 @@
+/*===-- runtime/Float128Math/complex-math.c -------------------------*- C -*-===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ * ===-----------------------------------------------------------------------===
+ */
+
+#include "complex-math.h"
+
+#if LDBL_MANT_DIG == 113 || HAS_FLOAT128
+
+CFloat128Type RTDEF(CAbsF128)(CFloat128ComplexType x) { return CAbs(x); }
+CFloat128ComplexType RTDEF(CAcosF128)(CFloat128ComplexType x) {
+  return CAcos(x);
+}
+CFloat128ComplexType RTDEF(CAcoshF128)(CFloat128ComplexType x) {
+  return CAcosh(x);
+}
+CFloat128ComplexType RTDEF(CAsinF128)(CFloat128ComplexType x) {
+  return CAsin(x);
+}
+CFloat128ComplexType RTDEF(CAsinhF128)(CFloat128ComplexType x) {
+  return CAsinh(x);
+}
+CFloat128ComplexType RTDEF(CAtanF128)(CFloat128ComplexType x) {
+  return CAtan(x);
+}
+CFloat128ComplexType RTDEF(CAtanhF128)(CFloat128ComplexType x) {
+  return CAtanh(x);
+}
+CFloat128ComplexType RTDEF(CCosF128)(CFloat128ComplexType x) { return CCos(x); }
+CFloat128ComplexType RTDEF(CCoshF128)(CFloat128ComplexType x) {
+  return CCosh(x);
+}
+CFloat128ComplexType RTDEF(CExpF128)(CFloat128ComplexType x) { return CExp(x); }
+CFloat128ComplexType RTDEF(CLogF128)(CFloat128ComplexType x) { return CLog(x); }
+CFloat128ComplexType RTDEF(CPowF128)(
+    CFloat128ComplexType x, CFloat128ComplexType p) {
+  return CPow(x, p);
+}
+CFloat128ComplexType RTDEF(CSinF128)(CFloat128ComplexType x) { return CSin(x); }
+CFloat128ComplexType RTDEF(CSinhF128)(CFloat128ComplexType x) {
+  return CSinh(x);
+}
+CFloat128ComplexType RTDEF(CSqrtF128)(CFloat128ComplexType x) {
+  return CSqrt(x);
+}
+CFloat128ComplexType RTDEF(CTanF128)(CFloat128ComplexType x) { return CTan(x); }
+CFloat128ComplexType RTDEF(CTanhF128)(CFloat128ComplexType x) {
+  return CTanh(x);
+}
+
+#endif // LDBL_MANT_DIG == 113 || HAS_FLOAT128
diff --git a/flang/runtime/Float128Math/complex-math.h b/flang/runtime/Float128Math/complex-math.h
new file mode 100644
index 0000000..81dd53a
--- /dev/null
+++ b/flang/runtime/Float128Math/complex-math.h
@@ -0,0 +1,62 @@
+/*===-- runtime/Float128Math/complex-math.h -------------------------*- C -*-===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===----------------------------------------------------------------------===*/
+
+#ifndef FORTRAN_RUNTIME_FLOAT128MATH_COMPLEX_MATH_H_
+#define FORTRAN_RUNTIME_FLOAT128MATH_COMPLEX_MATH_H_
+
+#include "flang/Common/float128.h"
+#include "flang/Runtime/entry-names.h"
+
+#if HAS_QUADMATHLIB
+#include "quadmath.h"
+#define CAbs(x) cabsq(x)
+#define CAcos(x) cacosq(x)
+#define CAcosh(x) cacoshq(x)
+#define CAsin(x) casinq(x)
+#define CAsinh(x) casinhq(x)
+#define CAtan(x) catanq(x)
+#define CAtanh(x) catanhq(x)
+#define CCos(x) ccosq(x)
+#define CCosh(x) ccoshq(x)
+#define CExp(x) cexpq(x)
+#define CLog(x) clogq(x)
+#define CPow(x, p) cpowq(x, p)
+#define CSin(x) csinq(x)
+#define CSinh(x) csinhq(x)
+#define CSqrt(x) csqrtq(x)
+#define CTan(x) ctanq(x)
+#define CTanh(x) ctanhq(x)
+#elif LDBL_MANT_DIG == 113
+/* Use 'long double' versions of libm functions. */
+#include <complex.h>
+
+#define CAbs(x) cabsl(x)
+#define CAcos(x) cacosl(x)
+#define CAcosh(x) cacoshl(x)
+#define CAsin(x) casinl(x)
+#define CAsinh(x) casinhl(x)
+#define CAtan(x) catanl(x)
+#define CAtanh(x) catanhl(x)
+#define CCos(x) ccosl(x)
+#define CCosh(x) ccoshl(x)
+#define CExp(x) cexpl(x)
+#define CLog(x) clogl(x)
+#define CPow(x, p) cpowl(x, p)
+#define CSin(x) csinl(x)
+#define CSinh(x) csinhl(x)
+#define CSqrt(x) csqrtl(x)
+#define CTan(x) ctanl(x)
+#define CTanh(x) ctanhl(x)
+#elif HAS_LIBMF128
+/* We can use __float128 versions of libm functions.
+ * __STDC_WANT_IEC_60559_TYPES_EXT__ needs to be defined
+ * before including math.h to enable the *f128 prototypes. */
+#error "Float128Math build with glibc>=2.26 is unsupported yet"
+#endif
+
+#endif /* FORTRAN_RUNTIME_FLOAT128MATH_COMPLEX_MATH_H_ */
diff --git a/flang/runtime/Float128Math/exponent.cpp b/flang/runtime/Float128Math/exponent.cpp
index c0e43c0..1be1dd0 100644
--- a/flang/runtime/Float128Math/exponent.cpp
+++ b/flang/runtime/Float128Math/exponent.cpp
@@ -12,7 +12,7 @@
 namespace Fortran::runtime {
 extern "C" {
 
-#if LDBL_MANT_DIG != 113 && HAS_FLOAT128
+#if LDBL_MANT_DIG == 113 || HAS_FLOAT128
 // EXPONENT (16.9.75)
 CppTypeFor<TypeCategory::Integer, 4> RTDEF(Exponent16_4)(F128Type x) {
   return Exponent<CppTypeFor<TypeCategory::Integer, 4>>(x);
diff --git a/flang/runtime/Float128Math/fraction.cpp b/flang/runtime/Float128Math/fraction.cpp
index 8de6d3c..8c9889b 100644
--- a/flang/runtime/Float128Math/fraction.cpp
+++ b/flang/runtime/Float128Math/fraction.cpp
@@ -12,7 +12,7 @@
 namespace Fortran::runtime {
 extern "C" {
 
-#if LDBL_MANT_DIG != 113 && HAS_FLOAT128
+#if LDBL_MANT_DIG == 113 || HAS_FLOAT128
 // FRACTION (16.9.80)
 F128Type RTDEF(Fraction16)(F128Type x) { return Fraction(x); }
 #endif
diff --git a/flang/runtime/Float128Math/math-entries.h b/flang/runtime/Float128Math/math-entries.h
index ad3f6aa..1eab7c8 100644
--- a/flang/runtime/Float128Math/math-entries.h
+++ b/flang/runtime/Float128Math/math-entries.h
@@ -106,11 +106,59 @@ DEFINE_FALLBACK_F128(Y0)
 DEFINE_FALLBACK_F128(Y1)
 DEFINE_FALLBACK_F128(Yn)
 
-#if HAS_LIBM
-#include <limits>
+#if HAS_QUADMATHLIB
+// Define wrapper callers for libquadmath.
+#include "quadmath.h"
+DEFINE_SIMPLE_ALIAS(Abs, fabsq)
+DEFINE_SIMPLE_ALIAS(Acos, acosq)
+DEFINE_SIMPLE_ALIAS(Acosh, acoshq)
+DEFINE_SIMPLE_ALIAS(Asin, asinq)
+DEFINE_SIMPLE_ALIAS(Asinh, asinhq)
+DEFINE_SIMPLE_ALIAS(Atan, atanq)
+DEFINE_SIMPLE_ALIAS(Atan2, atan2q)
+DEFINE_SIMPLE_ALIAS(Atanh, atanhq)
+DEFINE_SIMPLE_ALIAS(Ceil, ceilq)
+DEFINE_SIMPLE_ALIAS(Cos, cosq)
+DEFINE_SIMPLE_ALIAS(Cosh, coshq)
+DEFINE_SIMPLE_ALIAS(Erf, erfq)
+DEFINE_SIMPLE_ALIAS(Erfc, erfcq)
+DEFINE_SIMPLE_ALIAS(Exp, expq)
+DEFINE_SIMPLE_ALIAS(Floor, floorq)
+DEFINE_SIMPLE_ALIAS(Frexp, frexpq)
+DEFINE_SIMPLE_ALIAS(Hypot, hypotq)
+DEFINE_SIMPLE_ALIAS(Ilogb, ilogbq)
+DEFINE_SIMPLE_ALIAS(Isinf, isinfq)
+DEFINE_SIMPLE_ALIAS(Isnan, isnanq)
+DEFINE_SIMPLE_ALIAS(J0, j0q)
+DEFINE_SIMPLE_ALIAS(J1, j1q)
+DEFINE_SIMPLE_ALIAS(Jn, jnq)
+DEFINE_SIMPLE_ALIAS(Ldexp, ldexpq)
+DEFINE_SIMPLE_ALIAS(Lgamma, lgammaq)
+DEFINE_SIMPLE_ALIAS(Llround, llroundq)
+DEFINE_SIMPLE_ALIAS(Log, logq)
+DEFINE_SIMPLE_ALIAS(Log10, log10q)
+DEFINE_SIMPLE_ALIAS(Lround, lroundq)
+DEFINE_SIMPLE_ALIAS(Nextafter, nextafterq)
+DEFINE_SIMPLE_ALIAS(Pow, powq)
+DEFINE_SIMPLE_ALIAS(Round, roundq)
+DEFINE_SIMPLE_ALIAS(Sin, sinq)
+DEFINE_SIMPLE_ALIAS(Sinh, sinhq)
+DEFINE_SIMPLE_ALIAS(Sqrt, sqrtq)
+DEFINE_SIMPLE_ALIAS(Tan, tanq)
+DEFINE_SIMPLE_ALIAS(Tanh, tanhq)
+DEFINE_SIMPLE_ALIAS(Tgamma, tgammaq)
+DEFINE_SIMPLE_ALIAS(Trunc, truncq)
+DEFINE_SIMPLE_ALIAS(Y0, y0q)
+DEFINE_SIMPLE_ALIAS(Y1, y1q)
+DEFINE_SIMPLE_ALIAS(Yn, ynq)
 
+// Use cmath INFINITY/NAN definition. Rely on C implicit conversions.
+#define F128_RT_INFINITY (INFINITY)
+#define F128_RT_QNAN (NAN)
+#elif LDBL_MANT_DIG == 113
 // Define wrapper callers for libm.
-#if LDBL_MANT_DIG == 113
+#include <limits>
+
 // Use STD math functions. They provide IEEE-754 128-bit float
 // support either via 'long double' or __float128.
 // The Bessel's functions are not present in STD namespace.
@@ -122,9 +170,6 @@ DEFINE_SIMPLE_ALIAS(Asinh, std::asinh)
 DEFINE_SIMPLE_ALIAS(Atan, std::atan)
 DEFINE_SIMPLE_ALIAS(Atan2, std::atan2)
 DEFINE_SIMPLE_ALIAS(Atanh, std::atanh)
-// TODO: enable complex abs, when ABI adjustment for complex
-// data type is resolved.
-// DEFINE_SIMPLE_ALIAS(CAbs, std::abs)
 DEFINE_SIMPLE_ALIAS(Ceil, std::ceil)
 DEFINE_SIMPLE_ALIAS(Cos, std::cos)
 DEFINE_SIMPLE_ALIAS(Cosh, std::cosh)
@@ -165,70 +210,11 @@ DEFINE_SIMPLE_ALIAS(Yn, ynl)
   (std::numeric_limits<CppTypeFor<TypeCategory::Real, 16>>::infinity())
 #define F128_RT_QNAN \
   (std::numeric_limits<CppTypeFor<TypeCategory::Real, 16>>::quiet_NaN())
-#else // LDBL_MANT_DIG != 113
-#if !HAS_LIBMF128
-// glibc >=2.26 seems to have complete support for __float128
-// versions of the math functions.
-#error "FLANG_RUNTIME_F128_MATH_LIB=libm build requires libm >=2.26"
-#endif
-
+#elif HAS_LIBMF128
 // We can use __float128 versions of libm functions.
 // __STDC_WANT_IEC_60559_TYPES_EXT__ needs to be defined
 // before including cmath to enable the *f128 prototypes.
-// TODO: this needs to be enabled separately, especially
-// for complex data types that require C++ complex to C complex
-// adjustment to match the ABIs.
-#error "Unsupported FLANG_RUNTIME_F128_MATH_LIB=libm build"
-#endif // LDBL_MANT_DIG != 113
-#elif HAS_QUADMATHLIB
-// Define wrapper callers for libquadmath.
-#include "quadmath.h"
-DEFINE_SIMPLE_ALIAS(Abs, fabsq)
-DEFINE_SIMPLE_ALIAS(Acos, acosq)
-DEFINE_SIMPLE_ALIAS(Acosh, acoshq)
-DEFINE_SIMPLE_ALIAS(Asin, asinq)
-DEFINE_SIMPLE_ALIAS(Asinh, asinhq)
-DEFINE_SIMPLE_ALIAS(Atan, atanq)
-DEFINE_SIMPLE_ALIAS(Atan2, atan2q)
-DEFINE_SIMPLE_ALIAS(Atanh, atanhq)
-DEFINE_SIMPLE_ALIAS(Ceil, ceilq)
-DEFINE_SIMPLE_ALIAS(Cos, cosq)
-DEFINE_SIMPLE_ALIAS(Cosh, coshq)
-DEFINE_SIMPLE_ALIAS(Erf, erfq)
-DEFINE_SIMPLE_ALIAS(Erfc, erfcq)
-DEFINE_SIMPLE_ALIAS(Exp, expq)
-DEFINE_SIMPLE_ALIAS(Floor, floorq)
-DEFINE_SIMPLE_ALIAS(Frexp, frexpq)
-DEFINE_SIMPLE_ALIAS(Hypot, hypotq)
-DEFINE_SIMPLE_ALIAS(Ilogb, ilogbq)
-DEFINE_SIMPLE_ALIAS(Isinf, isinfq)
-DEFINE_SIMPLE_ALIAS(Isnan, isnanq)
-DEFINE_SIMPLE_ALIAS(J0, j0q)
-DEFINE_SIMPLE_ALIAS(J1, j1q)
-DEFINE_SIMPLE_ALIAS(Jn, jnq)
-DEFINE_SIMPLE_ALIAS(Ldexp, ldexpq)
-DEFINE_SIMPLE_ALIAS(Lgamma, lgammaq)
-DEFINE_SIMPLE_ALIAS(Llround, llroundq)
-DEFINE_SIMPLE_ALIAS(Log, logq)
-DEFINE_SIMPLE_ALIAS(Log10, log10q)
-DEFINE_SIMPLE_ALIAS(Lround, lroundq)
-DEFINE_SIMPLE_ALIAS(Nextafter, nextafterq)
-DEFINE_SIMPLE_ALIAS(Pow, powq)
-DEFINE_SIMPLE_ALIAS(Round, roundq)
-DEFINE_SIMPLE_ALIAS(Sin, sinq)
-DEFINE_SIMPLE_ALIAS(Sinh, sinhq)
-DEFINE_SIMPLE_ALIAS(Sqrt, sqrtq)
-DEFINE_SIMPLE_ALIAS(Tan, tanq)
-DEFINE_SIMPLE_ALIAS(Tanh, tanhq)
-DEFINE_SIMPLE_ALIAS(Tgamma, tgammaq)
-DEFINE_SIMPLE_ALIAS(Trunc, truncq)
-DEFINE_SIMPLE_ALIAS(Y0, y0q)
-DEFINE_SIMPLE_ALIAS(Y1, y1q)
-DEFINE_SIMPLE_ALIAS(Yn, ynq)
-
-// Use cmath INFINITY/NAN definition. Rely on C implicit conversions.
-#define F128_RT_INFINITY (INFINITY)
-#define F128_RT_QNAN (NAN)
+#error "Float128Math build with glibc>=2.26 is unsupported yet"
 #endif
 
 } // namespace Fortran::runtime
diff --git a/flang/runtime/Float128Math/mod-real.cpp b/flang/runtime/Float128Math/mod-real.cpp
index 9cc2926..42e6ce7 100644
--- a/flang/runtime/Float128Math/mod-real.cpp
+++ b/flang/runtime/Float128Math/mod-real.cpp
@@ -12,7 +12,7 @@
 namespace Fortran::runtime {
 extern "C" {
 
-#if LDBL_MANT_DIG != 113 && HAS_FLOAT128
+#if LDBL_MANT_DIG == 113 || HAS_FLOAT128
 // MOD (16.9.135)
 F128Type RTDEF(ModReal16)(
     F128Type x, F128Type p, const char *sourceFile, int sourceLine) {
diff --git a/flang/runtime/Float128Math/modulo-real.cpp b/flang/runtime/Float128Math/modulo-real.cpp
index b25797fd..13000ab 100644
--- a/flang/runtime/Float128Math/modulo-real.cpp
+++ b/flang/runtime/Float128Math/modulo-real.cpp
@@ -12,7 +12,7 @@
 namespace Fortran::runtime {
 extern "C" {
 
-#if LDBL_MANT_DIG != 113 && HAS_FLOAT128
+#if LDBL_MANT_DIG == 113 || HAS_FLOAT128
 // MODULO (16.9.136)
 F128Type RTDEF(ModuloReal16)(
     F128Type x, F128Type p, const char *sourceFile, int sourceLine) {
diff --git a/flang/runtime/Float128Math/nearest.cpp b/flang/runtime/Float128Math/nearest.cpp
index fd99053..148ac4e 100644
--- a/flang/runtime/Float128Math/nearest.cpp
+++ b/flang/runtime/Float128Math/nearest.cpp
@@ -11,7 +11,7 @@
 namespace Fortran::runtime {
 extern "C" {
 
-#if LDBL_MANT_DIG != 113 && HAS_FLOAT128
+#if LDBL_MANT_DIG == 113 || HAS_FLOAT128
 CppTypeFor<TypeCategory::Real, 16> RTDEF(Nearest16)(
     CppTypeFor<TypeCategory::Real, 16> x, bool positive) {
   return Nextafter<true>::invoke(
diff --git a/flang/runtime/Float128Math/rrspacing.cpp b/flang/runtime/Float128Math/rrspacing.cpp
index f2187f4..feddac4 100644
--- a/flang/runtime/Float128Math/rrspacing.cpp
+++ b/flang/runtime/Float128Math/rrspacing.cpp
@@ -12,7 +12,7 @@
 namespace Fortran::runtime {
 extern "C" {
 
-#if LDBL_MANT_DIG != 113 && HAS_FLOAT128
+#if LDBL_MANT_DIG == 113 || HAS_FLOAT128
 // FRACTION (16.9.80)
 F128Type RTDEF(RRSpacing16)(F128Type x) { return RRSpacing<113>(x); }
 #endif
diff --git a/flang/runtime/Float128Math/scale.cpp b/flang/runtime/Float128Math/scale.cpp
index d6b8431..0be958b 100644
--- a/flang/runtime/Float128Math/scale.cpp
+++ b/flang/runtime/Float128Math/scale.cpp
@@ -13,7 +13,7 @@
 namespace Fortran::runtime {
 extern "C" {
 
-#if LDBL_MANT_DIG != 113 && HAS_FLOAT128
+#if LDBL_MANT_DIG == 113 || HAS_FLOAT128
 F128Type RTDEF(Scale16)(F128Type x, std::int64_t p) {
   auto ip{static_cast<int>(p)};
   if (ip != p) {
diff --git a/flang/runtime/Float128Math/set-exponent.cpp b/flang/runtime/Float128Math/set-exponent.cpp
index 0f942d2..99c34af 100644
--- a/flang/runtime/Float128Math/set-exponent.cpp
+++ b/flang/runtime/Float128Math/set-exponent.cpp
@@ -12,7 +12,7 @@
 namespace Fortran::runtime {
 extern "C" {
 
-#if LDBL_MANT_DIG != 113 && HAS_FLOAT128
+#if LDBL_MANT_DIG == 113 || HAS_FLOAT128
 // SET_EXPONENT (16.9.171)
 F128Type RTDEF(SetExponent16)(F128Type x, std::int64_t p) {
   return SetExponent(x, p);
diff --git a/flang/runtime/Float128Math/spacing.cpp b/flang/runtime/Float128Math/spacing.cpp
index d00e746..a86c0b30 100644
--- a/flang/runtime/Float128Math/spacing.cpp
+++ b/flang/runtime/Float128Math/spacing.cpp
@@ -12,7 +12,7 @@
 namespace Fortran::runtime {
 extern "C" {
 
-#if LDBL_MANT_DIG != 113 && HAS_FLOAT128
+#if LDBL_MANT_DIG == 113 || HAS_FLOAT128
 // SPACING (16.9.180)
 F128Type RTDEF(Spacing16)(F128Type x) { return Spacing<113>(x); }
 #endif
diff --git a/flang/runtime/numeric.cpp b/flang/runtime/numeric.cpp
index d61f32e..abd3e50 100644
--- a/flang/runtime/numeric.cpp
+++ b/flang/runtime/numeric.cpp
@@ -324,16 +324,6 @@ CppTypeFor<TypeCategory::Integer, 8> RTDEF(Exponent10_8)(
     CppTypeFor<TypeCategory::Real, 10> x) {
   return Exponent<CppTypeFor<TypeCategory::Integer, 8>>(x);
 }
-#elif LDBL_MANT_DIG == 113
-// The __float128 implementation resides in FortranFloat128Math library.
-CppTypeFor<TypeCategory::Integer, 4> RTDEF(Exponent16_4)(
-    CppTypeFor<TypeCategory::Real, 16> x) {
-  return Exponent<CppTypeFor<TypeCategory::Integer, 4>>(x);
-}
-CppTypeFor<TypeCategory::Integer, 8> RTDEF(Exponent16_8)(
-    CppTypeFor<TypeCategory::Real, 16> x) {
-  return Exponent<CppTypeFor<TypeCategory::Integer, 8>>(x);
-}
 #endif
 
 CppTypeFor<TypeCategory::Integer, 1> RTDEF(Floor4_1)(
@@ -441,12 +431,6 @@ CppTypeFor<TypeCategory::Real, 10> RTDEF(Fraction10)(
     CppTypeFor<TypeCategory::Real, 10> x) {
   return Fraction(x);
 }
-#elif LDBL_MANT_DIG == 113
-// The __float128 implementation resides in FortranFloat128Math library.
-CppTypeFor<TypeCategory::Real, 16> RTDEF(Fraction16)(
-    CppTypeFor<TypeCategory::Real, 16> x) {
-  return Fraction(x);
-}
 #endif
 
 bool RTDEF(IsFinite4)(CppTypeFor<TypeCategory::Real, 4> x) {
@@ -529,13 +513,6 @@ CppTypeFor<TypeCategory::Real, 10> RTDEF(ModReal10)(
     const char *sourceFile, int sourceLine) {
   return RealMod<false>(x, p, sourceFile, sourceLine);
 }
-#elif LDBL_MANT_DIG == 113
-// The __float128 implementation resides in FortranFloat128Math library.
-CppTypeFor<TypeCategory::Real, 16> RTDEF(ModReal16)(
-    CppTypeFor<TypeCategory::Real, 16> x, CppTypeFor<TypeCategory::Real, 16> p,
-    const char *sourceFile, int sourceLine) {
-  return RealMod<false>(x, p, sourceFile, sourceLine);
-}
 #endif
 
 CppTypeFor<TypeCategory::Integer, 1> RTDEF(ModuloInteger1)(
@@ -586,13 +563,6 @@ CppTypeFor<TypeCategory::Real, 10> RTDEF(ModuloReal10)(
     const char *sourceFile, int sourceLine) {
   return RealMod<true>(x, p, sourceFile, sourceLine);
 }
-#elif LDBL_MANT_DIG == 113
-// The __float128 implementation resides in FortranFloat128Math library.
-CppTypeFor<TypeCategory::Real, 16> RTDEF(ModuloReal16)(
-    CppTypeFor<TypeCategory::Real, 16> x, CppTypeFor<TypeCategory::Real, 16> p,
-    const char *sourceFile, int sourceLine) {
-  return RealMod<true>(x, p, sourceFile, sourceLine);
-}
 #endif
 
 CppTypeFor<TypeCategory::Real, 4> RTDEF(Nearest4)(
@@ -608,12 +578,6 @@ CppTypeFor<TypeCategory::Real, 10> RTDEF(Nearest10)(
     CppTypeFor<TypeCategory::Real, 10> x, bool positive) {
   return Nearest<64>(x, positive);
 }
-#elif LDBL_MANT_DIG == 113
-// The __float128 implementation resides in FortranFloat128Math library.
-CppTypeFor<TypeCategory::Real, 16> RTDEF(Nearest16)(
-    CppTypeFor<TypeCategory::Real, 16> x, bool positive) {
-  return Nearest<113>(x, positive);
-}
 #endif
 
 CppTypeFor<TypeCategory::Integer, 1> RTDEF(Nint4_1)(
@@ -721,12 +685,6 @@ CppTypeFor<TypeCategory::Real, 10> RTDEF(RRSpacing10)(
     CppTypeFor<TypeCategory::Real, 10> x) {
   return RRSpacing<64>(x);
 }
-#elif LDBL_MANT_DIG == 113
-// The __float128 implementation resides in FortranFloat128Math library.
-CppTypeFor<TypeCategory::Real, 16> RTDEF(RRSpacing16)(
-    CppTypeFor<TypeCategory::Real, 16> x) {
-  return RRSpacing<113>(x);
-}
 #endif
 
 CppTypeFor<TypeCategory::Real, 4> RTDEF(SetExponent4)(
@@ -742,12 +700,6 @@ CppTypeFor<TypeCategory::Real, 10> RTDEF(SetExponent10)(
     CppTypeFor<TypeCategory::Real, 10> x, std::int64_t p) {
   return SetExponent(x, p);
 }
-#elif LDBL_MANT_DIG == 113
-// The __float128 implementation resides in FortranFloat128Math library.
-CppTypeFor<TypeCategory::Real, 16> RTDEF(SetExponent16)(
-    CppTypeFor<TypeCategory::Real, 16> x, std::int64_t p) {
-  return SetExponent(x, p);
-}
 #endif
 
 CppTypeFor<TypeCategory::Real, 4> RTDEF(Scale4)(
@@ -763,12 +715,6 @@ CppTypeFor<TypeCategory::Real, 10> RTDEF(Scale10)(
     CppTypeFor<TypeCategory::Real, 10> x, std::int64_t p) {
   return Scale(x, p);
 }
-#elif LDBL_MANT_DIG == 113
-// The __float128 implementation resides in FortranFloat128Math library.
-CppTypeFor<TypeCategory::Real, 16> RTDEF(Scale16)(
-    CppTypeFor<TypeCategory::Real, 16> x, std::int64_t p) {
-  return Scale(x, p);
-}
 #endif
 
 // SELECTED_INT_KIND
@@ -823,12 +769,6 @@ CppTypeFor<TypeCategory::Real, 10> RTDEF(Spacing10)(
     CppTypeFor<TypeCategory::Real, 10> x) {
   return Spacing<64>(x);
 }
-#elif LDBL_MANT_DIG == 113
-// The __float128 implementation resides in FortranFloat128Math library.
-CppTypeFor<TypeCategory::Real, 16> RTDEF(Spacing16)(
-    CppTypeFor<TypeCategory::Real, 16> x) {
-  return Spacing<113>(x);
-}
 #endif
 
 CppTypeFor<TypeCategory::Real, 4> RTDEF(FPow4i)(
diff --git a/flang/test/Lower/Intrinsics/acos_complex16.f90 b/flang/test/Lower/Intrinsics/acos_complex16.f90
new file mode 100644
index 0000000..5b3545e
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/acos_complex16.f90
@@ -0,0 +1,8 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+
+! CHECK: fir.call @_FortranACAcosF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+  complex(16) :: a, b
+  b = acos(a)
+end
diff --git a/flang/test/Lower/Intrinsics/acosh_complex16.f90 b/flang/test/Lower/Intrinsics/acosh_complex16.f90
new file mode 100644
index 0000000..a80238d
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/acosh_complex16.f90
@@ -0,0 +1,8 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+
+! CHECK: fir.call @_FortranACAcoshF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+  complex(16) :: a, b
+  b = acosh(a)
+end
diff --git a/flang/test/Lower/Intrinsics/asin_complex16.f90 b/flang/test/Lower/Intrinsics/asin_complex16.f90
new file mode 100644
index 0000000..982bf6f
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/asin_complex16.f90
@@ -0,0 +1,8 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+
+! CHECK: fir.call @_FortranACAsinF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+  complex(16) :: a, b
+  b = asin(a)
+end
diff --git a/flang/test/Lower/Intrinsics/asinh_complex16.f90 b/flang/test/Lower/Intrinsics/asinh_complex16.f90
new file mode 100644
index 0000000..2d658a6
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/asinh_complex16.f90
@@ -0,0 +1,8 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+
+! CHECK: fir.call @_FortranACAsinhF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+  complex(16) :: a, b
+  b = asinh(a)
+end
diff --git a/flang/test/Lower/Intrinsics/atan_complex16.f90 b/flang/test/Lower/Intrinsics/atan_complex16.f90
new file mode 100644
index 0000000..315928d
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/atan_complex16.f90
@@ -0,0 +1,8 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+
+! CHECK: fir.call @_FortranACAtanF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+  complex(16) :: a, b
+  b = atan(a)
+end
diff --git a/flang/test/Lower/Intrinsics/atanh_complex16.f90 b/flang/test/Lower/Intrinsics/atanh_complex16.f90
new file mode 100644
index 0000000..0d9f798
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/atanh_complex16.f90
@@ -0,0 +1,8 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+
+! CHECK: fir.call @_FortranACAtanhF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+  complex(16) :: a, b
+  b = atanh(a)
+end
diff --git a/flang/test/Lower/Intrinsics/cos_complex16.f90 b/flang/test/Lower/Intrinsics/cos_complex16.f90
new file mode 100644
index 0000000..79b89ce
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/cos_complex16.f90
@@ -0,0 +1,8 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+
+! CHECK: fir.call @_FortranACCosF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+  complex(16) :: a, b
+  b = cos(a)
+end
diff --git a/flang/test/Lower/Intrinsics/cosh_complex16.f90 b/flang/test/Lower/Intrinsics/cosh_complex16.f90
new file mode 100644
index 0000000..dc1723b
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/cosh_complex16.f90
@@ -0,0 +1,8 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+
+! CHECK: fir.call @_FortranACCoshF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+  complex(16) :: a, b
+  b = cosh(a)
+end
diff --git a/flang/test/Lower/Intrinsics/exp_complex16.f90 b/flang/test/Lower/Intrinsics/exp_complex16.f90
new file mode 100644
index 0000000..972285c
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/exp_complex16.f90
@@ -0,0 +1,8 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+
+! CHECK: fir.call @_FortranACExpF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+  complex(16) :: a, b
+  b = exp(a)
+end
diff --git a/flang/test/Lower/Intrinsics/log_complex16.f90 b/flang/test/Lower/Intrinsics/log_complex16.f90
new file mode 100644
index 0000000..d5d0eb3
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/log_complex16.f90
@@ -0,0 +1,8 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+
+! CHECK: fir.call @_FortranACLogF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+  complex(16) :: a, b
+  b = log(a)
+end
diff --git a/flang/test/Lower/Intrinsics/missing-math-runtime.f90 b/flang/test/Lower/Intrinsics/missing-math-runtime.f90
deleted file mode 100644
index 699678f..0000000
--- a/flang/test/Lower/Intrinsics/missing-math-runtime.f90
+++ /dev/null
@@ -1,12 +0,0 @@
-! If the compiler is built without 128-bit float math
-! support, an appropriate error message is emitted.
-! UNSUPPORTED: flang-supports-f128-math
-! RUN: bbc -emit-fir %s -o /dev/null >%t 2>&1 || echo
-! RUN: FileCheck %s --input-file=%t
-
- complex(16) :: a
- real(16) :: b
-! CHECK: compiler is built without support for 'ABS(COMPLEX(KIND=16))'
- b = abs(a)
-end
-
diff --git a/flang/test/Lower/Intrinsics/pow_complex16.f90 b/flang/test/Lower/Intrinsics/pow_complex16.f90
new file mode 100644
index 0000000..db6b207
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/pow_complex16.f90
@@ -0,0 +1,8 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+
+! CHECK: fir.call @_FortranACPowF128({{.*}}){{.*}}: (!fir.complex<16>, !fir.complex<16>) -> !fir.complex<16>
+  complex(16) :: a, b
+  b = a ** b
+end
diff --git a/flang/test/Lower/Intrinsics/sin_complex16.f90 b/flang/test/Lower/Intrinsics/sin_complex16.f90
new file mode 100644
index 0000000..5114501
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/sin_complex16.f90
@@ -0,0 +1,8 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+
+! CHECK: fir.call @_FortranACSinF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+  complex(16) :: a, b
+  b = sin(a)
+end
diff --git a/flang/test/Lower/Intrinsics/sinh_complex16.f90 b/flang/test/Lower/Intrinsics/sinh_complex16.f90
new file mode 100644
index 0000000..5baf9ea
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/sinh_complex16.f90
@@ -0,0 +1,8 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+
+! CHECK: fir.call @_FortranACSinhF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+  complex(16) :: a, b
+  b = sinh(a)
+end
diff --git a/flang/test/Lower/Intrinsics/sqrt_complex16.f90 b/flang/test/Lower/Intrinsics/sqrt_complex16.f90
new file mode 100644
index 0000000..75ffa22
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/sqrt_complex16.f90
@@ -0,0 +1,8 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+
+! CHECK: fir.call @_FortranACSqrtF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+  complex(16) :: a, b
+  b = sqrt(a)
+end
diff --git a/flang/test/Lower/Intrinsics/tan_complex16.f90 b/flang/test/Lower/Intrinsics/tan_complex16.f90
new file mode 100644
index 0000000..7217145f
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/tan_complex16.f90
@@ -0,0 +1,8 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+
+! CHECK: fir.call @_FortranACTanF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+  complex(16) :: a, b
+  b = tan(a)
+end
diff --git a/flang/test/Lower/Intrinsics/tanh_complex16.f90 b/flang/test/Lower/Intrinsics/tanh_complex16.f90
new file mode 100644
index 0000000..1965094
--- /dev/null
+++ b/flang/test/Lower/Intrinsics/tanh_complex16.f90
@@ -0,0 +1,8 @@
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
+! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
+
+! CHECK: fir.call @_FortranACTanhF128({{.*}}){{.*}}: (!fir.complex<16>) -> !fir.complex<16>
+  complex(16) :: a, b
+  b = tanh(a)
+end