diff options
Diffstat (limited to 'flang/lib')
| -rw-r--r-- | flang/lib/Lower/Bridge.cpp | 128 | ||||
| -rw-r--r-- | flang/lib/Lower/ConvertCall.cpp | 13 | ||||
| -rw-r--r-- | flang/lib/Lower/Support/PrivateReductionUtils.cpp | 35 | ||||
| -rw-r--r-- | flang/lib/Optimizer/CodeGen/CodeGen.cpp | 12 | ||||
| -rw-r--r-- | flang/lib/Optimizer/Dialect/MIF/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | flang/lib/Optimizer/Support/CMakeLists.txt | 2 | ||||
| -rw-r--r-- | flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp | 3 | ||||
| -rw-r--r-- | flang/lib/Parser/Fortran-parsers.cpp | 8 | ||||
| -rw-r--r-- | flang/lib/Parser/unparse.cpp | 7 | ||||
| -rw-r--r-- | flang/lib/Semantics/canonicalize-directives.cpp | 6 | ||||
| -rw-r--r-- | flang/lib/Semantics/check-omp-structure.cpp | 2 | ||||
| -rw-r--r-- | flang/lib/Semantics/resolve-names.cpp | 13 |
12 files changed, 201 insertions, 29 deletions
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp index a516a44..6e72987 100644 --- a/flang/lib/Lower/Bridge.cpp +++ b/flang/lib/Lower/Bridge.cpp @@ -1884,6 +1884,26 @@ private: setCurrentPosition(stmt.source); assert(stmt.typedCall && "Call was not analyzed"); mlir::Value res{}; + + // Set 'no_inline', 'inline_hint' or 'always_inline' to true on the + // ProcedureRef. The NoInline and AlwaysInline attribute will be set in + // genProcedureRef later. + for (const auto *dir : eval.dirs) { + Fortran::common::visit( + Fortran::common::visitors{ + [&](const Fortran::parser::CompilerDirective::ForceInline &) { + stmt.typedCall->setAlwaysInline(true); + }, + [&](const Fortran::parser::CompilerDirective::Inline &) { + stmt.typedCall->setInlineHint(true); + }, + [&](const Fortran::parser::CompilerDirective::NoInline &) { + stmt.typedCall->setNoInline(true); + }, + [&](const auto &) {}}, + dir->u); + } + if (lowerToHighLevelFIR()) { std::optional<mlir::Type> resultType; if (stmt.typedCall->hasAlternateReturns()) @@ -2200,6 +2220,50 @@ private: // so no clean-up needs to be generated for these entities. } + void attachInlineAttributes( + mlir::Operation &op, + const llvm::ArrayRef<const Fortran::parser::CompilerDirective *> &dirs) { + if (dirs.empty()) + return; + + for (mlir::Value operand : op.getOperands()) { + if (operand.getDefiningOp()) + attachInlineAttributes(*operand.getDefiningOp(), dirs); + } + + if (fir::CallOp callOp = mlir::dyn_cast<fir::CallOp>(op)) { + for (const auto *dir : dirs) { + Fortran::common::visit( + Fortran::common::visitors{ + [&](const Fortran::parser::CompilerDirective::NoInline &) { + callOp.setInlineAttr(fir::FortranInlineEnum::no_inline); + }, + [&](const Fortran::parser::CompilerDirective::Inline &) { + callOp.setInlineAttr(fir::FortranInlineEnum::inline_hint); + }, + [&](const Fortran::parser::CompilerDirective::ForceInline &) { + callOp.setInlineAttr(fir::FortranInlineEnum::always_inline); + }, + [&](const auto &) {}}, + dir->u); + } + } + } + + void attachAttributesToDoLoopOperations( + fir::DoLoopOp &doLoop, + llvm::SmallVectorImpl<const Fortran::parser::CompilerDirective *> &dirs) { + if (!doLoop.getOperation() || dirs.empty()) + return; + + for (mlir::Block &block : doLoop.getRegion()) { + for (mlir::Operation &op : block.getOperations()) { + if (!dirs.empty()) + attachInlineAttributes(op, dirs); + } + } + } + /// Generate FIR for a DO construct. There are six variants: /// - unstructured infinite and while loops /// - structured and unstructured increment loops @@ -2351,6 +2415,11 @@ private: if (!incrementLoopNestInfo.empty() && incrementLoopNestInfo.back().isConcurrent) localSymbols.popScope(); + + // Add attribute(s) on operations in fir::DoLoopOp if necessary + for (IncrementLoopInfo &info : incrementLoopNestInfo) + if (auto loopOp = mlir::dyn_cast_if_present<fir::DoLoopOp>(info.loopOp)) + attachAttributesToDoLoopOperations(loopOp, doStmtEval.dirs); } /// Generate FIR to evaluate loop control values (lower, upper and step). @@ -3154,6 +3223,26 @@ private: e->dirs.push_back(&dir); } + void + attachInliningDirectiveToStmt(const Fortran::parser::CompilerDirective &dir, + Fortran::lower::pft::Evaluation *e) { + while (e->isDirective()) + e = e->lexicalSuccessor; + + // If the successor is a statement or a do loop, the compiler + // will perform inlining. + if (e->isA<Fortran::parser::CallStmt>() || + e->isA<Fortran::parser::NonLabelDoStmt>() || + e->isA<Fortran::parser::AssignmentStmt>()) { + e->dirs.push_back(&dir); + } else { + mlir::Location loc = toLocation(); + mlir::emitWarning(loc, + "Inlining directive not in front of loops, function" + "call or assignment.\n"); + } + } + void genFIR(const Fortran::parser::CompilerDirective &dir) { Fortran::lower::pft::Evaluation &eval = getEval(); @@ -3177,6 +3266,15 @@ private: [&](const Fortran::parser::CompilerDirective::NoUnrollAndJam &) { attachDirectiveToLoop(dir, &eval); }, + [&](const Fortran::parser::CompilerDirective::ForceInline &) { + attachInliningDirectiveToStmt(dir, &eval); + }, + [&](const Fortran::parser::CompilerDirective::Inline &) { + attachInliningDirectiveToStmt(dir, &eval); + }, + [&](const Fortran::parser::CompilerDirective::NoInline &) { + attachInliningDirectiveToStmt(dir, &eval); + }, [&](const auto &) {}}, dir.u); } @@ -5086,7 +5184,9 @@ private: void genDataAssignment( const Fortran::evaluate::Assignment &assign, - const Fortran::evaluate::ProcedureRef *userDefinedAssignment) { + const Fortran::evaluate::ProcedureRef *userDefinedAssignment, + const llvm::ArrayRef<const Fortran::parser::CompilerDirective *> &dirs = + {}) { mlir::Location loc = getCurrentLocation(); fir::FirOpBuilder &builder = getFirOpBuilder(); @@ -5166,10 +5266,20 @@ private: genCUDADataTransfer(builder, loc, assign, lhs, rhs, isWholeAllocatableAssignment, keepLhsLengthInAllocatableAssignment); - else + else { + // If RHS or LHS have a CallOp in their expression, this operation will + // have the 'no_inline' or 'always_inline' attribute if there is a + // directive just before the assignement. + if (!dirs.empty()) { + if (rhs.getDefiningOp()) + attachInlineAttributes(*rhs.getDefiningOp(), dirs); + if (lhs.getDefiningOp()) + attachInlineAttributes(*lhs.getDefiningOp(), dirs); + } hlfir::AssignOp::create(builder, loc, rhs, lhs, isWholeAllocatableAssignment, keepLhsLengthInAllocatableAssignment); + } if (hasCUDAImplicitTransfer && !isInDeviceContext) { localSymbols.popScope(); for (mlir::Value temp : implicitTemps) @@ -5237,16 +5347,21 @@ private: } /// Shared for both assignments and pointer assignments. - void genAssignment(const Fortran::evaluate::Assignment &assign) { + void + genAssignment(const Fortran::evaluate::Assignment &assign, + const llvm::ArrayRef<const Fortran::parser::CompilerDirective *> + &dirs = {}) { mlir::Location loc = toLocation(); if (lowerToHighLevelFIR()) { Fortran::common::visit( Fortran::common::visitors{ [&](const Fortran::evaluate::Assignment::Intrinsic &) { - genDataAssignment(assign, /*userDefinedAssignment=*/nullptr); + genDataAssignment(assign, /*userDefinedAssignment=*/nullptr, + dirs); }, [&](const Fortran::evaluate::ProcedureRef &procRef) { - genDataAssignment(assign, /*userDefinedAssignment=*/&procRef); + genDataAssignment(assign, /*userDefinedAssignment=*/&procRef, + dirs); }, [&](const Fortran::evaluate::Assignment::BoundsSpec &lbExprs) { if (isInsideHlfirForallOrWhere()) @@ -5651,7 +5766,8 @@ private: } void genFIR(const Fortran::parser::AssignmentStmt &stmt) { - genAssignment(*stmt.typedAssignment->v); + Fortran::lower::pft::Evaluation &eval = getEval(); + genAssignment(*stmt.typedAssignment->v, eval.dirs); } void genFIR(const Fortran::parser::SyncAllStmt &stmt) { diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp index fb72040..9bf994e 100644 --- a/flang/lib/Lower/ConvertCall.cpp +++ b/flang/lib/Lower/ConvertCall.cpp @@ -700,9 +700,20 @@ Fortran::lower::genCallOpAndResult( callResult = dispatch.getResult(0); } else { // Standard procedure call with fir.call. + fir::FortranInlineEnumAttr inlineAttr; + + if (caller.getCallDescription().hasNoInline()) + inlineAttr = fir::FortranInlineEnumAttr::get( + builder.getContext(), fir::FortranInlineEnum::no_inline); + else if (caller.getCallDescription().hasInlineHint()) + inlineAttr = fir::FortranInlineEnumAttr::get( + builder.getContext(), fir::FortranInlineEnum::inline_hint); + else if (caller.getCallDescription().hasAlwaysInline()) + inlineAttr = fir::FortranInlineEnumAttr::get( + builder.getContext(), fir::FortranInlineEnum::always_inline); auto call = fir::CallOp::create( builder, loc, funcType.getResults(), funcSymbolAttr, operands, - /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr, procAttrs); + /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr, procAttrs, inlineAttr); callNumResults = call.getNumResults(); if (callNumResults != 0) diff --git a/flang/lib/Lower/Support/PrivateReductionUtils.cpp b/flang/lib/Lower/Support/PrivateReductionUtils.cpp index d433ce3..c6c4288 100644 --- a/flang/lib/Lower/Support/PrivateReductionUtils.cpp +++ b/flang/lib/Lower/Support/PrivateReductionUtils.cpp @@ -376,6 +376,8 @@ private: loadedMoldArg = builder.loadIfRef(loc, moldArg); return loadedMoldArg; } + + bool shouldAllocateTempOnStack() const; }; } // namespace @@ -438,8 +440,14 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedScalar( builder.setInsertionPointToStart(&ifUnallocated.getElseRegion().front()); } - mlir::Value valAlloc = builder.createHeapTemporary(loc, innerTy, /*name=*/{}, - /*shape=*/{}, lenParams); + bool shouldAllocateOnStack = shouldAllocateTempOnStack(); + mlir::Value valAlloc = + (shouldAllocateOnStack) + ? builder.createTemporary(loc, innerTy, /*name=*/{}, + /*shape=*/{}, lenParams) + : builder.createHeapTemporary(loc, innerTy, /*name=*/{}, + /*shape=*/{}, lenParams); + if (scalarInitValue) builder.createStoreWithConvert(loc, scalarInitValue, valAlloc); mlir::Value box = fir::EmboxOp::create(builder, loc, valType, valAlloc, @@ -451,8 +459,9 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedScalar( fir::StoreOp lastOp = fir::StoreOp::create(builder, loc, box, allocatedPrivVarArg); - createCleanupRegion(converter, loc, argType, cleanupRegion, sym, - isDoConcurrent); + if (!shouldAllocateOnStack) + createCleanupRegion(converter, loc, argType, cleanupRegion, sym, + isDoConcurrent); if (ifUnallocated) builder.setInsertionPointAfter(ifUnallocated); @@ -462,6 +471,14 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedScalar( createYield(allocatedPrivVarArg); } +bool PopulateInitAndCleanupRegionsHelper::shouldAllocateTempOnStack() const { + // On the GPU, always allocate on the stack since heap allocatins are very + // expensive. + auto offloadMod = + llvm::dyn_cast<mlir::omp::OffloadModuleInterface>(*builder.getModule()); + return offloadMod && offloadMod.getIsGPU(); +} + void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedArray( fir::BaseBoxType boxTy, bool needsInitialization) { bool isAllocatableOrPointer = @@ -504,15 +521,7 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedArray( // Allocating on the heap in case the whole reduction/privatization is nested // inside of a loop auto temp = [&]() { - bool shouldAllocateOnStack = false; - - // On the GPU, always allocate on the stack since heap allocatins are very - // expensive. - if (auto offloadMod = llvm::dyn_cast<mlir::omp::OffloadModuleInterface>( - *builder.getModule())) - shouldAllocateOnStack = offloadMod.getIsGPU(); - - if (shouldAllocateOnStack) + if (shouldAllocateTempOnStack()) return createStackTempFromMold(loc, builder, source); auto [temp, needsDealloc] = createTempFromMold(loc, builder, source); diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp index 478ab15..ca4aefb 100644 --- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp +++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp @@ -680,6 +680,18 @@ struct CallOpConversion : public fir::FIROpConversion<fir::CallOp> { if (mlir::ArrayAttr resAttrs = call.getResAttrsAttr()) llvmCall.setResAttrsAttr(resAttrs); + if (auto inlineAttr = call.getInlineAttrAttr()) { + llvmCall->removeAttr("inline_attr"); + if (inlineAttr.getValue() == fir::FortranInlineEnum::no_inline) { + llvmCall.setNoInlineAttr(rewriter.getUnitAttr()); + } else if (inlineAttr.getValue() == fir::FortranInlineEnum::inline_hint) { + llvmCall.setInlineHintAttr(rewriter.getUnitAttr()); + } else if (inlineAttr.getValue() == + fir::FortranInlineEnum::always_inline) { + llvmCall.setAlwaysInlineAttr(rewriter.getUnitAttr()); + } + } + if (memAttr) llvmCall.setMemoryEffectsAttr( mlir::cast<mlir::LLVM::MemoryEffectsAttr>(memAttr)); diff --git a/flang/lib/Optimizer/Dialect/MIF/CMakeLists.txt b/flang/lib/Optimizer/Dialect/MIF/CMakeLists.txt index ed8463e..d53937eb 100644 --- a/flang/lib/Optimizer/Dialect/MIF/CMakeLists.txt +++ b/flang/lib/Optimizer/Dialect/MIF/CMakeLists.txt @@ -8,7 +8,6 @@ add_flang_library(MIFDialect LINK_LIBS FIRDialect FIRDialectSupport - FIRSupport LINK_COMPONENTS AsmParser diff --git a/flang/lib/Optimizer/Support/CMakeLists.txt b/flang/lib/Optimizer/Support/CMakeLists.txt index 38038e1..6f3652b 100644 --- a/flang/lib/Optimizer/Support/CMakeLists.txt +++ b/flang/lib/Optimizer/Support/CMakeLists.txt @@ -7,9 +7,11 @@ add_flang_library(FIRSupport DEPENDS FIROpsIncGen HLFIROpsIncGen + MIFOpsIncGen LINK_LIBS FIRDialect + MIFDialect LINK_COMPONENTS TargetParser diff --git a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp index 25a8f7a..8c0acc5 100644 --- a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp +++ b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp @@ -246,7 +246,8 @@ struct DispatchOpConv : public OpConversionPattern<fir::DispatchOp> { args.append(dispatch.getArgs().begin(), dispatch.getArgs().end()); rewriter.replaceOpWithNewOp<fir::CallOp>( dispatch, resTypes, nullptr, args, dispatch.getArgAttrsAttr(), - dispatch.getResAttrsAttr(), dispatch.getProcedureAttrsAttr()); + dispatch.getResAttrsAttr(), dispatch.getProcedureAttrsAttr(), + /*inline_attr*/ fir::FortranInlineEnumAttr{}); return mlir::success(); } diff --git a/flang/lib/Parser/Fortran-parsers.cpp b/flang/lib/Parser/Fortran-parsers.cpp index d33a18f..59fe7d8 100644 --- a/flang/lib/Parser/Fortran-parsers.cpp +++ b/flang/lib/Parser/Fortran-parsers.cpp @@ -1314,6 +1314,11 @@ constexpr auto novector{"NOVECTOR" >> construct<CompilerDirective::NoVector>()}; constexpr auto nounroll{"NOUNROLL" >> construct<CompilerDirective::NoUnroll>()}; constexpr auto nounrollAndJam{ "NOUNROLL_AND_JAM" >> construct<CompilerDirective::NoUnrollAndJam>()}; +constexpr auto forceinlineDir{ + "FORCEINLINE" >> construct<CompilerDirective::ForceInline>()}; +constexpr auto noinlineDir{ + "NOINLINE" >> construct<CompilerDirective::NoInline>()}; +constexpr auto inlineDir{"INLINE" >> construct<CompilerDirective::Inline>()}; TYPE_PARSER(beginDirective >> "DIR$ "_tok >> sourced((construct<CompilerDirective>(ignore_tkr) || construct<CompilerDirective>(loopCount) || @@ -1324,6 +1329,9 @@ TYPE_PARSER(beginDirective >> "DIR$ "_tok >> construct<CompilerDirective>(novector) || construct<CompilerDirective>(nounrollAndJam) || construct<CompilerDirective>(nounroll) || + construct<CompilerDirective>(noinlineDir) || + construct<CompilerDirective>(forceinlineDir) || + construct<CompilerDirective>(inlineDir) || construct<CompilerDirective>( many(construct<CompilerDirective::NameValue>( name, maybe(("="_tok || ":"_tok) >> digitString64))))) / diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp index 2f86c76..20a8d2a 100644 --- a/flang/lib/Parser/unparse.cpp +++ b/flang/lib/Parser/unparse.cpp @@ -1867,6 +1867,13 @@ public: [&](const CompilerDirective::NoUnrollAndJam &) { Word("!DIR$ NOUNROLL_AND_JAM"); }, + [&](const CompilerDirective::ForceInline &) { + Word("!DIR$ FORCEINLINE"); + }, + [&](const CompilerDirective::Inline &) { Word("!DIR$ INLINE"); }, + [&](const CompilerDirective::NoInline &) { + Word("!DIR$ NOINLINE"); + }, [&](const CompilerDirective::Unrecognized &) { Word("!DIR$ "); Word(x.source.ToString()); diff --git a/flang/lib/Semantics/canonicalize-directives.cpp b/flang/lib/Semantics/canonicalize-directives.cpp index 104df25..a651a87 100644 --- a/flang/lib/Semantics/canonicalize-directives.cpp +++ b/flang/lib/Semantics/canonicalize-directives.cpp @@ -60,7 +60,11 @@ static bool IsExecutionDirective(const parser::CompilerDirective &dir) { std::holds_alternative<parser::CompilerDirective::UnrollAndJam>(dir.u) || std::holds_alternative<parser::CompilerDirective::NoVector>(dir.u) || std::holds_alternative<parser::CompilerDirective::NoUnroll>(dir.u) || - std::holds_alternative<parser::CompilerDirective::NoUnrollAndJam>(dir.u); + std::holds_alternative<parser::CompilerDirective::NoUnrollAndJam>( + dir.u) || + std::holds_alternative<parser::CompilerDirective::ForceInline>(dir.u) || + std::holds_alternative<parser::CompilerDirective::Inline>(dir.u) || + std::holds_alternative<parser::CompilerDirective::NoInline>(dir.u); } void CanonicalizationOfDirectives::Post(parser::SpecificationPart &spec) { diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp index 4141630..e094458f 100644 --- a/flang/lib/Semantics/check-omp-structure.cpp +++ b/flang/lib/Semantics/check-omp-structure.cpp @@ -98,7 +98,7 @@ bool OmpStructureChecker::Enter(const parser::BlockData &x) { } else { for (const Scope &scope : context_.globalScope().children()) { if (scope.kind() == Scope::Kind::BlockData) { - if (scope.symbol()->name().empty()) { + if (auto *s{scope.symbol()}; !s || s->name().empty()) { scopeStack_.push_back(&scope); break; } diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp index 4af6cf6..93faba7 100644 --- a/flang/lib/Semantics/resolve-names.cpp +++ b/flang/lib/Semantics/resolve-names.cpp @@ -9194,11 +9194,11 @@ bool DeclarationVisitor::CheckNonPointerInitialization( "'%s' has already been initialized"_err_en_US); } else if (IsAllocatable(ultimate)) { Say(name, "Allocatable object '%s' cannot be initialized"_err_en_US); + } else if (details->isCDefined()) { + // CDEFINED variables cannot have initializer, because their storage + // may come outside of Fortran. + Say(name, "CDEFINED variable cannot be initialized"_err_en_US); } else { - if (details->isCDefined()) { - context().Warn(common::UsageWarning::CdefinedInit, name.source, - "CDEFINED variable should not have an initializer"_warn_en_US); - } return true; } } else { @@ -10078,7 +10078,10 @@ void ResolveNamesVisitor::Post(const parser::CompilerDirective &x) { std::holds_alternative<parser::CompilerDirective::UnrollAndJam>(x.u) || std::holds_alternative<parser::CompilerDirective::NoVector>(x.u) || std::holds_alternative<parser::CompilerDirective::NoUnroll>(x.u) || - std::holds_alternative<parser::CompilerDirective::NoUnrollAndJam>(x.u)) { + std::holds_alternative<parser::CompilerDirective::NoUnrollAndJam>(x.u) || + std::holds_alternative<parser::CompilerDirective::ForceInline>(x.u) || + std::holds_alternative<parser::CompilerDirective::Inline>(x.u) || + std::holds_alternative<parser::CompilerDirective::NoInline>(x.u)) { return; } if (const auto *tkr{ |
