diff options
Diffstat (limited to 'llvm/lib/CodeGen')
57 files changed, 754 insertions, 629 deletions
diff --git a/llvm/lib/CodeGen/AsmPrinter/AIXException.cpp b/llvm/lib/CodeGen/AsmPrinter/AIXException.cpp index 6356d71..873ac8f 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AIXException.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AIXException.cpp @@ -20,7 +20,7 @@ #include "llvm/Target/TargetLoweringObjectFile.h" #include "llvm/Target/TargetMachine.h" -namespace llvm { +using namespace llvm; AIXException::AIXException(AsmPrinter *A) : EHStreamer(A) {} @@ -90,5 +90,3 @@ void AIXException::endFunction(const MachineFunction *MF) { emitExceptionInfoTable(LSDALabel, PerSym); } - -} // End of namespace llvm diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 10df9c1..e2af0c5 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -20,6 +20,7 @@ #include "WinException.h" #include "llvm/ADT/APFloat.h" #include "llvm/ADT/APInt.h" +#include "llvm/ADT/BitmaskEnum.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" @@ -118,6 +119,7 @@ #include "llvm/Support/MathExtras.h" #include "llvm/Support/Path.h" #include "llvm/Support/VCSRevision.h" +#include "llvm/Support/VirtualFileSystem.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetLoweringObjectFile.h" #include "llvm/Target/TargetMachine.h" @@ -205,6 +207,17 @@ public: }; } // namespace +namespace callgraph { +LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(); +enum Flags : uint8_t { + None = 0, + IsIndirectTarget = 1u << 0, + HasDirectCallees = 1u << 1, + HasIndirectCallees = 1u << 2, + LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue*/ HasIndirectCallees) +}; +} // namespace callgraph + class llvm::AddrLabelMap { MCContext &Context; struct AddrLabelSymEntry { @@ -464,6 +477,7 @@ void AsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const { } bool AsmPrinter::doInitialization(Module &M) { + VFS = vfs::getRealFileSystem(); auto *MMIWP = getAnalysisIfAvailable<MachineModuleInfoWrapperPass>(); MMI = MMIWP ? &MMIWP->getMMI() : nullptr; HasSplitStack = false; @@ -1425,7 +1439,8 @@ getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges, BrProbEnabled, MF.hasBBSections() && NumMBBSectionRanges > 1, static_cast<bool>(BBAddrMapSkipEmitBBEntries), - HasCalls}; + HasCalls, + false}; } void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) { @@ -1670,7 +1685,7 @@ static ConstantInt *extractNumericCGTypeId(const Function &F) { return nullptr; } -/// Emits .callgraph section. +/// Emits .llvm.callgraph section. void AsmPrinter::emitCallGraphSection(const MachineFunction &MF, FunctionCallGraphInfo &FuncCGInfo) { if (!MF.getTarget().Options.EmitCallGraphSection) @@ -1683,63 +1698,65 @@ void AsmPrinter::emitCallGraphSection(const MachineFunction &MF, OutStreamer->pushSection(); OutStreamer->switchSection(FuncCGSection); - // Emit format version number. - OutStreamer->emitInt64(CallGraphSectionFormatVersion::V_0); - - // Emit function's self information, which is composed of: - // 1) FunctionEntryPc - // 2) FunctionKind: Whether the function is indirect target, and if so, - // whether its type id is known. - // 3) FunctionTypeId: Emit only when the function is an indirect target - // and its type id is known. - - // Emit function entry pc. const MCSymbol *FunctionSymbol = getFunctionBegin(); - OutStreamer->emitSymbolValue(FunctionSymbol, TM.getProgramPointerSize()); - + const Function &F = MF.getFunction(); // If this function has external linkage or has its address taken and // it is not a callback, then anything could call it. - const Function &F = MF.getFunction(); bool IsIndirectTarget = !F.hasLocalLinkage() || F.hasAddressTaken(nullptr, /*IgnoreCallbackUses=*/true, /*IgnoreAssumeLikeCalls=*/true, /*IgnoreLLVMUsed=*/false); - // FIXME: FunctionKind takes a few values but emitted as a 64-bit value. - // Can be optimized to occupy 2 bits instead. - // Emit function kind, and type id if available. - if (!IsIndirectTarget) { - OutStreamer->emitInt64( - static_cast<uint64_t>(FunctionKind::NOT_INDIRECT_TARGET)); - } else { - if (const auto *TypeId = extractNumericCGTypeId(F)) { - OutStreamer->emitInt64( - static_cast<uint64_t>(FunctionKind::INDIRECT_TARGET_KNOWN_TID)); - OutStreamer->emitInt64(TypeId->getZExtValue()); - } else { - OutStreamer->emitInt64( - static_cast<uint64_t>(FunctionKind::INDIRECT_TARGET_UNKNOWN_TID)); - } - } + const auto &DirectCallees = FuncCGInfo.DirectCallees; + const auto &IndirectCalleeTypeIDs = FuncCGInfo.IndirectCalleeTypeIDs; + + using namespace callgraph; + Flags CGFlags = Flags::None; + if (IsIndirectTarget) + CGFlags |= Flags::IsIndirectTarget; + if (DirectCallees.size() > 0) + CGFlags |= Flags::HasDirectCallees; + if (IndirectCalleeTypeIDs.size() > 0) + CGFlags |= Flags::HasIndirectCallees; + + // Emit function's call graph information. + // 1) CallGraphSectionFormatVersion + // 2) Flags + // a. LSB bit 0 is set to 1 if the function is a potential indirect + // target. + // b. LSB bit 1 is set to 1 if there are direct callees. + // c. LSB bit 2 is set to 1 if there are indirect callees. + // d. Rest of the 5 bits in Flags are reserved for any future use. + // 3) Function entry PC. + // 4) FunctionTypeID if the function is indirect target and its type id + // is known, otherwise it is set to 0. + // 5) Number of unique direct callees, if at least one exists. + // 6) For each unique direct callee, the callee's PC. + // 7) Number of unique indirect target type IDs, if at least one exists. + // 8) Each unique indirect target type id. + OutStreamer->emitInt8(CallGraphSectionFormatVersion::V_0); + OutStreamer->emitInt8(static_cast<uint8_t>(CGFlags)); + OutStreamer->emitSymbolValue(FunctionSymbol, TM.getProgramPointerSize()); + const auto *TypeId = extractNumericCGTypeId(F); + if (IsIndirectTarget && TypeId) + OutStreamer->emitInt64(TypeId->getZExtValue()); + else + OutStreamer->emitInt64(0); - // Emit callsite labels, where each element is a pair of type id and - // indirect callsite pc. - const auto &CallSiteLabels = FuncCGInfo.CallSiteLabels; - OutStreamer->emitInt64(CallSiteLabels.size()); - for (const auto &[TypeId, Label] : CallSiteLabels) { - OutStreamer->emitInt64(TypeId); - OutStreamer->emitSymbolValue(Label, TM.getProgramPointerSize()); + if (DirectCallees.size() > 0) { + OutStreamer->emitULEB128IntValue(DirectCallees.size()); + for (const auto &CalleeSymbol : DirectCallees) + OutStreamer->emitSymbolValue(CalleeSymbol, TM.getProgramPointerSize()); + FuncCGInfo.DirectCallees.clear(); } - FuncCGInfo.CallSiteLabels.clear(); - - const auto &DirectCallees = FuncCGInfo.DirectCallees; - OutStreamer->emitInt64(DirectCallees.size()); - for (const auto &CalleeSymbol : DirectCallees) { - OutStreamer->emitSymbolValue(CalleeSymbol, TM.getProgramPointerSize()); + if (IndirectCalleeTypeIDs.size() > 0) { + OutStreamer->emitULEB128IntValue(IndirectCalleeTypeIDs.size()); + for (const auto &CalleeTypeId : IndirectCalleeTypeIDs) + OutStreamer->emitInt64(CalleeTypeId); + FuncCGInfo.IndirectCalleeTypeIDs.clear(); } - FuncCGInfo.DirectCallees.clear(); - + // End of emitting call graph section contents. OutStreamer->popSection(); } @@ -1877,8 +1894,7 @@ void AsmPrinter::handleCallsiteForCallgraph( FunctionCallGraphInfo &FuncCGInfo, const MachineFunction::CallSiteInfoMap &CallSitesInfoMap, const MachineInstr &MI) { - assert(MI.isCall() && - "Callsite labels are meant for call instructions only."); + assert(MI.isCall() && "This method is meant for call instructions only."); const MachineOperand &CalleeOperand = MI.getOperand(0); if (CalleeOperand.isGlobal() || CalleeOperand.isSymbol()) { // Handle direct calls. @@ -1903,10 +1919,8 @@ void AsmPrinter::handleCallsiteForCallgraph( // Handle indirect callsite info. // Only indirect calls have type identifiers set. for (ConstantInt *CalleeTypeId : CallSiteInfo->second.CalleeTypeIds) { - MCSymbol *S = MF->getContext().createTempSymbol(); - OutStreamer->emitLabel(S); uint64_t CalleeTypeIdVal = CalleeTypeId->getZExtValue(); - FuncCGInfo.CallSiteLabels.emplace_back(CalleeTypeIdVal, S); + FuncCGInfo.IndirectCalleeTypeIDs.insert(CalleeTypeIdVal); } } diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp index c364ffc..8dd8b9da 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp @@ -36,6 +36,7 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/SourceMgr.h" +#include "llvm/Support/VirtualFileSystem.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" using namespace llvm; @@ -98,6 +99,7 @@ void AsmPrinter::emitInlineAsm(StringRef Str, const MCSubtargetInfo &STI, unsigned BufNum = addInlineAsmDiagBuffer(Str, LocMDNode); SourceMgr &SrcMgr = *MMI->getContext().getInlineSourceManager(); SrcMgr.setIncludeDirs(MCOptions.IASSearchPaths); + SrcMgr.setVirtualFileSystem(VFS); std::unique_ptr<MCAsmParser> Parser( createMCAsmParser(SrcMgr, OutContext, *OutStreamer, *MAI, BufNum)); diff --git a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp index 12d749c..e57ed24 100644 --- a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp @@ -569,40 +569,30 @@ void CodeViewDebug::emitCodeViewMagicVersion() { OS.emitInt32(COFF::DEBUG_SECTION_MAGIC); } -static SourceLanguage MapDWLangToCVLang(unsigned DWLang) { - switch (DWLang) { - case dwarf::DW_LANG_C: - case dwarf::DW_LANG_C89: - case dwarf::DW_LANG_C99: - case dwarf::DW_LANG_C11: +static SourceLanguage +MapDWARFLanguageToCVLang(dwarf::SourceLanguageName DWLName) { + switch (DWLName) { + case dwarf::DW_LNAME_C: return SourceLanguage::C; - case dwarf::DW_LANG_C_plus_plus: - case dwarf::DW_LANG_C_plus_plus_03: - case dwarf::DW_LANG_C_plus_plus_11: - case dwarf::DW_LANG_C_plus_plus_14: + case dwarf::DW_LNAME_C_plus_plus: return SourceLanguage::Cpp; - case dwarf::DW_LANG_Fortran77: - case dwarf::DW_LANG_Fortran90: - case dwarf::DW_LANG_Fortran95: - case dwarf::DW_LANG_Fortran03: - case dwarf::DW_LANG_Fortran08: + case dwarf::DW_LNAME_Fortran: return SourceLanguage::Fortran; - case dwarf::DW_LANG_Pascal83: + case dwarf::DW_LNAME_Pascal: return SourceLanguage::Pascal; - case dwarf::DW_LANG_Cobol74: - case dwarf::DW_LANG_Cobol85: + case dwarf::DW_LNAME_Cobol: return SourceLanguage::Cobol; - case dwarf::DW_LANG_Java: + case dwarf::DW_LNAME_Java: return SourceLanguage::Java; - case dwarf::DW_LANG_D: + case dwarf::DW_LNAME_D: return SourceLanguage::D; - case dwarf::DW_LANG_Swift: + case dwarf::DW_LNAME_Swift: return SourceLanguage::Swift; - case dwarf::DW_LANG_Rust: + case dwarf::DW_LNAME_Rust: return SourceLanguage::Rust; - case dwarf::DW_LANG_ObjC: + case dwarf::DW_LNAME_ObjC: return SourceLanguage::ObjC; - case dwarf::DW_LANG_ObjC_plus_plus: + case dwarf::DW_LNAME_ObjC_plus_plus: return SourceLanguage::ObjCpp; default: // There's no CodeView representation for this language, and CV doesn't @@ -612,6 +602,14 @@ static SourceLanguage MapDWLangToCVLang(unsigned DWLang) { } } +static SourceLanguage MapDWARFLanguageToCVLang(dwarf::SourceLanguage DWLang) { + auto MaybeLName = dwarf::toDW_LNAME(DWLang); + if (!MaybeLName) + return MapDWARFLanguageToCVLang(static_cast<dwarf::SourceLanguageName>(0)); + + return MapDWARFLanguageToCVLang(MaybeLName->first); +} + void CodeViewDebug::beginModule(Module *M) { // If COFF debug section is not available, skip any debug info related stuff. if (!Asm->getObjFileLowering().getCOFFDebugSymbolsSection()) { @@ -633,8 +631,13 @@ void CodeViewDebug::beginModule(Module *M) { Node = *CUs->operands().begin(); } const auto *CU = cast<DICompileUnit>(Node); + DISourceLanguageName Lang = CU->getSourceLanguage(); CurrentSourceLanguage = - MapDWLangToCVLang(CU->getSourceLanguage().getUnversionedName()); + Lang.hasVersionedName() + ? MapDWARFLanguageToCVLang( + static_cast<dwarf::SourceLanguageName>(Lang.getName())) + : MapDWARFLanguageToCVLang( + static_cast<dwarf::SourceLanguage>(Lang.getName())); if (!M->getCodeViewFlag() || CU->getEmissionKind() == DICompileUnit::NoDebug) { Asm = nullptr; diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp index d751a7f..567acf7 100644 --- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp @@ -1039,8 +1039,17 @@ void DwarfDebug::finishUnitAttributes(const DICompileUnit *DIUnit, } else NewCU.addString(Die, dwarf::DW_AT_producer, Producer); - NewCU.addUInt(Die, dwarf::DW_AT_language, dwarf::DW_FORM_data2, - DIUnit->getSourceLanguage().getUnversionedName()); + if (auto Lang = DIUnit->getSourceLanguage(); Lang.hasVersionedName()) { + NewCU.addUInt(Die, dwarf::DW_AT_language_name, dwarf::DW_FORM_data2, + Lang.getName()); + + if (uint32_t LangVersion = Lang.getVersion(); LangVersion != 0) + NewCU.addUInt(Die, dwarf::DW_AT_language_version, /*Form=*/std::nullopt, + LangVersion); + } else { + NewCU.addUInt(Die, dwarf::DW_AT_language, dwarf::DW_FORM_data2, + Lang.getName()); + } NewCU.addString(Die, dwarf::DW_AT_name, FN); StringRef SysRoot = DIUnit->getSysRoot(); @@ -2062,11 +2071,36 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) { if (NoDebug) return; + auto RecordLineZero = [&]() { + // Preserve the file and column numbers, if we can, to save space in + // the encoded line table. + // Do not update PrevInstLoc, it remembers the last non-0 line. + const MDNode *Scope = nullptr; + unsigned Column = 0; + if (PrevInstLoc) { + Scope = PrevInstLoc.getScope(); + Column = PrevInstLoc.getCol(); + } + recordSourceLine(/*Line=*/0, Column, Scope, /*Flags=*/0); + }; + + // When we emit a line-0 record, we don't update PrevInstLoc; so look at + // the last line number actually emitted, to see if it was line 0. + unsigned LastAsmLine = + Asm->OutStreamer->getContext().getCurrentDwarfLoc().getLine(); + // Check if source location changes, but ignore DBG_VALUE and CFI locations. // If the instruction is part of the function frame setup code, do not emit // any line record, as there is no correspondence with any user code. - if (MI->isMetaInstruction() || MI->getFlag(MachineInstr::FrameSetup)) + if (MI->isMetaInstruction()) return; + if (MI->getFlag(MachineInstr::FrameSetup)) { + // Prevent a loc from the previous block leaking into frame setup instrs. + if (LastAsmLine && PrevInstBB && PrevInstBB != MI->getParent()) + RecordLineZero(); + return; + } + const DebugLoc &DL = MI->getDebugLoc(); unsigned Flags = 0; @@ -2089,11 +2123,6 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) { LocationString); }; - // When we emit a line-0 record, we don't update PrevInstLoc; so look at - // the last line number actually emitted, to see if it was line 0. - unsigned LastAsmLine = - Asm->OutStreamer->getContext().getCurrentDwarfLoc().getLine(); - // There may be a mixture of scopes using and not using Key Instructions. // Not-Key-Instructions functions inlined into Key Instructions functions // should use not-key is_stmt handling. Key Instructions functions inlined @@ -2159,18 +2188,8 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) { // - Instruction is at the top of a block; we don't want to inherit the // location from the physically previous (maybe unrelated) block. if (UnknownLocations == Enable || PrevLabel || - (PrevInstBB && PrevInstBB != MI->getParent())) { - // Preserve the file and column numbers, if we can, to save space in - // the encoded line table. - // Do not update PrevInstLoc, it remembers the last non-0 line. - const MDNode *Scope = nullptr; - unsigned Column = 0; - if (PrevInstLoc) { - Scope = PrevInstLoc.getScope(); - Column = PrevInstLoc.getCol(); - } - recordSourceLine(/*Line=*/0, Column, Scope, /*Flags=*/0); - } + (PrevInstBB && PrevInstBB != MI->getParent())) + RecordLineZero(); return; } diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp index bc0bb34..f0f0861 100644 --- a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp @@ -587,10 +587,12 @@ bool DwarfExpression::addExpression( emitUnsigned(LeftShift); emitOp(dwarf::DW_OP_shl); } - emitOp(dwarf::DW_OP_constu); - emitUnsigned(RightShift); - emitOp(OpNum == dwarf::DW_OP_LLVM_extract_bits_sext ? dwarf::DW_OP_shra - : dwarf::DW_OP_shr); + if (RightShift) { + emitOp(dwarf::DW_OP_constu); + emitUnsigned(RightShift); + emitOp(OpNum == dwarf::DW_OP_LLVM_extract_bits_sext ? dwarf::DW_OP_shra + : dwarf::DW_OP_shr); + } // The value is now at the top of the stack, so set the location to // implicit so that we get a stack_value at the end. diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp index aa078f3..e40fb76 100644 --- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp @@ -704,9 +704,17 @@ void DwarfUnit::addType(DIE &Entity, const DIType *Ty, addDIEEntry(Entity, Attribute, DIEEntry(*getOrCreateTypeDIE(Ty))); } +// FIXME: change callsites to use the new DW_LNAME_ language codes. llvm::dwarf::SourceLanguage DwarfUnit::getSourceLanguage() const { - return static_cast<llvm::dwarf::SourceLanguage>( - getLanguage().getUnversionedName()); + const auto &Lang = getLanguage(); + + if (!Lang.hasVersionedName()) + return static_cast<llvm::dwarf::SourceLanguage>(Lang.getName()); + + return llvm::dwarf::toDW_LANG( + static_cast<llvm::dwarf::SourceLanguageName>(Lang.getName()), + Lang.getVersion()) + .value_or(llvm::dwarf::DW_LANG_hi_user); } std::string DwarfUnit::getParentContextString(const DIScope *Context) const { diff --git a/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp b/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp index 260ce8f..93ae548 100644 --- a/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp +++ b/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp @@ -85,8 +85,7 @@ template <> struct llvm::DenseMapInfo<VariableID> { using VarLocInsertPt = PointerUnion<const Instruction *, const DbgRecord *>; -namespace std { -template <> struct hash<VarLocInsertPt> { +template <> struct std::hash<VarLocInsertPt> { using argument_type = VarLocInsertPt; using result_type = std::size_t; @@ -94,7 +93,6 @@ template <> struct hash<VarLocInsertPt> { return std::hash<void *>()(Arg.getOpaqueValue()); } }; -} // namespace std /// Helper class to build FunctionVarLocs, since that class isn't easy to /// modify. TODO: There's not a great deal of value in the split, it could be diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp index 4931403..53f1cfe2 100644 --- a/llvm/lib/CodeGen/AtomicExpandPass.cpp +++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp @@ -770,7 +770,7 @@ struct PartwordMaskValues { Value *Inv_Mask = nullptr; }; -LLVM_ATTRIBUTE_UNUSED +[[maybe_unused]] raw_ostream &operator<<(raw_ostream &O, const PartwordMaskValues &PMV) { auto PrintObj = [&O](auto *V) { if (V) diff --git a/llvm/lib/CodeGen/BasicBlockPathCloning.cpp b/llvm/lib/CodeGen/BasicBlockPathCloning.cpp index fd7df6b..47b7a88 100644 --- a/llvm/lib/CodeGen/BasicBlockPathCloning.cpp +++ b/llvm/lib/CodeGen/BasicBlockPathCloning.cpp @@ -207,9 +207,7 @@ bool ApplyCloning(MachineFunction &MF, } return AnyPathsCloned; } -} // end anonymous namespace -namespace llvm { class BasicBlockPathCloning : public MachineFunctionPass { public: static char ID; @@ -229,7 +227,7 @@ public: bool runOnMachineFunction(MachineFunction &MF) override; }; -} // namespace llvm +} // namespace char BasicBlockPathCloning::ID = 0; INITIALIZE_PASS_BEGIN( diff --git a/llvm/lib/CodeGen/BranchRelaxation.cpp b/llvm/lib/CodeGen/BranchRelaxation.cpp index 2d50167..fae952e 100644 --- a/llvm/lib/CodeGen/BranchRelaxation.cpp +++ b/llvm/lib/CodeGen/BranchRelaxation.cpp @@ -491,6 +491,20 @@ bool BranchRelaxation::fixupConditionalBranch(MachineInstr &MI) { return true; } if (FBB) { + // If we get here with a MBB which ends like this: + // + // bb.1: + // successors: %bb.2; + // ... + // BNE $x1, $x0, %bb.2 + // PseudoBR %bb.2 + // + // Just remove conditional branch. + if (TBB == FBB) { + removeBranch(MBB); + insertUncondBranch(MBB, TBB); + return true; + } // We need to split the basic block here to obtain two long-range // unconditional branches. NewBB = createNewBlockAfter(*MBB); diff --git a/llvm/lib/CodeGen/BreakFalseDeps.cpp b/llvm/lib/CodeGen/BreakFalseDeps.cpp index 28e6728..1846880 100644 --- a/llvm/lib/CodeGen/BreakFalseDeps.cpp +++ b/llvm/lib/CodeGen/BreakFalseDeps.cpp @@ -31,7 +31,7 @@ using namespace llvm; -namespace llvm { +namespace { class BreakFalseDeps : public MachineFunctionPass { private: @@ -95,7 +95,7 @@ private: void processUndefReads(MachineBasicBlock *); }; -} // namespace llvm +} // namespace #define DEBUG_TYPE "break-false-deps" diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index eb73d01b..9e78ec9 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -819,7 +819,7 @@ void CodeGenPrepare::removeAllAssertingVHReferences(Value *V) { } // Verify BFI has been updated correctly by recomputing BFI and comparing them. -void LLVM_ATTRIBUTE_UNUSED CodeGenPrepare::verifyBFIUpdates(Function &F) { +[[maybe_unused]] void CodeGenPrepare::verifyBFIUpdates(Function &F) { DominatorTree NewDT(F); LoopInfo NewLI(NewDT); BranchProbabilityInfo NewBPI(F, NewLI, TLInfo); @@ -3194,7 +3194,7 @@ struct ExtAddrMode : public TargetLowering::AddrMode { case ScaledRegField: return ScaledReg; case BaseOffsField: - return ConstantInt::get(IntPtrTy, BaseOffs); + return ConstantInt::getSigned(IntPtrTy, BaseOffs); } } @@ -6100,7 +6100,7 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, // Add in the Base Offset if present. if (AddrMode.BaseOffs) { - Value *V = ConstantInt::get(IntPtrTy, AddrMode.BaseOffs); + Value *V = ConstantInt::getSigned(IntPtrTy, AddrMode.BaseOffs); if (ResultIndex) { // We need to add this separately from the scale above to help with // SDAG consecutive load/store merging. @@ -6226,7 +6226,7 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, // Add in the Base Offset if present. if (AddrMode.BaseOffs) { - Value *V = ConstantInt::get(IntPtrTy, AddrMode.BaseOffs); + Value *V = ConstantInt::getSigned(IntPtrTy, AddrMode.BaseOffs); if (Result) Result = Builder.CreateAdd(Result, V, "sunkaddr"); else diff --git a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp index 6c2a5a7..87ada87 100644 --- a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp +++ b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp @@ -126,8 +126,7 @@ hash_code hash_value(const ComplexValue &Arg) { } // end namespace typedef SmallVector<struct ComplexValue, 2> ComplexValues; -namespace llvm { -template <> struct DenseMapInfo<ComplexValue> { +template <> struct llvm::DenseMapInfo<ComplexValue> { static inline ComplexValue getEmptyKey() { return {DenseMapInfo<Value *>::getEmptyKey(), DenseMapInfo<Value *>::getEmptyKey()}; @@ -144,7 +143,6 @@ template <> struct DenseMapInfo<ComplexValue> { return LHS.Real == RHS.Real && LHS.Imag == RHS.Imag; } }; -} // end namespace llvm namespace { template <typename T, typename IterT> diff --git a/llvm/lib/CodeGen/EdgeBundles.cpp b/llvm/lib/CodeGen/EdgeBundles.cpp index f4335396..50dd66f 100644 --- a/llvm/lib/CodeGen/EdgeBundles.cpp +++ b/llvm/lib/CodeGen/EdgeBundles.cpp @@ -81,13 +81,10 @@ void EdgeBundles::init() { } } -namespace llvm { - /// Specialize WriteGraph, the standard implementation won't work. -template<> -raw_ostream &WriteGraph<>(raw_ostream &O, const EdgeBundles &G, - bool ShortNames, - const Twine &Title) { +template <> +raw_ostream &llvm::WriteGraph<>(raw_ostream &O, const EdgeBundles &G, + bool ShortNames, const Twine &Title) { const MachineFunction *MF = G.getMachineFunction(); O << "digraph {\n"; @@ -107,8 +104,6 @@ raw_ostream &WriteGraph<>(raw_ostream &O, const EdgeBundles &G, return O; } -} // end namespace llvm - /// view - Visualize the annotated bipartite CFG with Graphviz. void EdgeBundles::view() const { ViewGraph(*this, "EdgeBundles"); diff --git a/llvm/lib/CodeGen/ExpandFp.cpp b/llvm/lib/CodeGen/ExpandFp.cpp index 9cc6c6a..04c7008 100644 --- a/llvm/lib/CodeGen/ExpandFp.cpp +++ b/llvm/lib/CodeGen/ExpandFp.cpp @@ -82,7 +82,7 @@ public: } static FRemExpander create(IRBuilder<> &B, Type *Ty) { - assert(canExpandType(Ty)); + assert(canExpandType(Ty) && "Expected supported floating point type"); // The type to use for the computation of the remainder. This may be // wider than the input/result type which affects the ... @@ -356,8 +356,9 @@ Value *FRemExpander::buildFRem(Value *X, Value *Y, static bool expandFRem(BinaryOperator &I, std::optional<SimplifyQuery> &SQ) { LLVM_DEBUG(dbgs() << "Expanding instruction: " << I << '\n'); - Type *ReturnTy = I.getType(); - assert(FRemExpander::canExpandType(ReturnTy->getScalarType())); + Type *Ty = I.getType(); + assert(FRemExpander::canExpandType(Ty) && + "Expected supported floating point type"); FastMathFlags FMF = I.getFastMathFlags(); // TODO Make use of those flags for optimization? @@ -368,32 +369,10 @@ static bool expandFRem(BinaryOperator &I, std::optional<SimplifyQuery> &SQ) { B.setFastMathFlags(FMF); B.SetCurrentDebugLocation(I.getDebugLoc()); - Type *ElemTy = ReturnTy->getScalarType(); - const FRemExpander Expander = FRemExpander::create(B, ElemTy); - - Value *Ret; - if (ReturnTy->isFloatingPointTy()) - Ret = FMF.approxFunc() - ? Expander.buildApproxFRem(I.getOperand(0), I.getOperand(1)) - : Expander.buildFRem(I.getOperand(0), I.getOperand(1), SQ); - else { - auto *VecTy = cast<FixedVectorType>(ReturnTy); - - // This could use SplitBlockAndInsertForEachLane but the interface - // is a bit awkward for a constant number of elements and it will - // boil down to the same code. - // TODO Expand the FRem instruction only once and reuse the code. - Value *Nums = I.getOperand(0); - Value *Denums = I.getOperand(1); - Ret = PoisonValue::get(I.getType()); - for (int I = 0, E = VecTy->getNumElements(); I != E; ++I) { - Value *Num = B.CreateExtractElement(Nums, I); - Value *Denum = B.CreateExtractElement(Denums, I); - Value *Rem = FMF.approxFunc() ? Expander.buildApproxFRem(Num, Denum) - : Expander.buildFRem(Num, Denum, SQ); - Ret = B.CreateInsertElement(Ret, Rem, I); - } - } + const FRemExpander Expander = FRemExpander::create(B, Ty); + Value *Ret = FMF.approxFunc() + ? Expander.buildApproxFRem(I.getOperand(0), I.getOperand(1)) + : Expander.buildFRem(I.getOperand(0), I.getOperand(1), SQ); I.replaceAllUsesWith(Ret); Ret->takeName(&I); @@ -939,7 +918,8 @@ static void expandIToFP(Instruction *IToFP) { IToFP->eraseFromParent(); } -static void scalarize(Instruction *I, SmallVectorImpl<Instruction *> &Replace) { +static void scalarize(Instruction *I, + SmallVectorImpl<Instruction *> &Worklist) { VectorType *VTy = cast<FixedVectorType>(I->getType()); IRBuilder<> Builder(I); @@ -948,12 +928,25 @@ static void scalarize(Instruction *I, SmallVectorImpl<Instruction *> &Replace) { Value *Result = PoisonValue::get(VTy); for (unsigned Idx = 0; Idx < NumElements; ++Idx) { Value *Ext = Builder.CreateExtractElement(I->getOperand(0), Idx); - Value *Cast = Builder.CreateCast(cast<CastInst>(I)->getOpcode(), Ext, - I->getType()->getScalarType()); - Result = Builder.CreateInsertElement(Result, Cast, Idx); - if (isa<Instruction>(Cast)) - Replace.push_back(cast<Instruction>(Cast)); + + Value *NewOp = nullptr; + if (auto *BinOp = dyn_cast<BinaryOperator>(I)) + NewOp = Builder.CreateBinOp( + BinOp->getOpcode(), Ext, + Builder.CreateExtractElement(I->getOperand(1), Idx)); + else if (auto *CastI = dyn_cast<CastInst>(I)) + NewOp = Builder.CreateCast(CastI->getOpcode(), Ext, + I->getType()->getScalarType()); + else + llvm_unreachable("Unsupported instruction type"); + + Result = Builder.CreateInsertElement(Result, NewOp, Idx); + if (auto *ScalarizedI = dyn_cast<Instruction>(NewOp)) { + ScalarizedI->copyIRFlags(I, true); + Worklist.push_back(ScalarizedI); + } } + I->replaceAllUsesWith(Result); I->dropAllReferences(); I->eraseFromParent(); @@ -989,10 +982,17 @@ static bool targetSupportsFrem(const TargetLowering &TLI, Type *Ty) { return TLI.getLibcallName(fremToLibcall(Ty->getScalarType())); } +static void addToWorklist(Instruction &I, + SmallVector<Instruction *, 4> &Worklist) { + if (I.getOperand(0)->getType()->isVectorTy()) + scalarize(&I, Worklist); + else + Worklist.push_back(&I); +} + static bool runImpl(Function &F, const TargetLowering &TLI, AssumptionCache *AC) { - SmallVector<Instruction *, 4> Replace; - SmallVector<Instruction *, 4> ReplaceVector; + SmallVector<Instruction *, 4> Worklist; bool Modified = false; unsigned MaxLegalFpConvertBitWidth = @@ -1003,55 +1003,39 @@ static bool runImpl(Function &F, const TargetLowering &TLI, if (MaxLegalFpConvertBitWidth >= llvm::IntegerType::MAX_INT_BITS) return false; - for (auto &I : instructions(F)) { - switch (I.getOpcode()) { - case Instruction::FRem: { - Type *Ty = I.getType(); - // TODO: This pass doesn't handle scalable vectors. - if (Ty->isScalableTy()) - continue; - - if (targetSupportsFrem(TLI, Ty) || - !FRemExpander::canExpandType(Ty->getScalarType())) - continue; - - Replace.push_back(&I); - Modified = true; + for (auto It = inst_begin(&F), End = inst_end(F); It != End;) { + Instruction &I = *It++; + Type *Ty = I.getType(); + // TODO: This pass doesn't handle scalable vectors. + if (Ty->isScalableTy()) + continue; + switch (I.getOpcode()) { + case Instruction::FRem: + if (!targetSupportsFrem(TLI, Ty) && + FRemExpander::canExpandType(Ty->getScalarType())) { + addToWorklist(I, Worklist); + Modified = true; + } break; - } case Instruction::FPToUI: case Instruction::FPToSI: { - // TODO: This pass doesn't handle scalable vectors. - if (I.getOperand(0)->getType()->isScalableTy()) - continue; - - auto *IntTy = cast<IntegerType>(I.getType()->getScalarType()); + auto *IntTy = cast<IntegerType>(Ty->getScalarType()); if (IntTy->getIntegerBitWidth() <= MaxLegalFpConvertBitWidth) continue; - if (I.getOperand(0)->getType()->isVectorTy()) - ReplaceVector.push_back(&I); - else - Replace.push_back(&I); + addToWorklist(I, Worklist); Modified = true; break; } case Instruction::UIToFP: case Instruction::SIToFP: { - // TODO: This pass doesn't handle scalable vectors. - if (I.getOperand(0)->getType()->isScalableTy()) - continue; - auto *IntTy = cast<IntegerType>(I.getOperand(0)->getType()->getScalarType()); if (IntTy->getIntegerBitWidth() <= MaxLegalFpConvertBitWidth) continue; - if (I.getOperand(0)->getType()->isVectorTy()) - ReplaceVector.push_back(&I); - else - Replace.push_back(&I); + addToWorklist(I, Worklist); Modified = true; break; } @@ -1060,16 +1044,8 @@ static bool runImpl(Function &F, const TargetLowering &TLI, } } - while (!ReplaceVector.empty()) { - Instruction *I = ReplaceVector.pop_back_val(); - scalarize(I, Replace); - } - - if (Replace.empty()) - return false; - - while (!Replace.empty()) { - Instruction *I = Replace.pop_back_val(); + while (!Worklist.empty()) { + Instruction *I = Worklist.pop_back_val(); if (I->getOpcode() == Instruction::FRem) { auto SQ = [&]() -> std::optional<SimplifyQuery> { if (AC) { diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp index 906d62a3..b425b95 100644 --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -1728,6 +1728,7 @@ static APFloat constantFoldFpUnary(const MachineInstr &MI, Result.clearSign(); return Result; } + case TargetOpcode::G_FPEXT: case TargetOpcode::G_FPTRUNC: { bool Unused; LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); diff --git a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp index 3f6813e..04d9309 100644 --- a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp +++ b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp @@ -112,7 +112,7 @@ APInt GISelValueTracking::getKnownOnes(Register R) { return getKnownBits(R).One; } -LLVM_ATTRIBUTE_UNUSED static void +[[maybe_unused]] static void dumpResult(const MachineInstr &MI, const KnownBits &Known, unsigned Depth) { dbgs() << "[" << Depth << "] Compute known bits: " << MI << "[" << Depth << "] Computed for: " << MI << "[" << Depth << "] Known: 0x" @@ -344,6 +344,22 @@ void GISelValueTracking::computeKnownBitsImpl(Register R, KnownBits &Known, Known = KnownBits::mul(Known, Known2); break; } + case TargetOpcode::G_UMULH: { + computeKnownBitsImpl(MI.getOperand(2).getReg(), Known, DemandedElts, + Depth + 1); + computeKnownBitsImpl(MI.getOperand(1).getReg(), Known2, DemandedElts, + Depth + 1); + Known = KnownBits::mulhu(Known, Known2); + break; + } + case TargetOpcode::G_SMULH: { + computeKnownBitsImpl(MI.getOperand(2).getReg(), Known, DemandedElts, + Depth + 1); + computeKnownBitsImpl(MI.getOperand(1).getReg(), Known2, DemandedElts, + Depth + 1); + Known = KnownBits::mulhs(Known, Known2); + break; + } case TargetOpcode::G_SELECT: { computeKnownBitsMin(MI.getOperand(2).getReg(), MI.getOperand(3).getReg(), Known, DemandedElts, Depth + 1); @@ -1959,6 +1975,81 @@ unsigned GISelValueTracking::computeNumSignBits(Register R, break; } + case TargetOpcode::G_SUB: { + Register Src2 = MI.getOperand(2).getReg(); + unsigned Src2NumSignBits = + computeNumSignBits(Src2, DemandedElts, Depth + 1); + if (Src2NumSignBits == 1) + return 1; // Early out. + + // Handle NEG. + Register Src1 = MI.getOperand(1).getReg(); + KnownBits Known1 = getKnownBits(Src1, DemandedElts, Depth); + if (Known1.isZero()) { + KnownBits Known2 = getKnownBits(Src2, DemandedElts, Depth); + // If the input is known to be 0 or 1, the output is 0/-1, which is all + // sign bits set. + if ((Known2.Zero | 1).isAllOnes()) + return TyBits; + + // If the input is known to be positive (the sign bit is known clear), + // the output of the NEG has, at worst, the same number of sign bits as + // the input. + if (Known2.isNonNegative()) { + FirstAnswer = Src2NumSignBits; + break; + } + + // Otherwise, we treat this like a SUB. + } + + unsigned Src1NumSignBits = + computeNumSignBits(Src1, DemandedElts, Depth + 1); + if (Src1NumSignBits == 1) + return 1; // Early Out. + + // Sub can have at most one carry bit. Thus we know that the output + // is, at worst, one more bit than the inputs. + FirstAnswer = std::min(Src1NumSignBits, Src2NumSignBits) - 1; + break; + } + case TargetOpcode::G_ADD: { + Register Src2 = MI.getOperand(2).getReg(); + unsigned Src2NumSignBits = + computeNumSignBits(Src2, DemandedElts, Depth + 1); + if (Src2NumSignBits <= 2) + return 1; // Early out. + + Register Src1 = MI.getOperand(1).getReg(); + unsigned Src1NumSignBits = + computeNumSignBits(Src1, DemandedElts, Depth + 1); + if (Src1NumSignBits == 1) + return 1; // Early Out. + + // Special case decrementing a value (ADD X, -1): + KnownBits Known2 = getKnownBits(Src2, DemandedElts, Depth); + if (Known2.isAllOnes()) { + KnownBits Known1 = getKnownBits(Src1, DemandedElts, Depth); + // If the input is known to be 0 or 1, the output is 0/-1, which is all + // sign bits set. + if ((Known1.Zero | 1).isAllOnes()) + return TyBits; + + // If we are subtracting one from a positive number, there is no carry + // out of the result. + if (Known1.isNonNegative()) { + FirstAnswer = Src1NumSignBits; + break; + } + + // Otherwise, we treat this like an ADD. + } + + // Add can have at most one carry bit. Thus we know that the output + // is, at worst, one more bit than the inputs. + FirstAnswer = std::min(Src1NumSignBits, Src2NumSignBits) - 1; + break; + } case TargetOpcode::G_FCMP: case TargetOpcode::G_ICMP: { bool IsFP = Opcode == TargetOpcode::G_FCMP; diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizeMutations.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizeMutations.cpp index 25c1db9..ded4df4 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizeMutations.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizeMutations.cpp @@ -55,12 +55,10 @@ LegalizeMutation LegalizeMutations::changeElementCountTo(unsigned TypeIdx, } LegalizeMutation LegalizeMutations::changeElementCountTo(unsigned TypeIdx, - LLT NewEltTy) { + ElementCount EC) { return [=](const LegalityQuery &Query) { const LLT OldTy = Query.Types[TypeIdx]; - ElementCount NewEltCount = NewEltTy.isVector() ? NewEltTy.getElementCount() - : ElementCount::getFixed(1); - return std::make_pair(TypeIdx, OldTy.changeElementCount(NewEltCount)); + return std::make_pair(TypeIdx, OldTy.changeElementCount(EC)); }; } diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index cffaf7c..38ec83f 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -3292,8 +3292,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { if (TypeIdx != 2) return UnableToLegalize; Observer.changingInstr(MI); - // TODO: Probably should be zext - widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); + widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); Observer.changedInstr(MI); return Legalized; } @@ -3325,8 +3324,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { if (TypeIdx == 2) { Observer.changingInstr(MI); - // TODO: Probably should be zext - widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_SEXT); + widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ZEXT); Observer.changedInstr(MI); return Legalized; } diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp index 055fdc6..ca82857 100644 --- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp +++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp @@ -818,8 +818,7 @@ bool llvm::isKnownNeverNaN(Register Val, const MachineRegisterInfo &MRI, if (!DefMI) return false; - const TargetMachine& TM = DefMI->getMF()->getTarget(); - if (DefMI->getFlag(MachineInstr::FmNoNans) || TM.Options.NoNaNsFPMath) + if (DefMI->getFlag(MachineInstr::FmNoNans)) return true; // If the value is a constant, we can obviously see if it is a NaN or not. diff --git a/llvm/lib/CodeGen/GlobalMergeFunctions.cpp b/llvm/lib/CodeGen/GlobalMergeFunctions.cpp index 47640c4a..81ab317 100644 --- a/llvm/lib/CodeGen/GlobalMergeFunctions.cpp +++ b/llvm/lib/CodeGen/GlobalMergeFunctions.cpp @@ -587,16 +587,12 @@ public: } // namespace char GlobalMergeFuncPassWrapper::ID = 0; -INITIALIZE_PASS_BEGIN(GlobalMergeFuncPassWrapper, "global-merge-func", - "Global merge function pass", false, false) -INITIALIZE_PASS_END(GlobalMergeFuncPassWrapper, "global-merge-func", - "Global merge function pass", false, false) +INITIALIZE_PASS(GlobalMergeFuncPassWrapper, "global-merge-func", + "Global merge function pass", false, false) -namespace llvm { -ModulePass *createGlobalMergeFuncPass() { +ModulePass *llvm::createGlobalMergeFuncPass() { return new GlobalMergeFuncPassWrapper(); } -} // namespace llvm GlobalMergeFuncPassWrapper::GlobalMergeFuncPassWrapper() : ModulePass(ID) { initializeGlobalMergeFuncPassWrapperPass( diff --git a/llvm/lib/CodeGen/LiveIntervals.cpp b/llvm/lib/CodeGen/LiveIntervals.cpp index 3485a27..d2f2c3e 100644 --- a/llvm/lib/CodeGen/LiveIntervals.cpp +++ b/llvm/lib/CodeGen/LiveIntervals.cpp @@ -101,15 +101,11 @@ static cl::opt<bool> EnablePrecomputePhysRegs( static bool EnablePrecomputePhysRegs = false; #endif // NDEBUG -namespace llvm { - -cl::opt<bool> UseSegmentSetForPhysRegs( +cl::opt<bool> llvm::UseSegmentSetForPhysRegs( "use-segment-set-for-physregs", cl::Hidden, cl::init(true), cl::desc( "Use segment set for the computation of the live ranges of physregs.")); -} // end namespace llvm - void LiveIntervalsWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesCFG(); AU.addPreserved<LiveVariablesWrapperPass>(); @@ -665,7 +661,10 @@ void LiveIntervals::extendToIndices(LiveRange &LR, void LiveIntervals::pruneValue(LiveRange &LR, SlotIndex Kill, SmallVectorImpl<SlotIndex> *EndPoints) { LiveQueryResult LRQ = LR.Query(Kill); - VNInfo *VNI = LRQ.valueOutOrDead(); + // LR may have liveness reachable from early clobber slot, which may be + // only live-in instead of live-out of the instruction. + // For example, LR =[1r, 3r), Kill = 3e, we have to prune [3e, 3r) of LR. + VNInfo *VNI = LRQ.valueOutOrDead() ? LRQ.valueOutOrDead() : LRQ.valueIn(); if (!VNI) return; diff --git a/llvm/lib/CodeGen/MIR2Vec.cpp b/llvm/lib/CodeGen/MIR2Vec.cpp index e859765..5c78d98 100644 --- a/llvm/lib/CodeGen/MIR2Vec.cpp +++ b/llvm/lib/CodeGen/MIR2Vec.cpp @@ -29,20 +29,17 @@ using namespace mir2vec; STATISTIC(MIRVocabMissCounter, "Number of lookups to MIR entities not present in the vocabulary"); -namespace llvm { -namespace mir2vec { -cl::OptionCategory MIR2VecCategory("MIR2Vec Options"); +cl::OptionCategory llvm::mir2vec::MIR2VecCategory("MIR2Vec Options"); // FIXME: Use a default vocab when not specified static cl::opt<std::string> VocabFile("mir2vec-vocab-path", cl::Optional, cl::desc("Path to the vocabulary file for MIR2Vec"), cl::init(""), cl::cat(MIR2VecCategory)); -cl::opt<float> OpcWeight("mir2vec-opc-weight", cl::Optional, cl::init(1.0), - cl::desc("Weight for machine opcode embeddings"), - cl::cat(MIR2VecCategory)); -} // namespace mir2vec -} // namespace llvm +cl::opt<float> + llvm::mir2vec::OpcWeight("mir2vec-opc-weight", cl::Optional, cl::init(1.0), + cl::desc("Weight for machine opcode embeddings"), + cl::cat(MIR2VecCategory)); //===----------------------------------------------------------------------===// // Vocabulary Implementation diff --git a/llvm/lib/CodeGen/MIRFSDiscriminator.cpp b/llvm/lib/CodeGen/MIRFSDiscriminator.cpp index f5146f5..d988a2a 100644 --- a/llvm/lib/CodeGen/MIRFSDiscriminator.cpp +++ b/llvm/lib/CodeGen/MIRFSDiscriminator.cpp @@ -40,7 +40,7 @@ cl::opt<bool> ImprovedFSDiscriminator( "improved-fs-discriminator", cl::Hidden, cl::init(false), cl::desc("New FS discriminators encoding (incompatible with the original " "encoding)")); -} +} // namespace llvm char MIRAddFSDiscriminators::ID = 0; diff --git a/llvm/lib/CodeGen/MIRNamerPass.cpp b/llvm/lib/CodeGen/MIRNamerPass.cpp index bc65700..cbf8867 100644 --- a/llvm/lib/CodeGen/MIRNamerPass.cpp +++ b/llvm/lib/CodeGen/MIRNamerPass.cpp @@ -23,10 +23,6 @@ using namespace llvm; -namespace llvm { -extern char &MIRNamerID; -} // namespace llvm - #define DEBUG_TYPE "mir-namer" namespace { @@ -53,10 +49,9 @@ public: VRegRenamer Renamer(MF.getRegInfo()); - unsigned BBIndex = 0; ReversePostOrderTraversal<MachineBasicBlock *> RPOT(&*MF.begin()); - for (auto &MBB : RPOT) - Changed |= Renamer.renameVRegs(MBB, BBIndex++); + for (const auto &[BBIndex, MBB] : enumerate(RPOT)) + Changed |= Renamer.renameVRegs(MBB, BBIndex); return Changed; } @@ -66,10 +61,4 @@ public: char MIRNamer::ID; -char &llvm::MIRNamerID = MIRNamer::ID; - -INITIALIZE_PASS_BEGIN(MIRNamer, "mir-namer", "Rename Register Operands", false, - false) - -INITIALIZE_PASS_END(MIRNamer, "mir-namer", "Rename Register Operands", false, - false) +INITIALIZE_PASS(MIRNamer, "mir-namer", "Rename Register Operands", false, false) diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp index bf8a6cd..1d54d72 100644 --- a/llvm/lib/CodeGen/MIRPrinter.cpp +++ b/llvm/lib/CodeGen/MIRPrinter.cpp @@ -107,10 +107,8 @@ struct MFPrintState { } // end anonymous namespace -namespace llvm::yaml { - /// This struct serializes the LLVM IR module. -template <> struct BlockScalarTraits<Module> { +template <> struct yaml::BlockScalarTraits<Module> { static void output(const Module &Mod, void *Ctxt, raw_ostream &OS) { Mod.print(OS, nullptr); } @@ -121,8 +119,6 @@ template <> struct BlockScalarTraits<Module> { } }; -} // end namespace llvm::yaml - static void printRegMIR(Register Reg, yaml::StringValue &Dest, const TargetRegisterInfo *TRI) { raw_string_ostream OS(Dest.Value); @@ -866,48 +862,46 @@ static void printMI(raw_ostream &OS, MFPrintState &State, OS << TII->getName(MI.getOpcode()); - LS = ListSeparator(); + // Print a space after the opcode if any additional tokens are printed. + LS = ListSeparator(", ", " "); - if (I < E) { - OS << ' '; - for (; I < E; ++I) { - OS << LS; - printMIOperand(OS, State, MI, I, TRI, TII, ShouldPrintRegisterTies, - PrintedTypes, MRI, /*PrintDef=*/true); - } + for (; I < E; ++I) { + OS << LS; + printMIOperand(OS, State, MI, I, TRI, TII, ShouldPrintRegisterTies, + PrintedTypes, MRI, /*PrintDef=*/true); } // Print any optional symbols attached to this instruction as-if they were // operands. if (MCSymbol *PreInstrSymbol = MI.getPreInstrSymbol()) { - OS << LS << " pre-instr-symbol "; + OS << LS << "pre-instr-symbol "; MachineOperand::printSymbol(OS, *PreInstrSymbol); } if (MCSymbol *PostInstrSymbol = MI.getPostInstrSymbol()) { - OS << LS << " post-instr-symbol "; + OS << LS << "post-instr-symbol "; MachineOperand::printSymbol(OS, *PostInstrSymbol); } if (MDNode *HeapAllocMarker = MI.getHeapAllocMarker()) { - OS << LS << " heap-alloc-marker "; + OS << LS << "heap-alloc-marker "; HeapAllocMarker->printAsOperand(OS, State.MST); } if (MDNode *PCSections = MI.getPCSections()) { - OS << LS << " pcsections "; + OS << LS << "pcsections "; PCSections->printAsOperand(OS, State.MST); } if (MDNode *MMRA = MI.getMMRAMetadata()) { - OS << LS << " mmra "; + OS << LS << "mmra "; MMRA->printAsOperand(OS, State.MST); } if (uint32_t CFIType = MI.getCFIType()) - OS << LS << " cfi-type " << CFIType; + OS << LS << "cfi-type " << CFIType; if (auto Num = MI.peekDebugInstrNum()) - OS << LS << " debug-instr-number " << Num; + OS << LS << "debug-instr-number " << Num; if (PrintLocations) { if (const DebugLoc &DL = MI.getDebugLoc()) { - OS << LS << " debug-location "; + OS << LS << "debug-location "; DL->printAsOperand(OS, State.MST); } } diff --git a/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp b/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp index b2731b69..a72c2c4 100644 --- a/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp +++ b/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp @@ -97,7 +97,9 @@ static const bool EnableDevelopmentFeatures = false; /// this happens only in development mode. It's a no-op otherwise. namespace llvm { extern cl::opt<unsigned> EvictInterferenceCutoff; +} // namespace llvm +namespace { class RegAllocScoring : public MachineFunctionPass { public: static char ID; @@ -124,11 +126,12 @@ public: /// Performs this pass bool runOnMachineFunction(MachineFunction &) override; }; +} // namespace char RegAllocScoring::ID = 0; -FunctionPass *createRegAllocScoringPass() { return new RegAllocScoring(); } - -} // namespace llvm +FunctionPass *llvm::createRegAllocScoringPass() { + return new RegAllocScoring(); +} INITIALIZE_PASS(RegAllocScoring, "regallocscoringpass", "Register Allocation Scoring Pass", false, false) diff --git a/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp b/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp index e7fa082..26eb10f 100644 --- a/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp +++ b/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp @@ -29,7 +29,6 @@ using namespace llvm; #define DEBUG_TYPE "machine-block-freq" -namespace llvm { static cl::opt<GVDAGType> ViewMachineBlockFreqPropagationDAG( "view-machine-block-freq-propagation-dags", cl::Hidden, cl::desc("Pop up a window to show a dag displaying how machine block " @@ -44,6 +43,7 @@ static cl::opt<GVDAGType> ViewMachineBlockFreqPropagationDAG( clEnumValN(GVDT_Count, "count", "display a graph using the real " "profile count if available."))); +namespace llvm { // Similar option above, but used to control BFI display only after MBP pass cl::opt<GVDAGType> ViewBlockLayoutWithBFI( "view-block-layout-with-bfi", cl::Hidden, @@ -69,15 +69,15 @@ extern cl::opt<std::string> ViewBlockFreqFuncName; // Defined in Analysis/BlockFrequencyInfo.cpp: -view-hot-freq-perc= extern cl::opt<unsigned> ViewHotFreqPercent; -static cl::opt<bool> PrintMachineBlockFreq( - "print-machine-bfi", cl::init(false), cl::Hidden, - cl::desc("Print the machine block frequency info.")); - // Command line option to specify the name of the function for block frequency // dump. Defined in Analysis/BlockFrequencyInfo.cpp. extern cl::opt<std::string> PrintBFIFuncName; } // namespace llvm +static cl::opt<bool> + PrintMachineBlockFreq("print-machine-bfi", cl::init(false), cl::Hidden, + cl::desc("Print the machine block frequency info.")); + static GVDAGType getGVDT() { if (ViewBlockLayoutWithBFI != GVDT_None) return ViewBlockLayoutWithBFI; @@ -85,9 +85,7 @@ static GVDAGType getGVDT() { return ViewMachineBlockFreqPropagationDAG; } -namespace llvm { - -template <> struct GraphTraits<MachineBlockFrequencyInfo *> { +template <> struct llvm::GraphTraits<MachineBlockFrequencyInfo *> { using NodeRef = const MachineBasicBlock *; using ChildIteratorType = MachineBasicBlock::const_succ_iterator; using nodes_iterator = pointer_iterator<MachineFunction::const_iterator>; @@ -116,7 +114,7 @@ using MBFIDOTGraphTraitsBase = MachineBranchProbabilityInfo>; template <> -struct DOTGraphTraits<MachineBlockFrequencyInfo *> +struct llvm::DOTGraphTraits<MachineBlockFrequencyInfo *> : public MBFIDOTGraphTraitsBase { const MachineFunction *CurFunc = nullptr; DenseMap<const MachineBasicBlock *, int> LayoutOrderMap; @@ -159,8 +157,6 @@ struct DOTGraphTraits<MachineBlockFrequencyInfo *> } }; -} // end namespace llvm - AnalysisKey MachineBlockFrequencyAnalysis::Key; MachineBlockFrequencyAnalysis::Result diff --git a/llvm/lib/CodeGen/MachineBranchProbabilityInfo.cpp b/llvm/lib/CodeGen/MachineBranchProbabilityInfo.cpp index 2e92dd8..7ca4582 100644 --- a/llvm/lib/CodeGen/MachineBranchProbabilityInfo.cpp +++ b/llvm/lib/CodeGen/MachineBranchProbabilityInfo.cpp @@ -18,13 +18,8 @@ using namespace llvm; -INITIALIZE_PASS_BEGIN(MachineBranchProbabilityInfoWrapperPass, - "machine-branch-prob", - "Machine Branch Probability Analysis", false, true) -INITIALIZE_PASS_END(MachineBranchProbabilityInfoWrapperPass, - "machine-branch-prob", - "Machine Branch Probability Analysis", false, true) - +INITIALIZE_PASS(MachineBranchProbabilityInfoWrapperPass, "machine-branch-prob", + "Machine Branch Probability Analysis", false, true) namespace llvm { cl::opt<unsigned> StaticLikelyProb("static-likely-prob", diff --git a/llvm/lib/CodeGen/MachineCopyPropagation.cpp b/llvm/lib/CodeGen/MachineCopyPropagation.cpp index e359831..ea08365 100644 --- a/llvm/lib/CodeGen/MachineCopyPropagation.cpp +++ b/llvm/lib/CodeGen/MachineCopyPropagation.cpp @@ -1257,7 +1257,7 @@ void MachineCopyPropagation::BackwardCopyPropagateBlock( Tracker.clear(); } -static void LLVM_ATTRIBUTE_UNUSED printSpillReloadChain( +[[maybe_unused]] static void printSpillReloadChain( DenseMap<MachineInstr *, SmallVector<MachineInstr *>> &SpillChain, DenseMap<MachineInstr *, SmallVector<MachineInstr *>> &ReloadChain, MachineInstr *Leader) { diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp index 224231c..bfa5ab2 100644 --- a/llvm/lib/CodeGen/MachineFunction.cpp +++ b/llvm/lib/CodeGen/MachineFunction.cpp @@ -719,43 +719,41 @@ MachineFunction::CallSiteInfo::CallSiteInfo(const CallBase &CB) { } } -namespace llvm { +template <> +struct llvm::DOTGraphTraits<const MachineFunction *> + : public DefaultDOTGraphTraits { + DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {} - template<> - struct DOTGraphTraits<const MachineFunction*> : public DefaultDOTGraphTraits { - DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {} + static std::string getGraphName(const MachineFunction *F) { + return ("CFG for '" + F->getName() + "' function").str(); + } - static std::string getGraphName(const MachineFunction *F) { - return ("CFG for '" + F->getName() + "' function").str(); + std::string getNodeLabel(const MachineBasicBlock *Node, + const MachineFunction *Graph) { + std::string OutStr; + { + raw_string_ostream OSS(OutStr); + + if (isSimple()) { + OSS << printMBBReference(*Node); + if (const BasicBlock *BB = Node->getBasicBlock()) + OSS << ": " << BB->getName(); + } else + Node->print(OSS); } - std::string getNodeLabel(const MachineBasicBlock *Node, - const MachineFunction *Graph) { - std::string OutStr; - { - raw_string_ostream OSS(OutStr); - - if (isSimple()) { - OSS << printMBBReference(*Node); - if (const BasicBlock *BB = Node->getBasicBlock()) - OSS << ": " << BB->getName(); - } else - Node->print(OSS); - } - - if (OutStr[0] == '\n') OutStr.erase(OutStr.begin()); - - // Process string output to make it nicer... - for (unsigned i = 0; i != OutStr.length(); ++i) - if (OutStr[i] == '\n') { // Left justify - OutStr[i] = '\\'; - OutStr.insert(OutStr.begin()+i+1, 'l'); - } - return OutStr; - } - }; + if (OutStr[0] == '\n') + OutStr.erase(OutStr.begin()); -} // end namespace llvm + // Process string output to make it nicer... + for (unsigned i = 0; i != OutStr.length(); ++i) + if (OutStr[i] == '\n') { // Left justify + OutStr[i] = '\\'; + OutStr.insert(OutStr.begin() + i + 1, 'l'); + } + return OutStr; + } +}; void MachineFunction::viewCFG() const { diff --git a/llvm/lib/CodeGen/MachineFunctionPrinterPass.cpp b/llvm/lib/CodeGen/MachineFunctionPrinterPass.cpp index 0f88a7b..5111322 100644 --- a/llvm/lib/CodeGen/MachineFunctionPrinterPass.cpp +++ b/llvm/lib/CodeGen/MachineFunctionPrinterPass.cpp @@ -60,13 +60,11 @@ char &llvm::MachineFunctionPrinterPassID = MachineFunctionPrinterPass::ID; INITIALIZE_PASS(MachineFunctionPrinterPass, "machineinstr-printer", "Machine Function Printer", false, false) -namespace llvm { /// Returns a newly-created MachineFunction Printer pass. The /// default banner is empty. /// -MachineFunctionPass *createMachineFunctionPrinterPass(raw_ostream &OS, - const std::string &Banner){ +MachineFunctionPass * +llvm::createMachineFunctionPrinterPass(raw_ostream &OS, + const std::string &Banner) { return new MachineFunctionPrinterPass(OS, Banner); } - -} diff --git a/llvm/lib/CodeGen/MachineOutliner.cpp b/llvm/lib/CodeGen/MachineOutliner.cpp index fdae3b4..9feb974 100644 --- a/llvm/lib/CodeGen/MachineOutliner.cpp +++ b/llvm/lib/CodeGen/MachineOutliner.cpp @@ -593,15 +593,12 @@ struct MachineOutliner : public ModulePass { char MachineOutliner::ID = 0; -namespace llvm { -ModulePass *createMachineOutlinerPass(RunOutliner RunOutlinerMode) { +ModulePass *llvm::createMachineOutlinerPass(RunOutliner RunOutlinerMode) { MachineOutliner *OL = new MachineOutliner(); OL->RunOutlinerMode = RunOutlinerMode; return OL; } -} // namespace llvm - INITIALIZE_PASS(MachineOutliner, DEBUG_TYPE, "Machine Function Outliner", false, false) diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp index 89ed4da..a717d9e 100644 --- a/llvm/lib/CodeGen/MachinePipeliner.cpp +++ b/llvm/lib/CodeGen/MachinePipeliner.cpp @@ -201,16 +201,15 @@ static cl::opt<unsigned> SwpMaxNumStores( cl::desc("Maximum number of stores allwed in the target loop."), cl::Hidden, cl::init(200)); -namespace llvm { - // A command line option to enable the CopyToPhi DAG mutation. -cl::opt<bool> SwpEnableCopyToPhi("pipeliner-enable-copytophi", cl::ReallyHidden, - cl::init(true), - cl::desc("Enable CopyToPhi DAG Mutation")); +cl::opt<bool> + llvm::SwpEnableCopyToPhi("pipeliner-enable-copytophi", cl::ReallyHidden, + cl::init(true), + cl::desc("Enable CopyToPhi DAG Mutation")); /// A command line argument to force pipeliner to use specified issue /// width. -cl::opt<int> SwpForceIssueWidth( +cl::opt<int> llvm::SwpForceIssueWidth( "pipeliner-force-issue-width", cl::desc("Force pipeliner to use specified issue width."), cl::Hidden, cl::init(-1)); @@ -226,8 +225,6 @@ static cl::opt<WindowSchedulingFlag> WindowSchedulingOption( clEnumValN(WindowSchedulingFlag::WS_Force, "force", "Use window algorithm instead of SMS algorithm."))); -} // end namespace llvm - unsigned SwingSchedulerDAG::Circuits::MaxPaths = 5; char MachinePipeliner::ID = 0; #ifndef NDEBUG diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp index 299bcc4..3ed1045 100644 --- a/llvm/lib/CodeGen/MachineScheduler.cpp +++ b/llvm/lib/CodeGen/MachineScheduler.cpp @@ -176,9 +176,7 @@ STATISTIC(NumNodeOrderPostRA, STATISTIC(NumFirstValidPostRA, "Number of scheduling units chosen for FirstValid heuristic post-RA"); -namespace llvm { - -cl::opt<MISched::Direction> PreRADirection( +cl::opt<MISched::Direction> llvm::PreRADirection( "misched-prera-direction", cl::Hidden, cl::desc("Pre reg-alloc list scheduling direction"), cl::init(MISched::Unspecified), @@ -206,33 +204,31 @@ static cl::opt<bool> DumpCriticalPathLength("misched-dcpl", cl::Hidden, cl::desc("Print critical path length to stdout")); -cl::opt<bool> VerifyScheduling( +cl::opt<bool> llvm::VerifyScheduling( "verify-misched", cl::Hidden, cl::desc("Verify machine instrs before and after machine scheduling")); #ifndef NDEBUG -cl::opt<bool> ViewMISchedDAGs( +cl::opt<bool> llvm::ViewMISchedDAGs( "view-misched-dags", cl::Hidden, cl::desc("Pop up a window to show MISched dags after they are processed")); -cl::opt<bool> PrintDAGs("misched-print-dags", cl::Hidden, - cl::desc("Print schedule DAGs")); -cl::opt<bool> MISchedDumpReservedCycles( +cl::opt<bool> llvm::PrintDAGs("misched-print-dags", cl::Hidden, + cl::desc("Print schedule DAGs")); +static cl::opt<bool> MISchedDumpReservedCycles( "misched-dump-reserved-cycles", cl::Hidden, cl::init(false), cl::desc("Dump resource usage at schedule boundary.")); -cl::opt<bool> MischedDetailResourceBooking( +static cl::opt<bool> MischedDetailResourceBooking( "misched-detail-resource-booking", cl::Hidden, cl::init(false), cl::desc("Show details of invoking getNextResoufceCycle.")); #else -const bool ViewMISchedDAGs = false; -const bool PrintDAGs = false; -const bool MischedDetailResourceBooking = false; +const bool llvm::ViewMISchedDAGs = false; +const bool llvm::PrintDAGs = false; +static const bool MischedDetailResourceBooking = false; #ifdef LLVM_ENABLE_DUMP -const bool MISchedDumpReservedCycles = false; +static const bool MISchedDumpReservedCycles = false; #endif // LLVM_ENABLE_DUMP #endif // NDEBUG -} // end namespace llvm - #ifndef NDEBUG /// In some situations a few uninteresting nodes depend on nearly all other /// nodes in the graph, provide a cutoff to hide them. @@ -2053,28 +2049,24 @@ public: } // end anonymous namespace -namespace llvm { - std::unique_ptr<ScheduleDAGMutation> -createLoadClusterDAGMutation(const TargetInstrInfo *TII, - const TargetRegisterInfo *TRI, - bool ReorderWhileClustering) { +llvm::createLoadClusterDAGMutation(const TargetInstrInfo *TII, + const TargetRegisterInfo *TRI, + bool ReorderWhileClustering) { return EnableMemOpCluster ? std::make_unique<LoadClusterMutation>( TII, TRI, ReorderWhileClustering) : nullptr; } std::unique_ptr<ScheduleDAGMutation> -createStoreClusterDAGMutation(const TargetInstrInfo *TII, - const TargetRegisterInfo *TRI, - bool ReorderWhileClustering) { +llvm::createStoreClusterDAGMutation(const TargetInstrInfo *TII, + const TargetRegisterInfo *TRI, + bool ReorderWhileClustering) { return EnableMemOpCluster ? std::make_unique<StoreClusterMutation>( TII, TRI, ReorderWhileClustering) : nullptr; } -} // end namespace llvm - // Sorting all the loads/stores first, then for each load/store, checking the // following load/store one by one, until reach the first non-dependent one and // call target hook to see if they can cluster. @@ -2304,16 +2296,12 @@ protected: } // end anonymous namespace -namespace llvm { - std::unique_ptr<ScheduleDAGMutation> -createCopyConstrainDAGMutation(const TargetInstrInfo *TII, - const TargetRegisterInfo *TRI) { +llvm::createCopyConstrainDAGMutation(const TargetInstrInfo *TII, + const TargetRegisterInfo *TRI) { return std::make_unique<CopyConstrain>(TII, TRI); } -} // end namespace llvm - /// constrainLocalCopy handles two possibilities: /// 1) Local src: /// I0: = dst @@ -3445,14 +3433,13 @@ void GenericSchedulerBase::traceCandidate(const SchedCandidate &Cand) { } #endif -namespace llvm { /// Return true if this heuristic determines order. /// TODO: Consider refactor return type of these functions as integer or enum, /// as we may need to differentiate whether TryCand is better than Cand. -bool tryLess(int TryVal, int CandVal, - GenericSchedulerBase::SchedCandidate &TryCand, - GenericSchedulerBase::SchedCandidate &Cand, - GenericSchedulerBase::CandReason Reason) { +bool llvm::tryLess(int TryVal, int CandVal, + GenericSchedulerBase::SchedCandidate &TryCand, + GenericSchedulerBase::SchedCandidate &Cand, + GenericSchedulerBase::CandReason Reason) { if (TryVal < CandVal) { TryCand.Reason = Reason; return true; @@ -3465,10 +3452,10 @@ bool tryLess(int TryVal, int CandVal, return false; } -bool tryGreater(int TryVal, int CandVal, - GenericSchedulerBase::SchedCandidate &TryCand, - GenericSchedulerBase::SchedCandidate &Cand, - GenericSchedulerBase::CandReason Reason) { +bool llvm::tryGreater(int TryVal, int CandVal, + GenericSchedulerBase::SchedCandidate &TryCand, + GenericSchedulerBase::SchedCandidate &Cand, + GenericSchedulerBase::CandReason Reason) { if (TryVal > CandVal) { TryCand.Reason = Reason; return true; @@ -3481,9 +3468,9 @@ bool tryGreater(int TryVal, int CandVal, return false; } -bool tryLatency(GenericSchedulerBase::SchedCandidate &TryCand, - GenericSchedulerBase::SchedCandidate &Cand, - SchedBoundary &Zone) { +bool llvm::tryLatency(GenericSchedulerBase::SchedCandidate &TryCand, + GenericSchedulerBase::SchedCandidate &Cand, + SchedBoundary &Zone) { if (Zone.isTop()) { // Prefer the candidate with the lesser depth, but only if one of them has // depth greater than the total latency scheduled so far, otherwise either @@ -3513,7 +3500,6 @@ bool tryLatency(GenericSchedulerBase::SchedCandidate &TryCand, } return false; } -} // end namespace llvm static void tracePick(GenericSchedulerBase::CandReason Reason, bool IsTop, bool IsPostRA = false) { @@ -3798,14 +3784,12 @@ void GenericScheduler::registerRoots() { } } -namespace llvm { -bool tryPressure(const PressureChange &TryP, - const PressureChange &CandP, - GenericSchedulerBase::SchedCandidate &TryCand, - GenericSchedulerBase::SchedCandidate &Cand, - GenericSchedulerBase::CandReason Reason, - const TargetRegisterInfo *TRI, - const MachineFunction &MF) { +bool llvm::tryPressure(const PressureChange &TryP, const PressureChange &CandP, + GenericSchedulerBase::SchedCandidate &TryCand, + GenericSchedulerBase::SchedCandidate &Cand, + GenericSchedulerBase::CandReason Reason, + const TargetRegisterInfo *TRI, + const MachineFunction &MF) { // If one candidate decreases and the other increases, go with it. // Invalid candidates have UnitInc==0. if (tryGreater(TryP.getUnitInc() < 0, CandP.getUnitInc() < 0, TryCand, Cand, @@ -3838,7 +3822,7 @@ bool tryPressure(const PressureChange &TryP, return tryGreater(TryRank, CandRank, TryCand, Cand, Reason); } -unsigned getWeakLeft(const SUnit *SU, bool isTop) { +unsigned llvm::getWeakLeft(const SUnit *SU, bool isTop) { return (isTop) ? SU->WeakPredsLeft : SU->WeakSuccsLeft; } @@ -3849,7 +3833,7 @@ unsigned getWeakLeft(const SUnit *SU, bool isTop) { /// copies which can be prescheduled. The rest (e.g. x86 MUL) could be bundled /// with the operation that produces or consumes the physreg. We'll do this when /// regalloc has support for parallel copies. -int biasPhysReg(const SUnit *SU, bool isTop) { +int llvm::biasPhysReg(const SUnit *SU, bool isTop) { const MachineInstr *MI = SU->getInstr(); if (MI->isCopy()) { @@ -3884,7 +3868,6 @@ int biasPhysReg(const SUnit *SU, bool isTop) { return 0; } -} // end namespace llvm void GenericScheduler::initCandidate(SchedCandidate &Cand, SUnit *SU, bool AtTop, @@ -4812,13 +4795,13 @@ static MachineSchedRegistry ShufflerRegistry( //===----------------------------------------------------------------------===// #ifndef NDEBUG -namespace llvm { -template<> struct GraphTraits< - ScheduleDAGMI*> : public GraphTraits<ScheduleDAG*> {}; +template <> +struct llvm::GraphTraits<ScheduleDAGMI *> : public GraphTraits<ScheduleDAG *> { +}; -template<> -struct DOTGraphTraits<ScheduleDAGMI*> : public DefaultDOTGraphTraits { +template <> +struct llvm::DOTGraphTraits<ScheduleDAGMI *> : public DefaultDOTGraphTraits { DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {} static std::string getGraphName(const ScheduleDAG *G) { @@ -4878,7 +4861,6 @@ struct DOTGraphTraits<ScheduleDAGMI*> : public DefaultDOTGraphTraits { } }; -} // end namespace llvm #endif // NDEBUG /// viewGraph - Pop up a ghostview window with the reachable parts of the DAG diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp index d5153b7..cdcb29d9 100644 --- a/llvm/lib/CodeGen/MachineSink.cpp +++ b/llvm/lib/CodeGen/MachineSink.cpp @@ -1209,7 +1209,7 @@ MachineSinking::getBBRegisterPressure(const MachineBasicBlock &MBB, MIE = MBB.instr_begin(); MII != MIE; --MII) { const MachineInstr &MI = *std::prev(MII); - if (MI.isDebugInstr() || MI.isPseudoProbe()) + if (MI.isDebugOrPseudoInstr()) continue; RegisterOperands RegOpers; RegOpers.collect(MI, *TRI, *MRI, false, false); diff --git a/llvm/lib/CodeGen/MachineTraceMetrics.cpp b/llvm/lib/CodeGen/MachineTraceMetrics.cpp index c2d4aa0..9ac3f741 100644 --- a/llvm/lib/CodeGen/MachineTraceMetrics.cpp +++ b/llvm/lib/CodeGen/MachineTraceMetrics.cpp @@ -485,10 +485,7 @@ struct LoopBounds { // Specialize po_iterator_storage in order to prune the post-order traversal so // it is limited to the current loop and doesn't traverse the loop back edges. -namespace llvm { - -template<> -class po_iterator_storage<LoopBounds, true> { +template <> class llvm::po_iterator_storage<LoopBounds, true> { LoopBounds &LB; public: @@ -519,8 +516,6 @@ public: } }; -} // end namespace llvm - /// Compute the trace through MBB. void MachineTraceMetrics::Ensemble::computeTrace(const MachineBasicBlock *MBB) { LLVM_DEBUG(dbgs() << "Computing " << getName() << " trace through " diff --git a/llvm/lib/CodeGen/NonRelocatableStringpool.cpp b/llvm/lib/CodeGen/NonRelocatableStringpool.cpp index 087ac62..59c587c 100644 --- a/llvm/lib/CodeGen/NonRelocatableStringpool.cpp +++ b/llvm/lib/CodeGen/NonRelocatableStringpool.cpp @@ -9,7 +9,7 @@ #include "llvm/CodeGen/NonRelocatableStringpool.h" #include "llvm/ADT/STLExtras.h" -namespace llvm { +using namespace llvm; DwarfStringPoolEntryRef NonRelocatableStringpool::getEntry(StringRef S) { auto I = Strings.try_emplace(S); @@ -43,5 +43,3 @@ NonRelocatableStringpool::getEntriesForEmission() const { }); return Result; } - -} // namespace llvm diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp index ebfea8e..e17a214b 100644 --- a/llvm/lib/CodeGen/RegisterCoalescer.cpp +++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp @@ -2051,6 +2051,12 @@ bool RegisterCoalescer::joinCopy( } if (CP.getNewRC()) { + if (RegClassInfo.getNumAllocatableRegs(CP.getNewRC()) == 0) { + LLVM_DEBUG(dbgs() << "\tNo " << TRI->getRegClassName(CP.getNewRC()) + << "are available for allocation\n"); + return false; + } + auto SrcRC = MRI->getRegClass(CP.getSrcReg()); auto DstRC = MRI->getRegClass(CP.getDstReg()); unsigned SrcIdx = CP.getSrcIdx(); diff --git a/llvm/lib/CodeGen/RegisterPressure.cpp b/llvm/lib/CodeGen/RegisterPressure.cpp index 5f37890..7d4674b 100644 --- a/llvm/lib/CodeGen/RegisterPressure.cpp +++ b/llvm/lib/CodeGen/RegisterPressure.cpp @@ -858,7 +858,7 @@ void RegPressureTracker::recedeSkipDebugValues() { void RegPressureTracker::recede(SmallVectorImpl<VRegMaskOrUnit> *LiveUses) { recedeSkipDebugValues(); - if (CurrPos->isDebugInstr() || CurrPos->isPseudoProbe()) { + if (CurrPos->isDebugOrPseudoInstr()) { // It's possible to only have debug_value and pseudo probe instructions and // hit the start of the block. assert(CurrPos == MBB->begin()); diff --git a/llvm/lib/CodeGen/SafeStack.cpp b/llvm/lib/CodeGen/SafeStack.cpp index 6f373a5..e9ffa85 100644 --- a/llvm/lib/CodeGen/SafeStack.cpp +++ b/llvm/lib/CodeGen/SafeStack.cpp @@ -76,8 +76,6 @@ using namespace llvm::safestack; #define DEBUG_TYPE "safe-stack" -namespace llvm { - STATISTIC(NumFunctions, "Total number of functions"); STATISTIC(NumUnsafeStackFunctions, "Number of functions with unsafe stack"); STATISTIC(NumUnsafeStackRestorePointsFunctions, @@ -89,8 +87,6 @@ STATISTIC(NumUnsafeDynamicAllocas, "Number of unsafe dynamic allocas"); STATISTIC(NumUnsafeByValArguments, "Number of unsafe byval arguments"); STATISTIC(NumUnsafeStackRestorePoints, "Number of setjmps and landingpads"); -} // namespace llvm - /// Use __safestack_pointer_address even if the platform has a faster way of /// access safe stack pointer. static cl::opt<bool> diff --git a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp index eae2e8c..9662511 100644 --- a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -1551,14 +1551,10 @@ LLVM_DUMP_METHOD void ILPValue::dump() const { dbgs() << *this << '\n'; } -namespace llvm { - -LLVM_ATTRIBUTE_UNUSED -raw_ostream &operator<<(raw_ostream &OS, const ILPValue &Val) { +[[maybe_unused]] +raw_ostream &llvm::operator<<(raw_ostream &OS, const ILPValue &Val) { Val.print(OS); return OS; } -} // end namespace llvm - #endif diff --git a/llvm/lib/CodeGen/ScheduleDAGPrinter.cpp b/llvm/lib/CodeGen/ScheduleDAGPrinter.cpp index e7b1494..c80eade 100644 --- a/llvm/lib/CodeGen/ScheduleDAGPrinter.cpp +++ b/llvm/lib/CodeGen/ScheduleDAGPrinter.cpp @@ -16,57 +16,51 @@ #include "llvm/Support/raw_ostream.h" using namespace llvm; -namespace llvm { - template<> - struct DOTGraphTraits<ScheduleDAG*> : public DefaultDOTGraphTraits { +template <> +struct llvm::DOTGraphTraits<ScheduleDAG *> : public DefaultDOTGraphTraits { - DOTGraphTraits (bool isSimple=false) : DefaultDOTGraphTraits(isSimple) {} + DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {} - static std::string getGraphName(const ScheduleDAG *G) { - return std::string(G->MF.getName()); - } + static std::string getGraphName(const ScheduleDAG *G) { + return std::string(G->MF.getName()); + } - static bool renderGraphFromBottomUp() { - return true; - } + static bool renderGraphFromBottomUp() { return true; } - static bool isNodeHidden(const SUnit *Node, const ScheduleDAG *G) { - return (Node->NumPreds > 10 || Node->NumSuccs > 10); - } + static bool isNodeHidden(const SUnit *Node, const ScheduleDAG *G) { + return (Node->NumPreds > 10 || Node->NumSuccs > 10); + } - static std::string getNodeIdentifierLabel(const SUnit *Node, - const ScheduleDAG *Graph) { - std::string R; - raw_string_ostream OS(R); - OS << static_cast<const void *>(Node); - return R; - } + static std::string getNodeIdentifierLabel(const SUnit *Node, + const ScheduleDAG *Graph) { + std::string R; + raw_string_ostream OS(R); + OS << static_cast<const void *>(Node); + return R; + } - /// If you want to override the dot attributes printed for a particular - /// edge, override this method. - static std::string getEdgeAttributes(const SUnit *Node, - SUnitIterator EI, - const ScheduleDAG *Graph) { - if (EI.isArtificialDep()) - return "color=cyan,style=dashed"; - if (EI.isCtrlDep()) - return "color=blue,style=dashed"; - return ""; - } + /// If you want to override the dot attributes printed for a particular + /// edge, override this method. + static std::string getEdgeAttributes(const SUnit *Node, SUnitIterator EI, + const ScheduleDAG *Graph) { + if (EI.isArtificialDep()) + return "color=cyan,style=dashed"; + if (EI.isCtrlDep()) + return "color=blue,style=dashed"; + return ""; + } + std::string getNodeLabel(const SUnit *SU, const ScheduleDAG *Graph); + static std::string getNodeAttributes(const SUnit *N, + const ScheduleDAG *Graph) { + return "shape=Mrecord"; + } - std::string getNodeLabel(const SUnit *SU, const ScheduleDAG *Graph); - static std::string getNodeAttributes(const SUnit *N, - const ScheduleDAG *Graph) { - return "shape=Mrecord"; - } - - static void addCustomGraphFeatures(ScheduleDAG *G, - GraphWriter<ScheduleDAG*> &GW) { - return G->addCustomGraphFeatures(GW); - } - }; -} + static void addCustomGraphFeatures(ScheduleDAG *G, + GraphWriter<ScheduleDAG *> &GW) { + return G->addCustomGraphFeatures(GW); + } +}; std::string DOTGraphTraits<ScheduleDAG*>::getNodeLabel(const SUnit *SU, const ScheduleDAG *G) { diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index c5c3866..c97300d 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -509,6 +509,7 @@ namespace { SDValue visitFMUL(SDNode *N); template <class MatchContextClass> SDValue visitFMA(SDNode *N); SDValue visitFMAD(SDNode *N); + SDValue visitFMULADD(SDNode *N); SDValue visitFDIV(SDNode *N); SDValue visitFREM(SDNode *N); SDValue visitFSQRT(SDNode *N); @@ -657,13 +658,13 @@ namespace { bool InexpensiveOnly = false, std::optional<EVT> OutVT = std::nullopt); SDValue BuildDivEstimate(SDValue N, SDValue Op, SDNodeFlags Flags); - SDValue buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags); - SDValue buildSqrtEstimate(SDValue Op, SDNodeFlags Flags); - SDValue buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, bool Recip); + SDValue buildRsqrtEstimate(SDValue Op); + SDValue buildSqrtEstimate(SDValue Op); + SDValue buildSqrtEstimateImpl(SDValue Op, bool Recip); SDValue buildSqrtNROneConst(SDValue Arg, SDValue Est, unsigned Iterations, - SDNodeFlags Flags, bool Reciprocal); + bool Reciprocal); SDValue buildSqrtNRTwoConst(SDValue Arg, SDValue Est, unsigned Iterations, - SDNodeFlags Flags, bool Reciprocal); + bool Reciprocal); SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1, bool DemandHighBits = true); SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1); @@ -1991,6 +1992,7 @@ SDValue DAGCombiner::visit(SDNode *N) { case ISD::FMUL: return visitFMUL(N); case ISD::FMA: return visitFMA<EmptyMatchContext>(N); case ISD::FMAD: return visitFMAD(N); + case ISD::FMULADD: return visitFMULADD(N); case ISD::FDIV: return visitFDIV(N); case ISD::FREM: return visitFREM(N); case ISD::FSQRT: return visitFSQRT(N); @@ -5042,7 +5044,6 @@ static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) { unsigned Opc = N->getOpcode(); bool IsDiv = (ISD::SDIV == Opc) || (ISD::UDIV == Opc); - ConstantSDNode *N1C = isConstOrConstSplat(N1); // X / undef -> undef // X % undef -> undef @@ -5074,7 +5075,7 @@ static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) { // division-by-zero or remainder-by-zero, so assume the divisor is 1. // TODO: Similarly, if we're zero-extending a boolean divisor, then assume // it's a 1. - if ((N1C && N1C->isOne()) || (VT.getScalarType() == MVT::i1)) + if (isOneOrOneSplat(N1) || (VT.getScalarType() == MVT::i1)) return IsDiv ? N0 : DAG.getConstant(0, DL, VT); return SDValue(); @@ -10628,7 +10629,7 @@ SDValue DAGCombiner::visitSHL(SDNode *N) { // folding this will increase the total number of instructions. if (N0.getOpcode() == ISD::SRL && (N0.getOperand(1) == N1 || N0.hasOneUse()) && - TLI.shouldFoldConstantShiftPairToMask(N, Level)) { + TLI.shouldFoldConstantShiftPairToMask(N)) { if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount, /*AllowUndefs*/ false, /*AllowTypeMismatch*/ true)) { @@ -11207,7 +11208,7 @@ SDValue DAGCombiner::visitSRL(SDNode *N) { // fold (srl (shl x, c1), c2) -> (and (shl x, (sub c1, c2), MASK) or // (and (srl x, (sub c2, c1), MASK) if ((N0.getOperand(1) == N1 || N0->hasOneUse()) && - TLI.shouldFoldConstantShiftPairToMask(N, Level)) { + TLI.shouldFoldConstantShiftPairToMask(N)) { auto MatchShiftAmount = [OpSizeInBits](ConstantSDNode *LHS, ConstantSDNode *RHS) { const APInt &LHSC = LHS->getAPIntValue(); @@ -17086,11 +17087,6 @@ static bool isContractableFMUL(const TargetOptions &Options, SDValue N) { N->getFlags().hasAllowContract(); } -// Returns true if `N` can assume no infinities involved in its computation. -static bool hasNoInfs(const TargetOptions &Options, SDValue N) { - return Options.NoInfsFPMath || N->getFlags().hasNoInfs(); -} - /// Try to perform FMA combining on a given FADD node. template <class MatchContextClass> SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) { @@ -17666,7 +17662,7 @@ SDValue DAGCombiner::visitFMULForFMADistributiveCombine(SDNode *N) { // The transforms below are incorrect when x == 0 and y == inf, because the // intermediate multiplication produces a nan. SDValue FAdd = N0.getOpcode() == ISD::FADD ? N0 : N1; - if (!hasNoInfs(Options, FAdd)) + if (!FAdd->getFlags().hasNoInfs()) return SDValue(); // Floating-point multiply-add without intermediate rounding. @@ -17763,7 +17759,6 @@ SDValue DAGCombiner::visitFADD(SDNode *N) { bool N1CFP = DAG.isConstantFPBuildVectorOrConstantFP(N1); EVT VT = N->getValueType(0); SDLoc DL(N); - const TargetOptions &Options = DAG.getTarget().Options; SDNodeFlags Flags = N->getFlags(); SelectionDAG::FlagInserter FlagsInserter(DAG, N); @@ -17829,7 +17824,7 @@ SDValue DAGCombiner::visitFADD(SDNode *N) { bool AllowNewConst = (Level < AfterLegalizeDAG); // If nnan is enabled, fold lots of things. - if ((Options.NoNaNsFPMath || Flags.hasNoNaNs()) && AllowNewConst) { + if (Flags.hasNoNaNs() && AllowNewConst) { // If allowed, fold (fadd (fneg x), x) -> 0.0 if (N0.getOpcode() == ISD::FNEG && N0.getOperand(0) == N1) return DAG.getConstantFP(0.0, DL, VT); @@ -17978,7 +17973,6 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) { ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1, true); EVT VT = N->getValueType(0); SDLoc DL(N); - const TargetOptions &Options = DAG.getTarget().Options; const SDNodeFlags Flags = N->getFlags(); SelectionDAG::FlagInserter FlagsInserter(DAG, N); @@ -18006,7 +18000,7 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) { if (N0 == N1) { // (fsub x, x) -> 0.0 - if (Options.NoNaNsFPMath || Flags.hasNoNaNs()) + if (Flags.hasNoNaNs()) return DAG.getConstantFP(0.0f, DL, VT); } @@ -18317,7 +18311,6 @@ template <class MatchContextClass> SDValue DAGCombiner::visitFMA(SDNode *N) { ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2); EVT VT = N->getValueType(0); SDLoc DL(N); - const TargetOptions &Options = DAG.getTarget().Options; // FMA nodes have flags that propagate to the created nodes. SelectionDAG::FlagInserter FlagsInserter(DAG, N); MatchContextClass matcher(DAG, TLI, N); @@ -18343,8 +18336,7 @@ template <class MatchContextClass> SDValue DAGCombiner::visitFMA(SDNode *N) { return matcher.getNode(ISD::FMA, DL, VT, NegN0, NegN1, N2); } - if ((Options.NoNaNsFPMath && Options.NoInfsFPMath) || - (N->getFlags().hasNoNaNs() && N->getFlags().hasNoInfs())) { + if (N->getFlags().hasNoNaNs() && N->getFlags().hasNoInfs()) { if (N->getFlags().hasNoSignedZeros() || (N2CFP && !N2CFP->isExactlyValue(-0.0))) { if (N0CFP && N0CFP->isZero()) @@ -18449,6 +18441,21 @@ SDValue DAGCombiner::visitFMAD(SDNode *N) { return SDValue(); } +SDValue DAGCombiner::visitFMULADD(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + SDValue N2 = N->getOperand(2); + EVT VT = N->getValueType(0); + SDLoc DL(N); + + // Constant fold FMULADD. + if (SDValue C = + DAG.FoldConstantArithmetic(ISD::FMULADD, DL, VT, {N0, N1, N2})) + return C; + + return SDValue(); +} + // Combine multiple FDIVs with the same divisor into multiple FMULs by the // reciprocal. // E.g., (a / D; b / D;) -> (recip = 1.0 / D; a * recip; b * recip) @@ -18533,7 +18540,6 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) { SDValue N1 = N->getOperand(1); EVT VT = N->getValueType(0); SDLoc DL(N); - const TargetOptions &Options = DAG.getTarget().Options; SDNodeFlags Flags = N->getFlags(); SelectionDAG::FlagInserter FlagsInserter(DAG, N); @@ -18580,20 +18586,18 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) { // If this FDIV is part of a reciprocal square root, it may be folded // into a target-specific square root estimate instruction. if (N1.getOpcode() == ISD::FSQRT) { - if (SDValue RV = buildRsqrtEstimate(N1.getOperand(0), Flags)) + if (SDValue RV = buildRsqrtEstimate(N1.getOperand(0))) return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); } else if (N1.getOpcode() == ISD::FP_EXTEND && N1.getOperand(0).getOpcode() == ISD::FSQRT) { - if (SDValue RV = - buildRsqrtEstimate(N1.getOperand(0).getOperand(0), Flags)) { + if (SDValue RV = buildRsqrtEstimate(N1.getOperand(0).getOperand(0))) { RV = DAG.getNode(ISD::FP_EXTEND, SDLoc(N1), VT, RV); AddToWorklist(RV.getNode()); return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); } } else if (N1.getOpcode() == ISD::FP_ROUND && N1.getOperand(0).getOpcode() == ISD::FSQRT) { - if (SDValue RV = - buildRsqrtEstimate(N1.getOperand(0).getOperand(0), Flags)) { + if (SDValue RV = buildRsqrtEstimate(N1.getOperand(0).getOperand(0))) { RV = DAG.getNode(ISD::FP_ROUND, SDLoc(N1), VT, RV, N1.getOperand(1)); AddToWorklist(RV.getNode()); return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); @@ -18625,7 +18629,7 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) { SDValue AA = DAG.getNode(ISD::FMUL, DL, VT, A, A); SDValue AAZ = DAG.getNode(ISD::FMUL, DL, VT, AA, Sqrt.getOperand(0)); - if (SDValue Rsqrt = buildRsqrtEstimate(AAZ, Flags)) + if (SDValue Rsqrt = buildRsqrtEstimate(AAZ)) return DAG.getNode(ISD::FMUL, DL, VT, N0, Rsqrt); // Estimate creation failed. Clean up speculatively created nodes. @@ -18635,7 +18639,7 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) { // We found a FSQRT, so try to make this fold: // X / (Y * sqrt(Z)) -> X * (rsqrt(Z) / Y) - if (SDValue Rsqrt = buildRsqrtEstimate(Sqrt.getOperand(0), Flags)) { + if (SDValue Rsqrt = buildRsqrtEstimate(Sqrt.getOperand(0))) { SDValue Div = DAG.getNode(ISD::FDIV, SDLoc(N1), VT, Rsqrt, Y); AddToWorklist(Div.getNode()); return DAG.getNode(ISD::FMUL, DL, VT, N0, Div); @@ -18644,7 +18648,7 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) { } // Fold into a reciprocal estimate and multiply instead of a real divide. - if (Options.NoInfsFPMath || Flags.hasNoInfs()) + if (Flags.hasNoInfs()) if (SDValue RV = BuildDivEstimate(N0, N1, Flags)) return RV; } @@ -18721,12 +18725,10 @@ SDValue DAGCombiner::visitFREM(SDNode *N) { SDValue DAGCombiner::visitFSQRT(SDNode *N) { SDNodeFlags Flags = N->getFlags(); - const TargetOptions &Options = DAG.getTarget().Options; // Require 'ninf' flag since sqrt(+Inf) = +Inf, but the estimation goes as: // sqrt(+Inf) == rsqrt(+Inf) * +Inf = 0 * +Inf = NaN - if (!Flags.hasApproximateFuncs() || - (!Options.NoInfsFPMath && !Flags.hasNoInfs())) + if (!Flags.hasApproximateFuncs() || !Flags.hasNoInfs()) return SDValue(); SDValue N0 = N->getOperand(0); @@ -18734,11 +18736,12 @@ SDValue DAGCombiner::visitFSQRT(SDNode *N) { return SDValue(); // FSQRT nodes have flags that propagate to the created nodes. + SelectionDAG::FlagInserter FlagInserter(DAG, Flags); // TODO: If this is N0/sqrt(N0), and we reach this node before trying to // transform the fdiv, we may produce a sub-optimal estimate sequence // because the reciprocal calculation may not have to filter out a // 0.0 input. - return buildSqrtEstimate(N0, Flags); + return buildSqrtEstimate(N0); } /// copysign(x, fp_extend(y)) -> copysign(x, y) @@ -18870,27 +18873,38 @@ SDValue DAGCombiner::visitFPOW(SDNode *N) { static SDValue foldFPToIntToFP(SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const TargetLowering &TLI) { - // We only do this if the target has legal ftrunc. Otherwise, we'd likely be - // replacing casts with a libcall. We also must be allowed to ignore -0.0 - // because FTRUNC will return -0.0 for (-1.0, -0.0), but using integer - // conversions would return +0.0. + // We can fold the fpto[us]i -> [us]itofp pattern into a single ftrunc. + // If NoSignedZerosFPMath is enabled, this is a direct replacement. + // Otherwise, for strict math, we must handle edge cases: + // 1. For unsigned conversions, use FABS to handle negative cases. Take -0.0 + // as example, it first becomes integer 0, and is converted back to +0.0. + // FTRUNC on its own could produce -0.0. + // FIXME: We should be able to use node-level FMF here. - // TODO: If strict math, should we use FABS (+ range check for signed cast)? EVT VT = N->getValueType(0); - if (!TLI.isOperationLegal(ISD::FTRUNC, VT) || - !DAG.getTarget().Options.NoSignedZerosFPMath) + if (!TLI.isOperationLegal(ISD::FTRUNC, VT)) return SDValue(); // fptosi/fptoui round towards zero, so converting from FP to integer and // back is the same as an 'ftrunc': [us]itofp (fpto[us]i X) --> ftrunc X SDValue N0 = N->getOperand(0); if (N->getOpcode() == ISD::SINT_TO_FP && N0.getOpcode() == ISD::FP_TO_SINT && - N0.getOperand(0).getValueType() == VT) - return DAG.getNode(ISD::FTRUNC, DL, VT, N0.getOperand(0)); + N0.getOperand(0).getValueType() == VT) { + if (DAG.getTarget().Options.NoSignedZerosFPMath) + return DAG.getNode(ISD::FTRUNC, DL, VT, N0.getOperand(0)); + } if (N->getOpcode() == ISD::UINT_TO_FP && N0.getOpcode() == ISD::FP_TO_UINT && - N0.getOperand(0).getValueType() == VT) - return DAG.getNode(ISD::FTRUNC, DL, VT, N0.getOperand(0)); + N0.getOperand(0).getValueType() == VT) { + if (DAG.getTarget().Options.NoSignedZerosFPMath) + return DAG.getNode(ISD::FTRUNC, DL, VT, N0.getOperand(0)); + + // Strict math: use FABS to handle negative inputs correctly. + if (TLI.isFAbsFree(VT)) { + SDValue Abs = DAG.getNode(ISD::FABS, DL, VT, N0.getOperand(0)); + return DAG.getNode(ISD::FTRUNC, DL, VT, Abs); + } + } return SDValue(); } @@ -19340,8 +19354,10 @@ SDValue DAGCombiner::visitFMinMax(SDNode *N) { EVT VT = N->getValueType(0); const SDNodeFlags Flags = N->getFlags(); unsigned Opc = N->getOpcode(); - bool PropagatesNaN = Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM; - bool IsMin = Opc == ISD::FMINNUM || Opc == ISD::FMINIMUM; + bool PropAllNaNsToQNaNs = Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM; + bool PropOnlySNaNsToQNaNs = Opc == ISD::FMINNUM || Opc == ISD::FMAXNUM; + bool IsMin = + Opc == ISD::FMINNUM || Opc == ISD::FMINIMUM || Opc == ISD::FMINIMUMNUM; SelectionDAG::FlagInserter FlagsInserter(DAG, N); // Constant fold. @@ -19356,34 +19372,53 @@ SDValue DAGCombiner::visitFMinMax(SDNode *N) { if (const ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1)) { const APFloat &AF = N1CFP->getValueAPF(); - // minnum(X, nan) -> X - // maxnum(X, nan) -> X - // minimum(X, nan) -> nan - // maximum(X, nan) -> nan - if (AF.isNaN()) - return PropagatesNaN ? N->getOperand(1) : N->getOperand(0); + // minnum(X, qnan) -> X + // maxnum(X, qnan) -> X + // minnum(X, snan) -> qnan + // maxnum(X, snan) -> qnan + // minimum(X, nan) -> qnan + // maximum(X, nan) -> qnan + // minimumnum(X, nan) -> X + // maximumnum(X, nan) -> X + if (AF.isNaN()) { + if (PropAllNaNsToQNaNs || (AF.isSignaling() && PropOnlySNaNsToQNaNs)) { + if (AF.isSignaling()) + return DAG.getConstantFP(AF.makeQuiet(), SDLoc(N), VT); + return N->getOperand(1); + } + return N->getOperand(0); + } // In the following folds, inf can be replaced with the largest finite // float, if the ninf flag is set. if (AF.isInfinity() || (Flags.hasNoInfs() && AF.isLargest())) { - // minnum(X, -inf) -> -inf - // maxnum(X, +inf) -> +inf + // minnum(X, -inf) -> -inf (ignoring sNaN -> qNaN propagation) + // maxnum(X, +inf) -> +inf (ignoring sNaN -> qNaN propagation) // minimum(X, -inf) -> -inf if nnan // maximum(X, +inf) -> +inf if nnan - if (IsMin == AF.isNegative() && (!PropagatesNaN || Flags.hasNoNaNs())) + // minimumnum(X, -inf) -> -inf + // maximumnum(X, +inf) -> +inf + if (IsMin == AF.isNegative() && + (!PropAllNaNsToQNaNs || Flags.hasNoNaNs())) return N->getOperand(1); // minnum(X, +inf) -> X if nnan // maxnum(X, -inf) -> X if nnan - // minimum(X, +inf) -> X - // maximum(X, -inf) -> X - if (IsMin != AF.isNegative() && (PropagatesNaN || Flags.hasNoNaNs())) + // minimum(X, +inf) -> X (ignoring quieting of sNaNs) + // maximum(X, -inf) -> X (ignoring quieting of sNaNs) + // minimumnum(X, +inf) -> X if nnan + // maximumnum(X, -inf) -> X if nnan + if (IsMin != AF.isNegative() && (PropAllNaNsToQNaNs || Flags.hasNoNaNs())) return N->getOperand(0); } } + // There are no VECREDUCE variants of FMINIMUMNUM or FMAXIMUMNUM + if (Opc == ISD::FMINIMUMNUM || Opc == ISD::FMAXIMUMNUM) + return SDValue(); + if (SDValue SD = reassociateReduction( - PropagatesNaN + PropAllNaNsToQNaNs ? (IsMin ? ISD::VECREDUCE_FMINIMUM : ISD::VECREDUCE_FMAXIMUM) : (IsMin ? ISD::VECREDUCE_FMIN : ISD::VECREDUCE_FMAX), Opc, SDLoc(N), VT, N0, N1, Flags)) @@ -29703,28 +29738,27 @@ SDValue DAGCombiner::BuildDivEstimate(SDValue N, SDValue Op, /// X_{i+1} = X_i (1.5 - A X_i^2 / 2) /// As a result, we precompute A/2 prior to the iteration loop. SDValue DAGCombiner::buildSqrtNROneConst(SDValue Arg, SDValue Est, - unsigned Iterations, - SDNodeFlags Flags, bool Reciprocal) { + unsigned Iterations, bool Reciprocal) { EVT VT = Arg.getValueType(); SDLoc DL(Arg); SDValue ThreeHalves = DAG.getConstantFP(1.5, DL, VT); // We now need 0.5 * Arg which we can write as (1.5 * Arg - Arg) so that // this entire sequence requires only one FP constant. - SDValue HalfArg = DAG.getNode(ISD::FMUL, DL, VT, ThreeHalves, Arg, Flags); - HalfArg = DAG.getNode(ISD::FSUB, DL, VT, HalfArg, Arg, Flags); + SDValue HalfArg = DAG.getNode(ISD::FMUL, DL, VT, ThreeHalves, Arg); + HalfArg = DAG.getNode(ISD::FSUB, DL, VT, HalfArg, Arg); // Newton iterations: Est = Est * (1.5 - HalfArg * Est * Est) for (unsigned i = 0; i < Iterations; ++i) { - SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, Est, Flags); - NewEst = DAG.getNode(ISD::FMUL, DL, VT, HalfArg, NewEst, Flags); - NewEst = DAG.getNode(ISD::FSUB, DL, VT, ThreeHalves, NewEst, Flags); - Est = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags); + SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, Est); + NewEst = DAG.getNode(ISD::FMUL, DL, VT, HalfArg, NewEst); + NewEst = DAG.getNode(ISD::FSUB, DL, VT, ThreeHalves, NewEst); + Est = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst); } // If non-reciprocal square root is requested, multiply the result by Arg. if (!Reciprocal) - Est = DAG.getNode(ISD::FMUL, DL, VT, Est, Arg, Flags); + Est = DAG.getNode(ISD::FMUL, DL, VT, Est, Arg); return Est; } @@ -29735,8 +29769,7 @@ SDValue DAGCombiner::buildSqrtNROneConst(SDValue Arg, SDValue Est, /// => /// X_{i+1} = (-0.5 * X_i) * (A * X_i * X_i + (-3.0)) SDValue DAGCombiner::buildSqrtNRTwoConst(SDValue Arg, SDValue Est, - unsigned Iterations, - SDNodeFlags Flags, bool Reciprocal) { + unsigned Iterations, bool Reciprocal) { EVT VT = Arg.getValueType(); SDLoc DL(Arg); SDValue MinusThree = DAG.getConstantFP(-3.0, DL, VT); @@ -29749,9 +29782,9 @@ SDValue DAGCombiner::buildSqrtNRTwoConst(SDValue Arg, SDValue Est, // Newton iterations for reciprocal square root: // E = (E * -0.5) * ((A * E) * E + -3.0) for (unsigned i = 0; i < Iterations; ++i) { - SDValue AE = DAG.getNode(ISD::FMUL, DL, VT, Arg, Est, Flags); - SDValue AEE = DAG.getNode(ISD::FMUL, DL, VT, AE, Est, Flags); - SDValue RHS = DAG.getNode(ISD::FADD, DL, VT, AEE, MinusThree, Flags); + SDValue AE = DAG.getNode(ISD::FMUL, DL, VT, Arg, Est); + SDValue AEE = DAG.getNode(ISD::FMUL, DL, VT, AE, Est); + SDValue RHS = DAG.getNode(ISD::FADD, DL, VT, AEE, MinusThree); // When calculating a square root at the last iteration build: // S = ((A * E) * -0.5) * ((A * E) * E + -3.0) @@ -29759,13 +29792,13 @@ SDValue DAGCombiner::buildSqrtNRTwoConst(SDValue Arg, SDValue Est, SDValue LHS; if (Reciprocal || (i + 1) < Iterations) { // RSQRT: LHS = (E * -0.5) - LHS = DAG.getNode(ISD::FMUL, DL, VT, Est, MinusHalf, Flags); + LHS = DAG.getNode(ISD::FMUL, DL, VT, Est, MinusHalf); } else { // SQRT: LHS = (A * E) * -0.5 - LHS = DAG.getNode(ISD::FMUL, DL, VT, AE, MinusHalf, Flags); + LHS = DAG.getNode(ISD::FMUL, DL, VT, AE, MinusHalf); } - Est = DAG.getNode(ISD::FMUL, DL, VT, LHS, RHS, Flags); + Est = DAG.getNode(ISD::FMUL, DL, VT, LHS, RHS); } return Est; @@ -29774,8 +29807,7 @@ SDValue DAGCombiner::buildSqrtNRTwoConst(SDValue Arg, SDValue Est, /// Build code to calculate either rsqrt(Op) or sqrt(Op). In the latter case /// Op*rsqrt(Op) is actually computed, so additional postprocessing is needed if /// Op can be zero. -SDValue DAGCombiner::buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, - bool Reciprocal) { +SDValue DAGCombiner::buildSqrtEstimateImpl(SDValue Op, bool Reciprocal) { if (LegalDAG) return SDValue(); @@ -29803,8 +29835,8 @@ SDValue DAGCombiner::buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, if (Iterations > 0) Est = UseOneConstNR - ? buildSqrtNROneConst(Op, Est, Iterations, Flags, Reciprocal) - : buildSqrtNRTwoConst(Op, Est, Iterations, Flags, Reciprocal); + ? buildSqrtNROneConst(Op, Est, Iterations, Reciprocal) + : buildSqrtNRTwoConst(Op, Est, Iterations, Reciprocal); if (!Reciprocal) { SDLoc DL(Op); // Try the target specific test first. @@ -29822,12 +29854,12 @@ SDValue DAGCombiner::buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, return SDValue(); } -SDValue DAGCombiner::buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags) { - return buildSqrtEstimateImpl(Op, Flags, true); +SDValue DAGCombiner::buildRsqrtEstimate(SDValue Op) { + return buildSqrtEstimateImpl(Op, true); } -SDValue DAGCombiner::buildSqrtEstimate(SDValue Op, SDNodeFlags Flags) { - return buildSqrtEstimateImpl(Op, Flags, false); +SDValue DAGCombiner::buildSqrtEstimate(SDValue Op) { + return buildSqrtEstimateImpl(Op, false); } /// Return true if there is any possibility that the two addresses overlap. diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp index b5f8a61..437d0f4 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -3313,7 +3313,6 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) { case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break; // Unary FP Operations - case ISD::FABS: case ISD::FACOS: case ISD::FASIN: case ISD::FATAN: @@ -3329,7 +3328,6 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) { case ISD::FLOG2: case ISD::FLOG10: case ISD::FNEARBYINT: - case ISD::FNEG: case ISD::FREEZE: case ISD::FRINT: case ISD::FROUND: @@ -3341,6 +3339,12 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) { case ISD::FTAN: case ISD::FTANH: case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break; + case ISD::FABS: + R = SoftPromoteHalfRes_FABS(N); + break; + case ISD::FNEG: + R = SoftPromoteHalfRes_FNEG(N); + break; case ISD::AssertNoFPClass: R = SoftPromoteHalfRes_AssertNoFPClass(N); break; @@ -3670,6 +3674,24 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) { return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); } +SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FABS(SDNode *N) { + SDValue Op = GetSoftPromotedHalf(N->getOperand(0)); + SDLoc dl(N); + + // Clear the sign bit. + return DAG.getNode(ISD::AND, dl, MVT::i16, Op, + DAG.getConstant(0x7fff, dl, MVT::i16)); +} + +SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FNEG(SDNode *N) { + SDValue Op = GetSoftPromotedHalf(N->getOperand(0)); + SDLoc dl(N); + + // Invert the sign bit. + return DAG.getNode(ISD::XOR, dl, MVT::i16, Op, + DAG.getConstant(0x8000, dl, MVT::i16)); +} + SDValue DAGTypeLegalizer::SoftPromoteHalfRes_AssertNoFPClass(SDNode *N) { return GetSoftPromotedHalf(N->getOperand(0)); } diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h index d580ce0..603dc34 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -832,6 +832,8 @@ private: SDValue SoftPromoteHalfRes_SELECT(SDNode *N); SDValue SoftPromoteHalfRes_SELECT_CC(SDNode *N); SDValue SoftPromoteHalfRes_UnaryOp(SDNode *N); + SDValue SoftPromoteHalfRes_FABS(SDNode *N); + SDValue SoftPromoteHalfRes_FNEG(SDNode *N); SDValue SoftPromoteHalfRes_AssertNoFPClass(SDNode *N); SDValue SoftPromoteHalfRes_XINT_TO_FP(SDNode *N); SDValue SoftPromoteHalfRes_UNDEF(SDNode *N); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 6ea2e27..90edaf3 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -5063,8 +5063,7 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, break; case ISD::ADD: case ISD::ADDC: - // Add can have at most one carry bit. Thus we know that the output - // is, at worst, one more bit than the inputs. + // TODO: Move Operand 1 check before Operand 0 check Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1); if (Tmp == 1) return 1; // Early out. @@ -5088,6 +5087,9 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); if (Tmp2 == 1) return 1; // Early out. + + // Add can have at most one carry bit. Thus we know that the output + // is, at worst, one more bit than the inputs. return std::min(Tmp, Tmp2) - 1; case ISD::SUB: Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1); @@ -5767,11 +5769,7 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, // even if the nonan flag is dropped somewhere. unsigned CCOp = Opcode == ISD::SETCC ? 2 : 4; ISD::CondCode CCCode = cast<CondCodeSDNode>(Op.getOperand(CCOp))->get(); - if (((unsigned)CCCode & 0x10U)) - return true; - - const TargetOptions &Options = getTarget().Options; - return Options.NoNaNsFPMath || Options.NoInfsFPMath; + return (unsigned)CCCode & 0x10U; } case ISD::OR: @@ -5790,6 +5788,7 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, case ISD::FCOPYSIGN: case ISD::FMA: case ISD::FMAD: + case ISD::FMULADD: case ISD::FP_EXTEND: case ISD::FP_TO_SINT_SAT: case ISD::FP_TO_UINT_SAT: @@ -5908,6 +5907,7 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op, const APInt &DemandedElts, case ISD::FCOSH: case ISD::FTANH: case ISD::FMA: + case ISD::FMULADD: case ISD::FMAD: { if (SNaN) return true; @@ -6405,8 +6405,9 @@ static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT, if (VT.isScalableVector()) return SDValue(); - // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be - // simplified to one big BUILD_VECTOR. + // A CONCAT_VECTOR of scalar sources, such as UNDEF, BUILD_VECTOR and + // single-element INSERT_VECTOR_ELT operands can be simplified to one big + // BUILD_VECTOR. // FIXME: Add support for SCALAR_TO_VECTOR as well. EVT SVT = VT.getScalarType(); SmallVector<SDValue, 16> Elts; @@ -6416,6 +6417,10 @@ static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT, Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT)); else if (Op.getOpcode() == ISD::BUILD_VECTOR) Elts.append(Op->op_begin(), Op->op_end()); + else if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT && + OpVT.getVectorNumElements() == 1 && + isNullConstant(Op.getOperand(2))) + Elts.push_back(Op.getOperand(1)); else return SDValue(); } @@ -7235,7 +7240,7 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, } // Handle fma/fmad special cases. - if (Opcode == ISD::FMA || Opcode == ISD::FMAD) { + if (Opcode == ISD::FMA || Opcode == ISD::FMAD || Opcode == ISD::FMULADD) { assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); assert(Ops[0].getValueType() == VT && Ops[1].getValueType() == VT && Ops[2].getValueType() == VT && "FMA types must match!"); @@ -7246,7 +7251,7 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, APFloat V1 = C1->getValueAPF(); const APFloat &V2 = C2->getValueAPF(); const APFloat &V3 = C3->getValueAPF(); - if (Opcode == ISD::FMAD) { + if (Opcode == ISD::FMAD || Opcode == ISD::FMULADD) { V1.multiply(V2, APFloat::rmNearestTiesToEven); V1.add(V3, APFloat::rmNearestTiesToEven); } else @@ -8785,7 +8790,7 @@ static SDValue getMemcpyLoadsAndStores( if (Value.getNode()) { Store = DAG.getStore( Chain, dl, Value, - DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl), + DAG.getObjectPtrOffset(dl, Dst, TypeSize::getFixed(DstOff)), DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo); OutChains.push_back(Store); } @@ -8801,7 +8806,7 @@ static SDValue getMemcpyLoadsAndStores( assert(NVT.bitsGE(VT)); bool isDereferenceable = - SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL); + SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL); MachineMemOperand::Flags SrcMMOFlags = MMOFlags; if (isDereferenceable) SrcMMOFlags |= MachineMemOperand::MODereferenceable; @@ -8810,14 +8815,14 @@ static SDValue getMemcpyLoadsAndStores( Value = DAG.getExtLoad( ISD::EXTLOAD, dl, NVT, Chain, - DAG.getMemBasePlusOffset(Src, TypeSize::getFixed(SrcOff), dl), + DAG.getObjectPtrOffset(dl, Src, TypeSize::getFixed(SrcOff)), SrcPtrInfo.getWithOffset(SrcOff), VT, commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo); OutLoadChains.push_back(Value.getValue(1)); Store = DAG.getTruncStore( Chain, dl, Value, - DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl), + DAG.getObjectPtrOffset(dl, Dst, TypeSize::getFixed(DstOff)), DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags, NewAAInfo); OutStoreChains.push_back(Store); } @@ -8947,14 +8952,14 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Value; bool isDereferenceable = - SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL); + SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL); MachineMemOperand::Flags SrcMMOFlags = MMOFlags; if (isDereferenceable) SrcMMOFlags |= MachineMemOperand::MODereferenceable; Value = DAG.getLoad( VT, dl, Chain, - DAG.getMemBasePlusOffset(Src, TypeSize::getFixed(SrcOff), dl), + DAG.getObjectPtrOffset(dl, Src, TypeSize::getFixed(SrcOff)), SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags, NewAAInfo); LoadValues.push_back(Value); LoadChains.push_back(Value.getValue(1)); @@ -8969,7 +8974,7 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, Store = DAG.getStore( Chain, dl, LoadValues[i], - DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl), + DAG.getObjectPtrOffset(dl, Dst, TypeSize::getFixed(DstOff)), DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo); OutChains.push_back(Store); DstOff += VTSize; @@ -9101,7 +9106,7 @@ static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, assert(Value.getValueType() == VT && "Value with wrong type."); SDValue Store = DAG.getStore( Chain, dl, Value, - DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl), + DAG.getObjectPtrOffset(dl, Dst, TypeSize::getFixed(DstOff)), DstPtrInfo.getWithOffset(DstOff), Alignment, isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone, NewAAInfo); @@ -11848,25 +11853,38 @@ SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, /// getNodeIfExists - Get the specified node if it's already available, or /// else return NULL. SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, - ArrayRef<SDValue> Ops) { + ArrayRef<SDValue> Ops, + bool AllowCommute) { SDNodeFlags Flags; if (Inserter) Flags = Inserter->getFlags(); - return getNodeIfExists(Opcode, VTList, Ops, Flags); + return getNodeIfExists(Opcode, VTList, Ops, Flags, AllowCommute); } SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef<SDValue> Ops, - const SDNodeFlags Flags) { - if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) { + const SDNodeFlags Flags, + bool AllowCommute) { + if (VTList.VTs[VTList.NumVTs - 1] == MVT::Glue) + return nullptr; + + auto Lookup = [&](ArrayRef<SDValue> LookupOps) -> SDNode * { FoldingSetNodeID ID; - AddNodeIDNode(ID, Opcode, VTList, Ops); + AddNodeIDNode(ID, Opcode, VTList, LookupOps); void *IP = nullptr; - if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) { + if (SDNode *E = FindNodeOrInsertPos(ID, IP)) { E->intersectFlagsWith(Flags); return E; } - } + return nullptr; + }; + + if (SDNode *Existing = Lookup(Ops)) + return Existing; + + if (AllowCommute && TLI->isCommutativeBinOp(Opcode)) + return Lookup({Ops[1], Ops[0]}); + return nullptr; } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index c21890a..cb0038c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3971,8 +3971,14 @@ void SelectionDAGBuilder::visitSIToFP(const User &I) { } void SelectionDAGBuilder::visitPtrToAddr(const User &I) { - // FIXME: this is not correct for pointers with addr width != pointer width - visitPtrToInt(I); + SDValue N = getValue(I.getOperand(0)); + // By definition the type of the ptrtoaddr must be equal to the address type. + const auto &TLI = DAG.getTargetLoweringInfo(); + EVT AddrVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); + // The address width must be smaller or equal to the pointer representation + // width, so we lower ptrtoaddr as a truncate (possibly folded to a no-op). + N = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), AddrVT, N); + setValue(&I, N); } void SelectionDAGBuilder::visitPtrToInt(const User &I) { @@ -6996,6 +7002,13 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), getValue(I.getArgOperand(2)), Flags)); + } else if (TLI.isOperationLegalOrCustom(ISD::FMULADD, VT)) { + // TODO: Support splitting the vector. + setValue(&I, DAG.getNode(ISD::FMULADD, sdl, + getValue(I.getArgOperand(0)).getValueType(), + getValue(I.getArgOperand(0)), + getValue(I.getArgOperand(1)), + getValue(I.getArgOperand(2)), Flags)); } else { // TODO: Intrinsic calls should have fast-math-flags. SDValue Mul = DAG.getNode( diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp index fcfbfe6..39cbfad 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -310,6 +310,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::FMA: return "fma"; case ISD::STRICT_FMA: return "strict_fma"; case ISD::FMAD: return "fmad"; + case ISD::FMULADD: return "fmuladd"; case ISD::FREM: return "frem"; case ISD::STRICT_FREM: return "strict_frem"; case ISD::FCOPYSIGN: return "fcopysign"; diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 175753f..6c11c5b 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -234,6 +234,19 @@ static bool dontUseFastISelFor(const Function &Fn) { }); } +static bool maintainPGOProfile(const TargetMachine &TM, + CodeGenOptLevel OptLevel) { + if (OptLevel != CodeGenOptLevel::None) + return true; + if (TM.getPGOOption()) { + const PGOOptions &Options = *TM.getPGOOption(); + return Options.Action == PGOOptions::PGOAction::IRUse || + Options.Action == PGOOptions::PGOAction::SampleUse || + Options.CSAction == PGOOptions::CSPGOAction::CSIRUse; + } + return false; +} + namespace llvm { //===--------------------------------------------------------------------===// @@ -395,6 +408,7 @@ SelectionDAGISel::~SelectionDAGISel() { delete CurDAG; } void SelectionDAGISelLegacy::getAnalysisUsage(AnalysisUsage &AU) const { CodeGenOptLevel OptLevel = Selector->OptLevel; + bool RegisterPGOPasses = maintainPGOProfile(Selector->TM, Selector->OptLevel); if (OptLevel != CodeGenOptLevel::None) AU.addRequired<AAResultsWrapperPass>(); AU.addRequired<GCModuleInfo>(); @@ -403,15 +417,15 @@ void SelectionDAGISelLegacy::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired<TargetLibraryInfoWrapperPass>(); AU.addRequired<TargetTransformInfoWrapperPass>(); AU.addRequired<AssumptionCacheTracker>(); - if (UseMBPI && OptLevel != CodeGenOptLevel::None) - AU.addRequired<BranchProbabilityInfoWrapperPass>(); + if (UseMBPI && RegisterPGOPasses) + AU.addRequired<BranchProbabilityInfoWrapperPass>(); AU.addRequired<ProfileSummaryInfoWrapperPass>(); // AssignmentTrackingAnalysis only runs if assignment tracking is enabled for // the module. AU.addRequired<AssignmentTrackingAnalysis>(); AU.addPreserved<AssignmentTrackingAnalysis>(); - if (OptLevel != CodeGenOptLevel::None) - LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU); + if (RegisterPGOPasses) + LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU); MachineFunctionPass::getAnalysisUsage(AU); } @@ -464,6 +478,7 @@ void SelectionDAGISel::initializeAnalysisResults( (void)MatchFilterFuncName; #endif + bool RegisterPGOPasses = maintainPGOProfile(TM, OptLevel); TII = MF->getSubtarget().getInstrInfo(); TLI = MF->getSubtarget().getTargetLowering(); RegInfo = &MF->getRegInfo(); @@ -474,7 +489,7 @@ void SelectionDAGISel::initializeAnalysisResults( auto *PSI = MAMP.getCachedResult<ProfileSummaryAnalysis>(*Fn.getParent()); BlockFrequencyInfo *BFI = nullptr; FAM.getResult<BlockFrequencyAnalysis>(Fn); - if (PSI && PSI->hasProfileSummary() && OptLevel != CodeGenOptLevel::None) + if (PSI && PSI->hasProfileSummary() && RegisterPGOPasses) BFI = &FAM.getResult<BlockFrequencyAnalysis>(Fn); FunctionVarLocs const *FnVarLocs = nullptr; @@ -492,7 +507,7 @@ void SelectionDAGISel::initializeAnalysisResults( // into account). That's unfortunate but OK because it just means we won't // ask for passes that have been required anyway. - if (UseMBPI && OptLevel != CodeGenOptLevel::None) + if (UseMBPI && RegisterPGOPasses) FuncInfo->BPI = &FAM.getResult<BranchProbabilityAnalysis>(Fn); else FuncInfo->BPI = nullptr; @@ -518,6 +533,7 @@ void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) { (void)MatchFilterFuncName; #endif + bool RegisterPGOPasses = maintainPGOProfile(TM, OptLevel); TII = MF->getSubtarget().getInstrInfo(); TLI = MF->getSubtarget().getTargetLowering(); RegInfo = &MF->getRegInfo(); @@ -528,7 +544,7 @@ void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) { AC = &MFP.getAnalysis<AssumptionCacheTracker>().getAssumptionCache(Fn); auto *PSI = &MFP.getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); BlockFrequencyInfo *BFI = nullptr; - if (PSI && PSI->hasProfileSummary() && OptLevel != CodeGenOptLevel::None) + if (PSI && PSI->hasProfileSummary() && RegisterPGOPasses) BFI = &MFP.getAnalysis<LazyBlockFrequencyInfoPass>().getBFI(); FunctionVarLocs const *FnVarLocs = nullptr; @@ -549,7 +565,7 @@ void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) { // into account). That's unfortunate but OK because it just means we won't // ask for passes that have been required anyway. - if (UseMBPI && OptLevel != CodeGenOptLevel::None) + if (UseMBPI && RegisterPGOPasses) FuncInfo->BPI = &MFP.getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI(); else diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index cc503d3..920dff9 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -7676,6 +7676,7 @@ SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG, break; } case ISD::FMA: + case ISD::FMULADD: case ISD::FMAD: { if (!Flags.hasNoSignedZeros()) break; diff --git a/llvm/lib/CodeGen/StackFrameLayoutAnalysisPass.cpp b/llvm/lib/CodeGen/StackFrameLayoutAnalysisPass.cpp index 64e5cd5..95a9c3f 100644 --- a/llvm/lib/CodeGen/StackFrameLayoutAnalysisPass.cpp +++ b/llvm/lib/CodeGen/StackFrameLayoutAnalysisPass.cpp @@ -306,10 +306,7 @@ char &llvm::StackFrameLayoutAnalysisPassID = StackFrameLayoutAnalysisLegacy::ID; INITIALIZE_PASS(StackFrameLayoutAnalysisLegacy, "stack-frame-layout", "Stack Frame Layout", false, false) -namespace llvm { /// Returns a newly-created StackFrameLayout pass. -MachineFunctionPass *createStackFrameLayoutAnalysisPass() { +MachineFunctionPass *llvm::createStackFrameLayoutAnalysisPass() { return new StackFrameLayoutAnalysisLegacy(); } - -} // namespace llvm diff --git a/llvm/lib/CodeGen/StaticDataAnnotator.cpp b/llvm/lib/CodeGen/StaticDataAnnotator.cpp index 53a9ab4..eac20120 100644 --- a/llvm/lib/CodeGen/StaticDataAnnotator.cpp +++ b/llvm/lib/CodeGen/StaticDataAnnotator.cpp @@ -75,22 +75,11 @@ bool StaticDataAnnotator::runOnModule(Module &M) { bool Changed = false; for (auto &GV : M.globals()) { - if (GV.isDeclarationForLinker()) + if (!llvm::memprof::IsAnnotationOK(GV)) continue; - // The implementation below assumes prior passes don't set section prefixes, - // and specifically do 'assign' rather than 'update'. So report error if a - // section prefix is already set. - if (auto maybeSectionPrefix = GV.getSectionPrefix(); - maybeSectionPrefix && !maybeSectionPrefix->empty()) - llvm::report_fatal_error("Global variable " + GV.getName() + - " already has a section prefix " + - *maybeSectionPrefix); - StringRef SectionPrefix = SDPI->getConstantSectionPrefix(&GV, PSI); - if (SectionPrefix.empty()) - continue; - + // setSectionPrefix returns true if the section prefix is updated. Changed |= GV.setSectionPrefix(SectionPrefix); } diff --git a/llvm/lib/CodeGen/StaticDataSplitter.cpp b/llvm/lib/CodeGen/StaticDataSplitter.cpp index e22dc25..1593a40 100644 --- a/llvm/lib/CodeGen/StaticDataSplitter.cpp +++ b/llvm/lib/CodeGen/StaticDataSplitter.cpp @@ -130,10 +130,8 @@ StaticDataSplitter::getConstant(const MachineOperand &Op, if (Op.isGlobal()) { // Find global variables with local linkage. const GlobalVariable *GV = getLocalLinkageGlobalVariable(Op.getGlobal()); - // Skip 'llvm.'-prefixed global variables conservatively because they are - // often handled specially, and skip those not in static data - // sections. - if (!GV || GV->getName().starts_with("llvm.") || + // Skip those not eligible for annotation or not in static data sections. + if (!GV || !llvm::memprof::IsAnnotationOK(*GV) || !inStaticDataSection(*GV, TM)) return nullptr; return GV; diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp index c23281a..060b1dd 100644 --- a/llvm/lib/CodeGen/TargetLoweringBase.cpp +++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp @@ -815,7 +815,8 @@ void TargetLoweringBase::initActions() { ISD::FTAN, ISD::FACOS, ISD::FASIN, ISD::FATAN, ISD::FCOSH, ISD::FSINH, - ISD::FTANH, ISD::FATAN2}, + ISD::FTANH, ISD::FATAN2, + ISD::FMULADD}, VT, Expand); // Overflow operations default to expand diff --git a/llvm/lib/CodeGen/TargetRegisterInfo.cpp b/llvm/lib/CodeGen/TargetRegisterInfo.cpp index c9e4618..971f822 100644 --- a/llvm/lib/CodeGen/TargetRegisterInfo.cpp +++ b/llvm/lib/CodeGen/TargetRegisterInfo.cpp @@ -102,10 +102,8 @@ bool TargetRegisterInfo::checkAllSuperRegsMarked(const BitVector &RegisterSet, return true; } -namespace llvm { - -Printable printReg(Register Reg, const TargetRegisterInfo *TRI, - unsigned SubIdx, const MachineRegisterInfo *MRI) { +Printable llvm::printReg(Register Reg, const TargetRegisterInfo *TRI, + unsigned SubIdx, const MachineRegisterInfo *MRI) { return Printable([Reg, TRI, SubIdx, MRI](raw_ostream &OS) { if (!Reg) OS << "$noreg"; @@ -135,7 +133,7 @@ Printable printReg(Register Reg, const TargetRegisterInfo *TRI, }); } -Printable printRegUnit(unsigned Unit, const TargetRegisterInfo *TRI) { +Printable llvm::printRegUnit(unsigned Unit, const TargetRegisterInfo *TRI) { return Printable([Unit, TRI](raw_ostream &OS) { // Generic printout when TRI is missing. if (!TRI) { @@ -158,7 +156,7 @@ Printable printRegUnit(unsigned Unit, const TargetRegisterInfo *TRI) { }); } -Printable printVRegOrUnit(unsigned Unit, const TargetRegisterInfo *TRI) { +Printable llvm::printVRegOrUnit(unsigned Unit, const TargetRegisterInfo *TRI) { return Printable([Unit, TRI](raw_ostream &OS) { if (Register::isVirtualRegister(Unit)) { OS << '%' << Register(Unit).virtRegIndex(); @@ -168,8 +166,9 @@ Printable printVRegOrUnit(unsigned Unit, const TargetRegisterInfo *TRI) { }); } -Printable printRegClassOrBank(Register Reg, const MachineRegisterInfo &RegInfo, - const TargetRegisterInfo *TRI) { +Printable llvm::printRegClassOrBank(Register Reg, + const MachineRegisterInfo &RegInfo, + const TargetRegisterInfo *TRI) { return Printable([Reg, &RegInfo, TRI](raw_ostream &OS) { if (RegInfo.getRegClassOrNull(Reg)) OS << StringRef(TRI->getRegClassName(RegInfo.getRegClass(Reg))).lower(); @@ -183,8 +182,6 @@ Printable printRegClassOrBank(Register Reg, const MachineRegisterInfo &RegInfo, }); } -} // end namespace llvm - /// getAllocatableClass - Return the maximal subclass of the given register /// class that is alloctable, or NULL. const TargetRegisterClass * |