diff options
Diffstat (limited to 'llvm/lib/CodeGen')
-rw-r--r-- | llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp | 2 | ||||
-rw-r--r-- | llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp | 3 | ||||
-rw-r--r-- | llvm/lib/CodeGen/MIR2Vec.cpp | 48 | ||||
-rw-r--r-- | llvm/lib/CodeGen/MachinePipeliner.cpp | 25 | ||||
-rw-r--r-- | llvm/lib/CodeGen/RegisterCoalescer.cpp | 6 | ||||
-rw-r--r-- | llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 51 | ||||
-rw-r--r-- | llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp | 10 | ||||
-rw-r--r-- | llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp | 32 |
8 files changed, 121 insertions, 56 deletions
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp index fa0ccd6..906d62a3 100644 --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -1215,7 +1215,7 @@ bool CombinerHelper::isIndexedLoadStoreLegal(GLoadStore &LdSt) const { LLT MemTy = LdSt.getMMO().getMemoryType(); SmallVector<LegalityQuery::MemDesc, 2> MemDescrs( {{MemTy, MemTy.getSizeInBits().getKnownMinValue(), - AtomicOrdering::NotAtomic}}); + AtomicOrdering::NotAtomic, AtomicOrdering::NotAtomic}}); unsigned IndexedOpc = getIndexedOpc(LdSt.getOpcode()); SmallVector<LLT> OpTys; if (IndexedOpc == TargetOpcode::G_INDEXED_STORE) diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp index b2f8435..cdc1f64 100644 --- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp @@ -958,7 +958,8 @@ void LoadStoreOpt::initializeStoreMergeTargetInfo(unsigned AddrSpace) { for (unsigned Size = 2; Size <= MaxStoreSizeToForm; Size *= 2) { LLT Ty = LLT::scalar(Size); SmallVector<LegalityQuery::MemDesc, 2> MemDescrs( - {{Ty, Ty.getSizeInBits(), AtomicOrdering::NotAtomic}}); + {{Ty, Ty.getSizeInBits(), AtomicOrdering::NotAtomic, + AtomicOrdering::NotAtomic}}); SmallVector<LLT> StoreTys({Ty, PtrTy}); LegalityQuery Q(TargetOpcode::G_STORE, StoreTys, MemDescrs); LegalizeActionStep ActionStep = LI.getAction(Q); diff --git a/llvm/lib/CodeGen/MIR2Vec.cpp b/llvm/lib/CodeGen/MIR2Vec.cpp index 87565c0..e859765 100644 --- a/llvm/lib/CodeGen/MIR2Vec.cpp +++ b/llvm/lib/CodeGen/MIR2Vec.cpp @@ -49,14 +49,8 @@ cl::opt<float> OpcWeight("mir2vec-opc-weight", cl::Optional, cl::init(1.0), //===----------------------------------------------------------------------===// MIRVocabulary::MIRVocabulary(VocabMap &&OpcodeEntries, - const TargetInstrInfo *TII) - : TII(*TII) { - // Fixme: Use static factory methods for creating vocabularies instead of - // public constructors - // Early return for invalid inputs - creates empty/invalid vocabulary - if (!TII || OpcodeEntries.empty()) - return; - + const TargetInstrInfo &TII) + : TII(TII) { buildCanonicalOpcodeMapping(); unsigned CanonicalOpcodeCount = UniqueBaseOpcodeNames.size(); @@ -67,6 +61,15 @@ MIRVocabulary::MIRVocabulary(VocabMap &&OpcodeEntries, Layout.TotalEntries = Storage.size(); } +Expected<MIRVocabulary> MIRVocabulary::create(VocabMap &&Entries, + const TargetInstrInfo &TII) { + if (Entries.empty()) + return createStringError(errc::invalid_argument, + "Empty vocabulary entries provided"); + + return MIRVocabulary(std::move(Entries), TII); +} + std::string MIRVocabulary::extractBaseOpcodeName(StringRef InstrName) { // Extract base instruction name using regex to capture letters and // underscores Examples: "ADD32rr" -> "ADD", "ARITH_FENCE" -> "ARITH_FENCE" @@ -107,13 +110,11 @@ unsigned MIRVocabulary::getCanonicalIndexForBaseName(StringRef BaseName) const { } unsigned MIRVocabulary::getCanonicalOpcodeIndex(unsigned Opcode) const { - assert(isValid() && "MIR2Vec Vocabulary is invalid"); auto BaseOpcode = extractBaseOpcodeName(TII.getName(Opcode)); return getCanonicalIndexForBaseName(BaseOpcode); } std::string MIRVocabulary::getStringKey(unsigned Pos) const { - assert(isValid() && "MIR2Vec Vocabulary is invalid"); assert(Pos < Layout.TotalEntries && "Position out of bounds in vocabulary"); // For now, all entries are opcodes since we only have one section @@ -232,16 +233,11 @@ Error MIR2VecVocabLegacyAnalysis::readVocabulary() { return Error::success(); } -void MIR2VecVocabLegacyAnalysis::emitError(Error Err, LLVMContext &Ctx) { - Ctx.emitError(toString(std::move(Err))); -} - -mir2vec::MIRVocabulary +Expected<mir2vec::MIRVocabulary> MIR2VecVocabLegacyAnalysis::getMIR2VecVocabulary(const Module &M) { if (StrVocabMap.empty()) { if (Error Err = readVocabulary()) { - emitError(std::move(Err), M.getContext()); - return mir2vec::MIRVocabulary(std::move(StrVocabMap), nullptr); + return std::move(Err); } } @@ -255,15 +251,13 @@ MIR2VecVocabLegacyAnalysis::getMIR2VecVocabulary(const Module &M) { if (auto *MF = MMI.getMachineFunction(F)) { const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); - return mir2vec::MIRVocabulary(std::move(StrVocabMap), TII); + return mir2vec::MIRVocabulary::create(std::move(StrVocabMap), *TII); } } - // No machine functions available - return invalid vocabulary - emitError(make_error<StringError>("No machine functions found in module", - inconvertibleErrorCode()), - M.getContext()); - return mir2vec::MIRVocabulary(std::move(StrVocabMap), nullptr); + // No machine functions available - return error + return createStringError(errc::invalid_argument, + "No machine functions found in module"); } //===----------------------------------------------------------------------===// @@ -284,13 +278,15 @@ bool MIR2VecVocabPrinterLegacyPass::runOnMachineFunction(MachineFunction &MF) { bool MIR2VecVocabPrinterLegacyPass::doFinalization(Module &M) { auto &Analysis = getAnalysis<MIR2VecVocabLegacyAnalysis>(); - auto MIR2VecVocab = Analysis.getMIR2VecVocabulary(M); + auto MIR2VecVocabOrErr = Analysis.getMIR2VecVocabulary(M); - if (!MIR2VecVocab.isValid()) { - OS << "MIR2Vec Vocabulary Printer: Invalid vocabulary\n"; + if (!MIR2VecVocabOrErr) { + OS << "MIR2Vec Vocabulary Printer: Failed to get vocabulary - " + << toString(MIR2VecVocabOrErr.takeError()) << "\n"; return false; } + auto &MIR2VecVocab = *MIR2VecVocabOrErr; unsigned Pos = 0; for (const auto &Entry : MIR2VecVocab) { OS << "Key: " << MIR2VecVocab.getStringKey(Pos++) << ": "; diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp index 3a9651c..89ed4da 100644 --- a/llvm/lib/CodeGen/MachinePipeliner.cpp +++ b/llvm/lib/CodeGen/MachinePipeliner.cpp @@ -110,6 +110,7 @@ STATISTIC(NumFailZeroMII, "Pipeliner abort due to zero MII"); STATISTIC(NumFailNoSchedule, "Pipeliner abort due to no schedule found"); STATISTIC(NumFailZeroStage, "Pipeliner abort due to zero stage"); STATISTIC(NumFailLargeMaxStage, "Pipeliner abort due to too many stages"); +STATISTIC(NumFailTooManyStores, "Pipeliner abort due to too many stores"); /// A command line option to turn software pipelining on or off. static cl::opt<bool> EnableSWP("enable-pipeliner", cl::Hidden, cl::init(true), @@ -193,6 +194,13 @@ static cl::opt<bool> MVECodeGen("pipeliner-mve-cg", cl::Hidden, cl::init(false), cl::desc("Use the MVE code generator for software pipelining")); +/// A command line argument to limit the number of store instructions in the +/// target basic block. +static cl::opt<unsigned> SwpMaxNumStores( + "pipeliner-max-num-stores", + cl::desc("Maximum number of stores allwed in the target loop."), cl::Hidden, + cl::init(200)); + namespace llvm { // A command line option to enable the CopyToPhi DAG mutation. @@ -544,6 +552,23 @@ bool MachinePipeliner::canPipelineLoop(MachineLoop &L) { return false; } + unsigned NumStores = 0; + for (MachineInstr &MI : *L.getHeader()) + if (MI.mayStore()) + ++NumStores; + if (NumStores > SwpMaxNumStores) { + LLVM_DEBUG(dbgs() << "Too many stores\n"); + NumFailTooManyStores++; + ORE->emit([&]() { + return MachineOptimizationRemarkAnalysis(DEBUG_TYPE, "canPipelineLoop", + L.getStartLoc(), L.getHeader()) + << "Too many store instructions in the loop: " + << ore::NV("NumStores", NumStores) << " > " + << ore::NV("SwpMaxNumStores", SwpMaxNumStores) << "."; + }); + return false; + } + // Remove any subregisters from inputs to phi nodes. preprocessPhiNodes(*L.getHeader()); return true; diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp index ebfea8e..e17a214 100644 --- a/llvm/lib/CodeGen/RegisterCoalescer.cpp +++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp @@ -2051,6 +2051,12 @@ bool RegisterCoalescer::joinCopy( } if (CP.getNewRC()) { + if (RegClassInfo.getNumAllocatableRegs(CP.getNewRC()) == 0) { + LLVM_DEBUG(dbgs() << "\tNo " << TRI->getRegClassName(CP.getNewRC()) + << "are available for allocation\n"); + return false; + } + auto SrcRC = MRI->getRegClass(CP.getSrcReg()); auto DstRC = MRI->getRegClass(CP.getDstReg()); unsigned SrcIdx = CP.getSrcIdx(); diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index c5c3866..5ffdc4e 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -19340,8 +19340,10 @@ SDValue DAGCombiner::visitFMinMax(SDNode *N) { EVT VT = N->getValueType(0); const SDNodeFlags Flags = N->getFlags(); unsigned Opc = N->getOpcode(); - bool PropagatesNaN = Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM; - bool IsMin = Opc == ISD::FMINNUM || Opc == ISD::FMINIMUM; + bool PropAllNaNsToQNaNs = Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM; + bool PropOnlySNaNsToQNaNs = Opc == ISD::FMINNUM || Opc == ISD::FMAXNUM; + bool IsMin = + Opc == ISD::FMINNUM || Opc == ISD::FMINIMUM || Opc == ISD::FMINIMUMNUM; SelectionDAG::FlagInserter FlagsInserter(DAG, N); // Constant fold. @@ -19356,34 +19358,53 @@ SDValue DAGCombiner::visitFMinMax(SDNode *N) { if (const ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1)) { const APFloat &AF = N1CFP->getValueAPF(); - // minnum(X, nan) -> X - // maxnum(X, nan) -> X - // minimum(X, nan) -> nan - // maximum(X, nan) -> nan - if (AF.isNaN()) - return PropagatesNaN ? N->getOperand(1) : N->getOperand(0); + // minnum(X, qnan) -> X + // maxnum(X, qnan) -> X + // minnum(X, snan) -> qnan + // maxnum(X, snan) -> qnan + // minimum(X, nan) -> qnan + // maximum(X, nan) -> qnan + // minimumnum(X, nan) -> X + // maximumnum(X, nan) -> X + if (AF.isNaN()) { + if (PropAllNaNsToQNaNs || (AF.isSignaling() && PropOnlySNaNsToQNaNs)) { + if (AF.isSignaling()) + return DAG.getConstantFP(AF.makeQuiet(), SDLoc(N), VT); + return N->getOperand(1); + } + return N->getOperand(0); + } // In the following folds, inf can be replaced with the largest finite // float, if the ninf flag is set. if (AF.isInfinity() || (Flags.hasNoInfs() && AF.isLargest())) { - // minnum(X, -inf) -> -inf - // maxnum(X, +inf) -> +inf + // minnum(X, -inf) -> -inf (ignoring sNaN -> qNaN propagation) + // maxnum(X, +inf) -> +inf (ignoring sNaN -> qNaN propagation) // minimum(X, -inf) -> -inf if nnan // maximum(X, +inf) -> +inf if nnan - if (IsMin == AF.isNegative() && (!PropagatesNaN || Flags.hasNoNaNs())) + // minimumnum(X, -inf) -> -inf + // maximumnum(X, +inf) -> +inf + if (IsMin == AF.isNegative() && + (!PropAllNaNsToQNaNs || Flags.hasNoNaNs())) return N->getOperand(1); // minnum(X, +inf) -> X if nnan // maxnum(X, -inf) -> X if nnan - // minimum(X, +inf) -> X - // maximum(X, -inf) -> X - if (IsMin != AF.isNegative() && (PropagatesNaN || Flags.hasNoNaNs())) + // minimum(X, +inf) -> X (ignoring quieting of sNaNs) + // maximum(X, -inf) -> X (ignoring quieting of sNaNs) + // minimumnum(X, +inf) -> X if nnan + // maximumnum(X, -inf) -> X if nnan + if (IsMin != AF.isNegative() && (PropAllNaNsToQNaNs || Flags.hasNoNaNs())) return N->getOperand(0); } } + // There are no VECREDUCE variants of FMINIMUMNUM or FMAXIMUMNUM + if (Opc == ISD::FMINIMUMNUM || Opc == ISD::FMAXIMUMNUM) + return SDValue(); + if (SDValue SD = reassociateReduction( - PropagatesNaN + PropAllNaNsToQNaNs ? (IsMin ? ISD::VECREDUCE_FMINIMUM : ISD::VECREDUCE_FMAXIMUM) : (IsMin ? ISD::VECREDUCE_FMIN : ISD::VECREDUCE_FMAX), Opc, SDLoc(N), VT, N0, N1, Flags)) diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index 87d5453..3b5f83f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -3416,7 +3416,7 @@ void DAGTypeLegalizer::SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo, SDValue Input2 = N->getOperand(2); SDValue AccLo, AccHi; - std::tie(AccLo, AccHi) = DAG.SplitVector(Acc, DL); + GetSplitVector(Acc, AccLo, AccHi); unsigned Opcode = N->getOpcode(); // If the input types don't need splitting, just accumulate into the @@ -3429,8 +3429,8 @@ void DAGTypeLegalizer::SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo, SDValue Input1Lo, Input1Hi; SDValue Input2Lo, Input2Hi; - std::tie(Input1Lo, Input1Hi) = DAG.SplitVector(Input1, DL); - std::tie(Input2Lo, Input2Hi) = DAG.SplitVector(Input2, DL); + GetSplitVector(Input1, Input1Lo, Input1Hi); + GetSplitVector(Input2, Input2Lo, Input2Hi); EVT ResultVT = AccLo.getValueType(); Lo = DAG.getNode(Opcode, DL, ResultVT, AccLo, Input1Lo, Input2Lo); @@ -4761,8 +4761,8 @@ SDValue DAGTypeLegalizer::SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N) { SDLoc DL(N); SDValue Input1Lo, Input1Hi, Input2Lo, Input2Hi; - std::tie(Input1Lo, Input1Hi) = DAG.SplitVector(N->getOperand(1), DL); - std::tie(Input2Lo, Input2Hi) = DAG.SplitVector(N->getOperand(2), DL); + GetSplitVector(N->getOperand(1), Input1Lo, Input1Hi); + GetSplitVector(N->getOperand(2), Input2Lo, Input2Hi); unsigned Opcode = N->getOpcode(); EVT ResultVT = Acc.getValueType(); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 175753f..6c11c5b 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -234,6 +234,19 @@ static bool dontUseFastISelFor(const Function &Fn) { }); } +static bool maintainPGOProfile(const TargetMachine &TM, + CodeGenOptLevel OptLevel) { + if (OptLevel != CodeGenOptLevel::None) + return true; + if (TM.getPGOOption()) { + const PGOOptions &Options = *TM.getPGOOption(); + return Options.Action == PGOOptions::PGOAction::IRUse || + Options.Action == PGOOptions::PGOAction::SampleUse || + Options.CSAction == PGOOptions::CSPGOAction::CSIRUse; + } + return false; +} + namespace llvm { //===--------------------------------------------------------------------===// @@ -395,6 +408,7 @@ SelectionDAGISel::~SelectionDAGISel() { delete CurDAG; } void SelectionDAGISelLegacy::getAnalysisUsage(AnalysisUsage &AU) const { CodeGenOptLevel OptLevel = Selector->OptLevel; + bool RegisterPGOPasses = maintainPGOProfile(Selector->TM, Selector->OptLevel); if (OptLevel != CodeGenOptLevel::None) AU.addRequired<AAResultsWrapperPass>(); AU.addRequired<GCModuleInfo>(); @@ -403,15 +417,15 @@ void SelectionDAGISelLegacy::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired<TargetLibraryInfoWrapperPass>(); AU.addRequired<TargetTransformInfoWrapperPass>(); AU.addRequired<AssumptionCacheTracker>(); - if (UseMBPI && OptLevel != CodeGenOptLevel::None) - AU.addRequired<BranchProbabilityInfoWrapperPass>(); + if (UseMBPI && RegisterPGOPasses) + AU.addRequired<BranchProbabilityInfoWrapperPass>(); AU.addRequired<ProfileSummaryInfoWrapperPass>(); // AssignmentTrackingAnalysis only runs if assignment tracking is enabled for // the module. AU.addRequired<AssignmentTrackingAnalysis>(); AU.addPreserved<AssignmentTrackingAnalysis>(); - if (OptLevel != CodeGenOptLevel::None) - LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU); + if (RegisterPGOPasses) + LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU); MachineFunctionPass::getAnalysisUsage(AU); } @@ -464,6 +478,7 @@ void SelectionDAGISel::initializeAnalysisResults( (void)MatchFilterFuncName; #endif + bool RegisterPGOPasses = maintainPGOProfile(TM, OptLevel); TII = MF->getSubtarget().getInstrInfo(); TLI = MF->getSubtarget().getTargetLowering(); RegInfo = &MF->getRegInfo(); @@ -474,7 +489,7 @@ void SelectionDAGISel::initializeAnalysisResults( auto *PSI = MAMP.getCachedResult<ProfileSummaryAnalysis>(*Fn.getParent()); BlockFrequencyInfo *BFI = nullptr; FAM.getResult<BlockFrequencyAnalysis>(Fn); - if (PSI && PSI->hasProfileSummary() && OptLevel != CodeGenOptLevel::None) + if (PSI && PSI->hasProfileSummary() && RegisterPGOPasses) BFI = &FAM.getResult<BlockFrequencyAnalysis>(Fn); FunctionVarLocs const *FnVarLocs = nullptr; @@ -492,7 +507,7 @@ void SelectionDAGISel::initializeAnalysisResults( // into account). That's unfortunate but OK because it just means we won't // ask for passes that have been required anyway. - if (UseMBPI && OptLevel != CodeGenOptLevel::None) + if (UseMBPI && RegisterPGOPasses) FuncInfo->BPI = &FAM.getResult<BranchProbabilityAnalysis>(Fn); else FuncInfo->BPI = nullptr; @@ -518,6 +533,7 @@ void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) { (void)MatchFilterFuncName; #endif + bool RegisterPGOPasses = maintainPGOProfile(TM, OptLevel); TII = MF->getSubtarget().getInstrInfo(); TLI = MF->getSubtarget().getTargetLowering(); RegInfo = &MF->getRegInfo(); @@ -528,7 +544,7 @@ void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) { AC = &MFP.getAnalysis<AssumptionCacheTracker>().getAssumptionCache(Fn); auto *PSI = &MFP.getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); BlockFrequencyInfo *BFI = nullptr; - if (PSI && PSI->hasProfileSummary() && OptLevel != CodeGenOptLevel::None) + if (PSI && PSI->hasProfileSummary() && RegisterPGOPasses) BFI = &MFP.getAnalysis<LazyBlockFrequencyInfoPass>().getBFI(); FunctionVarLocs const *FnVarLocs = nullptr; @@ -549,7 +565,7 @@ void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) { // into account). That's unfortunate but OK because it just means we won't // ask for passes that have been required anyway. - if (UseMBPI && OptLevel != CodeGenOptLevel::None) + if (UseMBPI && RegisterPGOPasses) FuncInfo->BPI = &MFP.getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI(); else |