diff options
| -rw-r--r-- | polly/include/polly/CodeGen/BlockGenerators.h | 155 | ||||
| -rw-r--r-- | polly/include/polly/ScopInfo.h | 10 | ||||
| -rw-r--r-- | polly/lib/Analysis/ScopInfo.cpp | 2 | ||||
| -rw-r--r-- | polly/lib/CodeGen/BlockGenerators.cpp | 188 | ||||
| -rw-r--r-- | polly/lib/CodeGen/IslCodeGeneration.cpp | 12 |
5 files changed, 186 insertions, 181 deletions
diff --git a/polly/include/polly/CodeGen/BlockGenerators.h b/polly/include/polly/CodeGen/BlockGenerators.h index 16e60be..6ccf801 100644 --- a/polly/include/polly/CodeGen/BlockGenerators.h +++ b/polly/include/polly/CodeGen/BlockGenerators.h @@ -67,41 +67,38 @@ bool isIgnoredIntrinsic(const llvm::Value *V); /// The only public function exposed is generate(). class BlockGenerator { public: - /// @brief Generate a new BasicBlock for a ScopStmt. + /// @brief Create a generator for basic blocks. /// - /// @param Builder The LLVM-IR Builder used to generate the statement. The - /// code is generated at the location, the Builder points to. - /// @param Stmt The statement to code generate. - /// @param GlobalMap A map that defines for certain Values referenced from the - /// original code new Values they should be replaced with. - /// @param P A reference to the pass this function is called from. - /// The pass is needed to update other analysis. - /// @param LI The loop info for the current function - /// @param SE The scalar evolution info for the current function - /// @param Build The AST build with the new schedule. + /// @param Builder The LLVM-IR Builder used to generate the statement. The + /// code is generated at the location, the Builder points + /// to. + /// @param P A reference to the pass this function is called from. + /// The pass is needed to update other analysis. + /// @param LI The loop info for the current function + /// @param SE The scalar evolution info for the current function /// @param ExprBuilder An expression builder to generate new access functions. - static void generate(PollyIRBuilder &Builder, ScopStmt &Stmt, - ValueMapT &GlobalMap, LoopToScevMapT <S, Pass *P, - LoopInfo &LI, ScalarEvolution &SE, - __isl_keep isl_ast_build *Build = nullptr, - IslExprBuilder *ExprBuilder = nullptr) { - BlockGenerator Generator(Builder, Stmt, P, LI, SE, Build, ExprBuilder); - Generator.copyBB(GlobalMap, LTS); - } + BlockGenerator(PollyIRBuilder &Builder, Pass *P, LoopInfo &LI, + ScalarEvolution &SE, IslExprBuilder *ExprBuilder = nullptr); + + /// @brief Copy the basic block. + /// + /// This copies the entire basic block and updates references to old values + /// with references to new values, as defined by GlobalMap. + /// + /// @param Stmt The statement to code generate. + /// @param GlobalMap A mapping from old values to their new values + /// (for values recalculated in the new ScoP, but not + /// within this basic block). + /// @param LTS A map from old loops to new induction variables as SCEVs. + void copyBB(ScopStmt &Stmt, ValueMapT &GlobalMap, LoopToScevMapT <S); protected: PollyIRBuilder &Builder; - ScopStmt &Statement; Pass *P; LoopInfo &LI; ScalarEvolution &SE; - isl_ast_build *Build; IslExprBuilder *ExprBuilder; - BlockGenerator(PollyIRBuilder &B, ScopStmt &Stmt, Pass *P, LoopInfo &LI, - ScalarEvolution &SE, __isl_keep isl_ast_build *Build, - IslExprBuilder *ExprBuilder); - /// @brief Get the new version of a value. /// /// Given an old value, we first check if a new version of this value is @@ -111,6 +108,7 @@ protected: /// we return the old value. If the value can still not be derived, this /// function will assert. /// + /// @param Stmt The statement to code generate. /// @param Old The old Value. /// @param BBMap A mapping from old values to their new values /// (for values recalculated within this basic block). @@ -128,10 +126,10 @@ protected: /// @returns o The old value, if it is still valid. /// o The new value, if available. /// o NULL, if no value is found. - Value *getNewValue(const Value *Old, ValueMapT &BBMap, ValueMapT &GlobalMap, - LoopToScevMapT <S, Loop *L) const; + Value *getNewValue(ScopStmt &Stmt, const Value *Old, ValueMapT &BBMap, + ValueMapT &GlobalMap, LoopToScevMapT <S, Loop *L) const; - void copyInstScalar(const Instruction *Inst, ValueMapT &BBMap, + void copyInstScalar(ScopStmt &Stmt, const Instruction *Inst, ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S); /// @brief Get the innermost loop that surrounds an instruction. @@ -141,41 +139,40 @@ protected: Loop *getLoopForInst(const Instruction *Inst); /// @brief Get the new operand address according to access relation of @p MA. - Value *getNewAccessOperand(const MemoryAccess &MA); + Value *getNewAccessOperand(ScopStmt &Stmt, const MemoryAccess &MA); /// @brief Generate the operand address - Value *generateLocationAccessed(const Instruction *Inst, const Value *Pointer, - ValueMapT &BBMap, ValueMapT &GlobalMap, - LoopToScevMapT <S); + Value *generateLocationAccessed(ScopStmt &Stmt, const Instruction *Inst, + const Value *Pointer, ValueMapT &BBMap, + ValueMapT &GlobalMap, LoopToScevMapT <S); - Value *generateScalarLoad(const LoadInst *load, ValueMapT &BBMap, - ValueMapT &GlobalMap, LoopToScevMapT <S); + Value *generateScalarLoad(ScopStmt &Stmt, const LoadInst *load, + ValueMapT &BBMap, ValueMapT &GlobalMap, + LoopToScevMapT <S); - Value *generateScalarStore(const StoreInst *store, ValueMapT &BBMap, - ValueMapT &GlobalMap, LoopToScevMapT <S); + Value *generateScalarStore(ScopStmt &Stmt, const StoreInst *store, + ValueMapT &BBMap, ValueMapT &GlobalMap, + LoopToScevMapT <S); /// @brief Copy a single Instruction. /// /// This copies a single Instruction and updates references to old values /// with references to new values, as defined by GlobalMap and BBMap. /// + /// @param Stmt The statement to code generate. + /// @param Inst The instruction to copy. /// @param BBMap A mapping from old values to their new values /// (for values recalculated within this basic block). /// @param GlobalMap A mapping from old values to their new values /// (for values recalculated in the new ScoP, but not /// within this basic block). - void copyInstruction(const Instruction *Inst, ValueMapT &BBMap, - ValueMapT &GlobalMap, LoopToScevMapT <S); - - /// @brief Copy the basic block. - /// - /// This copies the entire basic block and updates references to old values - /// with references to new values, as defined by GlobalMap. - /// - /// @param GlobalMap A mapping from old values to their new values + /// @param LTS A mapping from loops virtual canonical induction + /// variable to their new values /// (for values recalculated in the new ScoP, but not - /// within this basic block). - void copyBB(ValueMapT &GlobalMap, LoopToScevMapT <S); + /// within this basic block). + void copyInstruction(ScopStmt &Stmt, const Instruction *Inst, + ValueMapT &BBMap, ValueMapT &GlobalMap, + LoopToScevMapT <S); }; /// @brief Generate a new vector basic block for a polyhedral statement. @@ -191,6 +188,7 @@ public: /// instructions, but e.g. for address calculation instructions we currently /// generate scalar instructions for each vector lane. /// + /// @param BlockGen A block generator object used as parent. /// @param Stmt The statement to code generate. /// @param GlobalMaps A vector of maps that define for certain Values /// referenced from the original code new Values they should @@ -198,25 +196,19 @@ public: /// used for one vector lane. The number of elements in the /// vector defines the width of the generated vector /// instructions. + /// @param VLTS A mapping from loops virtual canonical induction + /// variable to their new values + /// (for values recalculated in the new ScoP, but not + /// within this basic block), one for each lane. /// @param Schedule A map from the statement to a schedule where the /// innermost dimension is the dimension of the innermost /// loop containing the statemenet. - /// @param P A reference to the pass this function is called from. - /// The pass is needed to update other analysis. - /// @param LI The loop info for the current function - /// @param SE The scalar evolution info for the current function - /// @param Build The AST build with the new schedule. - /// @param ExprBuilder An expression builder to generate new access functions. - static void generate(PollyIRBuilder &B, ScopStmt &Stmt, + static void generate(BlockGenerator &BlockGen, ScopStmt &Stmt, VectorValueMapT &GlobalMaps, std::vector<LoopToScevMapT> &VLTS, - __isl_keep isl_map *Schedule, Pass *P, LoopInfo &LI, - ScalarEvolution &SE, - __isl_keep isl_ast_build *Build = nullptr, - IslExprBuilder *ExprBuilder = nullptr) { - VectorBlockGenerator Generator(B, GlobalMaps, VLTS, Stmt, Schedule, P, LI, - SE, Build, ExprBuilder); - Generator.copyBB(); + __isl_keep isl_map *Schedule) { + VectorBlockGenerator Generator(BlockGen, GlobalMaps, VLTS, Schedule); + Generator.copyBB(Stmt); } private: @@ -249,16 +241,13 @@ private: // dimension of the innermost loop containing the statemenet. isl_map *Schedule; - VectorBlockGenerator(PollyIRBuilder &B, VectorValueMapT &GlobalMaps, - std::vector<LoopToScevMapT> &VLTS, ScopStmt &Stmt, - __isl_keep isl_map *Schedule, Pass *P, LoopInfo &LI, - ScalarEvolution &SE, - __isl_keep isl_ast_build *Build = nullptr, - IslExprBuilder *ExprBuilder = nullptr); + VectorBlockGenerator(BlockGenerator &BlockGen, VectorValueMapT &GlobalMaps, + std::vector<LoopToScevMapT> &VLTS, + __isl_keep isl_map *Schedule); int getVectorWidth(); - Value *getVectorValue(const Value *Old, ValueMapT &VectorMap, + Value *getVectorValue(ScopStmt &Stmt, const Value *Old, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps, Loop *L); Type *getVectorPtrTy(const Value *V, int Width); @@ -271,13 +260,14 @@ private: /// %vector_ptr= bitcast double* %p to <4 x double>* /// %vec_full = load <4 x double>* %vector_ptr /// + /// @param Stmt The statement to code generate. /// @param NegativeStride This is used to indicate a -1 stride. In such /// a case we load the end of a base address and /// shuffle the accesses in reverse order into the /// vector. By default we would do only positive /// strides. /// - Value *generateStrideOneLoad(const LoadInst *Load, + Value *generateStrideOneLoad(ScopStmt &Stmt, const LoadInst *Load, VectorValueMapT &ScalarMaps, bool NegativeStride); @@ -291,7 +281,8 @@ private: /// %splat = shufflevector <1 x double> %splat_one, <1 x /// double> %splat_one, <4 x i32> zeroinitializer /// - Value *generateStrideZeroLoad(const LoadInst *Load, ValueMapT &BBMap); + Value *generateStrideZeroLoad(ScopStmt &Stmt, const LoadInst *Load, + ValueMapT &BBMap); /// @brief Load a vector from scalars distributed in memory /// @@ -304,33 +295,33 @@ private: /// %scalar 2 = load double* %p_2 /// %vec_2 = insertelement <2 x double> %vec_1, double %scalar_1, i32 1 /// - Value *generateUnknownStrideLoad(const LoadInst *Load, + Value *generateUnknownStrideLoad(ScopStmt &Stmt, const LoadInst *Load, VectorValueMapT &ScalarMaps); - void generateLoad(const LoadInst *Load, ValueMapT &VectorMap, + void generateLoad(ScopStmt &Stmt, const LoadInst *Load, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); - void copyUnaryInst(const UnaryInstruction *Inst, ValueMapT &VectorMap, - VectorValueMapT &ScalarMaps); + void copyUnaryInst(ScopStmt &Stmt, const UnaryInstruction *Inst, + ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); - void copyBinaryInst(const BinaryOperator *Inst, ValueMapT &VectorMap, - VectorValueMapT &ScalarMaps); + void copyBinaryInst(ScopStmt &Stmt, const BinaryOperator *Inst, + ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); - void copyStore(const StoreInst *Store, ValueMapT &VectorMap, + void copyStore(ScopStmt &Stmt, const StoreInst *Store, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); - void copyInstScalarized(const Instruction *Inst, ValueMapT &VectorMap, - VectorValueMapT &ScalarMaps); + void copyInstScalarized(ScopStmt &Stmt, const Instruction *Inst, + ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); bool extractScalarValues(const Instruction *Inst, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); bool hasVectorOperands(const Instruction *Inst, ValueMapT &VectorMap); - void copyInstruction(const Instruction *Inst, ValueMapT &VectorMap, - VectorValueMapT &ScalarMaps); + void copyInstruction(ScopStmt &Stmt, const Instruction *Inst, + ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); - void copyBB(); + void copyBB(ScopStmt &Stmt); }; } #endif diff --git a/polly/include/polly/ScopInfo.h b/polly/include/polly/ScopInfo.h index df13781..b04a3f5 100644 --- a/polly/include/polly/ScopInfo.h +++ b/polly/include/polly/ScopInfo.h @@ -46,6 +46,7 @@ struct isl_set; struct isl_union_set; struct isl_union_map; struct isl_space; +struct isl_ast_build; struct isl_constraint; struct isl_pw_multi_aff; @@ -426,6 +427,9 @@ class ScopStmt { /// The BasicBlock represented by this statement. BasicBlock *BB; + /// @brief The isl AST build for the new generated AST. + isl_ast_build *Build; + std::vector<Loop *> NestLoops; std::string BaseName; @@ -559,6 +563,12 @@ public: const char *getBaseName() const; + /// @brief Set the isl AST build. + void setAstBuild(__isl_keep isl_ast_build *B) { Build = B; } + + /// @brief Get the isl AST build. + __isl_keep isl_ast_build *getAstBuild() const { return Build; } + /// @brief Restrict the domain of the statement. /// /// @param NewDomain The new statement domain. diff --git a/polly/lib/Analysis/ScopInfo.cpp b/polly/lib/Analysis/ScopInfo.cpp index f33b3e0..829c11a 100644 --- a/polly/lib/Analysis/ScopInfo.cpp +++ b/polly/lib/Analysis/ScopInfo.cpp @@ -907,7 +907,7 @@ void ScopStmt::deriveAssumptions() { ScopStmt::ScopStmt(Scop &parent, TempScop &tempScop, const Region &CurRegion, BasicBlock &bb, SmallVectorImpl<Loop *> &Nest, SmallVectorImpl<unsigned> &Scatter) - : Parent(parent), BB(&bb), NestLoops(Nest.size()) { + : Parent(parent), BB(&bb), Build(nullptr), NestLoops(Nest.size()) { // Setup the induction variables. for (unsigned i = 0, e = Nest.size(); i < e; ++i) NestLoops[i] = Nest[i]; diff --git a/polly/lib/CodeGen/BlockGenerators.cpp b/polly/lib/CodeGen/BlockGenerators.cpp index d7ccea6..6790b9d 100644 --- a/polly/lib/CodeGen/BlockGenerators.cpp +++ b/polly/lib/CodeGen/BlockGenerators.cpp @@ -76,16 +76,13 @@ bool polly::isIgnoredIntrinsic(const Value *V) { return false; } -BlockGenerator::BlockGenerator(PollyIRBuilder &B, ScopStmt &Stmt, Pass *P, - LoopInfo &LI, ScalarEvolution &SE, - isl_ast_build *Build, - IslExprBuilder *ExprBuilder) - : Builder(B), Statement(Stmt), P(P), LI(LI), SE(SE), Build(Build), - ExprBuilder(ExprBuilder) {} - -Value *BlockGenerator::getNewValue(const Value *Old, ValueMapT &BBMap, - ValueMapT &GlobalMap, LoopToScevMapT <S, - Loop *L) const { +BlockGenerator::BlockGenerator(PollyIRBuilder &B, Pass *P, LoopInfo &LI, + ScalarEvolution &SE, IslExprBuilder *ExprBuilder) + : Builder(B), P(P), LI(LI), SE(SE), ExprBuilder(ExprBuilder) {} + +Value *BlockGenerator::getNewValue(ScopStmt &Stmt, const Value *Old, + ValueMapT &BBMap, ValueMapT &GlobalMap, + LoopToScevMapT <S, Loop *L) const { // We assume constants never change. // This avoids map lookups for many calls to this function. if (isa<Constant>(Old)) @@ -125,7 +122,7 @@ Value *BlockGenerator::getNewValue(const Value *Old, ValueMapT &BBMap, // A scop-constant value defined by an instruction executed outside the scop. if (const Instruction *Inst = dyn_cast<Instruction>(Old)) - if (!Statement.getParent()->getRegion().contains(Inst->getParent())) + if (!Stmt.getParent()->getRegion().contains(Inst->getParent())) return const_cast<Value *>(Old); // The scalar dependence is neither available nor SCEVCodegenable. @@ -133,8 +130,9 @@ Value *BlockGenerator::getNewValue(const Value *Old, ValueMapT &BBMap, return nullptr; } -void BlockGenerator::copyInstScalar(const Instruction *Inst, ValueMapT &BBMap, - ValueMapT &GlobalMap, LoopToScevMapT <S) { +void BlockGenerator::copyInstScalar(ScopStmt &Stmt, const Instruction *Inst, + ValueMapT &BBMap, ValueMapT &GlobalMap, + LoopToScevMapT <S) { // We do not generate debug intrinsics as we did not investigate how to // copy them correctly. At the current state, they just crash the code // generation as the meta-data operands are not correctly copied. @@ -145,8 +143,8 @@ void BlockGenerator::copyInstScalar(const Instruction *Inst, ValueMapT &BBMap, // Replace old operands with the new ones. for (Value *OldOperand : Inst->operands()) { - Value *NewOperand = - getNewValue(OldOperand, BBMap, GlobalMap, LTS, getLoopForInst(Inst)); + Value *NewOperand = getNewValue(Stmt, OldOperand, BBMap, GlobalMap, LTS, + getLoopForInst(Inst)); if (!NewOperand) { assert(!isa<StoreInst>(NewInst) && @@ -165,10 +163,12 @@ void BlockGenerator::copyInstScalar(const Instruction *Inst, ValueMapT &BBMap, NewInst->setName("p_" + Inst->getName()); } -Value *BlockGenerator::getNewAccessOperand(const MemoryAccess &MA) { +Value *BlockGenerator::getNewAccessOperand(ScopStmt &Stmt, + const MemoryAccess &MA) { isl_pw_multi_aff *PWAccRel; isl_union_map *Schedule; isl_ast_expr *Expr; + isl_ast_build *Build = Stmt.getAstBuild(); assert(ExprBuilder && Build && "Cannot generate new value without IslExprBuilder!"); @@ -182,19 +182,17 @@ Value *BlockGenerator::getNewAccessOperand(const MemoryAccess &MA) { return ExprBuilder->create(Expr); } -Value *BlockGenerator::generateLocationAccessed(const Instruction *Inst, - const Value *Pointer, - ValueMapT &BBMap, - ValueMapT &GlobalMap, - LoopToScevMapT <S) { - const MemoryAccess &MA = Statement.getAccessFor(Inst); +Value *BlockGenerator::generateLocationAccessed( + ScopStmt &Stmt, const Instruction *Inst, const Value *Pointer, + ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S) { + const MemoryAccess &MA = Stmt.getAccessFor(Inst); Value *NewPointer; if (MA.hasNewAccessRelation()) - NewPointer = getNewAccessOperand(MA); + NewPointer = getNewAccessOperand(Stmt, MA); else NewPointer = - getNewValue(Pointer, BBMap, GlobalMap, LTS, getLoopForInst(Inst)); + getNewValue(Stmt, Pointer, BBMap, GlobalMap, LTS, getLoopForInst(Inst)); return NewPointer; } @@ -203,35 +201,36 @@ Loop *BlockGenerator::getLoopForInst(const llvm::Instruction *Inst) { return LI.getLoopFor(Inst->getParent()); } -Value *BlockGenerator::generateScalarLoad(const LoadInst *Load, +Value *BlockGenerator::generateScalarLoad(ScopStmt &Stmt, const LoadInst *Load, ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S) { const Value *Pointer = Load->getPointerOperand(); Value *NewPointer = - generateLocationAccessed(Load, Pointer, BBMap, GlobalMap, LTS); + generateLocationAccessed(Stmt, Load, Pointer, BBMap, GlobalMap, LTS); Value *ScalarLoad = Builder.CreateAlignedLoad( NewPointer, Load->getAlignment(), Load->getName() + "_p_scalar_"); return ScalarLoad; } -Value *BlockGenerator::generateScalarStore(const StoreInst *Store, +Value *BlockGenerator::generateScalarStore(ScopStmt &Stmt, + const StoreInst *Store, ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S) { const Value *Pointer = Store->getPointerOperand(); Value *NewPointer = - generateLocationAccessed(Store, Pointer, BBMap, GlobalMap, LTS); - Value *ValueOperand = getNewValue(Store->getValueOperand(), BBMap, GlobalMap, - LTS, getLoopForInst(Store)); + generateLocationAccessed(Stmt, Store, Pointer, BBMap, GlobalMap, LTS); + Value *ValueOperand = getNewValue(Stmt, Store->getValueOperand(), BBMap, + GlobalMap, LTS, getLoopForInst(Store)); Value *NewStore = Builder.CreateAlignedStore(ValueOperand, NewPointer, Store->getAlignment()); return NewStore; } -void BlockGenerator::copyInstruction(const Instruction *Inst, ValueMapT &BBMap, - ValueMapT &GlobalMap, +void BlockGenerator::copyInstruction(ScopStmt &Stmt, const Instruction *Inst, + ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S) { // Terminator instructions control the control flow. They are explicitly // expressed in the clast and do not need to be copied. @@ -239,11 +238,11 @@ void BlockGenerator::copyInstruction(const Instruction *Inst, ValueMapT &BBMap, return; if (canSynthesize(Inst, &P->getAnalysis<LoopInfoWrapperPass>().getLoopInfo(), - &SE, &Statement.getParent()->getRegion())) + &SE, &Stmt.getParent()->getRegion())) return; if (const LoadInst *Load = dyn_cast<LoadInst>(Inst)) { - Value *NewLoad = generateScalarLoad(Load, BBMap, GlobalMap, LTS); + Value *NewLoad = generateScalarLoad(Stmt, Load, BBMap, GlobalMap, LTS); // Compute NewLoad before its insertion in BBMap to make the insertion // deterministic. BBMap[Load] = NewLoad; @@ -251,7 +250,7 @@ void BlockGenerator::copyInstruction(const Instruction *Inst, ValueMapT &BBMap, } if (const StoreInst *Store = dyn_cast<StoreInst>(Inst)) { - Value *NewStore = generateScalarStore(Store, BBMap, GlobalMap, LTS); + Value *NewStore = generateScalarStore(Stmt, Store, BBMap, GlobalMap, LTS); // Compute NewStore before its insertion in BBMap to make the insertion // deterministic. BBMap[Store] = NewStore; @@ -282,16 +281,17 @@ void BlockGenerator::copyInstruction(const Instruction *Inst, ValueMapT &BBMap, } } - copyInstScalar(Inst, BBMap, GlobalMap, LTS); + copyInstScalar(Stmt, Inst, BBMap, GlobalMap, LTS); } -void BlockGenerator::copyBB(ValueMapT &GlobalMap, LoopToScevMapT <S) { +void BlockGenerator::copyBB(ScopStmt &Stmt, ValueMapT &GlobalMap, + LoopToScevMapT <S) { auto *DTWP = P->getAnalysisIfAvailable<DominatorTreeWrapperPass>(); auto *DT = DTWP ? &DTWP->getDomTree() : nullptr; auto *LIWP = P->getAnalysisIfAvailable<LoopInfoWrapperPass>(); auto *LI = LIWP ? &LIWP->getLoopInfo() : nullptr; - BasicBlock *BB = Statement.getBasicBlock(); + BasicBlock *BB = Stmt.getBasicBlock(); BasicBlock *CopyBB = SplitBlock(Builder.GetInsertBlock(), Builder.GetInsertPoint(), DT, LI); CopyBB->setName("polly.stmt." + BB->getName()); @@ -300,21 +300,20 @@ void BlockGenerator::copyBB(ValueMapT &GlobalMap, LoopToScevMapT <S) { ValueMapT BBMap; for (Instruction &Inst : *BB) - copyInstruction(&Inst, BBMap, GlobalMap, LTS); + copyInstruction(Stmt, &Inst, BBMap, GlobalMap, LTS); } -VectorBlockGenerator::VectorBlockGenerator( - PollyIRBuilder &B, VectorValueMapT &GlobalMaps, - std::vector<LoopToScevMapT> &VLTS, ScopStmt &Stmt, - __isl_keep isl_map *Schedule, Pass *P, LoopInfo &LI, ScalarEvolution &SE, - __isl_keep isl_ast_build *Build, IslExprBuilder *ExprBuilder) - : BlockGenerator(B, Stmt, P, LI, SE, Build, ExprBuilder), - GlobalMaps(GlobalMaps), VLTS(VLTS), Schedule(Schedule) { +VectorBlockGenerator::VectorBlockGenerator(BlockGenerator &BlockGen, + VectorValueMapT &GlobalMaps, + std::vector<LoopToScevMapT> &VLTS, + isl_map *Schedule) + : BlockGenerator(BlockGen), GlobalMaps(GlobalMaps), VLTS(VLTS), + Schedule(Schedule) { assert(GlobalMaps.size() > 1 && "Only one vector lane found"); assert(Schedule && "No statement domain provided"); } -Value *VectorBlockGenerator::getVectorValue(const Value *Old, +Value *VectorBlockGenerator::getVectorValue(ScopStmt &Stmt, const Value *Old, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps, Loop *L) { @@ -327,8 +326,8 @@ Value *VectorBlockGenerator::getVectorValue(const Value *Old, for (int Lane = 0; Lane < Width; Lane++) Vector = Builder.CreateInsertElement( - Vector, - getNewValue(Old, ScalarMaps[Lane], GlobalMaps[Lane], VLTS[Lane], L), + Vector, getNewValue(Stmt, Old, ScalarMaps[Lane], GlobalMaps[Lane], + VLTS[Lane], L), Builder.getInt32(Lane)); VectorMap[Old] = Vector; @@ -346,17 +345,16 @@ Type *VectorBlockGenerator::getVectorPtrTy(const Value *Val, int Width) { return PointerType::getUnqual(VectorType); } -Value * -VectorBlockGenerator::generateStrideOneLoad(const LoadInst *Load, - VectorValueMapT &ScalarMaps, - bool NegativeStride = false) { +Value *VectorBlockGenerator::generateStrideOneLoad( + ScopStmt &Stmt, const LoadInst *Load, VectorValueMapT &ScalarMaps, + bool NegativeStride = false) { unsigned VectorWidth = getVectorWidth(); const Value *Pointer = Load->getPointerOperand(); Type *VectorPtrType = getVectorPtrTy(Pointer, VectorWidth); unsigned Offset = NegativeStride ? VectorWidth - 1 : 0; Value *NewPointer = nullptr; - NewPointer = generateLocationAccessed(Load, Pointer, ScalarMaps[Offset], + NewPointer = generateLocationAccessed(Stmt, Load, Pointer, ScalarMaps[Offset], GlobalMaps[Offset], VLTS[Offset]); Value *VectorPtr = Builder.CreateBitCast(NewPointer, VectorPtrType, "vector_ptr"); @@ -378,12 +376,13 @@ VectorBlockGenerator::generateStrideOneLoad(const LoadInst *Load, return VecLoad; } -Value *VectorBlockGenerator::generateStrideZeroLoad(const LoadInst *Load, +Value *VectorBlockGenerator::generateStrideZeroLoad(ScopStmt &Stmt, + const LoadInst *Load, ValueMapT &BBMap) { const Value *Pointer = Load->getPointerOperand(); Type *VectorPtrType = getVectorPtrTy(Pointer, 1); - Value *NewPointer = - generateLocationAccessed(Load, Pointer, BBMap, GlobalMaps[0], VLTS[0]); + Value *NewPointer = generateLocationAccessed(Stmt, Load, Pointer, BBMap, + GlobalMaps[0], VLTS[0]); Value *VectorPtr = Builder.CreateBitCast(NewPointer, VectorPtrType, Load->getName() + "_p_vec_p"); LoadInst *ScalarLoad = @@ -400,9 +399,8 @@ Value *VectorBlockGenerator::generateStrideZeroLoad(const LoadInst *Load, return VectorLoad; } -Value * -VectorBlockGenerator::generateUnknownStrideLoad(const LoadInst *Load, - VectorValueMapT &ScalarMaps) { +Value *VectorBlockGenerator::generateUnknownStrideLoad( + ScopStmt &Stmt, const LoadInst *Load, VectorValueMapT &ScalarMaps) { int VectorWidth = getVectorWidth(); const Value *Pointer = Load->getPointerOperand(); VectorType *VectorType = VectorType::get( @@ -411,8 +409,8 @@ VectorBlockGenerator::generateUnknownStrideLoad(const LoadInst *Load, Value *Vector = UndefValue::get(VectorType); for (int i = 0; i < VectorWidth; i++) { - Value *NewPointer = generateLocationAccessed(Load, Pointer, ScalarMaps[i], - GlobalMaps[i], VLTS[i]); + Value *NewPointer = generateLocationAccessed( + Stmt, Load, Pointer, ScalarMaps[i], GlobalMaps[i], VLTS[i]); Value *ScalarLoad = Builder.CreateLoad(NewPointer, Load->getName() + "_p_scalar_"); Vector = Builder.CreateInsertElement( @@ -422,18 +420,18 @@ VectorBlockGenerator::generateUnknownStrideLoad(const LoadInst *Load, return Vector; } -void VectorBlockGenerator::generateLoad(const LoadInst *Load, +void VectorBlockGenerator::generateLoad(ScopStmt &Stmt, const LoadInst *Load, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps) { if (PollyVectorizerChoice >= VECTORIZER_FIRST_NEED_GROUPED_UNROLL || !VectorType::isValidElementType(Load->getType())) { for (int i = 0; i < getVectorWidth(); i++) ScalarMaps[i][Load] = - generateScalarLoad(Load, ScalarMaps[i], GlobalMaps[i], VLTS[i]); + generateScalarLoad(Stmt, Load, ScalarMaps[i], GlobalMaps[i], VLTS[i]); return; } - const MemoryAccess &Access = Statement.getAccessFor(Load); + const MemoryAccess &Access = Stmt.getAccessFor(Load); // Make sure we have scalar values available to access the pointer to // the data location. @@ -441,23 +439,24 @@ void VectorBlockGenerator::generateLoad(const LoadInst *Load, Value *NewLoad; if (Access.isStrideZero(isl_map_copy(Schedule))) - NewLoad = generateStrideZeroLoad(Load, ScalarMaps[0]); + NewLoad = generateStrideZeroLoad(Stmt, Load, ScalarMaps[0]); else if (Access.isStrideOne(isl_map_copy(Schedule))) - NewLoad = generateStrideOneLoad(Load, ScalarMaps); + NewLoad = generateStrideOneLoad(Stmt, Load, ScalarMaps); else if (Access.isStrideX(isl_map_copy(Schedule), -1)) - NewLoad = generateStrideOneLoad(Load, ScalarMaps, true); + NewLoad = generateStrideOneLoad(Stmt, Load, ScalarMaps, true); else - NewLoad = generateUnknownStrideLoad(Load, ScalarMaps); + NewLoad = generateUnknownStrideLoad(Stmt, Load, ScalarMaps); VectorMap[Load] = NewLoad; } -void VectorBlockGenerator::copyUnaryInst(const UnaryInstruction *Inst, +void VectorBlockGenerator::copyUnaryInst(ScopStmt &Stmt, + const UnaryInstruction *Inst, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps) { int VectorWidth = getVectorWidth(); - Value *NewOperand = getVectorValue(Inst->getOperand(0), VectorMap, ScalarMaps, - getLoopForInst(Inst)); + Value *NewOperand = getVectorValue(Stmt, Inst->getOperand(0), VectorMap, + ScalarMaps, getLoopForInst(Inst)); assert(isa<CastInst>(Inst) && "Can not generate vector code for instruction"); @@ -466,7 +465,8 @@ void VectorBlockGenerator::copyUnaryInst(const UnaryInstruction *Inst, VectorMap[Inst] = Builder.CreateCast(Cast->getOpcode(), NewOperand, DestType); } -void VectorBlockGenerator::copyBinaryInst(const BinaryOperator *Inst, +void VectorBlockGenerator::copyBinaryInst(ScopStmt &Stmt, + const BinaryOperator *Inst, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps) { Loop *L = getLoopForInst(Inst); @@ -474,21 +474,21 @@ void VectorBlockGenerator::copyBinaryInst(const BinaryOperator *Inst, Value *OpOne = Inst->getOperand(1); Value *NewOpZero, *NewOpOne; - NewOpZero = getVectorValue(OpZero, VectorMap, ScalarMaps, L); - NewOpOne = getVectorValue(OpOne, VectorMap, ScalarMaps, L); + NewOpZero = getVectorValue(Stmt, OpZero, VectorMap, ScalarMaps, L); + NewOpOne = getVectorValue(Stmt, OpOne, VectorMap, ScalarMaps, L); Value *NewInst = Builder.CreateBinOp(Inst->getOpcode(), NewOpZero, NewOpOne, Inst->getName() + "p_vec"); VectorMap[Inst] = NewInst; } -void VectorBlockGenerator::copyStore(const StoreInst *Store, +void VectorBlockGenerator::copyStore(ScopStmt &Stmt, const StoreInst *Store, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps) { - const MemoryAccess &Access = Statement.getAccessFor(Store); + const MemoryAccess &Access = Stmt.getAccessFor(Store); const Value *Pointer = Store->getPointerOperand(); - Value *Vector = getVectorValue(Store->getValueOperand(), VectorMap, + Value *Vector = getVectorValue(Stmt, Store->getValueOperand(), VectorMap, ScalarMaps, getLoopForInst(Store)); // Make sure we have scalar values available to access the pointer to @@ -497,8 +497,8 @@ void VectorBlockGenerator::copyStore(const StoreInst *Store, if (Access.isStrideOne(isl_map_copy(Schedule))) { Type *VectorPtrType = getVectorPtrTy(Pointer, getVectorWidth()); - Value *NewPointer = generateLocationAccessed(Store, Pointer, ScalarMaps[0], - GlobalMaps[0], VLTS[0]); + Value *NewPointer = generateLocationAccessed( + Stmt, Store, Pointer, ScalarMaps[0], GlobalMaps[0], VLTS[0]); Value *VectorPtr = Builder.CreateBitCast(NewPointer, VectorPtrType, "vector_ptr"); @@ -510,7 +510,7 @@ void VectorBlockGenerator::copyStore(const StoreInst *Store, for (unsigned i = 0; i < ScalarMaps.size(); i++) { Value *Scalar = Builder.CreateExtractElement(Vector, Builder.getInt32(i)); Value *NewPointer = generateLocationAccessed( - Store, Pointer, ScalarMaps[i], GlobalMaps[i], VLTS[i]); + Stmt, Store, Pointer, ScalarMaps[i], GlobalMaps[i], VLTS[i]); Builder.CreateStore(Scalar, NewPointer); } } @@ -556,7 +556,8 @@ bool VectorBlockGenerator::extractScalarValues(const Instruction *Inst, return HasVectorOperand; } -void VectorBlockGenerator::copyInstScalarized(const Instruction *Inst, +void VectorBlockGenerator::copyInstScalarized(ScopStmt &Stmt, + const Instruction *Inst, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps) { bool HasVectorOperand; @@ -565,7 +566,7 @@ void VectorBlockGenerator::copyInstScalarized(const Instruction *Inst, HasVectorOperand = extractScalarValues(Inst, VectorMap, ScalarMaps); for (int VectorLane = 0; VectorLane < getVectorWidth(); VectorLane++) - BlockGenerator::copyInstruction(Inst, ScalarMaps[VectorLane], + BlockGenerator::copyInstruction(Stmt, Inst, ScalarMaps[VectorLane], GlobalMaps[VectorLane], VLTS[VectorLane]); if (!VectorType::isValidElementType(Inst->getType()) || !HasVectorOperand) @@ -584,7 +585,8 @@ void VectorBlockGenerator::copyInstScalarized(const Instruction *Inst, int VectorBlockGenerator::getVectorWidth() { return GlobalMaps.size(); } -void VectorBlockGenerator::copyInstruction(const Instruction *Inst, +void VectorBlockGenerator::copyInstruction(ScopStmt &Stmt, + const Instruction *Inst, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps) { // Terminator instructions control the control flow. They are explicitly @@ -593,27 +595,27 @@ void VectorBlockGenerator::copyInstruction(const Instruction *Inst, return; if (canSynthesize(Inst, &P->getAnalysis<LoopInfoWrapperPass>().getLoopInfo(), - &SE, &Statement.getParent()->getRegion())) + &SE, &Stmt.getParent()->getRegion())) return; if (const LoadInst *Load = dyn_cast<LoadInst>(Inst)) { - generateLoad(Load, VectorMap, ScalarMaps); + generateLoad(Stmt, Load, VectorMap, ScalarMaps); return; } if (hasVectorOperands(Inst, VectorMap)) { if (const StoreInst *Store = dyn_cast<StoreInst>(Inst)) { - copyStore(Store, VectorMap, ScalarMaps); + copyStore(Stmt, Store, VectorMap, ScalarMaps); return; } if (const UnaryInstruction *Unary = dyn_cast<UnaryInstruction>(Inst)) { - copyUnaryInst(Unary, VectorMap, ScalarMaps); + copyUnaryInst(Stmt, Unary, VectorMap, ScalarMaps); return; } if (const BinaryOperator *Binary = dyn_cast<BinaryOperator>(Inst)) { - copyBinaryInst(Binary, VectorMap, ScalarMaps); + copyBinaryInst(Stmt, Binary, VectorMap, ScalarMaps); return; } @@ -621,16 +623,16 @@ void VectorBlockGenerator::copyInstruction(const Instruction *Inst, // generate vector code. } - copyInstScalarized(Inst, VectorMap, ScalarMaps); + copyInstScalarized(Stmt, Inst, VectorMap, ScalarMaps); } -void VectorBlockGenerator::copyBB() { +void VectorBlockGenerator::copyBB(ScopStmt &Stmt) { auto *DTWP = P->getAnalysisIfAvailable<DominatorTreeWrapperPass>(); auto *DT = DTWP ? &DTWP->getDomTree() : nullptr; auto *LIWP = P->getAnalysisIfAvailable<LoopInfoWrapperPass>(); auto *LI = LIWP ? &LIWP->getLoopInfo() : nullptr; - BasicBlock *BB = Statement.getBasicBlock(); + BasicBlock *BB = Stmt.getBasicBlock(); BasicBlock *CopyBB = SplitBlock(Builder.GetInsertBlock(), Builder.GetInsertPoint(), DT, LI); CopyBB->setName("polly.stmt." + BB->getName()); @@ -654,5 +656,5 @@ void VectorBlockGenerator::copyBB() { ValueMapT VectorBlockMap; for (Instruction &Inst : *BB) - copyInstruction(&Inst, VectorBlockMap, ScalarBlockMap); + copyInstruction(Stmt, &Inst, VectorBlockMap, ScalarBlockMap); } diff --git a/polly/lib/CodeGen/IslCodeGeneration.cpp b/polly/lib/CodeGen/IslCodeGeneration.cpp index db8712b..45e8e9d 100644 --- a/polly/lib/CodeGen/IslCodeGeneration.cpp +++ b/polly/lib/CodeGen/IslCodeGeneration.cpp @@ -64,7 +64,8 @@ public: DominatorTree &DT, Scop &S) : S(S), Builder(Builder), Annotator(Annotator), Rewriter(new SCEVExpander(SE, "polly")), - ExprBuilder(Builder, IDToValue, *Rewriter), P(P), DL(DL), LI(LI), + ExprBuilder(Builder, IDToValue, *Rewriter), + BlockGen(Builder, P, LI, SE, &ExprBuilder), P(P), DL(DL), LI(LI), SE(SE), DT(DT) {} ~IslNodeBuilder() { delete Rewriter; } @@ -82,6 +83,7 @@ private: SCEVExpander *Rewriter; IslExprBuilder ExprBuilder; + BlockGenerator BlockGen; Pass *P; const DataLayout &DL; LoopInfo &LI; @@ -398,6 +400,7 @@ void IslNodeBuilder::createUserVector(__isl_take isl_ast_node *User, isl_id *Id = isl_ast_expr_get_id(StmtExpr); isl_ast_expr_free(StmtExpr); ScopStmt *Stmt = (ScopStmt *)isl_id_get_user(Id); + Stmt->setAstBuild(IslAstInfo::getBuild(User)); VectorValueMapT VectorMap(IVS.size()); std::vector<LoopToScevMapT> VLTS(IVS.size()); @@ -406,8 +409,7 @@ void IslNodeBuilder::createUserVector(__isl_take isl_ast_node *User, isl_map *S = isl_map_from_union_map(Schedule); createSubstitutionsVector(Expr, Stmt, VectorMap, VLTS, IVS, IteratorID); - VectorBlockGenerator::generate(Builder, *Stmt, VectorMap, VLTS, S, P, LI, SE, - IslAstInfo::getBuild(User), &ExprBuilder); + VectorBlockGenerator::generate(BlockGen, *Stmt, VectorMap, VLTS, S); isl_map_free(S); isl_id_free(Id); @@ -795,10 +797,10 @@ void IslNodeBuilder::createUser(__isl_take isl_ast_node *User) { LTS.insert(OutsideLoopIterations.begin(), OutsideLoopIterations.end()); Stmt = (ScopStmt *)isl_id_get_user(Id); + Stmt->setAstBuild(IslAstInfo::getBuild(User)); createSubstitutions(Expr, Stmt, VMap, LTS); - BlockGenerator::generate(Builder, *Stmt, VMap, LTS, P, LI, SE, - IslAstInfo::getBuild(User), &ExprBuilder); + BlockGen.copyBB(*Stmt, VMap, LTS); isl_ast_node_free(User); isl_id_free(Id); |