diff options
author | Marco Elver <elver@google.com> | 2022-03-17 16:46:12 +0100 |
---|---|---|
committer | Marco Elver <elver@google.com> | 2022-03-17 16:58:16 +0100 |
commit | b09439e20b468be80a6e832f66ba7fff403574dd (patch) | |
tree | 2cb3991421b319f3348c2864a13ab25c0b7847b3 /llvm/lib/CodeGen/AtomicExpandPass.cpp | |
parent | 948d05324a150a5a24e93bad07c9090d5b8bd129 (diff) | |
download | llvm-b09439e20b468be80a6e832f66ba7fff403574dd.zip llvm-b09439e20b468be80a6e832f66ba7fff403574dd.tar.gz llvm-b09439e20b468be80a6e832f66ba7fff403574dd.tar.bz2 |
[AtomicExpandPass][NFC] Reformat with clang-format
NFCI.
Diffstat (limited to 'llvm/lib/CodeGen/AtomicExpandPass.cpp')
-rw-r--r-- | llvm/lib/CodeGen/AtomicExpandPass.cpp | 220 |
1 files changed, 112 insertions, 108 deletions
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp index 4537148..3a0eb96 100644 --- a/llvm/lib/CodeGen/AtomicExpandPass.cpp +++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp @@ -57,71 +57,71 @@ using namespace llvm; namespace { - class AtomicExpand: public FunctionPass { - const TargetLowering *TLI = nullptr; +class AtomicExpand : public FunctionPass { + const TargetLowering *TLI = nullptr; - public: - static char ID; // Pass identification, replacement for typeid +public: + static char ID; // Pass identification, replacement for typeid - AtomicExpand() : FunctionPass(ID) { - initializeAtomicExpandPass(*PassRegistry::getPassRegistry()); - } + AtomicExpand() : FunctionPass(ID) { + initializeAtomicExpandPass(*PassRegistry::getPassRegistry()); + } - bool runOnFunction(Function &F) override; - - private: - bool bracketInstWithFences(Instruction *I, AtomicOrdering Order); - IntegerType *getCorrespondingIntegerType(Type *T, const DataLayout &DL); - LoadInst *convertAtomicLoadToIntegerType(LoadInst *LI); - bool tryExpandAtomicLoad(LoadInst *LI); - bool expandAtomicLoadToLL(LoadInst *LI); - bool expandAtomicLoadToCmpXchg(LoadInst *LI); - StoreInst *convertAtomicStoreToIntegerType(StoreInst *SI); - bool expandAtomicStore(StoreInst *SI); - bool tryExpandAtomicRMW(AtomicRMWInst *AI); - AtomicRMWInst *convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI); - Value * - insertRMWLLSCLoop(IRBuilder<> &Builder, Type *ResultTy, Value *Addr, - Align AddrAlign, AtomicOrdering MemOpOrder, - function_ref<Value *(IRBuilder<> &, Value *)> PerformOp); - void expandAtomicOpToLLSC( - Instruction *I, Type *ResultTy, Value *Addr, Align AddrAlign, - AtomicOrdering MemOpOrder, - function_ref<Value *(IRBuilder<> &, Value *)> PerformOp); - void expandPartwordAtomicRMW( - AtomicRMWInst *I, - TargetLoweringBase::AtomicExpansionKind ExpansionKind); - AtomicRMWInst *widenPartwordAtomicRMW(AtomicRMWInst *AI); - bool expandPartwordCmpXchg(AtomicCmpXchgInst *I); - void expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI); - void expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI); - - AtomicCmpXchgInst *convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI); - static Value *insertRMWCmpXchgLoop( - IRBuilder<> &Builder, Type *ResultType, Value *Addr, Align AddrAlign, - AtomicOrdering MemOpOrder, SyncScope::ID SSID, - function_ref<Value *(IRBuilder<> &, Value *)> PerformOp, - CreateCmpXchgInstFun CreateCmpXchg); - bool tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI); - - bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI); - bool isIdempotentRMW(AtomicRMWInst *RMWI); - bool simplifyIdempotentRMW(AtomicRMWInst *RMWI); - - bool expandAtomicOpToLibcall(Instruction *I, unsigned Size, Align Alignment, - Value *PointerOperand, Value *ValueOperand, - Value *CASExpected, AtomicOrdering Ordering, - AtomicOrdering Ordering2, - ArrayRef<RTLIB::Libcall> Libcalls); - void expandAtomicLoadToLibcall(LoadInst *LI); - void expandAtomicStoreToLibcall(StoreInst *LI); - void expandAtomicRMWToLibcall(AtomicRMWInst *I); - void expandAtomicCASToLibcall(AtomicCmpXchgInst *I); - - friend bool - llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI, - CreateCmpXchgInstFun CreateCmpXchg); - }; + bool runOnFunction(Function &F) override; + +private: + bool bracketInstWithFences(Instruction *I, AtomicOrdering Order); + IntegerType *getCorrespondingIntegerType(Type *T, const DataLayout &DL); + LoadInst *convertAtomicLoadToIntegerType(LoadInst *LI); + bool tryExpandAtomicLoad(LoadInst *LI); + bool expandAtomicLoadToLL(LoadInst *LI); + bool expandAtomicLoadToCmpXchg(LoadInst *LI); + StoreInst *convertAtomicStoreToIntegerType(StoreInst *SI); + bool expandAtomicStore(StoreInst *SI); + bool tryExpandAtomicRMW(AtomicRMWInst *AI); + AtomicRMWInst *convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI); + Value * + insertRMWLLSCLoop(IRBuilder<> &Builder, Type *ResultTy, Value *Addr, + Align AddrAlign, AtomicOrdering MemOpOrder, + function_ref<Value *(IRBuilder<> &, Value *)> PerformOp); + void + expandAtomicOpToLLSC(Instruction *I, Type *ResultTy, Value *Addr, + Align AddrAlign, AtomicOrdering MemOpOrder, + function_ref<Value *(IRBuilder<> &, Value *)> PerformOp); + void expandPartwordAtomicRMW( + AtomicRMWInst *I, TargetLoweringBase::AtomicExpansionKind ExpansionKind); + AtomicRMWInst *widenPartwordAtomicRMW(AtomicRMWInst *AI); + bool expandPartwordCmpXchg(AtomicCmpXchgInst *I); + void expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI); + void expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI); + + AtomicCmpXchgInst *convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI); + static Value * + insertRMWCmpXchgLoop(IRBuilder<> &Builder, Type *ResultType, Value *Addr, + Align AddrAlign, AtomicOrdering MemOpOrder, + SyncScope::ID SSID, + function_ref<Value *(IRBuilder<> &, Value *)> PerformOp, + CreateCmpXchgInstFun CreateCmpXchg); + bool tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI); + + bool expandAtomicCmpXchg(AtomicCmpXchgInst *CI); + bool isIdempotentRMW(AtomicRMWInst *RMWI); + bool simplifyIdempotentRMW(AtomicRMWInst *RMWI); + + bool expandAtomicOpToLibcall(Instruction *I, unsigned Size, Align Alignment, + Value *PointerOperand, Value *ValueOperand, + Value *CASExpected, AtomicOrdering Ordering, + AtomicOrdering Ordering2, + ArrayRef<RTLIB::Libcall> Libcalls); + void expandAtomicLoadToLibcall(LoadInst *LI); + void expandAtomicStoreToLibcall(StoreInst *LI); + void expandAtomicRMWToLibcall(AtomicRMWInst *I); + void expandAtomicCASToLibcall(AtomicCmpXchgInst *I); + + friend bool + llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI, + CreateCmpXchgInstFun CreateCmpXchg); +}; } // end anonymous namespace @@ -129,8 +129,8 @@ char AtomicExpand::ID = 0; char &llvm::AtomicExpandID = AtomicExpand::ID; -INITIALIZE_PASS(AtomicExpand, DEBUG_TYPE, "Expand Atomic instructions", - false, false) +INITIALIZE_PASS(AtomicExpand, DEBUG_TYPE, "Expand Atomic instructions", false, + false) FunctionPass *llvm::createAtomicExpandPass() { return new AtomicExpand(); } @@ -308,7 +308,7 @@ bool AtomicExpand::runOnFunction(Function &F) { // extend convertCmpXchgToInteger for floating point too. assert(!CASI->getCompareOperand()->getType()->isFloatingPointTy() && "unimplemented - floating point not legal at IR level"); - if (CASI->getCompareOperand()->getType()->isPointerTy() ) { + if (CASI->getCompareOperand()->getType()->isPointerTy()) { // TODO: add a TLI hook to control this so that each target can // convert to lowering the original type one at a time. CASI = convertCmpXchgToIntegerType(CASI); @@ -351,14 +351,12 @@ IntegerType *AtomicExpand::getCorrespondingIntegerType(Type *T, /// convertAtomicStoreToIntegerType for background. LoadInst *AtomicExpand::convertAtomicLoadToIntegerType(LoadInst *LI) { auto *M = LI->getModule(); - Type *NewTy = getCorrespondingIntegerType(LI->getType(), - M->getDataLayout()); + Type *NewTy = getCorrespondingIntegerType(LI->getType(), M->getDataLayout()); IRBuilder<> Builder(LI); Value *Addr = LI->getPointerOperand(); - Type *PT = PointerType::get(NewTy, - Addr->getType()->getPointerAddressSpace()); + Type *PT = PointerType::get(NewTy, Addr->getType()->getPointerAddressSpace()); Value *NewAddr = Builder.CreateBitCast(Addr, PT); auto *NewLI = Builder.CreateLoad(NewTy, NewAddr); @@ -471,8 +469,7 @@ StoreInst *AtomicExpand::convertAtomicStoreToIntegerType(StoreInst *SI) { Value *NewVal = Builder.CreateBitCast(SI->getValueOperand(), NewTy); Value *Addr = SI->getPointerOperand(); - Type *PT = PointerType::get(NewTy, - Addr->getType()->getPointerAddressSpace()); + Type *PT = PointerType::get(NewTy, Addr->getType()->getPointerAddressSpace()); Value *NewAddr = Builder.CreateBitCast(Addr, PT); StoreInst *NewSI = Builder.CreateStore(NewVal, NewAddr); @@ -844,9 +841,8 @@ void AtomicExpand::expandPartwordAtomicRMW( Value *OldResult; if (ExpansionKind == TargetLoweringBase::AtomicExpansionKind::CmpXChg) { OldResult = insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr, - PMV.AlignedAddrAlignment, MemOpOrder, - SSID, PerformPartwordOp, - createCmpXchgInstFun); + PMV.AlignedAddrAlignment, MemOpOrder, SSID, + PerformPartwordOp, createCmpXchgInstFun); } else { assert(ExpansionKind == TargetLoweringBase::AtomicExpansionKind::LLSC); OldResult = insertRMWLLSCLoop(Builder, PMV.WordType, PMV.AlignedAddr, @@ -1110,7 +1106,7 @@ Value *AtomicExpand::insertRMWLLSCLoop( // [...] BasicBlock *ExitBB = BB->splitBasicBlock(Builder.GetInsertPoint(), "atomicrmw.end"); - BasicBlock *LoopBB = BasicBlock::Create(Ctx, "atomicrmw.start", F, ExitBB); + BasicBlock *LoopBB = BasicBlock::Create(Ctx, "atomicrmw.start", F, ExitBB); // The split call above "helpfully" added a branch at the end of BB (to the // wrong place). @@ -1139,7 +1135,8 @@ Value *AtomicExpand::insertRMWLLSCLoop( /// IR. As a migration step, we convert back to what use to be the standard /// way to represent a pointer cmpxchg so that we can update backends one by /// one. -AtomicCmpXchgInst *AtomicExpand::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) { +AtomicCmpXchgInst * +AtomicExpand::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) { auto *M = CI->getModule(); Type *NewTy = getCorrespondingIntegerType(CI->getCompareOperand()->getType(), M->getDataLayout()); @@ -1147,8 +1144,7 @@ AtomicCmpXchgInst *AtomicExpand::convertCmpXchgToIntegerType(AtomicCmpXchgInst * IRBuilder<> Builder(CI); Value *Addr = CI->getPointerOperand(); - Type *PT = PointerType::get(NewTy, - Addr->getType()->getPointerAddressSpace()); + Type *PT = PointerType::get(NewTy, Addr->getType()->getPointerAddressSpace()); Value *NewAddr = Builder.CreateBitCast(Addr, PT); Value *NewCmp = Builder.CreatePtrToInt(CI->getCompareOperand(), NewTy); @@ -1309,9 +1305,8 @@ bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) { LoadedTryStore->addIncoming(UnreleasedLoad, ReleasingStoreBB); Value *NewValueInsert = insertMaskedValue(Builder, LoadedTryStore, CI->getNewValOperand(), PMV); - Value *StoreSuccess = - TLI->emitStoreConditional(Builder, NewValueInsert, PMV.AlignedAddr, - MemOpOrder); + Value *StoreSuccess = TLI->emitStoreConditional(Builder, NewValueInsert, + PMV.AlignedAddr, MemOpOrder); StoreSuccess = Builder.CreateICmpEQ( StoreSuccess, ConstantInt::get(Type::getInt32Ty(Ctx), 0), "success"); BasicBlock *RetryBB = HasReleasedLoadBB ? ReleasedLoadBB : StartBB; @@ -1422,27 +1417,27 @@ bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) { return true; } -bool AtomicExpand::isIdempotentRMW(AtomicRMWInst* RMWI) { +bool AtomicExpand::isIdempotentRMW(AtomicRMWInst *RMWI) { auto C = dyn_cast<ConstantInt>(RMWI->getValOperand()); - if(!C) + if (!C) return false; AtomicRMWInst::BinOp Op = RMWI->getOperation(); - switch(Op) { - case AtomicRMWInst::Add: - case AtomicRMWInst::Sub: - case AtomicRMWInst::Or: - case AtomicRMWInst::Xor: - return C->isZero(); - case AtomicRMWInst::And: - return C->isMinusOne(); - // FIXME: we could also treat Min/Max/UMin/UMax by the INT_MIN/INT_MAX/... - default: - return false; + switch (Op) { + case AtomicRMWInst::Add: + case AtomicRMWInst::Sub: + case AtomicRMWInst::Or: + case AtomicRMWInst::Xor: + return C->isZero(); + case AtomicRMWInst::And: + return C->isMinusOne(); + // FIXME: we could also treat Min/Max/UMin/UMax by the INT_MIN/INT_MAX/... + default: + return false; } } -bool AtomicExpand::simplifyIdempotentRMW(AtomicRMWInst* RMWI) { +bool AtomicExpand::simplifyIdempotentRMW(AtomicRMWInst *RMWI) { if (auto ResultingLoad = TLI->lowerIdempotentRMWIntoFencedLoad(RMWI)) { tryExpandAtomicLoad(ResultingLoad); return true; @@ -1742,11 +1737,21 @@ bool AtomicExpand::expandAtomicOpToLibcall( RTLIB::Libcall RTLibType; if (UseSizedLibcall) { switch (Size) { - case 1: RTLibType = Libcalls[1]; break; - case 2: RTLibType = Libcalls[2]; break; - case 4: RTLibType = Libcalls[3]; break; - case 8: RTLibType = Libcalls[4]; break; - case 16: RTLibType = Libcalls[5]; break; + case 1: + RTLibType = Libcalls[1]; + break; + case 2: + RTLibType = Libcalls[2]; + break; + case 4: + RTLibType = Libcalls[3]; + break; + case 8: + RTLibType = Libcalls[4]; + break; + case 16: + RTLibType = Libcalls[5]; + break; } } else if (Libcalls[0] != RTLIB::UNKNOWN_LIBCALL) { RTLibType = Libcalls[0]; @@ -1810,8 +1815,8 @@ bool AtomicExpand::expandAtomicOpToLibcall( // that property, we'd need to extend this mechanism to support AS-specific // families of atomic intrinsics. auto PtrTypeAS = PointerOperand->getType()->getPointerAddressSpace(); - Value *PtrVal = Builder.CreateBitCast(PointerOperand, - Type::getInt8PtrTy(Ctx, PtrTypeAS)); + Value *PtrVal = + Builder.CreateBitCast(PointerOperand, Type::getInt8PtrTy(Ctx, PtrTypeAS)); PtrVal = Builder.CreateAddrSpaceCast(PtrVal, Type::getInt8PtrTy(Ctx)); Args.push_back(PtrVal); @@ -1819,11 +1824,10 @@ bool AtomicExpand::expandAtomicOpToLibcall( if (CASExpected) { AllocaCASExpected = AllocaBuilder.CreateAlloca(CASExpected->getType()); AllocaCASExpected->setAlignment(AllocaAlignment); - unsigned AllocaAS = AllocaCASExpected->getType()->getPointerAddressSpace(); + unsigned AllocaAS = AllocaCASExpected->getType()->getPointerAddressSpace(); - AllocaCASExpected_i8 = - Builder.CreateBitCast(AllocaCASExpected, - Type::getInt8PtrTy(Ctx, AllocaAS)); + AllocaCASExpected_i8 = Builder.CreateBitCast( + AllocaCASExpected, Type::getInt8PtrTy(Ctx, AllocaAS)); Builder.CreateLifetimeStart(AllocaCASExpected_i8, SizeVal64); Builder.CreateAlignedStore(CASExpected, AllocaCASExpected, AllocaAlignment); Args.push_back(AllocaCASExpected_i8); @@ -1850,9 +1854,9 @@ bool AtomicExpand::expandAtomicOpToLibcall( if (!CASExpected && HasResult && !UseSizedLibcall) { AllocaResult = AllocaBuilder.CreateAlloca(I->getType()); AllocaResult->setAlignment(AllocaAlignment); - unsigned AllocaAS = AllocaResult->getType()->getPointerAddressSpace(); + unsigned AllocaAS = AllocaResult->getType()->getPointerAddressSpace(); AllocaResult_i8 = - Builder.CreateBitCast(AllocaResult, Type::getInt8PtrTy(Ctx, AllocaAS)); + Builder.CreateBitCast(AllocaResult, Type::getInt8PtrTy(Ctx, AllocaAS)); Builder.CreateLifetimeStart(AllocaResult_i8, SizeVal64); Args.push_back(AllocaResult_i8); } |