From bdc77d1ecc9219dfc641aaaa4fb088233e9a32d1 Mon Sep 17 00:00:00 2001 From: Stephen Tozer Date: Wed, 20 Mar 2024 13:11:28 +0000 Subject: [PATCH 001/296] [RemoveDIs][NFC] Rename DPLabel->DbgLabelRecord (#85918) This patch renames DPLabel to DbgLabelRecord, in accordance with the ongoing DbgRecord rename. This rename was fairly trivial, since DPLabel isn't as widely used as DPValue and has no real conflicts in either its full or abbreviated name. As usual, the entire replacement was done automatically, with `s/DPLabel/DbgLabelRecord/` and `s/DPL/DLR/`. --- llvm/docs/RemoveDIsDebugInfo.md | 6 ++-- .../include/llvm/IR/DebugProgramInstruction.h | 22 +++++++------ llvm/lib/AsmParser/LLParser.cpp | 2 +- llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 5 +-- llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 6 ++-- llvm/lib/Bitcode/Writer/ValueEnumerator.cpp | 6 ++-- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 10 +++--- llvm/lib/CodeGen/SelectionDAG/FastISel.cpp | 10 +++--- .../SelectionDAG/SelectionDAGBuilder.cpp | 15 ++++----- llvm/lib/IR/AsmWriter.cpp | 20 ++++++------ llvm/lib/IR/AutoUpgrade.cpp | 2 +- llvm/lib/IR/BasicBlock.cpp | 3 +- llvm/lib/IR/DIBuilder.cpp | 8 ++--- llvm/lib/IR/DebugProgramInstruction.cpp | 31 ++++++++++--------- llvm/lib/IR/Verifier.cpp | 24 +++++++------- .../Scalar/SpeculativeExecution.cpp | 2 +- llvm/lib/Transforms/Utils/BasicBlockUtils.cpp | 2 +- llvm/lib/Transforms/Utils/CodeExtractor.cpp | 4 +-- llvm/lib/Transforms/Utils/ValueMapper.cpp | 4 +-- llvm/unittests/IR/IRBuilderTest.cpp | 4 +-- 20 files changed, 97 insertions(+), 89 deletions(-) diff --git a/llvm/docs/RemoveDIsDebugInfo.md b/llvm/docs/RemoveDIsDebugInfo.md index 2cb17e2b5e44..f8405767a579 100644 --- a/llvm/docs/RemoveDIsDebugInfo.md +++ b/llvm/docs/RemoveDIsDebugInfo.md @@ -65,9 +65,9 @@ We're using a dedicated C++ class called `DbgRecord` to store debug info, with a https://llvm.org/docs/doxygen/classllvm_1_1DbgRecord.html https://llvm.org/docs/doxygen/classllvm_1_1DbgVariableRecord.html - https://llvm.org/docs/doxygen/classllvm_1_1DPLabel.html + https://llvm.org/docs/doxygen/classllvm_1_1DbgLabelRecord.html -This allows you to treat a `DbgVariableRecord` as if it's a `dbg.value`/`dbg.declare`/`dbg.assign` intrinsic most of the time, for example in generic (auto-param) lambdas, and the same for `DPLabel` and `dbg.label`s. +This allows you to treat a `DbgVariableRecord` as if it's a `dbg.value`/`dbg.declare`/`dbg.assign` intrinsic most of the time, for example in generic (auto-param) lambdas, and the same for `DbgLabelRecord` and `dbg.label`s. ## How do these `DbgRecords` fit into the instruction stream? @@ -97,7 +97,7 @@ Each instruction has a pointer to a `DPMarker` (which will become optional), tha Not shown are the links from DbgRecord to other parts of the `Value`/`Metadata` hierachy: `DbgRecord` subclasses have tracking pointers to the DIMetadata that they use, and `DbgVariableRecord` has references to `Value`s that are stored in a `DebugValueUser` base class. This refers to a `ValueAsMetadata` object referring to `Value`s, via the `TrackingMetadata` facility. -The various kinds of debug intrinsic (value, declare, assign, label) are all stored in `DbgRecord` subclasses, with a "RecordKind" field distinguishing `DPLabel`s from `DbgVariableRecord`s, and a `LocationType` field in the `DbgVariableRecord` class further disambiguating the various debug variable intrinsics it can represent. +The various kinds of debug intrinsic (value, declare, assign, label) are all stored in `DbgRecord` subclasses, with a "RecordKind" field distinguishing `DbgLabelRecord`s from `DbgVariableRecord`s, and a `LocationType` field in the `DbgVariableRecord` class further disambiguating the various debug variable intrinsics it can represent. ## Finding debug info records diff --git a/llvm/include/llvm/IR/DebugProgramInstruction.h b/llvm/include/llvm/IR/DebugProgramInstruction.h index db41e9acc7be..8bd6331a9a3e 100644 --- a/llvm/include/llvm/IR/DebugProgramInstruction.h +++ b/llvm/include/llvm/IR/DebugProgramInstruction.h @@ -222,23 +222,25 @@ inline raw_ostream &operator<<(raw_ostream &OS, const DbgRecord &R) { /// llvm.dbg.label intrinsic. /// FIXME: Rename DbgLabelRecord when DbgVariableRecord is renamed to /// DbgVariableRecord. -class DPLabel : public DbgRecord { +class DbgLabelRecord : public DbgRecord { DbgRecordParamRef Label; /// This constructor intentionally left private, so that it is only called via - /// "createUnresolvedDPLabel", which clearly expresses that it is for parsing - /// only. - DPLabel(MDNode *Label, MDNode *DL); + /// "createUnresolvedDbgLabelRecord", which clearly expresses that it is for + /// parsing only. + DbgLabelRecord(MDNode *Label, MDNode *DL); public: - DPLabel(DILabel *Label, DebugLoc DL); + DbgLabelRecord(DILabel *Label, DebugLoc DL); - /// For use during parsing; creates a DPLabel from as-of-yet unresolved - /// MDNodes. Trying to access the resulting DPLabel's fields before they are - /// resolved, or if they resolve to the wrong type, will result in a crash. - static DPLabel *createUnresolvedDPLabel(MDNode *Label, MDNode *DL); + /// For use during parsing; creates a DbgLabelRecord from as-of-yet unresolved + /// MDNodes. Trying to access the resulting DbgLabelRecord's fields before + /// they are resolved, or if they resolve to the wrong type, will result in a + /// crash. + static DbgLabelRecord *createUnresolvedDbgLabelRecord(MDNode *Label, + MDNode *DL); - DPLabel *clone() const; + DbgLabelRecord *clone() const; void print(raw_ostream &O, bool IsForDebug = false) const; void print(raw_ostream &ROS, ModuleSlotTracker &MST, bool IsForDebug) const; DbgLabelInst *createDebugIntrinsic(Module *M, diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp index 8738cb47dd9c..f0be021668af 100644 --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -6607,7 +6607,7 @@ bool LLParser::parseDebugRecord(DbgRecord *&DR, PerFunctionState &PFS) { return true; if (parseToken(lltok::rparen, "Expected ')' here")) return true; - DR = DPLabel::createUnresolvedDPLabel(Label, DbgLoc); + DR = DbgLabelRecord::createUnresolvedDbgLabelRecord(Label, DbgLoc); return false; } diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index fa7038acc524..3fc8141381c6 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -6426,14 +6426,15 @@ Error BitcodeReader::parseFunctionBody(Function *F) { break; } case bitc::FUNC_CODE_DEBUG_RECORD_LABEL: { - // DPLabels are placed after the Instructions that they are attached to. + // DbgLabelRecords are placed after the Instructions that they are + // attached to. Instruction *Inst = getLastInstruction(); if (!Inst) return error("Invalid dbg record: missing instruction"); DILocation *DIL = cast(getFnMetadataByID(Record[0])); DILabel *Label = cast(getFnMetadataByID(Record[1])); Inst->getParent()->insertDbgRecordBefore( - new DPLabel(Label, DebugLoc(DIL)), Inst->getIterator()); + new DbgLabelRecord(Label, DebugLoc(DIL)), Inst->getIterator()); continue; // This isn't an instruction. } case bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE: diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index a1ee02918dfa..5addf0ac33c4 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -3570,9 +3570,9 @@ void ModuleBitcodeWriter::writeFunction( // instruction. Write it after the instruction so that it's easy to // re-attach to the instruction reading the records in. for (DbgRecord &DR : I.DbgMarker->getDbgRecordRange()) { - if (DPLabel *DPL = dyn_cast(&DR)) { - Vals.push_back(VE.getMetadataID(&*DPL->getDebugLoc())); - Vals.push_back(VE.getMetadataID(DPL->getLabel())); + if (DbgLabelRecord *DLR = dyn_cast(&DR)) { + Vals.push_back(VE.getMetadataID(&*DLR->getDebugLoc())); + Vals.push_back(VE.getMetadataID(DLR->getLabel())); Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_RECORD_LABEL, Vals); Vals.clear(); continue; diff --git a/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp b/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp index 3209dca253fe..e6787e245a49 100644 --- a/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp +++ b/llvm/lib/Bitcode/Writer/ValueEnumerator.cpp @@ -441,9 +441,9 @@ ValueEnumerator::ValueEnumerator(const Module &M, }; for (DbgRecord &DR : I.getDbgRecordRange()) { - if (DPLabel *DPL = dyn_cast(&DR)) { - EnumerateMetadata(&F, DPL->getLabel()); - EnumerateMetadata(&F, &*DPL->getDebugLoc()); + if (DbgLabelRecord *DLR = dyn_cast(&DR)) { + EnumerateMetadata(&F, DLR->getLabel()); + EnumerateMetadata(&F, &*DLR->getDebugLoc()); continue; } // Enumerate non-local location metadata. diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index c18574071bfb..757af3b1c4fe 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -3376,13 +3376,13 @@ void IRTranslator::translateDbgDeclareRecord(Value *Address, bool HasArgList, void IRTranslator::translateDbgInfo(const Instruction &Inst, MachineIRBuilder &MIRBuilder) { for (DbgRecord &DR : Inst.getDbgRecordRange()) { - if (DPLabel *DPL = dyn_cast(&DR)) { - MIRBuilder.setDebugLoc(DPL->getDebugLoc()); - assert(DPL->getLabel() && "Missing label"); - assert(DPL->getLabel()->isValidLocationForIntrinsic( + if (DbgLabelRecord *DLR = dyn_cast(&DR)) { + MIRBuilder.setDebugLoc(DLR->getDebugLoc()); + assert(DLR->getLabel() && "Missing label"); + assert(DLR->getLabel()->isValidLocationForIntrinsic( MIRBuilder.getDebugLoc()) && "Expected inlined-at fields to agree"); - MIRBuilder.buildDbgLabel(DPL->getLabel()); + MIRBuilder.buildDbgLabel(DLR->getLabel()); continue; } DbgVariableRecord &DVR = cast(DR); diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp index 8b834862fb4d..27b8472ddb73 100644 --- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -1192,16 +1192,16 @@ void FastISel::handleDbgInfo(const Instruction *II) { flushLocalValueMap(); recomputeInsertPt(); - if (DPLabel *DPL = dyn_cast(&DR)) { - assert(DPL->getLabel() && "Missing label"); + if (DbgLabelRecord *DLR = dyn_cast(&DR)) { + assert(DLR->getLabel() && "Missing label"); if (!FuncInfo.MF->getMMI().hasDebugInfo()) { - LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DPL << "\n"); + LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DLR << "\n"); continue; } - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DPL->getDebugLoc(), + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DLR->getDebugLoc(), TII.get(TargetOpcode::DBG_LABEL)) - .addMetadata(DPL->getLabel()); + .addMetadata(DLR->getLabel()); continue; } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index dd19ee16d1d6..2d63774c75e3 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -1248,18 +1248,19 @@ void SelectionDAGBuilder::visitDbgInfo(const Instruction &I) { // We must skip DbgVariableRecords if they've already been processed above as // we have just emitted the debug values resulting from assignment tracking // analysis, making any existing DbgVariableRecords redundant (and probably - // less correct). We still need to process DPLabels. This does sink DPLabels - // to the bottom of the group of debug records. That sholdn't be important - // as it does so deterministcally and ordering between DPLabels and - // DbgVariableRecords is immaterial (other than for MIR/IR printing). + // less correct). We still need to process DbgLabelRecords. This does sink + // DbgLabelRecords to the bottom of the group of debug records. That sholdn't + // be important as it does so deterministcally and ordering between + // DbgLabelRecords and DbgVariableRecords is immaterial (other than for MIR/IR + // printing). bool SkipDbgVariableRecords = DAG.getFunctionVarLocs(); // Is there is any debug-info attached to this instruction, in the form of // DbgRecord non-instruction debug-info records. for (DbgRecord &DR : I.getDbgRecordRange()) { - if (DPLabel *DPL = dyn_cast(&DR)) { - assert(DPL->getLabel() && "Missing label"); + if (DbgLabelRecord *DLR = dyn_cast(&DR)) { + assert(DLR->getLabel() && "Missing label"); SDDbgLabel *SDV = - DAG.getDbgLabel(DPL->getLabel(), DPL->getDebugLoc(), SDNodeOrder); + DAG.getDbgLabel(DLR->getLabel(), DLR->getDebugLoc(), SDNodeOrder); DAG.AddDbgLabel(SDV); continue; } diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index 5caed518e265..8ce6fabb30f2 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -1152,8 +1152,8 @@ void SlotTracker::processDbgRecordMetadata(const DbgRecord &DR) { if (auto *Empty = dyn_cast(DVR->getRawAddress())) CreateMetadataSlot(Empty); } - } else if (const DPLabel *DPL = dyn_cast(&DR)) { - CreateMetadataSlot(DPL->getRawLabel()); + } else if (const DbgLabelRecord *DLR = dyn_cast(&DR)) { + CreateMetadataSlot(DLR->getRawLabel()); } else { llvm_unreachable("unsupported DbgRecord kind"); } @@ -2719,7 +2719,7 @@ public: void printInstruction(const Instruction &I); void printDPMarker(const DPMarker &DPI); void printDbgVariableRecord(const DbgVariableRecord &DVR); - void printDPLabel(const DPLabel &DPL); + void printDbgLabelRecord(const DbgLabelRecord &DLR); void printDbgRecord(const DbgRecord &DR); void printDbgRecordLine(const DbgRecord &DR); @@ -4621,8 +4621,8 @@ void AssemblyWriter::printDPMarker(const DPMarker &Marker) { void AssemblyWriter::printDbgRecord(const DbgRecord &DR) { if (auto *DVR = dyn_cast(&DR)) printDbgVariableRecord(*DVR); - else if (auto *DPL = dyn_cast(&DR)) - printDPLabel(*DPL); + else if (auto *DLR = dyn_cast(&DR)) + printDbgLabelRecord(*DLR); else llvm_unreachable("Unexpected DbgRecord kind"); } @@ -4672,7 +4672,7 @@ void AssemblyWriter::printDbgRecordLine(const DbgRecord &DR) { Out << '\n'; } -void AssemblyWriter::printDPLabel(const DPLabel &Label) { +void AssemblyWriter::printDbgLabelRecord(const DbgLabelRecord &Label) { auto WriterCtx = getContext(); Out << "#dbg_label("; WriteAsOperandInternal(Out, Label.getRawLabel(), WriterCtx, true); @@ -4934,7 +4934,7 @@ void DPMarker::print(raw_ostream &ROS, ModuleSlotTracker &MST, W.printDPMarker(*this); } -void DPLabel::print(raw_ostream &ROS, bool IsForDebug) const { +void DbgLabelRecord::print(raw_ostream &ROS, bool IsForDebug) const { ModuleSlotTracker MST(getModuleFromDPI(this), true); print(ROS, MST, IsForDebug); @@ -4957,8 +4957,8 @@ void DbgVariableRecord::print(raw_ostream &ROS, ModuleSlotTracker &MST, W.printDbgVariableRecord(*this); } -void DPLabel::print(raw_ostream &ROS, ModuleSlotTracker &MST, - bool IsForDebug) const { +void DbgLabelRecord::print(raw_ostream &ROS, ModuleSlotTracker &MST, + bool IsForDebug) const { formatted_raw_ostream OS(ROS); SlotTracker EmptySlotTable(static_cast(nullptr)); SlotTracker &SlotTable = @@ -4970,7 +4970,7 @@ void DPLabel::print(raw_ostream &ROS, ModuleSlotTracker &MST, incorporateFunction(Marker->getParent() ? Marker->getParent()->getParent() : nullptr); AssemblyWriter W(OS, SlotTable, getModuleFromDPI(this), nullptr, IsForDebug); - W.printDPLabel(*this); + W.printDbgLabelRecord(*this); } void Value::print(raw_ostream &ROS, bool IsForDebug) const { diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp index 7d954f9d09ad..a44f6af4162f 100644 --- a/llvm/lib/IR/AutoUpgrade.cpp +++ b/llvm/lib/IR/AutoUpgrade.cpp @@ -2358,7 +2358,7 @@ static MDType *unwrapMAVOp(CallBase *CI, unsigned Op) { static void upgradeDbgIntrinsicToDbgRecord(StringRef Name, CallBase *CI) { DbgRecord *DR = nullptr; if (Name == "label") { - DR = new DPLabel(unwrapMAVOp(CI, 0), CI->getDebugLoc()); + DR = new DbgLabelRecord(unwrapMAVOp(CI, 0), CI->getDebugLoc()); } else if (Name == "assign") { DR = new DbgVariableRecord( unwrapMAVOp(CI, 0), unwrapMAVOp(CI, 1), diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp index 2dff6e4d6d9c..2fa9b3330d18 100644 --- a/llvm/lib/IR/BasicBlock.cpp +++ b/llvm/lib/IR/BasicBlock.cpp @@ -83,7 +83,8 @@ void BasicBlock::convertToNewDbgValues() { } if (DbgLabelInst *DLI = dyn_cast(&I)) { - DbgVarRecs.push_back(new DPLabel(DLI->getLabel(), DLI->getDebugLoc())); + DbgVarRecs.push_back( + new DbgLabelRecord(DLI->getLabel(), DLI->getDebugLoc())); DLI->eraseFromParent(); continue; } diff --git a/llvm/lib/IR/DIBuilder.cpp b/llvm/lib/IR/DIBuilder.cpp index f10b5acb980d..f86e557b8def 100644 --- a/llvm/lib/IR/DIBuilder.cpp +++ b/llvm/lib/IR/DIBuilder.cpp @@ -1155,12 +1155,12 @@ DbgInstPtr DIBuilder::insertLabel(DILabel *LabelInfo, const DILocation *DL, trackIfUnresolved(LabelInfo); if (M.IsNewDbgInfoFormat) { - DPLabel *DPL = new DPLabel(LabelInfo, DL); + DbgLabelRecord *DLR = new DbgLabelRecord(LabelInfo, DL); if (InsertBB && InsertBefore) - InsertBB->insertDbgRecordBefore(DPL, InsertBefore->getIterator()); + InsertBB->insertDbgRecordBefore(DLR, InsertBefore->getIterator()); else if (InsertBB) - InsertBB->insertDbgRecordBefore(DPL, InsertBB->end()); - return DPL; + InsertBB->insertDbgRecordBefore(DLR, InsertBB->end()); + return DLR; } if (!LabelFn) diff --git a/llvm/lib/IR/DebugProgramInstruction.cpp b/llvm/lib/IR/DebugProgramInstruction.cpp index 1fb435f46a5f..1c0be6f598cc 100644 --- a/llvm/lib/IR/DebugProgramInstruction.cpp +++ b/llvm/lib/IR/DebugProgramInstruction.cpp @@ -82,7 +82,7 @@ void DbgRecord::deleteRecord() { delete cast(this); return; case LabelKind: - delete cast(this); + delete cast(this); return; } llvm_unreachable("unsupported DbgRecord kind"); @@ -94,7 +94,7 @@ void DbgRecord::print(raw_ostream &O, bool IsForDebug) const { cast(this)->print(O, IsForDebug); return; case LabelKind: - cast(this)->print(O, IsForDebug); + cast(this)->print(O, IsForDebug); return; }; llvm_unreachable("unsupported DbgRecord kind"); @@ -107,7 +107,7 @@ void DbgRecord::print(raw_ostream &O, ModuleSlotTracker &MST, cast(this)->print(O, MST, IsForDebug); return; case LabelKind: - cast(this)->print(O, MST, IsForDebug); + cast(this)->print(O, MST, IsForDebug); return; }; llvm_unreachable("unsupported DbgRecord kind"); @@ -121,7 +121,8 @@ bool DbgRecord::isIdenticalToWhenDefined(const DbgRecord &R) const { return cast(this)->isIdenticalToWhenDefined( *cast(&R)); case LabelKind: - return cast(this)->getLabel() == cast(R).getLabel(); + return cast(this)->getLabel() == + cast(R).getLabel(); }; llvm_unreachable("unsupported DbgRecord kind"); } @@ -136,24 +137,25 @@ DbgRecord::createDebugIntrinsic(Module *M, Instruction *InsertBefore) const { case ValueKind: return cast(this)->createDebugIntrinsic(M, InsertBefore); case LabelKind: - return cast(this)->createDebugIntrinsic(M, InsertBefore); + return cast(this)->createDebugIntrinsic(M, InsertBefore); }; llvm_unreachable("unsupported DbgRecord kind"); } -DPLabel::DPLabel(MDNode *Label, MDNode *DL) +DbgLabelRecord::DbgLabelRecord(MDNode *Label, MDNode *DL) : DbgRecord(LabelKind, DebugLoc(DL)), Label(Label) { assert(Label && "Unexpected nullptr"); assert((isa(Label) || Label->isTemporary()) && "Label type must be or resolve to a DILabel"); } -DPLabel::DPLabel(DILabel *Label, DebugLoc DL) +DbgLabelRecord::DbgLabelRecord(DILabel *Label, DebugLoc DL) : DbgRecord(LabelKind, DL), Label(Label) { assert(Label && "Unexpected nullptr"); } -DPLabel *DPLabel::createUnresolvedDPLabel(MDNode *Label, MDNode *DL) { - return new DPLabel(Label, DL); +DbgLabelRecord *DbgLabelRecord::createUnresolvedDbgLabelRecord(MDNode *Label, + MDNode *DL) { + return new DbgLabelRecord(Label, DL); } DbgVariableRecord::DbgVariableRecord(DbgVariableRecord::LocationType Type, @@ -380,7 +382,7 @@ DbgRecord *DbgRecord::clone() const { case ValueKind: return cast(this)->clone(); case LabelKind: - return cast(this)->clone(); + return cast(this)->clone(); }; llvm_unreachable("unsupported DbgRecord kind"); } @@ -389,8 +391,8 @@ DbgVariableRecord *DbgVariableRecord::clone() const { return new DbgVariableRecord(*this); } -DPLabel *DPLabel::clone() const { - return new DPLabel(getLabel(), getDebugLoc()); +DbgLabelRecord *DbgLabelRecord::clone() const { + return new DbgLabelRecord(getLabel(), getDebugLoc()); } DbgVariableIntrinsic * @@ -450,8 +452,9 @@ DbgVariableRecord::createDebugIntrinsic(Module *M, return DVI; } -DbgLabelInst *DPLabel::createDebugIntrinsic(Module *M, - Instruction *InsertBefore) const { +DbgLabelInst * +DbgLabelRecord::createDebugIntrinsic(Module *M, + Instruction *InsertBefore) const { auto *LabelFn = Intrinsic::getDeclaration(M, Intrinsic::dbg_label); Value *Args[] = { MetadataAsValue::get(getDebugLoc()->getContext(), getLabel())}; diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 1e16e8648462..a99b307a3536 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -544,7 +544,7 @@ private: void visitTemplateParams(const MDNode &N, const Metadata &RawParams); - void visit(DPLabel &DPL); + void visit(DbgLabelRecord &DLR); void visit(DbgVariableRecord &DVR); // InstVisitor overrides... using InstVisitor::visit; @@ -696,8 +696,8 @@ void Verifier::visitDbgRecords(Instruction &I) { // intrinsic behaviour. verifyFragmentExpression(*DVR); verifyNotEntryValue(*DVR); - } else if (auto *DPL = dyn_cast(&DR)) { - visit(*DPL); + } else if (auto *DLR = dyn_cast(&DR)) { + visit(*DLR); } } } @@ -6244,22 +6244,22 @@ static DISubprogram *getSubprogram(Metadata *LocalScope) { return nullptr; } -void Verifier::visit(DPLabel &DPL) { - CheckDI(isa(DPL.getRawLabel()), - "invalid #dbg_label intrinsic variable", &DPL, DPL.getRawLabel()); +void Verifier::visit(DbgLabelRecord &DLR) { + CheckDI(isa(DLR.getRawLabel()), + "invalid #dbg_label intrinsic variable", &DLR, DLR.getRawLabel()); // Ignore broken !dbg attachments; they're checked elsewhere. - if (MDNode *N = DPL.getDebugLoc().getAsMDNode()) + if (MDNode *N = DLR.getDebugLoc().getAsMDNode()) if (!isa(N)) return; - BasicBlock *BB = DPL.getParent(); + BasicBlock *BB = DLR.getParent(); Function *F = BB ? BB->getParent() : nullptr; // The scopes for variables and !dbg attachments must agree. - DILabel *Label = DPL.getLabel(); - DILocation *Loc = DPL.getDebugLoc(); - CheckDI(Loc, "#dbg_label record requires a !dbg attachment", &DPL, BB, F); + DILabel *Label = DLR.getLabel(); + DILocation *Loc = DLR.getDebugLoc(); + CheckDI(Loc, "#dbg_label record requires a !dbg attachment", &DLR, BB, F); DISubprogram *LabelSP = getSubprogram(Label->getRawScope()); DISubprogram *LocSP = getSubprogram(Loc->getRawScope()); @@ -6268,7 +6268,7 @@ void Verifier::visit(DPLabel &DPL) { CheckDI(LabelSP == LocSP, "mismatched subprogram between #dbg_label label and !dbg attachment", - &DPL, BB, F, Label, Label->getScope()->getSubprogram(), Loc, + &DLR, BB, F, Label, Label->getScope()->getSubprogram(), Loc, Loc->getScope()->getSubprogram()); } diff --git a/llvm/lib/Transforms/Scalar/SpeculativeExecution.cpp b/llvm/lib/Transforms/Scalar/SpeculativeExecution.cpp index 400b56894174..5efc340da60b 100644 --- a/llvm/lib/Transforms/Scalar/SpeculativeExecution.cpp +++ b/llvm/lib/Transforms/Scalar/SpeculativeExecution.cpp @@ -292,7 +292,7 @@ bool SpeculativeExecutionPass::considerHoistingFromTo( InstructionCost TotalSpeculationCost = 0; unsigned NotHoistedInstCount = 0; for (const auto &I : FromBlock) { - // Make note of any DbgVariableRecords that need hoisting. DPLabels + // Make note of any DbgVariableRecords that need hoisting. DbgLabelRecords // get left behind just like llvm.dbg.labels. for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) { if (HasNoUnhoistedInstr(DVR.location_ops())) diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp index bf1de05a647d..915cd81661f0 100644 --- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp +++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp @@ -388,7 +388,7 @@ DbgVariableRecordsRemoveRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) { SmallDenseSet VariableSet; for (auto &I : reverse(*BB)) { for (DbgRecord &DR : reverse(I.getDbgRecordRange())) { - if (isa(DR)) { + if (isa(DR)) { // Emulate existing behaviour (see comment below for dbg.declares). // FIXME: Don't do this. VariableSet.clear(); diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp index 122b7a9747b6..3191751d92e1 100644 --- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp +++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp @@ -1619,8 +1619,8 @@ static void fixupDebugInfoPostExtraction(Function &OldFunc, Function &NewFunc, auto UpdateDbgRecordsOnInst = [&](Instruction &I) -> void { for (DbgRecord &DR : I.getDbgRecordRange()) { - if (DPLabel *DPL = dyn_cast(&DR)) { - UpdateDbgLabel(DPL); + if (DbgLabelRecord *DLR = dyn_cast(&DR)) { + UpdateDbgLabel(DLR); continue; } diff --git a/llvm/lib/Transforms/Utils/ValueMapper.cpp b/llvm/lib/Transforms/Utils/ValueMapper.cpp index 8c24599c8ce3..6ebdd85d37b4 100644 --- a/llvm/lib/Transforms/Utils/ValueMapper.cpp +++ b/llvm/lib/Transforms/Utils/ValueMapper.cpp @@ -538,8 +538,8 @@ Value *Mapper::mapValue(const Value *V) { } void Mapper::remapDbgRecord(DbgRecord &DR) { - if (DPLabel *DPL = dyn_cast(&DR)) { - DPL->setLabel(cast(mapMetadata(DPL->getLabel()))); + if (DbgLabelRecord *DLR = dyn_cast(&DR)) { + DLR->setLabel(cast(mapMetadata(DLR->getLabel()))); return; } diff --git a/llvm/unittests/IR/IRBuilderTest.cpp b/llvm/unittests/IR/IRBuilderTest.cpp index ec3059821230..2001df090aed 100644 --- a/llvm/unittests/IR/IRBuilderTest.cpp +++ b/llvm/unittests/IR/IRBuilderTest.cpp @@ -922,7 +922,7 @@ TEST_F(IRBuilderTest, DIBuilder) { DILabel *Label = DIB.createLabel(BarScope, "badger", File, 1, /*AlwaysPreserve*/ false); - { /* dbg.label | DPLabel */ + { /* dbg.label | DbgLabelRecord */ // Insert before I and check order. ExpectOrder(DIB.insertLabel(Label, LabelLoc, I), I->getIterator()); @@ -931,7 +931,7 @@ TEST_F(IRBuilderTest, DIBuilder) { // inserted into the block untill another instruction is added. DbgInstPtr LabelRecord = DIB.insertLabel(Label, LabelLoc, BB); // Specifically do not insert a terminator, to check this works. `I` - // should have absorbed the DPLabel in the new debug info mode. + // should have absorbed the DbgLabelRecord in the new debug info mode. I = Builder.CreateAlloca(Builder.getInt32Ty()); ExpectOrder(LabelRecord, I->getIterator()); } -- GitLab From 2ac85d8d200a9e1e0ced501c2d2f04404c400bd9 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 20 Mar 2024 11:56:02 +0000 Subject: [PATCH 002/296] [VectorCombine] foldBitcastShuf - add support for binary shuffles Generalise fold to "bitcast (shuf V0, V1, MaskC) --> shuf (bitcast V0), (bitcast V1), MaskC'". Further prep work for #67803 --- .../Transforms/Vectorize/VectorCombine.cpp | 21 ++++++----- .../Transforms/PhaseOrdering/X86/pr67803.ll | 35 +++++++++++++++++-- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp index 0b16a8b76769..23494314f132 100644 --- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp +++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp @@ -684,10 +684,10 @@ bool VectorCombine::foldInsExtFNeg(Instruction &I) { /// destination type followed by shuffle. This can enable further transforms by /// moving bitcasts or shuffles together. bool VectorCombine::foldBitcastShuffle(Instruction &I) { - Value *V0; + Value *V0, *V1; ArrayRef Mask; if (!match(&I, m_BitCast(m_OneUse( - m_Shuffle(m_Value(V0), m_Undef(), m_Mask(Mask)))))) + m_Shuffle(m_Value(V0), m_Value(V1), m_Mask(Mask)))))) return false; // 1) Do not fold bitcast shuffle for scalable type. First, shuffle cost for @@ -728,17 +728,21 @@ bool VectorCombine::foldBitcastShuffle(Instruction &I) { FixedVectorType::get(DestTy->getScalarType(), NumSrcElts); auto *OldShuffleTy = FixedVectorType::get(SrcTy->getScalarType(), Mask.size()); + bool IsUnary = isa(V1); + unsigned NumOps = IsUnary ? 1 : 2; // The new shuffle must not cost more than the old shuffle. TargetTransformInfo::TargetCostKind CK = TargetTransformInfo::TCK_RecipThroughput; TargetTransformInfo::ShuffleKind SK = - TargetTransformInfo::SK_PermuteSingleSrc; + IsUnary ? TargetTransformInfo::SK_PermuteSingleSrc + : TargetTransformInfo::SK_PermuteTwoSrc; InstructionCost DestCost = TTI.getShuffleCost(SK, NewShuffleTy, NewMask, CK) + - TTI.getCastInstrCost(Instruction::BitCast, NewShuffleTy, SrcTy, - TargetTransformInfo::CastContextHint::None, CK); + (NumOps * TTI.getCastInstrCost(Instruction::BitCast, NewShuffleTy, SrcTy, + TargetTransformInfo::CastContextHint::None, + CK)); InstructionCost SrcCost = TTI.getShuffleCost(SK, SrcTy, Mask, CK) + TTI.getCastInstrCost(Instruction::BitCast, DestTy, OldShuffleTy, @@ -746,10 +750,11 @@ bool VectorCombine::foldBitcastShuffle(Instruction &I) { if (DestCost > SrcCost || !DestCost.isValid()) return false; - // bitcast (shuf V0, MaskC) --> shuf (bitcast V0), MaskC' + // bitcast (shuf V0, V1, MaskC) --> shuf (bitcast V0), (bitcast V1), MaskC' ++NumShufOfBitcast; - Value *CastV = Builder.CreateBitCast(V0, NewShuffleTy); - Value *Shuf = Builder.CreateShuffleVector(CastV, NewMask); + Value *CastV0 = Builder.CreateBitCast(V0, NewShuffleTy); + Value *CastV1 = Builder.CreateBitCast(V1, NewShuffleTy); + Value *Shuf = Builder.CreateShuffleVector(CastV0, CastV1, NewMask); replaceValue(I, *Shuf); return true; } diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll index 211c90b5604e..e61b254b7a5f 100644 --- a/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll +++ b/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s -; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s -; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s +; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK +; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK +; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX512 define <4 x i64> @PR67803(<4 x i64> %x, <4 x i64> %y, <4 x i64> %a, <4 x i64> %b) { ; CHECK-LABEL: @PR67803( @@ -35,6 +35,35 @@ define <4 x i64> @PR67803(<4 x i64> %x, <4 x i64> %y, <4 x i64> %a, <4 x i64> %b ; CHECK-NEXT: [[SHUFFLE_I23:%.*]] = shufflevector <2 x i64> [[TMP12]], <2 x i64> [[TMP20]], <4 x i32> ; CHECK-NEXT: ret <4 x i64> [[SHUFFLE_I23]] ; +; AVX512-LABEL: @PR67803( +; AVX512-NEXT: entry: +; AVX512-NEXT: [[TMP0:%.*]] = bitcast <4 x i64> [[X:%.*]] to <8 x i32> +; AVX512-NEXT: [[TMP1:%.*]] = bitcast <4 x i64> [[Y:%.*]] to <8 x i32> +; AVX512-NEXT: [[TMP2:%.*]] = icmp sgt <8 x i32> [[TMP0]], [[TMP1]] +; AVX512-NEXT: [[CMP_I21:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> +; AVX512-NEXT: [[SEXT_I22:%.*]] = sext <4 x i1> [[CMP_I21]] to <4 x i32> +; AVX512-NEXT: [[CMP_I:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> +; AVX512-NEXT: [[SEXT_I:%.*]] = sext <4 x i1> [[CMP_I]] to <4 x i32> +; AVX512-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[SEXT_I22]], <4 x i32> [[SEXT_I]], <8 x i32> +; AVX512-NEXT: [[TMP4:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8> +; AVX512-NEXT: [[TMP5:%.*]] = shufflevector <32 x i8> [[TMP4]], <32 x i8> poison, <16 x i32> +; AVX512-NEXT: [[TMP6:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8> +; AVX512-NEXT: [[TMP7:%.*]] = shufflevector <32 x i8> [[TMP6]], <32 x i8> poison, <16 x i32> +; AVX512-NEXT: [[TMP8:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8> +; AVX512-NEXT: [[TMP9:%.*]] = shufflevector <32 x i8> [[TMP8]], <32 x i8> poison, <16 x i32> +; AVX512-NEXT: [[TMP10:%.*]] = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[TMP5]], <16 x i8> [[TMP7]], <16 x i8> [[TMP9]]) +; AVX512-NEXT: [[TMP11:%.*]] = bitcast <16 x i8> [[TMP10]] to <2 x i64> +; AVX512-NEXT: [[TMP12:%.*]] = bitcast <4 x i64> [[A]] to <32 x i8> +; AVX512-NEXT: [[TMP13:%.*]] = shufflevector <32 x i8> [[TMP12]], <32 x i8> poison, <16 x i32> +; AVX512-NEXT: [[TMP14:%.*]] = bitcast <4 x i64> [[B]] to <32 x i8> +; AVX512-NEXT: [[TMP15:%.*]] = shufflevector <32 x i8> [[TMP14]], <32 x i8> poison, <16 x i32> +; AVX512-NEXT: [[TMP16:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8> +; AVX512-NEXT: [[TMP17:%.*]] = shufflevector <32 x i8> [[TMP16]], <32 x i8> poison, <16 x i32> +; AVX512-NEXT: [[TMP18:%.*]] = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[TMP13]], <16 x i8> [[TMP15]], <16 x i8> [[TMP17]]) +; AVX512-NEXT: [[TMP19:%.*]] = bitcast <16 x i8> [[TMP18]] to <2 x i64> +; AVX512-NEXT: [[SHUFFLE_I23:%.*]] = shufflevector <2 x i64> [[TMP11]], <2 x i64> [[TMP19]], <4 x i32> +; AVX512-NEXT: ret <4 x i64> [[SHUFFLE_I23]] +; entry: %0 = bitcast <4 x i64> %x to <8 x i32> %extract = shufflevector <8 x i32> %0, <8 x i32> poison, <4 x i32> -- GitLab From 3eb806373e3164b242db65f8c900e4adb5a2eddf Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Wed, 20 Mar 2024 14:19:41 +0100 Subject: [PATCH 003/296] [CodeGen] Fix test on 32-bit targets (NFC) The range here will be different for 32-bit targets. Use a wildcard, just like all te other target-sensitive parts in this test. --- clang/test/CodeGenCXX/copy-constructor-synthesis-2.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/test/CodeGenCXX/copy-constructor-synthesis-2.cpp b/clang/test/CodeGenCXX/copy-constructor-synthesis-2.cpp index 4f96a3ae6707..ae0c3a26c597 100644 --- a/clang/test/CodeGenCXX/copy-constructor-synthesis-2.cpp +++ b/clang/test/CodeGenCXX/copy-constructor-synthesis-2.cpp @@ -24,4 +24,4 @@ struct A { virtual void a(); }; A x(A& y) { return y; } // CHECK: define linkonce_odr {{.*}} @_ZN1AC1ERKS_(ptr {{.*}}%this, ptr noundef nonnull align {{[0-9]+}} dereferenceable({{[0-9]+}}) %0) unnamed_addr -// CHECK: store ptr getelementptr inbounds inrange(-16, 8) ({ [3 x ptr] }, ptr @_ZTV1A, i32 0, i32 0, i32 2) +// CHECK: store ptr getelementptr inbounds inrange(-{{[0-9]+}}, {{[0-9]+}}) ({ [3 x ptr] }, ptr @_ZTV1A, i32 0, i32 0, i32 2) -- GitLab From 98c6bc531d091215896087b94e4e047c67f892c2 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Wed, 20 Mar 2024 14:21:53 +0100 Subject: [PATCH 004/296] [MLIR][SROA][Mem2Reg] Add data layout to interface methods (#85644) This commit expends the Mem2Reg and SROA interface methods with passed in handles to a `DataLayout` structure. This is done to avoid superfluous retreiving of data layouts during each conversion of intrinsics. This change, additionally, enables subsequent changes to make the LLVM dialect implementation of these interfaces type agnostic. --- .../mlir/Interfaces/MemorySlotInterfaces.td | 43 ++--- mlir/include/mlir/Transforms/Mem2Reg.h | 2 +- mlir/include/mlir/Transforms/SROA.h | 3 +- mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp | 153 +++++++++++------- .../Dialect/MemRef/IR/MemRefMemorySlot.cpp | 18 ++- mlir/lib/Transforms/Mem2Reg.cpp | 24 ++- mlir/lib/Transforms/SROA.cpp | 30 ++-- 7 files changed, 165 insertions(+), 108 deletions(-) diff --git a/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td b/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td index 9ffa709cc5bf..e10e2d4e104c 100644 --- a/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td +++ b/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td @@ -83,11 +83,10 @@ def PromotableAllocationOpInterface def PromotableMemOpInterface : OpInterface<"PromotableMemOpInterface"> { let description = [{ Describes an operation that can load from memory slots and/or store - to memory slots. Loads and stores must be of whole values of the same - type as the slot itself. + to memory slots. - For a memory operation on a slot to be valid, it must operate on the slot - pointer *only as a pointer to an element of the type of the slot*. + For a memory operation on a slot to be valid, it must strictly operate + within the bounds of the slot. If the same operation does both loads and stores on the same slot, the load must semantically happen first. @@ -142,7 +141,8 @@ def PromotableMemOpInterface : OpInterface<"PromotableMemOpInterface"> { }], "bool", "canUsesBeRemoved", (ins "const ::mlir::MemorySlot &":$slot, "const ::llvm::SmallPtrSetImpl<::mlir::OpOperand *> &":$blockingUses, - "::llvm::SmallVectorImpl<::mlir::OpOperand *> &":$newBlockingUses) + "::llvm::SmallVectorImpl<::mlir::OpOperand *> &":$newBlockingUses, + "const ::mlir::DataLayout &":$datalayout) >, InterfaceMethod<[{ Transforms IR to ensure that the current operation does not use the @@ -197,7 +197,8 @@ def PromotableOpInterface : OpInterface<"PromotableOpInterface"> { No IR mutation is allowed in this method. }], "bool", "canUsesBeRemoved", (ins "const ::llvm::SmallPtrSetImpl<::mlir::OpOperand *> &":$blockingUses, - "::llvm::SmallVectorImpl<::mlir::OpOperand *> &":$newBlockingUses) + "::llvm::SmallVectorImpl<::mlir::OpOperand *> &":$newBlockingUses, + "const ::mlir::DataLayout &":$datalayout) >, InterfaceMethod<[{ Transforms IR to ensure that the current operation does not use the @@ -285,29 +286,28 @@ def DestructurableAllocationOpInterface def SafeMemorySlotAccessOpInterface : OpInterface<"SafeMemorySlotAccessOpInterface"> { let description = [{ - Describes operations using memory slots in a type-safe manner. + Describes operations using memory slots in a safe manner. }]; let cppNamespace = "::mlir"; let methods = [ InterfaceMethod<[{ Returns whether all accesses in this operation to the provided slot are - done in a type-safe manner. To be type-safe, the access must only load - the value in this type as the type of the slot, and without assuming any - context around the slot. For example, a type-safe load must not load - outside the bounds of the slot. + done in a safe manner. To be safe, the access most only access the slot + inside the bounds that its type implies. - If the type-safety of the accesses depends on the type-safety of the - accesses to further memory slots, the result of this method will be - conditioned to the type-safety of the accesses to the slots added by - this method to `mustBeSafelyUsed`. + If the safety of the accesses depends on the safety of the accesses to + further memory slots, the result of this method will be conditioned to + the safety of the accesses to the slots added by this method to + `mustBeSafelyUsed`. No IR mutation is allowed in this method. }], "::mlir::LogicalResult", "ensureOnlySafeAccesses", (ins "const ::mlir::MemorySlot &":$slot, - "::mlir::SmallVectorImpl<::mlir::MemorySlot> &":$mustBeSafelyUsed) + "::mlir::SmallVectorImpl<::mlir::MemorySlot> &":$mustBeSafelyUsed, + "const ::mlir::DataLayout &":$dataLayout) > ]; } @@ -323,13 +323,12 @@ def DestructurableAccessorOpInterface InterfaceMethod<[{ For a given destructurable memory slot, returns whether this operation can rewire its uses of the slot to use the slots generated after - destructuring. This may involve creating new operations, and usually - amounts to checking if the pointer types match. + destructuring. This may involve creating new operations. This method must also register the indices it will access within the `usedIndices` set. If the accessor generates new slots mapping to subelements, they must be registered in `mustBeSafelyUsed` to ensure - they are used in a locally type-safe manner. + they are used in a safe manner. No IR mutation is allowed in this method. }], @@ -337,7 +336,8 @@ def DestructurableAccessorOpInterface "canRewire", (ins "const ::mlir::DestructurableMemorySlot &":$slot, "::llvm::SmallPtrSetImpl<::mlir::Attribute> &":$usedIndices, - "::mlir::SmallVectorImpl<::mlir::MemorySlot> &":$mustBeSafelyUsed) + "::mlir::SmallVectorImpl<::mlir::MemorySlot> &":$mustBeSafelyUsed, + "const ::mlir::DataLayout &":$dataLayout) >, InterfaceMethod<[{ Rewires the use of a slot to the generated subslots, without deleting @@ -351,7 +351,8 @@ def DestructurableAccessorOpInterface "rewire", (ins "const ::mlir::DestructurableMemorySlot &":$slot, "::llvm::DenseMap<::mlir::Attribute, ::mlir::MemorySlot> &":$subslots, - "::mlir::RewriterBase &":$rewriter) + "::mlir::RewriterBase &":$rewriter, + "const ::mlir::DataLayout &":$dataLayout) > ]; } diff --git a/mlir/include/mlir/Transforms/Mem2Reg.h b/mlir/include/mlir/Transforms/Mem2Reg.h index d145f7ed4375..ed10644e26a5 100644 --- a/mlir/include/mlir/Transforms/Mem2Reg.h +++ b/mlir/include/mlir/Transforms/Mem2Reg.h @@ -27,7 +27,7 @@ struct Mem2RegStatistics { /// at least one memory slot was promoted. LogicalResult tryToPromoteMemorySlots(ArrayRef allocators, - RewriterBase &rewriter, + RewriterBase &rewriter, const DataLayout &dataLayout, Mem2RegStatistics statistics = {}); } // namespace mlir diff --git a/mlir/include/mlir/Transforms/SROA.h b/mlir/include/mlir/Transforms/SROA.h index 1af1fe930723..d09a7989edea 100644 --- a/mlir/include/mlir/Transforms/SROA.h +++ b/mlir/include/mlir/Transforms/SROA.h @@ -31,7 +31,8 @@ struct SROAStatistics { /// failure if no slot was destructured. LogicalResult tryToDestructureMemorySlots( ArrayRef allocators, - RewriterBase &rewriter, SROAStatistics statistics = {}); + RewriterBase &rewriter, const DataLayout &dataLayout, + SROAStatistics statistics = {}); } // namespace mlir diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp index 81766b307d3d..00b4559658fd 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp @@ -132,7 +132,8 @@ Value LLVM::StoreOp::getStored(const MemorySlot &slot, RewriterBase &rewriter) { bool LLVM::LoadOp::canUsesBeRemoved( const MemorySlot &slot, const SmallPtrSetImpl &blockingUses, - SmallVectorImpl &newBlockingUses) { + SmallVectorImpl &newBlockingUses, + const DataLayout &dataLayout) { if (blockingUses.size() != 1) return false; Value blockingUse = (*blockingUses.begin())->get(); @@ -155,7 +156,8 @@ DeletionKind LLVM::LoadOp::removeBlockingUses( bool LLVM::StoreOp::canUsesBeRemoved( const MemorySlot &slot, const SmallPtrSetImpl &blockingUses, - SmallVectorImpl &newBlockingUses) { + SmallVectorImpl &newBlockingUses, + const DataLayout &dataLayout) { if (blockingUses.size() != 1) return false; Value blockingUse = (*blockingUses.begin())->get(); @@ -181,12 +183,14 @@ DeletionKind LLVM::StoreOp::removeBlockingUses( } LogicalResult LLVM::LoadOp::ensureOnlySafeAccesses( - const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed) { + const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { return success(getAddr() != slot.ptr || getType() == slot.elemType); } LogicalResult LLVM::StoreOp::ensureOnlySafeAccesses( - const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed) { + const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { return success(getAddr() != slot.ptr || getValue().getType() == slot.elemType); } @@ -206,7 +210,8 @@ static bool forwardToUsers(Operation *op, bool LLVM::BitcastOp::canUsesBeRemoved( const SmallPtrSetImpl &blockingUses, - SmallVectorImpl &newBlockingUses) { + SmallVectorImpl &newBlockingUses, + const DataLayout &dataLayout) { return forwardToUsers(*this, newBlockingUses); } @@ -217,7 +222,8 @@ DeletionKind LLVM::BitcastOp::removeBlockingUses( bool LLVM::AddrSpaceCastOp::canUsesBeRemoved( const SmallPtrSetImpl &blockingUses, - SmallVectorImpl &newBlockingUses) { + SmallVectorImpl &newBlockingUses, + const DataLayout &dataLayout) { return forwardToUsers(*this, newBlockingUses); } @@ -228,7 +234,8 @@ DeletionKind LLVM::AddrSpaceCastOp::removeBlockingUses( bool LLVM::LifetimeStartOp::canUsesBeRemoved( const SmallPtrSetImpl &blockingUses, - SmallVectorImpl &newBlockingUses) { + SmallVectorImpl &newBlockingUses, + const DataLayout &dataLayout) { return true; } @@ -239,7 +246,8 @@ DeletionKind LLVM::LifetimeStartOp::removeBlockingUses( bool LLVM::LifetimeEndOp::canUsesBeRemoved( const SmallPtrSetImpl &blockingUses, - SmallVectorImpl &newBlockingUses) { + SmallVectorImpl &newBlockingUses, + const DataLayout &dataLayout) { return true; } @@ -250,7 +258,8 @@ DeletionKind LLVM::LifetimeEndOp::removeBlockingUses( bool LLVM::InvariantStartOp::canUsesBeRemoved( const SmallPtrSetImpl &blockingUses, - SmallVectorImpl &newBlockingUses) { + SmallVectorImpl &newBlockingUses, + const DataLayout &dataLayout) { return true; } @@ -261,7 +270,8 @@ DeletionKind LLVM::InvariantStartOp::removeBlockingUses( bool LLVM::InvariantEndOp::canUsesBeRemoved( const SmallPtrSetImpl &blockingUses, - SmallVectorImpl &newBlockingUses) { + SmallVectorImpl &newBlockingUses, + const DataLayout &dataLayout) { return true; } @@ -272,7 +282,8 @@ DeletionKind LLVM::InvariantEndOp::removeBlockingUses( bool LLVM::DbgDeclareOp::canUsesBeRemoved( const SmallPtrSetImpl &blockingUses, - SmallVectorImpl &newBlockingUses) { + SmallVectorImpl &newBlockingUses, + const DataLayout &dataLayout) { return true; } @@ -283,7 +294,8 @@ DeletionKind LLVM::DbgDeclareOp::removeBlockingUses( bool LLVM::DbgValueOp::canUsesBeRemoved( const SmallPtrSetImpl &blockingUses, - SmallVectorImpl &newBlockingUses) { + SmallVectorImpl &newBlockingUses, + const DataLayout &dataLayout) { // There is only one operand that we can remove the use of. if (blockingUses.size() != 1) return false; @@ -318,7 +330,8 @@ static bool hasAllZeroIndices(LLVM::GEPOp gepOp) { bool LLVM::GEPOp::canUsesBeRemoved( const SmallPtrSetImpl &blockingUses, - SmallVectorImpl &newBlockingUses) { + SmallVectorImpl &newBlockingUses, + const DataLayout &dataLayout) { // GEP can be removed as long as it is a no-op and its users can be removed. if (!hasAllZeroIndices(*this)) return false; @@ -337,7 +350,8 @@ static bool isFirstIndexZero(LLVM::GEPOp gep) { } LogicalResult LLVM::GEPOp::ensureOnlySafeAccesses( - const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed) { + const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { if (getBase() != slot.ptr) return success(); if (slot.elemType != getElemType()) @@ -357,9 +371,9 @@ LogicalResult LLVM::GEPOp::ensureOnlySafeAccesses( bool LLVM::GEPOp::canRewire(const DestructurableMemorySlot &slot, SmallPtrSetImpl &usedIndices, - SmallVectorImpl &mustBeSafelyUsed) { - auto basePtrType = llvm::dyn_cast(getBase().getType()); - if (!basePtrType) + SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { + if (!isa(getBase().getType())) return false; if (getBase() != slot.ptr || slot.elemType != getElemType()) @@ -386,7 +400,8 @@ bool LLVM::GEPOp::canRewire(const DestructurableMemorySlot &slot, DeletionKind LLVM::GEPOp::rewire(const DestructurableMemorySlot &slot, DenseMap &subslots, - RewriterBase &rewriter) { + RewriterBase &rewriter, + const DataLayout &dataLayout) { IntegerAttr firstLevelIndex = llvm::dyn_cast_if_present(getIndices()[1]); const MemorySlot &newSlot = subslots.at(firstLevelIndex); @@ -455,7 +470,7 @@ std::optional getStaticMemIntrLen(LLVM::MemcpyInlineOp op) { /// of the bounds of the given slot, on a best-effort basis. template static bool definitelyWritesOnlyWithinSlot(MemIntr op, const MemorySlot &slot, - DataLayout &dataLayout) { + const DataLayout &dataLayout) { if (!isa(slot.ptr.getType()) || op.getDst() != slot.ptr) return false; @@ -520,7 +535,8 @@ Value LLVM::MemsetOp::getStored(const MemorySlot &slot, bool LLVM::MemsetOp::canUsesBeRemoved( const MemorySlot &slot, const SmallPtrSetImpl &blockingUses, - SmallVectorImpl &newBlockingUses) { + SmallVectorImpl &newBlockingUses, + const DataLayout &dataLayout) { // TODO: Support non-integer types. bool canConvertType = TypeSwitch(slot.elemType) @@ -534,8 +550,7 @@ bool LLVM::MemsetOp::canUsesBeRemoved( if (getIsVolatile()) return false; - DataLayout layout = DataLayout::closest(*this); - return getStaticMemIntrLen(*this) == layout.getTypeSize(slot.elemType); + return getStaticMemIntrLen(*this) == dataLayout.getTypeSize(slot.elemType); } DeletionKind LLVM::MemsetOp::removeBlockingUses( @@ -545,14 +560,15 @@ DeletionKind LLVM::MemsetOp::removeBlockingUses( } LogicalResult LLVM::MemsetOp::ensureOnlySafeAccesses( - const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed) { - DataLayout dataLayout = DataLayout::closest(*this); + const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { return success(definitelyWritesOnlyWithinSlot(*this, slot, dataLayout)); } bool LLVM::MemsetOp::canRewire(const DestructurableMemorySlot &slot, SmallPtrSetImpl &usedIndices, - SmallVectorImpl &mustBeSafelyUsed) { + SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { if (&slot.elemType.getDialect() != getOperation()->getDialect()) return false; @@ -566,13 +582,13 @@ bool LLVM::MemsetOp::canRewire(const DestructurableMemorySlot &slot, if (!areAllIndicesI32(slot)) return false; - DataLayout dataLayout = DataLayout::closest(*this); return definitelyWritesOnlyWithinSlot(*this, slot, dataLayout); } DeletionKind LLVM::MemsetOp::rewire(const DestructurableMemorySlot &slot, DenseMap &subslots, - RewriterBase &rewriter) { + RewriterBase &rewriter, + const DataLayout &dataLayout) { std::optional> types = slot.elemType.cast().getSubelementIndexMap(); @@ -587,7 +603,6 @@ DeletionKind LLVM::MemsetOp::rewire(const DestructurableMemorySlot &slot, packed = structType.isPacked(); Type i32 = IntegerType::get(getContext(), 32); - DataLayout dataLayout = DataLayout::closest(*this); uint64_t memsetLen = memsetLenAttr.getValue().getZExtValue(); uint64_t covered = 0; for (size_t i = 0; i < types->size(); i++) { @@ -650,7 +665,8 @@ template static bool memcpyCanUsesBeRemoved(MemcpyLike op, const MemorySlot &slot, const SmallPtrSetImpl &blockingUses, - SmallVectorImpl &newBlockingUses) { + SmallVectorImpl &newBlockingUses, + const DataLayout &dataLayout) { // If source and destination are the same, memcpy behavior is undefined and // memmove is a no-op. Because there is no memory change happening here, // simplifying such operations is left to canonicalization. @@ -660,8 +676,7 @@ memcpyCanUsesBeRemoved(MemcpyLike op, const MemorySlot &slot, if (op.getIsVolatile()) return false; - DataLayout layout = DataLayout::closest(op); - return getStaticMemIntrLen(op) == layout.getTypeSize(slot.elemType); + return getStaticMemIntrLen(op) == dataLayout.getTypeSize(slot.elemType); } template @@ -689,7 +704,8 @@ memcpyEnsureOnlySafeAccesses(MemcpyLike op, const MemorySlot &slot, template static bool memcpyCanRewire(MemcpyLike op, const DestructurableMemorySlot &slot, SmallPtrSetImpl &usedIndices, - SmallVectorImpl &mustBeSafelyUsed) { + SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { if (op.getIsVolatile()) return false; @@ -701,7 +717,6 @@ static bool memcpyCanRewire(MemcpyLike op, const DestructurableMemorySlot &slot, return false; // Only full copies are supported. - DataLayout dataLayout = DataLayout::closest(op); if (getStaticMemIntrLen(op) != dataLayout.getTypeSize(slot.elemType)) return false; @@ -741,15 +756,13 @@ void createMemcpyLikeToReplace(RewriterBase &rewriter, const DataLayout &layout, /// Rewires a memcpy-like operation. Only copies to or from the full slot are /// supported. template -static DeletionKind memcpyRewire(MemcpyLike op, - const DestructurableMemorySlot &slot, - DenseMap &subslots, - RewriterBase &rewriter) { +static DeletionKind +memcpyRewire(MemcpyLike op, const DestructurableMemorySlot &slot, + DenseMap &subslots, RewriterBase &rewriter, + const DataLayout &dataLayout) { if (subslots.empty()) return DeletionKind::Delete; - DataLayout layout = DataLayout::closest(op); - assert((slot.ptr == op.getDst()) != (slot.ptr == op.getSrc())); bool isDst = slot.ptr == op.getDst(); @@ -780,7 +793,7 @@ static DeletionKind memcpyRewire(MemcpyLike op, isDst ? op.getSrc() : op.getDst(), gepIndices); // Then create a new memcpy out of this source pointer. - createMemcpyLikeToReplace(rewriter, layout, op, + createMemcpyLikeToReplace(rewriter, dataLayout, op, isDst ? subslot.ptr : subslotPtrInOther, isDst ? subslotPtrInOther : subslot.ptr, subslot.elemType, op.getIsVolatile()); @@ -806,8 +819,10 @@ Value LLVM::MemcpyOp::getStored(const MemorySlot &slot, bool LLVM::MemcpyOp::canUsesBeRemoved( const MemorySlot &slot, const SmallPtrSetImpl &blockingUses, - SmallVectorImpl &newBlockingUses) { - return memcpyCanUsesBeRemoved(*this, slot, blockingUses, newBlockingUses); + SmallVectorImpl &newBlockingUses, + const DataLayout &dataLayout) { + return memcpyCanUsesBeRemoved(*this, slot, blockingUses, newBlockingUses, + dataLayout); } DeletionKind LLVM::MemcpyOp::removeBlockingUses( @@ -818,20 +833,24 @@ DeletionKind LLVM::MemcpyOp::removeBlockingUses( } LogicalResult LLVM::MemcpyOp::ensureOnlySafeAccesses( - const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed) { + const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { return memcpyEnsureOnlySafeAccesses(*this, slot, mustBeSafelyUsed); } bool LLVM::MemcpyOp::canRewire(const DestructurableMemorySlot &slot, SmallPtrSetImpl &usedIndices, - SmallVectorImpl &mustBeSafelyUsed) { - return memcpyCanRewire(*this, slot, usedIndices, mustBeSafelyUsed); + SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { + return memcpyCanRewire(*this, slot, usedIndices, mustBeSafelyUsed, + dataLayout); } DeletionKind LLVM::MemcpyOp::rewire(const DestructurableMemorySlot &slot, DenseMap &subslots, - RewriterBase &rewriter) { - return memcpyRewire(*this, slot, subslots, rewriter); + RewriterBase &rewriter, + const DataLayout &dataLayout) { + return memcpyRewire(*this, slot, subslots, rewriter, dataLayout); } bool LLVM::MemcpyInlineOp::loadsFrom(const MemorySlot &slot) { @@ -849,8 +868,10 @@ Value LLVM::MemcpyInlineOp::getStored(const MemorySlot &slot, bool LLVM::MemcpyInlineOp::canUsesBeRemoved( const MemorySlot &slot, const SmallPtrSetImpl &blockingUses, - SmallVectorImpl &newBlockingUses) { - return memcpyCanUsesBeRemoved(*this, slot, blockingUses, newBlockingUses); + SmallVectorImpl &newBlockingUses, + const DataLayout &dataLayout) { + return memcpyCanUsesBeRemoved(*this, slot, blockingUses, newBlockingUses, + dataLayout); } DeletionKind LLVM::MemcpyInlineOp::removeBlockingUses( @@ -861,22 +882,26 @@ DeletionKind LLVM::MemcpyInlineOp::removeBlockingUses( } LogicalResult LLVM::MemcpyInlineOp::ensureOnlySafeAccesses( - const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed) { + const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { return memcpyEnsureOnlySafeAccesses(*this, slot, mustBeSafelyUsed); } bool LLVM::MemcpyInlineOp::canRewire( const DestructurableMemorySlot &slot, SmallPtrSetImpl &usedIndices, - SmallVectorImpl &mustBeSafelyUsed) { - return memcpyCanRewire(*this, slot, usedIndices, mustBeSafelyUsed); + SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { + return memcpyCanRewire(*this, slot, usedIndices, mustBeSafelyUsed, + dataLayout); } DeletionKind LLVM::MemcpyInlineOp::rewire(const DestructurableMemorySlot &slot, DenseMap &subslots, - RewriterBase &rewriter) { - return memcpyRewire(*this, slot, subslots, rewriter); + RewriterBase &rewriter, + const DataLayout &dataLayout) { + return memcpyRewire(*this, slot, subslots, rewriter, dataLayout); } bool LLVM::MemmoveOp::loadsFrom(const MemorySlot &slot) { @@ -894,8 +919,10 @@ Value LLVM::MemmoveOp::getStored(const MemorySlot &slot, bool LLVM::MemmoveOp::canUsesBeRemoved( const MemorySlot &slot, const SmallPtrSetImpl &blockingUses, - SmallVectorImpl &newBlockingUses) { - return memcpyCanUsesBeRemoved(*this, slot, blockingUses, newBlockingUses); + SmallVectorImpl &newBlockingUses, + const DataLayout &dataLayout) { + return memcpyCanUsesBeRemoved(*this, slot, blockingUses, newBlockingUses, + dataLayout); } DeletionKind LLVM::MemmoveOp::removeBlockingUses( @@ -906,20 +933,24 @@ DeletionKind LLVM::MemmoveOp::removeBlockingUses( } LogicalResult LLVM::MemmoveOp::ensureOnlySafeAccesses( - const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed) { + const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { return memcpyEnsureOnlySafeAccesses(*this, slot, mustBeSafelyUsed); } bool LLVM::MemmoveOp::canRewire(const DestructurableMemorySlot &slot, SmallPtrSetImpl &usedIndices, - SmallVectorImpl &mustBeSafelyUsed) { - return memcpyCanRewire(*this, slot, usedIndices, mustBeSafelyUsed); + SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { + return memcpyCanRewire(*this, slot, usedIndices, mustBeSafelyUsed, + dataLayout); } DeletionKind LLVM::MemmoveOp::rewire(const DestructurableMemorySlot &slot, DenseMap &subslots, - RewriterBase &rewriter) { - return memcpyRewire(*this, slot, subslots, rewriter); + RewriterBase &rewriter, + const DataLayout &dataLayout) { + return memcpyRewire(*this, slot, subslots, rewriter, dataLayout); } //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp index 561b8619032c..7be4056fb2fc 100644 --- a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp +++ b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp @@ -170,7 +170,8 @@ Value memref::LoadOp::getStored(const MemorySlot &slot, bool memref::LoadOp::canUsesBeRemoved( const MemorySlot &slot, const SmallPtrSetImpl &blockingUses, - SmallVectorImpl &newBlockingUses) { + SmallVectorImpl &newBlockingUses, + const DataLayout &dataLayout) { if (blockingUses.size() != 1) return false; Value blockingUse = (*blockingUses.begin())->get(); @@ -210,7 +211,8 @@ static Attribute getAttributeIndexFromIndexOperands(MLIRContext *ctx, bool memref::LoadOp::canRewire(const DestructurableMemorySlot &slot, SmallPtrSetImpl &usedIndices, - SmallVectorImpl &mustBeSafelyUsed) { + SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { if (slot.ptr != getMemRef()) return false; Attribute index = getAttributeIndexFromIndexOperands( @@ -223,7 +225,8 @@ bool memref::LoadOp::canRewire(const DestructurableMemorySlot &slot, DeletionKind memref::LoadOp::rewire(const DestructurableMemorySlot &slot, DenseMap &subslots, - RewriterBase &rewriter) { + RewriterBase &rewriter, + const DataLayout &dataLayout) { Attribute index = getAttributeIndexFromIndexOperands( getContext(), getIndices(), getMemRefType()); const MemorySlot &memorySlot = subslots.at(index); @@ -247,7 +250,8 @@ Value memref::StoreOp::getStored(const MemorySlot &slot, bool memref::StoreOp::canUsesBeRemoved( const MemorySlot &slot, const SmallPtrSetImpl &blockingUses, - SmallVectorImpl &newBlockingUses) { + SmallVectorImpl &newBlockingUses, + const DataLayout &dataLayout) { if (blockingUses.size() != 1) return false; Value blockingUse = (*blockingUses.begin())->get(); @@ -263,7 +267,8 @@ DeletionKind memref::StoreOp::removeBlockingUses( bool memref::StoreOp::canRewire(const DestructurableMemorySlot &slot, SmallPtrSetImpl &usedIndices, - SmallVectorImpl &mustBeSafelyUsed) { + SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { if (slot.ptr != getMemRef() || getValue() == slot.ptr) return false; Attribute index = getAttributeIndexFromIndexOperands( @@ -276,7 +281,8 @@ bool memref::StoreOp::canRewire(const DestructurableMemorySlot &slot, DeletionKind memref::StoreOp::rewire(const DestructurableMemorySlot &slot, DenseMap &subslots, - RewriterBase &rewriter) { + RewriterBase &rewriter, + const DataLayout &dataLayout) { Attribute index = getAttributeIndexFromIndexOperands( getContext(), getIndices(), getMemRefType()); const MemorySlot &memorySlot = subslots.at(index); diff --git a/mlir/lib/Transforms/Mem2Reg.cpp b/mlir/lib/Transforms/Mem2Reg.cpp index 84ac69b4514b..80e3b7901632 100644 --- a/mlir/lib/Transforms/Mem2Reg.cpp +++ b/mlir/lib/Transforms/Mem2Reg.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "mlir/Transforms/Mem2Reg.h" +#include "mlir/Analysis/DataLayoutAnalysis.h" #include "mlir/Analysis/SliceAnalysis.h" #include "mlir/IR/Builders.h" #include "mlir/IR/Dominance.h" @@ -117,8 +118,9 @@ struct MemorySlotPromotionInfo { /// promotion. This does not mutate IR. class MemorySlotPromotionAnalyzer { public: - MemorySlotPromotionAnalyzer(MemorySlot slot, DominanceInfo &dominance) - : slot(slot), dominance(dominance) {} + MemorySlotPromotionAnalyzer(MemorySlot slot, DominanceInfo &dominance, + const DataLayout &dataLayout) + : slot(slot), dominance(dominance), dataLayout(dataLayout) {} /// Computes the information for slot promotion if promotion is possible, /// returns nothing otherwise. @@ -153,6 +155,7 @@ private: MemorySlot slot; DominanceInfo &dominance; + const DataLayout &dataLayout; }; /// The MemorySlotPromoter handles the state of promoting a memory slot. It @@ -267,10 +270,12 @@ LogicalResult MemorySlotPromotionAnalyzer::computeBlockingUses( // If the operation decides it cannot deal with removing the blocking uses, // promotion must fail. if (auto promotable = dyn_cast(user)) { - if (!promotable.canUsesBeRemoved(blockingUses, newBlockingUses)) + if (!promotable.canUsesBeRemoved(blockingUses, newBlockingUses, + dataLayout)) return failure(); } else if (auto promotable = dyn_cast(user)) { - if (!promotable.canUsesBeRemoved(slot, blockingUses, newBlockingUses)) + if (!promotable.canUsesBeRemoved(slot, blockingUses, newBlockingUses, + dataLayout)) return failure(); } else { // An operation that has blocking uses must be promoted. If it is not @@ -610,7 +615,8 @@ void MemorySlotPromoter::promoteSlot() { LogicalResult mlir::tryToPromoteMemorySlots( ArrayRef allocators, - RewriterBase &rewriter, Mem2RegStatistics statistics) { + RewriterBase &rewriter, const DataLayout &dataLayout, + Mem2RegStatistics statistics) { bool promotedAny = false; for (PromotableAllocationOpInterface allocator : allocators) { @@ -619,7 +625,7 @@ LogicalResult mlir::tryToPromoteMemorySlots( continue; DominanceInfo dominance; - MemorySlotPromotionAnalyzer analyzer(slot, dominance); + MemorySlotPromotionAnalyzer analyzer(slot, dominance, dataLayout); std::optional info = analyzer.computeInfo(); if (info) { MemorySlotPromoter(slot, allocator, rewriter, dominance, @@ -661,8 +667,12 @@ struct Mem2Reg : impl::Mem2RegBase { allocators.emplace_back(allocator); }); + auto &dataLayoutAnalysis = getAnalysis(); + const DataLayout &dataLayout = dataLayoutAnalysis.getAtOrAbove(scopeOp); + // Attempt promoting until no promotion succeeds. - if (failed(tryToPromoteMemorySlots(allocators, rewriter, statistics))) + if (failed(tryToPromoteMemorySlots(allocators, rewriter, dataLayout, + statistics))) break; changed = true; diff --git a/mlir/lib/Transforms/SROA.cpp b/mlir/lib/Transforms/SROA.cpp index 6111489bdebe..f24cbb7b1725 100644 --- a/mlir/lib/Transforms/SROA.cpp +++ b/mlir/lib/Transforms/SROA.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "mlir/Transforms/SROA.h" +#include "mlir/Analysis/DataLayoutAnalysis.h" #include "mlir/Analysis/SliceAnalysis.h" #include "mlir/Interfaces/MemorySlotInterfaces.h" #include "mlir/Transforms/Passes.h" @@ -42,7 +43,8 @@ struct MemorySlotDestructuringInfo { /// nothing if the slot cannot be destructured or if there is no useful work to /// be done. static std::optional -computeDestructuringInfo(DestructurableMemorySlot &slot) { +computeDestructuringInfo(DestructurableMemorySlot &slot, + const DataLayout &dataLayout) { assert(isa(slot.elemType)); if (slot.ptr.use_empty()) @@ -62,7 +64,8 @@ computeDestructuringInfo(DestructurableMemorySlot &slot) { for (OpOperand &use : slot.ptr.getUses()) { if (auto accessor = dyn_cast(use.getOwner())) { - if (accessor.canRewire(slot, info.usedIndices, usedSafelyWorklist)) { + if (accessor.canRewire(slot, info.usedIndices, usedSafelyWorklist, + dataLayout)) { info.accessors.push_back(accessor); continue; } @@ -82,8 +85,8 @@ computeDestructuringInfo(DestructurableMemorySlot &slot) { Operation *subslotUser = subslotUse.getOwner(); if (auto memOp = dyn_cast(subslotUser)) - if (succeeded(memOp.ensureOnlySafeAccesses(mustBeUsedSafely, - usedSafelyWorklist))) + if (succeeded(memOp.ensureOnlySafeAccesses( + mustBeUsedSafely, usedSafelyWorklist, dataLayout))) continue; // If it cannot be shown that the operation uses the slot safely, maybe it @@ -110,7 +113,7 @@ computeDestructuringInfo(DestructurableMemorySlot &slot) { SmallVector newBlockingUses; // If the operation decides it cannot deal with removing the blocking uses, // destructuring must fail. - if (!promotable.canUsesBeRemoved(blockingUses, newBlockingUses)) + if (!promotable.canUsesBeRemoved(blockingUses, newBlockingUses, dataLayout)) return {}; // Then, register any new blocking uses for coming operations. @@ -132,6 +135,7 @@ computeDestructuringInfo(DestructurableMemorySlot &slot) { static void destructureSlot(DestructurableMemorySlot &slot, DestructurableAllocationOpInterface allocator, RewriterBase &rewriter, + const DataLayout &dataLayout, MemorySlotDestructuringInfo &info, const SROAStatistics &statistics) { RewriterBase::InsertionGuard guard(rewriter); @@ -158,7 +162,8 @@ static void destructureSlot(DestructurableMemorySlot &slot, for (Operation *toRewire : llvm::reverse(usersToRewire)) { rewriter.setInsertionPointAfter(toRewire); if (auto accessor = dyn_cast(toRewire)) { - if (accessor.rewire(slot, subslots, rewriter) == DeletionKind::Delete) + if (accessor.rewire(slot, subslots, rewriter, dataLayout) == + DeletionKind::Delete) toErase.push_back(accessor); continue; } @@ -186,17 +191,18 @@ static void destructureSlot(DestructurableMemorySlot &slot, LogicalResult mlir::tryToDestructureMemorySlots( ArrayRef allocators, - RewriterBase &rewriter, SROAStatistics statistics) { + RewriterBase &rewriter, const DataLayout &dataLayout, + SROAStatistics statistics) { bool destructuredAny = false; for (DestructurableAllocationOpInterface allocator : allocators) { for (DestructurableMemorySlot slot : allocator.getDestructurableSlots()) { std::optional info = - computeDestructuringInfo(slot); + computeDestructuringInfo(slot, dataLayout); if (!info) continue; - destructureSlot(slot, allocator, rewriter, *info, statistics); + destructureSlot(slot, allocator, rewriter, dataLayout, *info, statistics); destructuredAny = true; } } @@ -215,6 +221,8 @@ struct SROA : public impl::SROABase { SROAStatistics statistics{&destructuredAmount, &slotsWithMemoryBenefit, &maxSubelementAmount}; + auto &dataLayoutAnalysis = getAnalysis(); + const DataLayout &dataLayout = dataLayoutAnalysis.getAtOrAbove(scopeOp); bool changed = false; for (Region ®ion : scopeOp->getRegions()) { @@ -235,8 +243,8 @@ struct SROA : public impl::SROABase { allocators.emplace_back(allocator); }); - if (failed( - tryToDestructureMemorySlots(allocators, rewriter, statistics))) + if (failed(tryToDestructureMemorySlots(allocators, rewriter, dataLayout, + statistics))) break; changed = true; -- GitLab From ada24ae5e6e3da1002fe4debe9d37a8279d11c11 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 20 Mar 2024 13:39:42 +0000 Subject: [PATCH 005/296] Revert 2ac85d8d200a9e1e0ced501c2d2f04404c400bd9 "[VectorCombine] foldBitcastShuf - add support for binary shuffles" Breaks some tests in other subprojects - will recommit with a fix later --- .../Transforms/Vectorize/VectorCombine.cpp | 21 +++++------ .../Transforms/PhaseOrdering/X86/pr67803.ll | 35 ++----------------- 2 files changed, 11 insertions(+), 45 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp index 23494314f132..0b16a8b76769 100644 --- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp +++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp @@ -684,10 +684,10 @@ bool VectorCombine::foldInsExtFNeg(Instruction &I) { /// destination type followed by shuffle. This can enable further transforms by /// moving bitcasts or shuffles together. bool VectorCombine::foldBitcastShuffle(Instruction &I) { - Value *V0, *V1; + Value *V0; ArrayRef Mask; if (!match(&I, m_BitCast(m_OneUse( - m_Shuffle(m_Value(V0), m_Value(V1), m_Mask(Mask)))))) + m_Shuffle(m_Value(V0), m_Undef(), m_Mask(Mask)))))) return false; // 1) Do not fold bitcast shuffle for scalable type. First, shuffle cost for @@ -728,21 +728,17 @@ bool VectorCombine::foldBitcastShuffle(Instruction &I) { FixedVectorType::get(DestTy->getScalarType(), NumSrcElts); auto *OldShuffleTy = FixedVectorType::get(SrcTy->getScalarType(), Mask.size()); - bool IsUnary = isa(V1); - unsigned NumOps = IsUnary ? 1 : 2; // The new shuffle must not cost more than the old shuffle. TargetTransformInfo::TargetCostKind CK = TargetTransformInfo::TCK_RecipThroughput; TargetTransformInfo::ShuffleKind SK = - IsUnary ? TargetTransformInfo::SK_PermuteSingleSrc - : TargetTransformInfo::SK_PermuteTwoSrc; + TargetTransformInfo::SK_PermuteSingleSrc; InstructionCost DestCost = TTI.getShuffleCost(SK, NewShuffleTy, NewMask, CK) + - (NumOps * TTI.getCastInstrCost(Instruction::BitCast, NewShuffleTy, SrcTy, - TargetTransformInfo::CastContextHint::None, - CK)); + TTI.getCastInstrCost(Instruction::BitCast, NewShuffleTy, SrcTy, + TargetTransformInfo::CastContextHint::None, CK); InstructionCost SrcCost = TTI.getShuffleCost(SK, SrcTy, Mask, CK) + TTI.getCastInstrCost(Instruction::BitCast, DestTy, OldShuffleTy, @@ -750,11 +746,10 @@ bool VectorCombine::foldBitcastShuffle(Instruction &I) { if (DestCost > SrcCost || !DestCost.isValid()) return false; - // bitcast (shuf V0, V1, MaskC) --> shuf (bitcast V0), (bitcast V1), MaskC' + // bitcast (shuf V0, MaskC) --> shuf (bitcast V0), MaskC' ++NumShufOfBitcast; - Value *CastV0 = Builder.CreateBitCast(V0, NewShuffleTy); - Value *CastV1 = Builder.CreateBitCast(V1, NewShuffleTy); - Value *Shuf = Builder.CreateShuffleVector(CastV0, CastV1, NewMask); + Value *CastV = Builder.CreateBitCast(V0, NewShuffleTy); + Value *Shuf = Builder.CreateShuffleVector(CastV, NewMask); replaceValue(I, *Shuf); return true; } diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll index e61b254b7a5f..211c90b5604e 100644 --- a/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll +++ b/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK -; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK -; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX512 +; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s +; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s +; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s define <4 x i64> @PR67803(<4 x i64> %x, <4 x i64> %y, <4 x i64> %a, <4 x i64> %b) { ; CHECK-LABEL: @PR67803( @@ -35,35 +35,6 @@ define <4 x i64> @PR67803(<4 x i64> %x, <4 x i64> %y, <4 x i64> %a, <4 x i64> %b ; CHECK-NEXT: [[SHUFFLE_I23:%.*]] = shufflevector <2 x i64> [[TMP12]], <2 x i64> [[TMP20]], <4 x i32> ; CHECK-NEXT: ret <4 x i64> [[SHUFFLE_I23]] ; -; AVX512-LABEL: @PR67803( -; AVX512-NEXT: entry: -; AVX512-NEXT: [[TMP0:%.*]] = bitcast <4 x i64> [[X:%.*]] to <8 x i32> -; AVX512-NEXT: [[TMP1:%.*]] = bitcast <4 x i64> [[Y:%.*]] to <8 x i32> -; AVX512-NEXT: [[TMP2:%.*]] = icmp sgt <8 x i32> [[TMP0]], [[TMP1]] -; AVX512-NEXT: [[CMP_I21:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> -; AVX512-NEXT: [[SEXT_I22:%.*]] = sext <4 x i1> [[CMP_I21]] to <4 x i32> -; AVX512-NEXT: [[CMP_I:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> -; AVX512-NEXT: [[SEXT_I:%.*]] = sext <4 x i1> [[CMP_I]] to <4 x i32> -; AVX512-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[SEXT_I22]], <4 x i32> [[SEXT_I]], <8 x i32> -; AVX512-NEXT: [[TMP4:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8> -; AVX512-NEXT: [[TMP5:%.*]] = shufflevector <32 x i8> [[TMP4]], <32 x i8> poison, <16 x i32> -; AVX512-NEXT: [[TMP6:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8> -; AVX512-NEXT: [[TMP7:%.*]] = shufflevector <32 x i8> [[TMP6]], <32 x i8> poison, <16 x i32> -; AVX512-NEXT: [[TMP8:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8> -; AVX512-NEXT: [[TMP9:%.*]] = shufflevector <32 x i8> [[TMP8]], <32 x i8> poison, <16 x i32> -; AVX512-NEXT: [[TMP10:%.*]] = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[TMP5]], <16 x i8> [[TMP7]], <16 x i8> [[TMP9]]) -; AVX512-NEXT: [[TMP11:%.*]] = bitcast <16 x i8> [[TMP10]] to <2 x i64> -; AVX512-NEXT: [[TMP12:%.*]] = bitcast <4 x i64> [[A]] to <32 x i8> -; AVX512-NEXT: [[TMP13:%.*]] = shufflevector <32 x i8> [[TMP12]], <32 x i8> poison, <16 x i32> -; AVX512-NEXT: [[TMP14:%.*]] = bitcast <4 x i64> [[B]] to <32 x i8> -; AVX512-NEXT: [[TMP15:%.*]] = shufflevector <32 x i8> [[TMP14]], <32 x i8> poison, <16 x i32> -; AVX512-NEXT: [[TMP16:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8> -; AVX512-NEXT: [[TMP17:%.*]] = shufflevector <32 x i8> [[TMP16]], <32 x i8> poison, <16 x i32> -; AVX512-NEXT: [[TMP18:%.*]] = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[TMP13]], <16 x i8> [[TMP15]], <16 x i8> [[TMP17]]) -; AVX512-NEXT: [[TMP19:%.*]] = bitcast <16 x i8> [[TMP18]] to <2 x i64> -; AVX512-NEXT: [[SHUFFLE_I23:%.*]] = shufflevector <2 x i64> [[TMP11]], <2 x i64> [[TMP19]], <4 x i32> -; AVX512-NEXT: ret <4 x i64> [[SHUFFLE_I23]] -; entry: %0 = bitcast <4 x i64> %x to <8 x i32> %extract = shufflevector <8 x i32> %0, <8 x i32> poison, <4 x i32> -- GitLab From 9d1cb18d19862fc0627e4a56e1e491a498e84c71 Mon Sep 17 00:00:00 2001 From: Hans Date: Wed, 20 Mar 2024 14:51:45 +0100 Subject: [PATCH 006/296] [Coroutines] Ignore instructions more aggressively in addMustTailToCoroResumes() (#85271) The old code used isInstructionTriviallyDead() and removed instructions when walking the path from a resume call to function return to check if the call is in tail position. However, since the code was walking forwards it was not able to get past instructions such as: %gep = getelementptr inbounds i64, ptr %alloc.var, i32 0 %foo = ptrtoint ptr %gep to i64 This patch instead ignores such instructions as long as their values are not needed. This enables the code to emit tail calls in more situations. --- .../coro-symmetric-transfer-04.cpp | 66 ++++++++++++++++ llvm/lib/Transforms/Coroutines/CoroSplit.cpp | 79 +++++++------------ .../Coroutines/coro-split-musttail7.ll | 12 ++- 3 files changed, 104 insertions(+), 53 deletions(-) create mode 100644 clang/test/CodeGenCoroutines/coro-symmetric-transfer-04.cpp diff --git a/clang/test/CodeGenCoroutines/coro-symmetric-transfer-04.cpp b/clang/test/CodeGenCoroutines/coro-symmetric-transfer-04.cpp new file mode 100644 index 000000000000..cf9170d7e711 --- /dev/null +++ b/clang/test/CodeGenCoroutines/coro-symmetric-transfer-04.cpp @@ -0,0 +1,66 @@ +// This tests that the symmetric transfer at the final suspend point could happen successfully. +// Based on https://github.com/llvm/llvm-project/pull/85271#issuecomment-2007554532 +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++20 -O2 -emit-llvm %s -o - | FileCheck %s + +#include "Inputs/coroutine.h" + +struct Task { + struct promise_type { + struct FinalAwaiter { + bool await_ready() const noexcept { return false; } + template + std::coroutine_handle<> await_suspend(std::coroutine_handle h) noexcept { + return h.promise().continuation; + } + void await_resume() noexcept {} + }; + Task get_return_object() noexcept { + return std::coroutine_handle::from_promise(*this); + } + std::suspend_always initial_suspend() noexcept { return {}; } + FinalAwaiter final_suspend() noexcept { return {}; } + void unhandled_exception() noexcept {} + void return_value(int x) noexcept { + _value = x; + } + std::coroutine_handle<> continuation; + int _value; + }; + + Task(std::coroutine_handle handle) : handle(handle), stuff(123) {} + + struct Awaiter { + std::coroutine_handle handle; + Awaiter(std::coroutine_handle handle) : handle(handle) {} + bool await_ready() const noexcept { return false; } + std::coroutine_handle await_suspend(std::coroutine_handle continuation) noexcept { + handle.promise().continuation = continuation; + return handle; + } + int await_resume() noexcept { + int ret = handle.promise()._value; + handle.destroy(); + return ret; + } + }; + + auto operator co_await() { + auto handle_ = handle; + handle = nullptr; + return Awaiter(handle_); + } + +private: + std::coroutine_handle handle; + int stuff; +}; + +Task task0() { + co_return 43; +} + +// CHECK-LABEL: define{{.*}} void @_Z5task0v.resume +// This checks we are still in the scope of the current function. +// CHECK-NOT: {{^}}} +// CHECK: musttail call fastcc void +// CHECK-NEXT: ret void diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp index 3f3d81474faf..3a43b1edcaba 100644 --- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp +++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp @@ -1198,22 +1198,6 @@ static bool simplifyTerminatorLeadingToRet(Instruction *InitialInst) { assert(InitialInst->getModule()); const DataLayout &DL = InitialInst->getModule()->getDataLayout(); - auto GetFirstValidInstruction = [](Instruction *I) { - while (I) { - // BitCastInst wouldn't generate actual code so that we could skip it. - if (isa(I) || I->isDebugOrPseudoInst() || - I->isLifetimeStartOrEnd()) - I = I->getNextNode(); - else if (isInstructionTriviallyDead(I)) - // Duing we are in the middle of the transformation, we need to erase - // the dead instruction manually. - I = &*I->eraseFromParent(); - else - break; - } - return I; - }; - auto TryResolveConstant = [&ResolvedValues](Value *V) { auto It = ResolvedValues.find(V); if (It != ResolvedValues.end()) @@ -1222,8 +1206,9 @@ static bool simplifyTerminatorLeadingToRet(Instruction *InitialInst) { }; Instruction *I = InitialInst; - while (I->isTerminator() || isa(I)) { + while (true) { if (isa(I)) { + assert(!cast(I)->getReturnValue()); ReplaceInstWithInst(InitialInst, I->clone()); return true; } @@ -1247,40 +1232,26 @@ static bool simplifyTerminatorLeadingToRet(Instruction *InitialInst) { BasicBlock *Succ = BR->getSuccessor(SuccIndex); scanPHIsAndUpdateValueMap(I, Succ, ResolvedValues); - I = GetFirstValidInstruction(Succ->getFirstNonPHIOrDbgOrLifetime()); - + I = Succ->getFirstNonPHIOrDbgOrLifetime(); continue; } - if (auto *CondCmp = dyn_cast(I)) { + if (auto *Cmp = dyn_cast(I)) { // If the case number of suspended switch instruction is reduced to // 1, then it is simplified to CmpInst in llvm::ConstantFoldTerminator. - auto *BR = dyn_cast( - GetFirstValidInstruction(CondCmp->getNextNode())); - if (!BR || !BR->isConditional() || CondCmp != BR->getCondition()) - return false; - - // And the comparsion looks like : %cond = icmp eq i8 %V, constant. - // So we try to resolve constant for the first operand only since the - // second operand should be literal constant by design. - ConstantInt *Cond0 = TryResolveConstant(CondCmp->getOperand(0)); - auto *Cond1 = dyn_cast(CondCmp->getOperand(1)); - if (!Cond0 || !Cond1) - return false; - - // Both operands of the CmpInst are Constant. So that we could evaluate - // it immediately to get the destination. - auto *ConstResult = - dyn_cast_or_null(ConstantFoldCompareInstOperands( - CondCmp->getPredicate(), Cond0, Cond1, DL)); - if (!ConstResult) - return false; - - ResolvedValues[BR->getCondition()] = ConstResult; - - // Handle this branch in next iteration. - I = BR; - continue; + // Try to constant fold it. + ConstantInt *Cond0 = TryResolveConstant(Cmp->getOperand(0)); + ConstantInt *Cond1 = TryResolveConstant(Cmp->getOperand(1)); + if (Cond0 && Cond1) { + ConstantInt *Result = + dyn_cast_or_null(ConstantFoldCompareInstOperands( + Cmp->getPredicate(), Cond0, Cond1, DL)); + if (Result) { + ResolvedValues[Cmp] = Result; + I = I->getNextNode(); + continue; + } + } } if (auto *SI = dyn_cast(I)) { @@ -1288,13 +1259,21 @@ static bool simplifyTerminatorLeadingToRet(Instruction *InitialInst) { if (!Cond) return false; - BasicBlock *BB = SI->findCaseValue(Cond)->getCaseSuccessor(); - scanPHIsAndUpdateValueMap(I, BB, ResolvedValues); - I = GetFirstValidInstruction(BB->getFirstNonPHIOrDbgOrLifetime()); + BasicBlock *Succ = SI->findCaseValue(Cond)->getCaseSuccessor(); + scanPHIsAndUpdateValueMap(I, Succ, ResolvedValues); + I = Succ->getFirstNonPHIOrDbgOrLifetime(); + continue; + } + + if (I->isDebugOrPseudoInst() || I->isLifetimeStartOrEnd() || + wouldInstructionBeTriviallyDead(I)) { + // We can skip instructions without side effects. If their values are + // needed, we'll notice later, e.g. when hitting a conditional branch. + I = I->getNextNode(); continue; } - return false; + break; } return false; diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail7.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail7.ll index 2257d5aee473..d0d5005587bd 100644 --- a/llvm/test/Transforms/Coroutines/coro-split-musttail7.ll +++ b/llvm/test/Transforms/Coroutines/coro-split-musttail7.ll @@ -1,6 +1,6 @@ ; Tests that sinked lifetime markers wouldn't provent optimization ; to convert a resuming call to a musttail call. -; The difference between this and coro-split-musttail5.ll and coro-split-musttail5.ll +; The difference between this and coro-split-musttail5.ll and coro-split-musttail6.ll ; is that this contains dead instruction generated during the transformation, ; which makes the optimization harder. ; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s @@ -8,7 +8,7 @@ declare void @fakeresume1(ptr align 8) -define void @g() #0 { +define i64 @g() #0 { entry: %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null) %alloc = call ptr @malloc(i64 16) #3 @@ -27,6 +27,11 @@ await.suspend: %save2 = call token @llvm.coro.save(ptr null) call fastcc void @fakeresume1(ptr align 8 null) %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false) + + ; These (non-trivially) dead instructions are in the way. + %gep = getelementptr inbounds i64, ptr %alloc.var, i32 0 + %foo = ptrtoint ptr %gep to i64 + switch i8 %suspend2, label %exit [ i8 0, label %await.ready i8 1, label %exit @@ -36,8 +41,9 @@ await.ready: call void @llvm.lifetime.end.p0(i64 1, ptr %alloc.var) br label %exit exit: + %result = phi i64 [0, %entry], [0, %entry], [%foo, %await.suspend], [%foo, %await.suspend], [%foo, %await.ready] call i1 @llvm.coro.end(ptr null, i1 false, token none) - ret void + ret i64 %result } ; Verify that in the resume part resume call is marked with musttail. -- GitLab From abed4b74764de7df2a40272699e304f43e118994 Mon Sep 17 00:00:00 2001 From: Jay Foad Date: Wed, 20 Mar 2024 13:53:33 +0000 Subject: [PATCH 007/296] [AMDGPU] Simplify definition of GLOBAL_LOAD_TR Real instructions --- llvm/lib/Target/AMDGPU/FLATInstructions.td | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td index c91fa1477098..db1f8c187265 100644 --- a/llvm/lib/Target/AMDGPU/FLATInstructions.td +++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td @@ -982,11 +982,15 @@ defm SCRATCH_LOAD_LDS_DWORD : FLAT_Scratch_Load_LDS_Pseudo <"scratch_load_lds_d let SubtargetPredicate = isGFX12Plus in { let WaveSizePredicate = isWave32 in { + let Mnemonic = "global_load_tr_b128" in defm GLOBAL_LOAD_TR_B128_w32 : FLAT_Global_Load_Pseudo <"global_load_tr_b128_w32", VReg_128>; + let Mnemonic = "global_load_tr_b64" in defm GLOBAL_LOAD_TR_B64_w32 : FLAT_Global_Load_Pseudo <"global_load_tr_b64_w32", VReg_64>; } let WaveSizePredicate = isWave64 in { + let Mnemonic = "global_load_tr_b128" in defm GLOBAL_LOAD_TR_B128_w64 : FLAT_Global_Load_Pseudo <"global_load_tr_b128_w64", VReg_64>; + let Mnemonic = "global_load_tr_b64" in defm GLOBAL_LOAD_TR_B64_w64 : FLAT_Global_Load_Pseudo <"global_load_tr_b64_w64", VGPR_32>; } } // End SubtargetPredicate = isGFX12Plus @@ -2710,11 +2714,11 @@ defm GLOBAL_ATOMIC_FMIN : VGLOBAL_Real_Atomics_gfx12<0x051, "global_a defm GLOBAL_ATOMIC_FMAX : VGLOBAL_Real_Atomics_gfx12<0x052, "global_atomic_max_num_f32", "global_atomic_max_f32">; defm GLOBAL_ATOMIC_ADD_F32 : VGLOBAL_Real_Atomics_gfx12<0x056>; -defm GLOBAL_LOAD_TR_B128_w32 : VGLOBAL_Real_AllAddr_gfx12<0x057, "global_load_tr_b128">; -defm GLOBAL_LOAD_TR_B64_w32 : VGLOBAL_Real_AllAddr_gfx12<0x058, "global_load_tr_b64">; +defm GLOBAL_LOAD_TR_B128_w32 : VGLOBAL_Real_AllAddr_gfx12<0x057>; +defm GLOBAL_LOAD_TR_B64_w32 : VGLOBAL_Real_AllAddr_gfx12<0x058>; -defm GLOBAL_LOAD_TR_B128_w64 : VGLOBAL_Real_AllAddr_gfx12_w64<0x057, "global_load_tr_b128">; -defm GLOBAL_LOAD_TR_B64_w64 : VGLOBAL_Real_AllAddr_gfx12_w64<0x058, "global_load_tr_b64">; +defm GLOBAL_LOAD_TR_B128_w64 : VGLOBAL_Real_AllAddr_gfx12_w64<0x057>; +defm GLOBAL_LOAD_TR_B64_w64 : VGLOBAL_Real_AllAddr_gfx12_w64<0x058>; defm GLOBAL_ATOMIC_ORDERED_ADD_B64 : VGLOBAL_Real_Atomics_gfx12<0x073>; defm GLOBAL_ATOMIC_PK_ADD_F16 : VGLOBAL_Real_Atomics_gfx12<0x059>; -- GitLab From 3fbac79064e405a54388d11370ab5a8f0f23914d Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Wed, 20 Mar 2024 14:54:02 +0100 Subject: [PATCH 008/296] [AArch64] Don't write to source directory in test --- clang/test/CodeGen/aarch64-soft-float-abi-errors.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clang/test/CodeGen/aarch64-soft-float-abi-errors.c b/clang/test/CodeGen/aarch64-soft-float-abi-errors.c index 3e5ab9e92a1d..551e53bcd63d 100644 --- a/clang/test/CodeGen/aarch64-soft-float-abi-errors.c +++ b/clang/test/CodeGen/aarch64-soft-float-abi-errors.c @@ -1,9 +1,9 @@ // REQUIRES: aarch64-registered-target -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +fp-armv8 -S -target-abi aapcs -verify=fp-hard %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature -fp-armv8 -S -target-abi aapcs-soft -verify=nofp-soft %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature -fp-armv8 -S -target-abi aapcs -verify=nofp-hard %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature -fp-armv8 -S -target-abi aapcs -O1 -verify=nofp-hard,nofp-hard-opt -emit-llvm %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +fp-armv8 -S -o /dev/null -target-abi aapcs -verify=fp-hard %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature -fp-armv8 -S -o /dev/null -target-abi aapcs-soft -verify=nofp-soft %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature -fp-armv8 -S -o /dev/null -target-abi aapcs -verify=nofp-hard %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature -fp-armv8 -S -o /dev/null -target-abi aapcs -O1 -verify=nofp-hard,nofp-hard-opt -emit-llvm %s // No run line needed for soft-float ABI with an FPU because that is rejected by the driver // With the hard-float ABI and a target with an FPU, FP arguments are passed in -- GitLab From 12329648e2c3f8651228f17d3619b1e1ddab80f0 Mon Sep 17 00:00:00 2001 From: Felipe de Azevedo Piovezan Date: Wed, 20 Mar 2024 07:03:24 -0700 Subject: [PATCH 009/296] [lldb] Omit --show-globals in `help target var` (#85855) This option doesn't exist. It is currently displayed by `help target var` due to a bug introduced by 41ae8e7445 in 2018. Some code for `target var` and `frame var` is shared, and some hard-code constants are used in order to filter out options that belong only to `frame var`. However, the aforementioned commit failed to update these constants properly. This patch addresses the issue by having a _single_ place where the filtering of options needs to be done. --- .../Interpreter/OptionGroupVariable.cpp | 26 +++++++++---------- .../target_var/TestTargetVar.py | 10 +++++++ 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/lldb/source/Interpreter/OptionGroupVariable.cpp b/lldb/source/Interpreter/OptionGroupVariable.cpp index 0e35a641361b..99644b3f423c 100644 --- a/lldb/source/Interpreter/OptionGroupVariable.cpp +++ b/lldb/source/Interpreter/OptionGroupVariable.cpp @@ -50,6 +50,11 @@ static constexpr OptionDefinition g_variable_options[] = { "Specify a summary string to use to format the variable output."}, }; +static constexpr auto g_num_frame_options = 4; +static const auto g_variable_options_noframe = + llvm::ArrayRef(g_variable_options) + .drop_front(g_num_frame_options); + static Status ValidateNamedSummary(const char *str, void *) { if (!str || !str[0]) return Status("must specify a valid named summary"); @@ -77,9 +82,9 @@ OptionGroupVariable::SetOptionValue(uint32_t option_idx, llvm::StringRef option_arg, ExecutionContext *execution_context) { Status error; - if (!include_frame_options) - option_idx += 3; - const int short_option = g_variable_options[option_idx].short_option; + llvm::ArrayRef variable_options = + include_frame_options ? g_variable_options : g_variable_options_noframe; + const int short_option = variable_options[option_idx].short_option; switch (short_option) { case 'r': use_regex = true; @@ -128,16 +133,9 @@ void OptionGroupVariable::OptionParsingStarting( summary_string.Clear(); } -#define NUM_FRAME_OPTS 3 - llvm::ArrayRef OptionGroupVariable::GetDefinitions() { - auto result = llvm::ArrayRef(g_variable_options); - // Show the "--no-args", "--no-locals" and "--show-globals" options if we are - // showing frame specific options - if (include_frame_options) - return result; - - // Skip the "--no-args", "--no-locals" and "--show-globals" options if we are - // not showing frame specific options (globals only) - return result.drop_front(NUM_FRAME_OPTS); + // Show the "--no-args", "--no-recognized-args", "--no-locals" and + // "--show-globals" options if we are showing frame specific options + return include_frame_options ? g_variable_options + : g_variable_options_noframe; } diff --git a/lldb/test/API/functionalities/target_var/TestTargetVar.py b/lldb/test/API/functionalities/target_var/TestTargetVar.py index a0f3663f0365..54b7b77b6773 100644 --- a/lldb/test/API/functionalities/target_var/TestTargetVar.py +++ b/lldb/test/API/functionalities/target_var/TestTargetVar.py @@ -15,6 +15,16 @@ class targetCommandTestCase(TestBase): def testTargetVarExpr(self): self.build() lldbutil.run_to_name_breakpoint(self, "main") + self.expect( + "help target variable", + substrs=[ + "--no-args", + "--no-recognized-args", + "--no-locals", + "--show-globals", + ], + matching=False, + ) self.expect("target variable i", substrs=["i", "42"]) self.expect( "target variable var", patterns=["\(incomplete \*\) var = 0[xX](0)*dead"] -- GitLab From 5f5a64134b679d0b97d8fbd4ea65da361bb22cae Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Wed, 20 Mar 2024 15:08:37 +0100 Subject: [PATCH 010/296] Revert "[DAGCombiner] Simplifying `{si|ui}tofp` when only signbit is needed" This reverts commit 353fbeb0a294d2c7cef6d88607fa0fd50ee81462. It crashes when it encounters an UINT_TO_FP. llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp:1618 in SDValue llvm::SelectionDAG::getConstant(const ConstantInt &, const SDLoc &, EVT, bool, bool): VT.isInteger() && "Cannot create FP integer constant!" --- .../CodeGen/SelectionDAG/TargetLowering.cpp | 30 ------------------- .../CodeGen/X86/combine-sse41-intrinsics.ll | 3 +- llvm/test/CodeGen/X86/int-to-fp-demanded.ll | 21 ++++++++----- 3 files changed, 16 insertions(+), 38 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 16069c6c0dc3..57f8fc409de4 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -611,25 +611,6 @@ bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth, return false; } -static SDValue simplifyUseOfIntToFP(SDValue Op, const APInt &DemandedBits, - SelectionDAG &DAG) { - unsigned Opc = Op.getOpcode(); - assert((Opc == ISD::SINT_TO_FP || Opc == ISD::UINT_TO_FP) && - "Invalid Int -> FP Opcode"); - if (!DemandedBits.isSignMask()) - return SDValue(); - - EVT VT = Op.getValueType(); - if (Opc == ISD::UINT_TO_FP) - return DAG.getConstant(0, SDLoc(Op), VT); - - EVT InnerVT = Op.getOperand(0).getValueType(); - if (VT.getScalarSizeInBits() == InnerVT.getScalarSizeInBits()) - return DAG.getBitcast(VT, Op.getOperand(0)); - - return SDValue(); -} - bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; @@ -835,11 +816,6 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBits( } break; } - case ISD::UINT_TO_FP: - case ISD::SINT_TO_FP: - if (SDValue R = simplifyUseOfIntToFP(Op, DemandedBits, DAG)) - return R; - break; case ISD::SIGN_EXTEND_INREG: { // If none of the extended bits are demanded, eliminate the sextinreg. SDValue Op0 = Op.getOperand(0); @@ -2337,12 +2313,6 @@ bool TargetLowering::SimplifyDemandedBits( Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth); break; } - case ISD::UINT_TO_FP: - case ISD::SINT_TO_FP: - if (SDValue R = simplifyUseOfIntToFP(Op, DemandedBits, TLO.DAG)) - return TLO.CombineTo(Op, R); - Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth); - break; case ISD::SIGN_EXTEND_INREG: { SDValue Op0 = Op.getOperand(0); EVT ExVT = cast(Op.getOperand(1))->getVT(); diff --git a/llvm/test/CodeGen/X86/combine-sse41-intrinsics.ll b/llvm/test/CodeGen/X86/combine-sse41-intrinsics.ll index a332b3e89080..cbb5bd09c239 100644 --- a/llvm/test/CodeGen/X86/combine-sse41-intrinsics.ll +++ b/llvm/test/CodeGen/X86/combine-sse41-intrinsics.ll @@ -164,13 +164,14 @@ define <4 x float> @demandedbits_sitofp_blendvps(<4 x float> %a0, <4 x float> %a ; SSE-LABEL: demandedbits_sitofp_blendvps: ; SSE: # %bb.0: ; SSE-NEXT: movaps %xmm0, %xmm3 -; SSE-NEXT: movaps %xmm2, %xmm0 +; SSE-NEXT: cvtdq2ps %xmm2, %xmm0 ; SSE-NEXT: blendvps %xmm0, %xmm1, %xmm3 ; SSE-NEXT: movaps %xmm3, %xmm0 ; SSE-NEXT: retq ; ; AVX-LABEL: demandedbits_sitofp_blendvps: ; AVX: # %bb.0: +; AVX-NEXT: vcvtdq2ps %xmm2, %xmm2 ; AVX-NEXT: vblendvps %xmm2, %xmm1, %xmm0, %xmm0 ; AVX-NEXT: retq %cvt = sitofp <4 x i32> %a2 to <4 x float> diff --git a/llvm/test/CodeGen/X86/int-to-fp-demanded.ll b/llvm/test/CodeGen/X86/int-to-fp-demanded.ll index 8652136ae5cd..cdde03fb0534 100644 --- a/llvm/test/CodeGen/X86/int-to-fp-demanded.ll +++ b/llvm/test/CodeGen/X86/int-to-fp-demanded.ll @@ -7,13 +7,19 @@ declare void @use.i32(i32) define i32 @sitofp_signbit_only(i32 %i_in) nounwind { ; X86-LABEL: sitofp_signbit_only: ; X86: # %bb.0: +; X86-NEXT: subl $8, %esp +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl %eax, (%esp) +; X86-NEXT: fildl (%esp) +; X86-NEXT: fstps {{[0-9]+}}(%esp) ; X86-NEXT: movl $-2147483648, %eax # imm = 0x80000000 ; X86-NEXT: andl {{[0-9]+}}(%esp), %eax +; X86-NEXT: addl $8, %esp ; X86-NEXT: retl ; ; X64-LABEL: sitofp_signbit_only: ; X64: # %bb.0: -; X64-NEXT: movd %edi, %xmm0 +; X64-NEXT: cvtsi2ss %edi, %xmm0 ; X64-NEXT: movmskps %xmm0, %eax ; X64-NEXT: shll $31, %eax ; X64-NEXT: retq @@ -38,8 +44,8 @@ define i32 @sitofp_signbit_only_okay_width(i16 %i_in) nounwind { ; ; X64-LABEL: sitofp_signbit_only_okay_width: ; X64: # %bb.0: -; X64-NEXT: shll $16, %edi -; X64-NEXT: movd %edi, %xmm0 +; X64-NEXT: movswl %di, %eax +; X64-NEXT: cvtsi2ss %eax, %xmm0 ; X64-NEXT: movmskps %xmm0, %eax ; X64-NEXT: shll $31, %eax ; X64-NEXT: retq @@ -76,14 +82,15 @@ define <2 x i16> @sitofp_signbit_only_fail_bad_width2(i32 %i_in) nounwind { ; X86-LABEL: sitofp_signbit_only_fail_bad_width2: ; X86: # %bb.0: ; X86-NEXT: subl $8, %esp -; X86-NEXT: movl {{[0-9]+}}(%esp), %edx -; X86-NEXT: movl %edx, (%esp) +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl %eax, (%esp) ; X86-NEXT: fildl (%esp) ; X86-NEXT: fstps {{[0-9]+}}(%esp) +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movl %eax, %edx ; X86-NEXT: shrl $16, %edx +; X86-NEXT: andl $32768, %eax # imm = 0x8000 ; X86-NEXT: andl $32768, %edx # imm = 0x8000 -; X86-NEXT: movl $32768, %eax # imm = 0x8000 -; X86-NEXT: andl {{[0-9]+}}(%esp), %eax ; X86-NEXT: # kill: def $ax killed $ax killed $eax ; X86-NEXT: # kill: def $dx killed $dx killed $edx ; X86-NEXT: addl $8, %esp -- GitLab From 2137894a6f5475e51c541b6d16e8902125a8f002 Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Wed, 20 Mar 2024 15:10:19 +0100 Subject: [PATCH 011/296] [libc][NFC] Move `Sign` type to separate header (#85930) --- libc/src/__support/CMakeLists.txt | 8 ++++ libc/src/__support/FPUtil/CMakeLists.txt | 1 + libc/src/__support/FPUtil/FPBits.h | 27 +------------ libc/src/__support/FPUtil/fpbits_str.h | 1 - libc/src/__support/sign.h | 40 +++++++++++++++++++ libc/src/__support/str_to_float.h | 4 +- libc/src/math/generic/acosf.cpp | 2 +- libc/src/math/generic/asinf.cpp | 2 +- libc/src/math/generic/atanf.cpp | 1 - libc/src/math/generic/atanhf.cpp | 2 +- libc/src/math/generic/cosf.cpp | 2 +- libc/src/math/generic/coshf.cpp | 2 +- libc/src/math/generic/exp.cpp | 2 +- libc/src/math/generic/exp10.cpp | 2 +- libc/src/math/generic/exp2.cpp | 2 +- libc/src/math/generic/expm1.cpp | 4 +- libc/src/math/generic/log.cpp | 4 +- libc/src/math/generic/log10.cpp | 4 +- libc/src/math/generic/log10f.cpp | 2 +- libc/src/math/generic/log1p.cpp | 4 +- libc/src/math/generic/log1pf.cpp | 2 +- libc/src/math/generic/log2.cpp | 4 +- libc/src/math/generic/log2f.cpp | 2 +- libc/src/math/generic/log_range_reduction.h | 1 - libc/src/math/generic/logf.cpp | 2 +- libc/src/math/generic/powf.cpp | 4 +- .../stdio/printf_core/float_dec_converter.h | 2 +- libc/test/UnitTest/FPMatcher.h | 3 +- libc/test/src/__support/FPUtil/CMakeLists.txt | 1 + .../__support/FPUtil/dyadic_float_test.cpp | 1 - .../test/src/__support/FPUtil/fpbits_test.cpp | 2 +- libc/test/src/math/FDimTest.h | 1 - libc/test/src/math/FmaTest.h | 1 - libc/test/src/math/HypotTest.h | 2 +- libc/test/src/math/ILogbTest.h | 2 +- libc/test/src/math/LdExpTest.h | 1 - libc/test/src/math/NextAfterTest.h | 1 - libc/test/src/math/RIntTest.h | 1 - libc/test/src/math/RemQuoTest.h | 1 - libc/test/src/math/RoundToIntegerTest.h | 1 - libc/test/src/math/atanhf_test.cpp | 2 +- libc/test/src/math/smoke/FDimTest.h | 1 - libc/test/src/math/smoke/FmaTest.h | 1 - libc/test/src/math/smoke/HypotTest.h | 2 +- libc/test/src/math/smoke/ILogbTest.h | 1 - libc/test/src/math/smoke/LdExpTest.h | 1 - libc/test/src/math/smoke/NextAfterTest.h | 1 - libc/test/src/math/smoke/NextTowardTest.h | 1 - libc/test/src/math/smoke/RIntTest.h | 1 - libc/test/src/math/smoke/RemQuoTest.h | 1 - libc/test/src/math/smoke/RoundToIntegerTest.h | 1 - libc/test/src/math/smoke/atanhf_test.cpp | 2 +- .../llvm-project-overlay/libc/BUILD.bazel | 9 +++++ .../test/src/__support/FPUtil/BUILD.bazel | 1 + 54 files changed, 94 insertions(+), 82 deletions(-) create mode 100644 libc/src/__support/sign.h diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt index 4c1f271e1df4..7b1820d9bf35 100644 --- a/libc/src/__support/CMakeLists.txt +++ b/libc/src/__support/CMakeLists.txt @@ -41,6 +41,14 @@ add_header_library( libc.src.__support.macros.config ) +add_header_library( + sign + HDRS + sign.h + DEPENDS + libc.src.__support.macros.attributes +) + add_header_library( error_or HDRS diff --git a/libc/src/__support/FPUtil/CMakeLists.txt b/libc/src/__support/FPUtil/CMakeLists.txt index f1c6fba22856..4ded70a675ea 100644 --- a/libc/src/__support/FPUtil/CMakeLists.txt +++ b/libc/src/__support/FPUtil/CMakeLists.txt @@ -35,6 +35,7 @@ add_header_library( libc.src.__support.macros.attributes libc.src.__support.macros.properties.types libc.src.__support.math_extras + libc.src.__support.sign libc.src.__support.uint128 ) diff --git a/libc/src/__support/FPUtil/FPBits.h b/libc/src/__support/FPUtil/FPBits.h index b06b3f7b7395..155bff2f5581 100644 --- a/libc/src/__support/FPUtil/FPBits.h +++ b/libc/src/__support/FPUtil/FPBits.h @@ -17,6 +17,7 @@ #include "src/__support/macros/attributes.h" // LIBC_INLINE, LIBC_INLINE_VAR #include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_FLOAT128 #include "src/__support/math_extras.h" // mask_trailing_ones +#include "src/__support/sign.h" // Sign #include @@ -32,32 +33,6 @@ enum class FPType { X86_Binary80, }; -// A type to interact with floating point type signs. -// This may be moved outside of 'fputil' if useful. -struct Sign { - LIBC_INLINE constexpr bool is_pos() const { return !is_negative; } - LIBC_INLINE constexpr bool is_neg() const { return is_negative; } - - LIBC_INLINE friend constexpr bool operator==(Sign a, Sign b) { - return a.is_negative == b.is_negative; - } - LIBC_INLINE friend constexpr bool operator!=(Sign a, Sign b) { - return !(a == b); - } - - static const Sign POS; - static const Sign NEG; - -private: - LIBC_INLINE constexpr explicit Sign(bool is_negative) - : is_negative(is_negative) {} - - bool is_negative; -}; - -LIBC_INLINE_VAR constexpr Sign Sign::NEG = Sign(true); -LIBC_INLINE_VAR constexpr Sign Sign::POS = Sign(false); - // The classes hierarchy is as follows: // // ┌───────────────────┐ diff --git a/libc/src/__support/FPUtil/fpbits_str.h b/libc/src/__support/FPUtil/fpbits_str.h index 212265bb9ad4..97689867da4d 100644 --- a/libc/src/__support/FPUtil/fpbits_str.h +++ b/libc/src/__support/FPUtil/fpbits_str.h @@ -35,7 +35,6 @@ using ZeroPaddedHexFmt = IntegerToString< // floating encoding. template LIBC_INLINE cpp::string str(fputil::FPBits x) { using StorageType = typename fputil::FPBits::StorageType; - using Sign = fputil::Sign; if (x.is_nan()) return "(NaN)"; diff --git a/libc/src/__support/sign.h b/libc/src/__support/sign.h new file mode 100644 index 000000000000..28cfae4bab1d --- /dev/null +++ b/libc/src/__support/sign.h @@ -0,0 +1,40 @@ +//===-- A simple sign type --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_SIGN_H +#define LLVM_LIBC_SRC___SUPPORT_SIGN_H + +#include "src/__support/macros/attributes.h" // LIBC_INLINE, LIBC_INLINE_VAR + +// A type to interact with signed arithmetic types. +struct Sign { + LIBC_INLINE constexpr bool is_pos() const { return !is_negative; } + LIBC_INLINE constexpr bool is_neg() const { return is_negative; } + + LIBC_INLINE friend constexpr bool operator==(Sign a, Sign b) { + return a.is_negative == b.is_negative; + } + + LIBC_INLINE friend constexpr bool operator!=(Sign a, Sign b) { + return !(a == b); + } + + static const Sign POS; + static const Sign NEG; + +private: + LIBC_INLINE constexpr explicit Sign(bool is_negative) + : is_negative(is_negative) {} + + bool is_negative; +}; + +LIBC_INLINE_VAR constexpr Sign Sign::NEG = Sign(true); +LIBC_INLINE_VAR constexpr Sign Sign::POS = Sign(false); + +#endif // LLVM_LIBC_SRC___SUPPORT_SIGN_H diff --git a/libc/src/__support/str_to_float.h b/libc/src/__support/str_to_float.h index 2cf2cfb02724..f622b7edaa8a 100644 --- a/libc/src/__support/str_to_float.h +++ b/libc/src/__support/str_to_float.h @@ -513,7 +513,6 @@ clinger_fast_path(ExpandedFloat init_num, RoundDirection round = RoundDirection::Nearest) { using FPBits = typename fputil::FPBits; using StorageType = typename FPBits::StorageType; - using Sign = fputil::Sign; StorageType mantissa = init_num.mantissa; int32_t exp10 = init_num.exponent; @@ -1085,7 +1084,6 @@ template LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { using FPBits = typename fputil::FPBits; using StorageType = typename FPBits::StorageType; - using Sign = fputil::Sign; FPBits result = FPBits(); bool seen_digit = false; @@ -1223,7 +1221,7 @@ template LIBC_INLINE StrToNumResult strtonan(const char *arg) { if (arg[index] == '\0') nan_mantissa = nan_mantissa_from_ncharseq(cpp::string_view(arg, index)); - result = FPBits::quiet_nan(fputil::Sign::POS, nan_mantissa); + result = FPBits::quiet_nan(Sign::POS, nan_mantissa); return {result.get_val(), 0, error}; } diff --git a/libc/src/math/generic/acosf.cpp b/libc/src/math/generic/acosf.cpp index 0c1fdbc68693..e6e28d43ef61 100644 --- a/libc/src/math/generic/acosf.cpp +++ b/libc/src/math/generic/acosf.cpp @@ -38,7 +38,7 @@ static constexpr fputil::ExceptValues ACOSF_EXCEPTS = {{ LLVM_LIBC_FUNCTION(float, acosf, (float x)) { using FPBits = typename fputil::FPBits; - using Sign = fputil::Sign; + FPBits xbits(x); uint32_t x_uint = xbits.uintval(); uint32_t x_abs = xbits.uintval() & 0x7fff'ffffU; diff --git a/libc/src/math/generic/asinf.cpp b/libc/src/math/generic/asinf.cpp index 6e3a27238ac9..d9133333d256 100644 --- a/libc/src/math/generic/asinf.cpp +++ b/libc/src/math/generic/asinf.cpp @@ -44,7 +44,7 @@ static constexpr fputil::ExceptValues ASINF_EXCEPTS_HI = {{ LLVM_LIBC_FUNCTION(float, asinf, (float x)) { using FPBits = typename fputil::FPBits; - using Sign = fputil::Sign; + FPBits xbits(x); uint32_t x_uint = xbits.uintval(); uint32_t x_abs = xbits.uintval() & 0x7fff'ffffU; diff --git a/libc/src/math/generic/atanf.cpp b/libc/src/math/generic/atanf.cpp index 5f66ea52d0d7..4adda429cc04 100644 --- a/libc/src/math/generic/atanf.cpp +++ b/libc/src/math/generic/atanf.cpp @@ -20,7 +20,6 @@ namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(float, atanf, (float x)) { using FPBits = typename fputil::FPBits; - using Sign = fputil::Sign; constexpr double FINAL_SIGN[2] = {1.0, -1.0}; constexpr double SIGNED_PI_OVER_2[2] = {0x1.921fb54442d18p0, diff --git a/libc/src/math/generic/atanhf.cpp b/libc/src/math/generic/atanhf.cpp index fe2c36494a72..97fd1b233600 100644 --- a/libc/src/math/generic/atanhf.cpp +++ b/libc/src/math/generic/atanhf.cpp @@ -15,7 +15,7 @@ namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(float, atanhf, (float x)) { using FPBits = typename fputil::FPBits; - using Sign = fputil::Sign; + FPBits xbits(x); Sign sign = xbits.sign(); uint32_t x_abs = xbits.abs().uintval(); diff --git a/libc/src/math/generic/cosf.cpp b/libc/src/math/generic/cosf.cpp index d59304933d60..180a44e947ea 100644 --- a/libc/src/math/generic/cosf.cpp +++ b/libc/src/math/generic/cosf.cpp @@ -42,7 +42,7 @@ static constexpr fputil::ExceptValues COSF_EXCEPTS{{ LLVM_LIBC_FUNCTION(float, cosf, (float x)) { using FPBits = typename fputil::FPBits; - using Sign = fputil::Sign; + FPBits xbits(x); xbits.set_sign(Sign::POS); diff --git a/libc/src/math/generic/coshf.cpp b/libc/src/math/generic/coshf.cpp index a618056a64dc..a8ea324c9505 100644 --- a/libc/src/math/generic/coshf.cpp +++ b/libc/src/math/generic/coshf.cpp @@ -17,7 +17,7 @@ namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(float, coshf, (float x)) { using FPBits = typename fputil::FPBits; - using Sign = fputil::Sign; + FPBits xbits(x); xbits.set_sign(Sign::POS); x = xbits.get_val(); diff --git a/libc/src/math/generic/exp.cpp b/libc/src/math/generic/exp.cpp index 42a4491131a0..3d060bcbd3be 100644 --- a/libc/src/math/generic/exp.cpp +++ b/libc/src/math/generic/exp.cpp @@ -31,7 +31,7 @@ namespace LIBC_NAMESPACE { using fputil::DoubleDouble; using fputil::TripleDouble; using Float128 = typename fputil::DyadicFloat<128>; -using Sign = fputil::Sign; + using LIBC_NAMESPACE::operator""_u128; // log2(e) diff --git a/libc/src/math/generic/exp10.cpp b/libc/src/math/generic/exp10.cpp index 72ece6697656..a4ae41407112 100644 --- a/libc/src/math/generic/exp10.cpp +++ b/libc/src/math/generic/exp10.cpp @@ -31,7 +31,7 @@ namespace LIBC_NAMESPACE { using fputil::DoubleDouble; using fputil::TripleDouble; using Float128 = typename fputil::DyadicFloat<128>; -using Sign = fputil::Sign; + using LIBC_NAMESPACE::operator""_u128; // log2(10) diff --git a/libc/src/math/generic/exp2.cpp b/libc/src/math/generic/exp2.cpp index 83f545eb116b..1a2fa3feb83e 100644 --- a/libc/src/math/generic/exp2.cpp +++ b/libc/src/math/generic/exp2.cpp @@ -31,7 +31,7 @@ namespace LIBC_NAMESPACE { using fputil::DoubleDouble; using fputil::TripleDouble; using Float128 = typename fputil::DyadicFloat<128>; -using Sign = fputil::Sign; + using LIBC_NAMESPACE::operator""_u128; // Error bounds: diff --git a/libc/src/math/generic/expm1.cpp b/libc/src/math/generic/expm1.cpp index 9f14a8c2068e..574c4b9aaf39 100644 --- a/libc/src/math/generic/expm1.cpp +++ b/libc/src/math/generic/expm1.cpp @@ -39,7 +39,7 @@ namespace LIBC_NAMESPACE { using fputil::DoubleDouble; using fputil::TripleDouble; using Float128 = typename fputil::DyadicFloat<128>; -using Sign = fputil::Sign; + using LIBC_NAMESPACE::operator""_u128; // log2(e) @@ -276,7 +276,7 @@ double set_exceptional(double x) { LLVM_LIBC_FUNCTION(double, expm1, (double x)) { using FPBits = typename fputil::FPBits; - using Sign = fputil::Sign; + FPBits xbits(x); bool x_is_neg = xbits.is_neg(); diff --git a/libc/src/math/generic/log.cpp b/libc/src/math/generic/log.cpp index 339e0297560f..6de0d90be80e 100644 --- a/libc/src/math/generic/log.cpp +++ b/libc/src/math/generic/log.cpp @@ -24,7 +24,7 @@ namespace LIBC_NAMESPACE { // 128-bit precision dyadic floating point numbers. using Float128 = typename fputil::DyadicFloat<128>; -using Sign = fputil::Sign; + using LIBC_NAMESPACE::operator""_u128; namespace { @@ -735,7 +735,7 @@ double log_accurate(int e_x, int index, double m_x) { LLVM_LIBC_FUNCTION(double, log, (double x)) { using FPBits_t = typename fputil::FPBits; - using Sign = fputil::Sign; + FPBits_t xbits(x); uint64_t x_u = xbits.uintval(); diff --git a/libc/src/math/generic/log10.cpp b/libc/src/math/generic/log10.cpp index c690ca287040..fb839c111e6a 100644 --- a/libc/src/math/generic/log10.cpp +++ b/libc/src/math/generic/log10.cpp @@ -24,7 +24,7 @@ namespace LIBC_NAMESPACE { // 128-bit precision dyadic floating point numbers. using Float128 = typename fputil::DyadicFloat<128>; -using Sign = fputil::Sign; + using LIBC_NAMESPACE::operator""_u128; namespace { @@ -737,7 +737,7 @@ double log10_accurate(int e_x, int index, double m_x) { LLVM_LIBC_FUNCTION(double, log10, (double x)) { using FPBits_t = typename fputil::FPBits; - using Sign = fputil::Sign; + FPBits_t xbits(x); uint64_t x_u = xbits.uintval(); diff --git a/libc/src/math/generic/log10f.cpp b/libc/src/math/generic/log10f.cpp index 0216bb2133f1..1b6979d4414a 100644 --- a/libc/src/math/generic/log10f.cpp +++ b/libc/src/math/generic/log10f.cpp @@ -106,7 +106,7 @@ LLVM_LIBC_FUNCTION(float, log10f, (float x)) { constexpr double LOG10_2 = 0x1.34413509f79ffp-2; using FPBits = typename fputil::FPBits; - using Sign = fputil::Sign; + FPBits xbits(x); uint32_t x_u = xbits.uintval(); diff --git a/libc/src/math/generic/log1p.cpp b/libc/src/math/generic/log1p.cpp index 26bb4d369278..83bd753cde5d 100644 --- a/libc/src/math/generic/log1p.cpp +++ b/libc/src/math/generic/log1p.cpp @@ -23,7 +23,7 @@ namespace LIBC_NAMESPACE { // 128-bit precision dyadic floating point numbers. using Float128 = typename fputil::DyadicFloat<128>; -using Sign = fputil::Sign; + using LIBC_NAMESPACE::operator""_u128; namespace { @@ -877,7 +877,7 @@ LIBC_INLINE double log1p_accurate(int e_x, int index, LLVM_LIBC_FUNCTION(double, log1p, (double x)) { using FPBits_t = typename fputil::FPBits; - using Sign = fputil::Sign; + constexpr int EXP_BIAS = FPBits_t::EXP_BIAS; constexpr int FRACTION_LEN = FPBits_t::FRACTION_LEN; constexpr uint64_t FRACTION_MASK = FPBits_t::FRACTION_MASK; diff --git a/libc/src/math/generic/log1pf.cpp b/libc/src/math/generic/log1pf.cpp index 28426a88e649..e3c7d95418b1 100644 --- a/libc/src/math/generic/log1pf.cpp +++ b/libc/src/math/generic/log1pf.cpp @@ -106,7 +106,7 @@ LLVM_LIBC_FUNCTION(float, log1pf, (float x)) { case 0xbf800000U: // x = -1.0 fputil::set_errno_if_required(ERANGE); fputil::raise_except_if_required(FE_DIVBYZERO); - return FPBits::inf(fputil::Sign::NEG).get_val(); + return FPBits::inf(Sign::NEG).get_val(); #ifndef LIBC_TARGET_CPU_HAS_FMA case 0x4cc1c80bU: // x = 0x1.839016p+26f return fputil::round_result_slightly_down(0x1.26fc04p+4f); diff --git a/libc/src/math/generic/log2.cpp b/libc/src/math/generic/log2.cpp index 648850b80b04..c68bc60e8468 100644 --- a/libc/src/math/generic/log2.cpp +++ b/libc/src/math/generic/log2.cpp @@ -24,7 +24,7 @@ namespace LIBC_NAMESPACE { // 128-bit precision dyadic floating point numbers. using Float128 = typename fputil::DyadicFloat<128>; -using Sign = fputil::Sign; + using LIBC_NAMESPACE::operator""_u128; namespace { @@ -857,7 +857,7 @@ double log2_accurate(int e_x, int index, double m_x) { LLVM_LIBC_FUNCTION(double, log2, (double x)) { using FPBits_t = typename fputil::FPBits; - using Sign = fputil::Sign; + FPBits_t xbits(x); uint64_t x_u = xbits.uintval(); diff --git a/libc/src/math/generic/log2f.cpp b/libc/src/math/generic/log2f.cpp index 8651316d282c..c9f7b2121519 100644 --- a/libc/src/math/generic/log2f.cpp +++ b/libc/src/math/generic/log2f.cpp @@ -55,7 +55,7 @@ namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(float, log2f, (float x)) { using FPBits = typename fputil::FPBits; - using Sign = fputil::Sign; + FPBits xbits(x); uint32_t x_u = xbits.uintval(); diff --git a/libc/src/math/generic/log_range_reduction.h b/libc/src/math/generic/log_range_reduction.h index 8c9b7d2eabeb..64c0fc3aa4f5 100644 --- a/libc/src/math/generic/log_range_reduction.h +++ b/libc/src/math/generic/log_range_reduction.h @@ -37,7 +37,6 @@ log_range_reduction(double m_x, const LogRR &log_table, fputil::DyadicFloat<128> &sum) { using Float128 = typename fputil::DyadicFloat<128>; using MType = typename Float128::MantissaType; - using Sign = fputil::Sign; int64_t v = static_cast(m_x * 0x1.0p60); // ulp = 2^-60 diff --git a/libc/src/math/generic/logf.cpp b/libc/src/math/generic/logf.cpp index 49d258ecc133..5296ba6bc13c 100644 --- a/libc/src/math/generic/logf.cpp +++ b/libc/src/math/generic/logf.cpp @@ -54,7 +54,7 @@ namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(float, logf, (float x)) { constexpr double LOG_2 = 0x1.62e42fefa39efp-1; using FPBits = typename fputil::FPBits; - using Sign = fputil::Sign; + FPBits xbits(x); uint32_t x_u = xbits.uintval(); diff --git a/libc/src/math/generic/powf.cpp b/libc/src/math/generic/powf.cpp index 2c666bab6d62..0450ffd711ff 100644 --- a/libc/src/math/generic/powf.cpp +++ b/libc/src/math/generic/powf.cpp @@ -424,7 +424,7 @@ LIBC_INLINE bool larger_exponent(double a, double b) { double powf_double_double(int idx_x, double dx, double y6, double lo6_hi, const DoubleDouble &exp2_hi_mid) { using DoubleBits = typename fputil::FPBits; - using Sign = fputil::Sign; + // Perform a second range reduction step: // idx2 = round(2^14 * (dx + 2^-8)) = round ( dx * 2^14 + 2^6) // dx2 = (1 + dx) * r2 - 1 @@ -513,7 +513,7 @@ double powf_double_double(int idx_x, double dx, double y6, double lo6_hi, LLVM_LIBC_FUNCTION(float, powf, (float x, float y)) { using FloatBits = typename fputil::FPBits; using DoubleBits = typename fputil::FPBits; - using Sign = fputil::Sign; + FloatBits xbits(x), ybits(y); uint32_t x_u = xbits.uintval(); diff --git a/libc/src/stdio/printf_core/float_dec_converter.h b/libc/src/stdio/printf_core/float_dec_converter.h index 5270fc9de037..c4e8aaa2f0e2 100644 --- a/libc/src/stdio/printf_core/float_dec_converter.h +++ b/libc/src/stdio/printf_core/float_dec_converter.h @@ -48,7 +48,7 @@ constexpr uint32_t MAX_BLOCK = 999999999; constexpr char DECIMAL_POINT = '.'; LIBC_INLINE RoundDirection get_round_direction(int last_digit, bool truncated, - fputil::Sign sign) { + Sign sign) { switch (fputil::quick_get_round()) { case FE_TONEAREST: // Round to nearest, if it's exactly halfway then round to even. diff --git a/libc/test/UnitTest/FPMatcher.h b/libc/test/UnitTest/FPMatcher.h index 43000efa09a3..ee618a623efe 100644 --- a/libc/test/UnitTest/FPMatcher.h +++ b/libc/test/UnitTest/FPMatcher.h @@ -63,7 +63,6 @@ template FPMatcher getMatcher(T expectedValue) { template struct FPTest : public Test { using FPBits = LIBC_NAMESPACE::fputil::FPBits; using StorageType = typename FPBits::StorageType; - using Sign = LIBC_NAMESPACE::fputil::Sign; static constexpr StorageType STORAGE_MAX = LIBC_NAMESPACE::cpp::numeric_limits::max(); static constexpr T zero = FPBits::zero(Sign::POS).get_val(); @@ -92,7 +91,7 @@ template struct FPTest : public Test { #define DECLARE_SPECIAL_CONSTANTS(T) \ using FPBits = LIBC_NAMESPACE::fputil::FPBits; \ using StorageType = typename FPBits::StorageType; \ - using Sign = LIBC_NAMESPACE::fputil::Sign; \ + \ static constexpr StorageType STORAGE_MAX = \ LIBC_NAMESPACE::cpp::numeric_limits::max(); \ const T zero = FPBits::zero(Sign::POS).get_val(); \ diff --git a/libc/test/src/__support/FPUtil/CMakeLists.txt b/libc/test/src/__support/FPUtil/CMakeLists.txt index f1a027a514ba..1cbeec0cc4eb 100644 --- a/libc/test/src/__support/FPUtil/CMakeLists.txt +++ b/libc/test/src/__support/FPUtil/CMakeLists.txt @@ -24,6 +24,7 @@ add_libc_test( libc.src.__support.FPUtil.fp_bits libc.src.__support.FPUtil.fpbits_str libc.src.__support.integer_literals + libc.src.__support.sign ) add_fp_unittest( diff --git a/libc/test/src/__support/FPUtil/dyadic_float_test.cpp b/libc/test/src/__support/FPUtil/dyadic_float_test.cpp index 625aa70973b9..5ee9aaad5638 100644 --- a/libc/test/src/__support/FPUtil/dyadic_float_test.cpp +++ b/libc/test/src/__support/FPUtil/dyadic_float_test.cpp @@ -15,7 +15,6 @@ using Float128 = LIBC_NAMESPACE::fputil::DyadicFloat<128>; using Float192 = LIBC_NAMESPACE::fputil::DyadicFloat<192>; using Float256 = LIBC_NAMESPACE::fputil::DyadicFloat<256>; -using Sign = LIBC_NAMESPACE::fputil::Sign; TEST(LlvmLibcDyadicFloatTest, BasicConversions) { Float128 x(Sign::POS, /*exponent*/ 0, diff --git a/libc/test/src/__support/FPUtil/fpbits_test.cpp b/libc/test/src/__support/FPUtil/fpbits_test.cpp index f5c27d4fc030..af20b1a0bdc7 100644 --- a/libc/test/src/__support/FPUtil/fpbits_test.cpp +++ b/libc/test/src/__support/FPUtil/fpbits_test.cpp @@ -9,11 +9,11 @@ #include "src/__support/FPUtil/FPBits.h" #include "src/__support/FPUtil/fpbits_str.h" #include "src/__support/integer_literals.h" +#include "src/__support/sign.h" // Sign #include "test/UnitTest/Test.h" using LIBC_NAMESPACE::fputil::FPBits; using LIBC_NAMESPACE::fputil::FPType; -using LIBC_NAMESPACE::fputil::Sign; using LIBC_NAMESPACE::fputil::internal::FPRep; using LIBC_NAMESPACE::operator""_u16; diff --git a/libc/test/src/math/FDimTest.h b/libc/test/src/math/FDimTest.h index 76f0f18bbc68..df8de91b4298 100644 --- a/libc/test/src/math/FDimTest.h +++ b/libc/test/src/math/FDimTest.h @@ -18,7 +18,6 @@ public: using FuncPtr = T (*)(T, T); using FPBits = LIBC_NAMESPACE::fputil::FPBits; using StorageType = typename FPBits::StorageType; - using Sign = LIBC_NAMESPACE::fputil::Sign; const T inf = FPBits::inf(Sign::POS).get_val(); const T neg_inf = FPBits::inf(Sign::NEG).get_val(); diff --git a/libc/test/src/math/FmaTest.h b/libc/test/src/math/FmaTest.h index 34c582c18242..0c93ec858a12 100644 --- a/libc/test/src/math/FmaTest.h +++ b/libc/test/src/math/FmaTest.h @@ -23,7 +23,6 @@ private: using Func = T (*)(T, T, T); using FPBits = LIBC_NAMESPACE::fputil::FPBits; using StorageType = typename FPBits::StorageType; - using Sign = LIBC_NAMESPACE::fputil::Sign; const T min_subnormal = FPBits::min_subnormal(Sign::POS).get_val(); const T min_normal = FPBits::min_normal(Sign::POS).get_val(); diff --git a/libc/test/src/math/HypotTest.h b/libc/test/src/math/HypotTest.h index 46fcc462a279..df69965d5dbc 100644 --- a/libc/test/src/math/HypotTest.h +++ b/libc/test/src/math/HypotTest.h @@ -23,7 +23,7 @@ class HypotTestTemplate : public LIBC_NAMESPACE::testing::Test { private: using Func = T (*)(T, T); using FPBits = LIBC_NAMESPACE::fputil::FPBits; - using Sign = LIBC_NAMESPACE::fputil::Sign; + using StorageType = typename FPBits::StorageType; const T nan = FPBits::quiet_nan().get_val(); const T inf = FPBits::inf().get_val(); diff --git a/libc/test/src/math/ILogbTest.h b/libc/test/src/math/ILogbTest.h index dcc9d554eb3c..ad47b9bb3961 100644 --- a/libc/test/src/math/ILogbTest.h +++ b/libc/test/src/math/ILogbTest.h @@ -24,7 +24,7 @@ public: template void test_special_numbers(typename ILogbFunc::Func func) { using FPBits = LIBC_NAMESPACE::fputil::FPBits; - using Sign = LIBC_NAMESPACE::fputil::Sign; + EXPECT_EQ(FP_ILOGB0, func(FPBits::zero(Sign::POS).get_val())); EXPECT_EQ(FP_ILOGB0, func(FPBits::zero(Sign::NEG).get_val())); EXPECT_EQ(FP_ILOGBNAN, func(FPBits::quiet_nan().get_val())); diff --git a/libc/test/src/math/LdExpTest.h b/libc/test/src/math/LdExpTest.h index 738135d6afe2..8bfd022973b4 100644 --- a/libc/test/src/math/LdExpTest.h +++ b/libc/test/src/math/LdExpTest.h @@ -23,7 +23,6 @@ class LdExpTestTemplate : public LIBC_NAMESPACE::testing::Test { using FPBits = LIBC_NAMESPACE::fputil::FPBits; using NormalFloat = LIBC_NAMESPACE::fputil::NormalFloat; using StorageType = typename FPBits::StorageType; - using Sign = LIBC_NAMESPACE::fputil::Sign; const T inf = FPBits::inf(Sign::POS).get_val(); const T neg_inf = FPBits::inf(Sign::NEG).get_val(); diff --git a/libc/test/src/math/NextAfterTest.h b/libc/test/src/math/NextAfterTest.h index d45d819bfdb6..05803fb45ee2 100644 --- a/libc/test/src/math/NextAfterTest.h +++ b/libc/test/src/math/NextAfterTest.h @@ -21,7 +21,6 @@ template class NextAfterTestTemplate : public LIBC_NAMESPACE::testing::Test { using FPBits = LIBC_NAMESPACE::fputil::FPBits; using StorageType = typename FPBits::StorageType; - using Sign = LIBC_NAMESPACE::fputil::Sign; const T inf = FPBits::inf(Sign::POS).get_val(); const T neg_inf = FPBits::inf(Sign::NEG).get_val(); diff --git a/libc/test/src/math/RIntTest.h b/libc/test/src/math/RIntTest.h index d392d4fb14a2..301655c64ed3 100644 --- a/libc/test/src/math/RIntTest.h +++ b/libc/test/src/math/RIntTest.h @@ -32,7 +32,6 @@ public: private: using FPBits = LIBC_NAMESPACE::fputil::FPBits; using StorageType = typename FPBits::StorageType; - using Sign = LIBC_NAMESPACE::fputil::Sign; const T inf = FPBits::inf(Sign::POS).get_val(); const T neg_inf = FPBits::inf(Sign::NEG).get_val(); diff --git a/libc/test/src/math/RemQuoTest.h b/libc/test/src/math/RemQuoTest.h index d61b97554199..1cb8cdbe81a2 100644 --- a/libc/test/src/math/RemQuoTest.h +++ b/libc/test/src/math/RemQuoTest.h @@ -22,7 +22,6 @@ template class RemQuoTestTemplate : public LIBC_NAMESPACE::testing::Test { using FPBits = LIBC_NAMESPACE::fputil::FPBits; using StorageType = typename FPBits::StorageType; - using Sign = LIBC_NAMESPACE::fputil::Sign; const T inf = FPBits::inf(Sign::POS).get_val(); const T neg_inf = FPBits::inf(Sign::NEG).get_val(); diff --git a/libc/test/src/math/RoundToIntegerTest.h b/libc/test/src/math/RoundToIntegerTest.h index 017f5867fc8d..d2fabd0b4c9c 100644 --- a/libc/test/src/math/RoundToIntegerTest.h +++ b/libc/test/src/math/RoundToIntegerTest.h @@ -31,7 +31,6 @@ public: private: using FPBits = LIBC_NAMESPACE::fputil::FPBits; using StorageType = typename FPBits::StorageType; - using Sign = LIBC_NAMESPACE::fputil::Sign; const F zero = FPBits::zero().get_val(); const F neg_zero = FPBits::zero(Sign::NEG).get_val(); diff --git a/libc/test/src/math/atanhf_test.cpp b/libc/test/src/math/atanhf_test.cpp index 39c067f3e764..c659f17d13b0 100644 --- a/libc/test/src/math/atanhf_test.cpp +++ b/libc/test/src/math/atanhf_test.cpp @@ -22,7 +22,7 @@ using LlvmLibcAtanhfTest = LIBC_NAMESPACE::testing::FPTest; namespace mpfr = LIBC_NAMESPACE::testing::mpfr; TEST_F(LlvmLibcAtanhfTest, SpecialNumbers) { - using Sign = LIBC_NAMESPACE::fputil::Sign; + LIBC_NAMESPACE::libc_errno = 0; LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT); EXPECT_FP_EQ_ALL_ROUNDING(aNaN, LIBC_NAMESPACE::atanhf(aNaN)); diff --git a/libc/test/src/math/smoke/FDimTest.h b/libc/test/src/math/smoke/FDimTest.h index 5cb3dd117348..e557b40d90ef 100644 --- a/libc/test/src/math/smoke/FDimTest.h +++ b/libc/test/src/math/smoke/FDimTest.h @@ -17,7 +17,6 @@ public: using FuncPtr = T (*)(T, T); using FPBits = LIBC_NAMESPACE::fputil::FPBits; using StorageType = typename FPBits::StorageType; - using Sign = LIBC_NAMESPACE::fputil::Sign; const T inf = FPBits::inf(Sign::POS).get_val(); const T neg_inf = FPBits::inf(Sign::NEG).get_val(); diff --git a/libc/test/src/math/smoke/FmaTest.h b/libc/test/src/math/smoke/FmaTest.h index d04f648c2d7d..c66035927d98 100644 --- a/libc/test/src/math/smoke/FmaTest.h +++ b/libc/test/src/math/smoke/FmaTest.h @@ -19,7 +19,6 @@ private: using Func = T (*)(T, T, T); using FPBits = LIBC_NAMESPACE::fputil::FPBits; using StorageType = typename FPBits::StorageType; - using Sign = LIBC_NAMESPACE::fputil::Sign; const T inf = FPBits::inf(Sign::POS).get_val(); const T neg_inf = FPBits::inf(Sign::NEG).get_val(); diff --git a/libc/test/src/math/smoke/HypotTest.h b/libc/test/src/math/smoke/HypotTest.h index 43499267b711..80816033f28f 100644 --- a/libc/test/src/math/smoke/HypotTest.h +++ b/libc/test/src/math/smoke/HypotTest.h @@ -21,7 +21,7 @@ private: using Func = T (*)(T, T); using FPBits = LIBC_NAMESPACE::fputil::FPBits; using StorageType = typename FPBits::StorageType; - using Sign = LIBC_NAMESPACE::fputil::Sign; + const T nan = FPBits::quiet_nan().get_val(); const T inf = FPBits::inf(Sign::POS).get_val(); const T neg_inf = FPBits::inf(Sign::NEG).get_val(); diff --git a/libc/test/src/math/smoke/ILogbTest.h b/libc/test/src/math/smoke/ILogbTest.h index cbee25b139d4..bb5bc33b6b3a 100644 --- a/libc/test/src/math/smoke/ILogbTest.h +++ b/libc/test/src/math/smoke/ILogbTest.h @@ -18,7 +18,6 @@ template class LlvmLibcILogbTest : public LIBC_NAMESPACE::testing::Test { using FPBits = LIBC_NAMESPACE::fputil::FPBits; using StorageType = typename FPBits::StorageType; - using Sign = LIBC_NAMESPACE::fputil::Sign; public: typedef OutType (*Func)(InType); diff --git a/libc/test/src/math/smoke/LdExpTest.h b/libc/test/src/math/smoke/LdExpTest.h index 7d17071f5b30..c3e852a2a473 100644 --- a/libc/test/src/math/smoke/LdExpTest.h +++ b/libc/test/src/math/smoke/LdExpTest.h @@ -22,7 +22,6 @@ class LdExpTestTemplate : public LIBC_NAMESPACE::testing::Test { using FPBits = LIBC_NAMESPACE::fputil::FPBits; using NormalFloat = LIBC_NAMESPACE::fputil::NormalFloat; using StorageType = typename FPBits::StorageType; - using Sign = LIBC_NAMESPACE::fputil::Sign; const T inf = FPBits::inf(Sign::POS).get_val(); const T neg_inf = FPBits::inf(Sign::NEG).get_val(); diff --git a/libc/test/src/math/smoke/NextAfterTest.h b/libc/test/src/math/smoke/NextAfterTest.h index 23b3b1534740..403ea6bd8df6 100644 --- a/libc/test/src/math/smoke/NextAfterTest.h +++ b/libc/test/src/math/smoke/NextAfterTest.h @@ -32,7 +32,6 @@ template class NextAfterTestTemplate : public LIBC_NAMESPACE::testing::Test { using FPBits = LIBC_NAMESPACE::fputil::FPBits; using StorageType = typename FPBits::StorageType; - using Sign = LIBC_NAMESPACE::fputil::Sign; const T inf = FPBits::inf(Sign::POS).get_val(); const T neg_inf = FPBits::inf(Sign::NEG).get_val(); diff --git a/libc/test/src/math/smoke/NextTowardTest.h b/libc/test/src/math/smoke/NextTowardTest.h index caf98262c5d1..0c2abf815c23 100644 --- a/libc/test/src/math/smoke/NextTowardTest.h +++ b/libc/test/src/math/smoke/NextTowardTest.h @@ -34,7 +34,6 @@ class NextTowardTestTemplate : public LIBC_NAMESPACE::testing::Test { using FPBits = LIBC_NAMESPACE::fputil::FPBits; using ToFPBits = LIBC_NAMESPACE::fputil::FPBits; using StorageType = typename FPBits::StorageType; - using Sign = LIBC_NAMESPACE::fputil::Sign; const T inf = FPBits::inf(Sign::POS).get_val(); const T neg_inf = FPBits::inf(Sign::NEG).get_val(); diff --git a/libc/test/src/math/smoke/RIntTest.h b/libc/test/src/math/smoke/RIntTest.h index 903fbe9ce343..5a283a8bc0b5 100644 --- a/libc/test/src/math/smoke/RIntTest.h +++ b/libc/test/src/math/smoke/RIntTest.h @@ -29,7 +29,6 @@ public: private: using FPBits = LIBC_NAMESPACE::fputil::FPBits; using StorageType = typename FPBits::StorageType; - using Sign = LIBC_NAMESPACE::fputil::Sign; const T inf = FPBits::inf(Sign::POS).get_val(); const T neg_inf = FPBits::inf(Sign::NEG).get_val(); diff --git a/libc/test/src/math/smoke/RemQuoTest.h b/libc/test/src/math/smoke/RemQuoTest.h index a9fa405b2700..cf56b1d6460f 100644 --- a/libc/test/src/math/smoke/RemQuoTest.h +++ b/libc/test/src/math/smoke/RemQuoTest.h @@ -19,7 +19,6 @@ template class RemQuoTestTemplate : public LIBC_NAMESPACE::testing::Test { using FPBits = LIBC_NAMESPACE::fputil::FPBits; using StorageType = typename FPBits::StorageType; - using Sign = LIBC_NAMESPACE::fputil::Sign; const T inf = FPBits::inf(Sign::POS).get_val(); const T neg_inf = FPBits::inf(Sign::NEG).get_val(); diff --git a/libc/test/src/math/smoke/RoundToIntegerTest.h b/libc/test/src/math/smoke/RoundToIntegerTest.h index 1b5135d016cc..44b3f8996df5 100644 --- a/libc/test/src/math/smoke/RoundToIntegerTest.h +++ b/libc/test/src/math/smoke/RoundToIntegerTest.h @@ -28,7 +28,6 @@ public: private: using FPBits = LIBC_NAMESPACE::fputil::FPBits; using StorageType = typename FPBits::StorageType; - using Sign = LIBC_NAMESPACE::fputil::Sign; const F zero = FPBits::zero(Sign::POS).get_val(); const F neg_zero = FPBits::zero(Sign::NEG).get_val(); diff --git a/libc/test/src/math/smoke/atanhf_test.cpp b/libc/test/src/math/smoke/atanhf_test.cpp index df0746e0c9c3..590a7ab60f04 100644 --- a/libc/test/src/math/smoke/atanhf_test.cpp +++ b/libc/test/src/math/smoke/atanhf_test.cpp @@ -19,7 +19,7 @@ using LlvmLibcAtanhfTest = LIBC_NAMESPACE::testing::FPTest; TEST_F(LlvmLibcAtanhfTest, SpecialNumbers) { - using Sign = LIBC_NAMESPACE::fputil::Sign; + LIBC_NAMESPACE::libc_errno = 0; LIBC_NAMESPACE::fputil::clear_except(FE_ALL_EXCEPT); diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 59b0bbbda2f5..c4f6eab06221 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -501,6 +501,14 @@ libc_support_library( ], ) +libc_support_library( + name = "__support_sign", + hdrs = ["src/__support/sign.h"], + deps = [ + ":__support_macros_properties_types", + ], +) + libc_support_library( name = "__support_uint128", hdrs = ["src/__support/UInt128.h"], @@ -734,6 +742,7 @@ libc_support_library( deps = [ ":__support_common", ":__support_cpp_bit", + ":__support_sign", ":__support_cpp_type_traits", ":__support_libc_assert", ":__support_macros_attributes", diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/__support/FPUtil/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/__support/FPUtil/BUILD.bazel index 76443fc5d9f8..18683e42724a 100644 --- a/utils/bazel/llvm-project-overlay/libc/test/src/__support/FPUtil/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/test/src/__support/FPUtil/BUILD.bazel @@ -17,6 +17,7 @@ libc_test( "//libc:__support_fputil_fp_bits", "//libc:__support_fputil_fpbits_str", "//libc:__support_integer_literals", + "//libc:__support_sign", ], ) -- GitLab From f872043e055f4163c3c4b1b86ca0354490174987 Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Wed, 20 Mar 2024 15:12:33 +0100 Subject: [PATCH 012/296] Revert "[VPlan] Replace disjoint or with add instead of dropping disjoint. (#83821)" This reverts commit c2c1e6ee4ce0df3d000ba880fa6cf58441da6462. It creates a use after free. ==8342==ERROR: AddressSanitizer: heap-use-after-free on address 0x50f000001760 at pc 0x55b9fb84a8fb bp 0x7ffc18468a10 sp 0x7ffc18468a08 READ of size 1 at 0x50f000001760 thread T0 #0 0x55b9fb84a8fa in dropPoisonGeneratingFlags llvm/lib/Transforms/Vectorize/VPlan.h:1040:13 #1 0x55b9fb84a8fa in llvm::VPlanTransforms::dropPoisonGeneratingRecipes(llvm::VPlan&, llvm::function_ref)::$_0::operator()(llvm::VPRecipeBase*) const llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp:1236:23 #2 0x55b9fb84a196 in llvm::VPlanTransforms::dropPoisonGeneratingRecipes(llvm::VPlan&, llvm::function_ref) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp Can be reproduced with asan on Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll Transforms/LoopVectorize/X86/pr81872.ll Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll --- .../Vectorize/LoopVectorizationPlanner.h | 3 --- llvm/lib/Transforms/Vectorize/VPlan.h | 6 ------ .../Transforms/Vectorize/VPlanPatternMatch.h | 5 ----- .../Transforms/Vectorize/VPlanTransforms.cpp | 17 ----------------- .../Transforms/LoopVectorize/X86/pr81872.ll | 2 +- 5 files changed, 1 insertion(+), 32 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index 5d03b66b0ce3..e86705e89889 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -68,9 +68,6 @@ class VPBuilder { public: VPBuilder() = default; VPBuilder(VPBasicBlock *InsertBB) { setInsertPoint(InsertBB); } - VPBuilder(VPRecipeBase *InsertPt) { - setInsertPoint(InsertPt->getParent(), InsertPt->getIterator()); - } /// Clear the insertion point: created instructions will not be inserted into /// a block. diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index d720f17a3a88..d77c7554d50e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1127,12 +1127,6 @@ public: return WrapFlags.HasNSW; } - bool isDisjoint() const { - assert(OpType == OperationType::DisjointOp && - "recipe cannot have a disjoing flag"); - return DisjointFlags.IsDisjoint; - } - #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void printFlags(raw_ostream &O) const; #endif diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index a03a408686ef..aa2535906945 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -261,11 +261,6 @@ m_Mul(const Op0_t &Op0, const Op1_t &Op1) { return m_Binary(Op0, Op1); } -template -inline AllBinaryRecipe_match -m_Or(const Op0_t &Op0, const Op1_t &Op1) { - return m_Binary(Op0, Op1); -} } // namespace VPlanPatternMatch } // namespace llvm diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index c6ec99fbbf0a..a91ccefe4b6d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1216,23 +1216,6 @@ void VPlanTransforms::dropPoisonGeneratingRecipes( // load/store. If the underlying instruction has poison-generating flags, // drop them directly. if (auto *RecWithFlags = dyn_cast(CurRec)) { - VPValue *A, *B; - using namespace llvm::VPlanPatternMatch; - // Dropping disjoint from an OR may yield incorrect results, as some - // analysis may have converted it to an Add implicitly (e.g. SCEV used - // for dependence analysis). Instead, replace it with an equivalent Add. - // This is possible as all users of the disjoint OR only access lanes - // where the operands are disjoint or poison otherwise. - if (match(RecWithFlags, m_Or(m_VPValue(A), m_VPValue(B))) && - RecWithFlags->isDisjoint()) { - VPBuilder Builder(RecWithFlags); - VPInstruction *New = Builder.createOverflowingOp( - Instruction::Add, {A, B}, {false, false}, - RecWithFlags->getDebugLoc()); - RecWithFlags->replaceAllUsesWith(New); - RecWithFlags->eraseFromParent(); - CurRec = New; - } RecWithFlags->dropPoisonGeneratingFlags(); } else { Instruction *Instr = dyn_cast_or_null( diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll b/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll index 3f38abc75a58..14acb6f57aa0 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll @@ -29,7 +29,7 @@ define void @test(ptr noundef align 8 dereferenceable_or_null(16) %arr) #0 { ; CHECK-NEXT: [[TMP2:%.*]] = and <4 x i64> [[VEC_IND]], ; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i64> [[TMP2]], zeroinitializer ; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP3]], <4 x i1> zeroinitializer -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP0]], 1 +; CHECK-NEXT: [[TMP5:%.*]] = or i64 [[TMP0]], 1 ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[ARR]], i64 [[TMP5]] ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[TMP6]], i32 0 ; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[TMP7]], i32 -3 -- GitLab From 9f168591f36952d8cca543a5ba67906dc07096ff Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Wed, 20 Mar 2024 07:26:08 -0700 Subject: [PATCH 013/296] [InstallAPI] Simplify & improve symbol printing for diagnostics (#85894) * Defer mangling of symbols until an error is ready to report * Pass around fewer parameters when reporting --- .../include/clang/InstallAPI/DylibVerifier.h | 4 + clang/include/clang/InstallAPI/MachO.h | 2 + clang/lib/InstallAPI/DylibVerifier.cpp | 134 +++++++++--------- clang/test/InstallAPI/diagnostics-cpp.test | 4 +- 4 files changed, 73 insertions(+), 71 deletions(-) diff --git a/clang/include/clang/InstallAPI/DylibVerifier.h b/clang/include/clang/InstallAPI/DylibVerifier.h index 8269715c7f23..bbfa8711313e 100644 --- a/clang/include/clang/InstallAPI/DylibVerifier.h +++ b/clang/include/clang/InstallAPI/DylibVerifier.h @@ -128,6 +128,10 @@ private: /// Find matching dylib slice for target triple that is being parsed. void assignSlice(const Target &T); + /// Gather annotations for symbol for error reporting. + std::string getAnnotatedName(const Record *R, SymbolContext &SymCtx, + bool ValidSourceLoc = true); + // Symbols in dylib. llvm::MachO::Records Dylib; diff --git a/clang/include/clang/InstallAPI/MachO.h b/clang/include/clang/InstallAPI/MachO.h index a77766511fa3..f0dea8bbd24c 100644 --- a/clang/include/clang/InstallAPI/MachO.h +++ b/clang/include/clang/InstallAPI/MachO.h @@ -26,11 +26,13 @@ using SymbolFlags = llvm::MachO::SymbolFlags; using RecordLinkage = llvm::MachO::RecordLinkage; using Record = llvm::MachO::Record; +using EncodeKind = llvm::MachO::EncodeKind; using GlobalRecord = llvm::MachO::GlobalRecord; using ObjCContainerRecord = llvm::MachO::ObjCContainerRecord; using ObjCInterfaceRecord = llvm::MachO::ObjCInterfaceRecord; using ObjCCategoryRecord = llvm::MachO::ObjCCategoryRecord; using ObjCIVarRecord = llvm::MachO::ObjCIVarRecord; +using ObjCIFSymbolKind = llvm::MachO::ObjCIFSymbolKind; using Records = llvm::MachO::Records; using RecordsSlice = llvm::MachO::RecordsSlice; using BinaryAttrs = llvm::MachO::RecordsSlice::BinaryAttrs; diff --git a/clang/lib/InstallAPI/DylibVerifier.cpp b/clang/lib/InstallAPI/DylibVerifier.cpp index 700763b3fee0..24e0d0addf2f 100644 --- a/clang/lib/InstallAPI/DylibVerifier.cpp +++ b/clang/lib/InstallAPI/DylibVerifier.cpp @@ -10,9 +10,6 @@ namespace installapi { /// Metadata stored about a mapping of a declaration to a symbol. struct DylibVerifier::SymbolContext { - // Name to use for printing in diagnostics. - std::string PrettyPrintName{""}; - // Name to use for all querying and verification // purposes. std::string SymbolName{""}; @@ -30,11 +27,35 @@ struct DylibVerifier::SymbolContext { bool Inlined = false; }; -static std::string -getAnnotatedName(const Record *R, EncodeKind Kind, StringRef Name, - bool ValidSourceLoc = true, - ObjCIFSymbolKind ObjCIF = ObjCIFSymbolKind::None) { - assert(!Name.empty() && "Need symbol name for printing"); +static bool isCppMangled(StringRef Name) { + // InstallAPI currently only supports itanium manglings. + return (Name.starts_with("_Z") || Name.starts_with("__Z") || + Name.starts_with("___Z")); +} + +static std::string demangle(StringRef Name) { + // InstallAPI currently only supports itanium manglings. + if (!isCppMangled(Name)) + return Name.str(); + char *Result = llvm::itaniumDemangle(Name); + if (!Result) + return Name.str(); + + std::string Demangled(Result); + free(Result); + return Demangled; +} + +std::string DylibVerifier::getAnnotatedName(const Record *R, + SymbolContext &SymCtx, + bool ValidSourceLoc) { + assert(!SymCtx.SymbolName.empty() && "Expected symbol name"); + + const StringRef SymbolName = SymCtx.SymbolName; + std::string PrettyName = + (Demangle && (SymCtx.Kind == EncodeKind::GlobalSymbol)) + ? demangle(SymbolName) + : SymbolName.str(); std::string Annotation; if (R->isWeakDefined()) @@ -45,15 +66,16 @@ getAnnotatedName(const Record *R, EncodeKind Kind, StringRef Name, Annotation += "(tlv) "; // Check if symbol represents only part of a @interface declaration. - const bool IsAnnotatedObjCClass = ((ObjCIF != ObjCIFSymbolKind::None) && - (ObjCIF <= ObjCIFSymbolKind::EHType)); + const bool IsAnnotatedObjCClass = + ((SymCtx.ObjCIFKind != ObjCIFSymbolKind::None) && + (SymCtx.ObjCIFKind <= ObjCIFSymbolKind::EHType)); if (IsAnnotatedObjCClass) { - if (ObjCIF == ObjCIFSymbolKind::EHType) + if (SymCtx.ObjCIFKind == ObjCIFSymbolKind::EHType) Annotation += "Exception Type of "; - if (ObjCIF == ObjCIFSymbolKind::MetaClass) + if (SymCtx.ObjCIFKind == ObjCIFSymbolKind::MetaClass) Annotation += "Metaclass of "; - if (ObjCIF == ObjCIFSymbolKind::Class) + if (SymCtx.ObjCIFKind == ObjCIFSymbolKind::Class) Annotation += "Class of "; } @@ -61,42 +83,30 @@ getAnnotatedName(const Record *R, EncodeKind Kind, StringRef Name, // tied to it. This can only ever happen when the location has to come from // debug info. if (ValidSourceLoc) { - if ((Kind == EncodeKind::GlobalSymbol) && Name.starts_with("_")) - return Annotation + Name.drop_front(1).str(); - return Annotation + Name.str(); + StringRef PrettyNameRef(PrettyName); + if ((SymCtx.Kind == EncodeKind::GlobalSymbol) && + !isCppMangled(SymbolName) && PrettyNameRef.starts_with("_")) + return Annotation + PrettyNameRef.drop_front(1).str(); + return Annotation + PrettyName; } if (IsAnnotatedObjCClass) - return Annotation + Name.str(); + return Annotation + PrettyName; - switch (Kind) { + switch (SymCtx.Kind) { case EncodeKind::GlobalSymbol: - return Annotation + Name.str(); + return Annotation + PrettyName; case EncodeKind::ObjectiveCInstanceVariable: - return Annotation + "(ObjC IVar) " + Name.str(); + return Annotation + "(ObjC IVar) " + PrettyName; case EncodeKind::ObjectiveCClass: - return Annotation + "(ObjC Class) " + Name.str(); + return Annotation + "(ObjC Class) " + PrettyName; case EncodeKind::ObjectiveCClassEHType: - return Annotation + "(ObjC Class EH) " + Name.str(); + return Annotation + "(ObjC Class EH) " + PrettyName; } llvm_unreachable("unexpected case for EncodeKind"); } -static std::string demangle(StringRef Name) { - // InstallAPI currently only supports itanium manglings. - if (!(Name.starts_with("_Z") || Name.starts_with("__Z") || - Name.starts_with("___Z"))) - return Name.str(); - char *Result = llvm::itaniumDemangle(Name); - if (!Result) - return Name.str(); - - std::string Demangled(Result); - free(Result); - return Demangled; -} - static DylibVerifier::Result updateResult(const DylibVerifier::Result Prev, const DylibVerifier::Result Curr) { if (Prev == Curr) @@ -193,19 +203,18 @@ bool DylibVerifier::compareObjCInterfaceSymbols(const Record *R, // The decl represents a complete ObjCInterface, but the symbols in the // dylib do not. Determine which symbol is missing. To keep older projects // building, treat this as a warning. - if (!DR->isExportedSymbol(ObjCIFSymbolKind::Class)) + if (!DR->isExportedSymbol(ObjCIFSymbolKind::Class)) { + SymCtx.ObjCIFKind = ObjCIFSymbolKind::Class; PrintDiagnostic(DR->getLinkageForSymbol(ObjCIFSymbolKind::Class), R, - getAnnotatedName(R, SymCtx.Kind, SymCtx.PrettyPrintName, - /*ValidSourceLoc=*/true, - ObjCIFSymbolKind::Class), + getAnnotatedName(R, SymCtx), /*PrintAsWarning=*/true); - - if (!DR->isExportedSymbol(ObjCIFSymbolKind::MetaClass)) + } + if (!DR->isExportedSymbol(ObjCIFSymbolKind::MetaClass)) { + SymCtx.ObjCIFKind = ObjCIFSymbolKind::MetaClass; PrintDiagnostic(DR->getLinkageForSymbol(ObjCIFSymbolKind::MetaClass), R, - getAnnotatedName(R, SymCtx.Kind, SymCtx.PrettyPrintName, - /*ValidSourceLoc=*/true, - ObjCIFSymbolKind::MetaClass), + getAnnotatedName(R, SymCtx), /*PrintAsWarning=*/true); + } return true; } @@ -221,7 +230,7 @@ bool DylibVerifier::compareObjCInterfaceSymbols(const Record *R, // At this point that means there was not a matching class symbol // to represent the one discovered as a declaration. PrintDiagnostic(DR->getLinkageForSymbol(SymCtx.ObjCIFKind), R, - SymCtx.PrettyPrintName); + SymCtx.SymbolName); return false; } @@ -234,7 +243,7 @@ DylibVerifier::Result DylibVerifier::compareVisibility(const Record *R, Ctx.emitDiag([&]() { Ctx.Diag->Report(SymCtx.FA->D->getLocation(), diag::err_library_missing_symbol) - << SymCtx.PrettyPrintName; + << getAnnotatedName(R, SymCtx); }); return Result::Invalid; } @@ -242,7 +251,7 @@ DylibVerifier::Result DylibVerifier::compareVisibility(const Record *R, Ctx.emitDiag([&]() { Ctx.Diag->Report(SymCtx.FA->D->getLocation(), diag::err_library_hidden_symbol) - << SymCtx.PrettyPrintName; + << getAnnotatedName(R, SymCtx); }); return Result::Invalid; } @@ -269,7 +278,7 @@ DylibVerifier::Result DylibVerifier::compareVisibility(const Record *R, } Ctx.emitDiag([&]() { Ctx.Diag->Report(SymCtx.FA->D->getLocation(), ID) - << SymCtx.PrettyPrintName; + << getAnnotatedName(R, SymCtx); }); return Outcome; } @@ -293,14 +302,14 @@ DylibVerifier::Result DylibVerifier::compareAvailability(const Record *R, Ctx.emitDiag([&]() { Ctx.Diag->Report(SymCtx.FA->D->getLocation(), diag::warn_header_availability_mismatch) - << SymCtx.PrettyPrintName << IsDeclAvailable << IsDeclAvailable; + << getAnnotatedName(R, SymCtx) << IsDeclAvailable << IsDeclAvailable; }); return Result::Ignore; case VerificationMode::Pedantic: Ctx.emitDiag([&]() { Ctx.Diag->Report(SymCtx.FA->D->getLocation(), diag::err_header_availability_mismatch) - << SymCtx.PrettyPrintName << IsDeclAvailable << IsDeclAvailable; + << getAnnotatedName(R, SymCtx) << IsDeclAvailable << IsDeclAvailable; }); return Result::Invalid; case VerificationMode::ErrorsOnly: @@ -313,15 +322,11 @@ DylibVerifier::Result DylibVerifier::compareAvailability(const Record *R, bool DylibVerifier::compareSymbolFlags(const Record *R, SymbolContext &SymCtx, const Record *DR) { - std::string DisplayName = - Demangle ? demangle(DR->getName()) : DR->getName().str(); - if (DR->isThreadLocalValue() && !R->isThreadLocalValue()) { Ctx.emitDiag([&]() { Ctx.Diag->Report(SymCtx.FA->D->getLocation(), diag::err_dylib_symbol_flags_mismatch) - << getAnnotatedName(DR, SymCtx.Kind, DisplayName) - << DR->isThreadLocalValue(); + << getAnnotatedName(DR, SymCtx) << DR->isThreadLocalValue(); }); return false; } @@ -329,7 +334,7 @@ bool DylibVerifier::compareSymbolFlags(const Record *R, SymbolContext &SymCtx, Ctx.emitDiag([&]() { SymCtx.FA->D->getLocation(), Ctx.Diag->Report(diag::err_header_symbol_flags_mismatch) - << SymCtx.PrettyPrintName << R->isThreadLocalValue(); + << getAnnotatedName(DR, SymCtx) << R->isThreadLocalValue(); }); return false; } @@ -338,8 +343,7 @@ bool DylibVerifier::compareSymbolFlags(const Record *R, SymbolContext &SymCtx, Ctx.emitDiag([&]() { Ctx.Diag->Report(SymCtx.FA->D->getLocation(), diag::err_dylib_symbol_flags_mismatch) - << getAnnotatedName(DR, SymCtx.Kind, DisplayName) - << R->isWeakDefined(); + << getAnnotatedName(DR, SymCtx) << R->isWeakDefined(); }); return false; } @@ -347,7 +351,7 @@ bool DylibVerifier::compareSymbolFlags(const Record *R, SymbolContext &SymCtx, Ctx.emitDiag([&]() { Ctx.Diag->Report(SymCtx.FA->D->getLocation(), diag::err_header_symbol_flags_mismatch) - << SymCtx.PrettyPrintName << R->isWeakDefined(); + << getAnnotatedName(R, SymCtx) << R->isWeakDefined(); }); return false; } @@ -457,10 +461,7 @@ DylibVerifier::Result DylibVerifier::verify(ObjCIVarRecord *R, std::string FullName = ObjCIVarRecord::createScopedName(SuperClass, R->getName()); - SymbolContext SymCtx{ - getAnnotatedName(R, EncodeKind::ObjectiveCInstanceVariable, - Demangle ? demangle(FullName) : FullName), - FullName, EncodeKind::ObjectiveCInstanceVariable, FA}; + SymbolContext SymCtx{FullName, EncodeKind::ObjectiveCInstanceVariable, FA}; return verifyImpl(R, SymCtx); } @@ -485,11 +486,8 @@ DylibVerifier::Result DylibVerifier::verify(ObjCInterfaceRecord *R, SymCtx.SymbolName = R->getName(); SymCtx.ObjCIFKind = assignObjCIFSymbolKind(R); - std::string DisplayName = - Demangle ? demangle(SymCtx.SymbolName) : SymCtx.SymbolName; SymCtx.Kind = R->hasExceptionAttribute() ? EncodeKind::ObjectiveCClassEHType : EncodeKind::ObjectiveCClass; - SymCtx.PrettyPrintName = getAnnotatedName(R, SymCtx.Kind, DisplayName); SymCtx.FA = FA; return verifyImpl(R, SymCtx); @@ -504,8 +502,6 @@ DylibVerifier::Result DylibVerifier::verify(GlobalRecord *R, SimpleSymbol Sym = parseSymbol(R->getName()); SymbolContext SymCtx; SymCtx.SymbolName = Sym.Name; - SymCtx.PrettyPrintName = - getAnnotatedName(R, Sym.Kind, Demangle ? demangle(Sym.Name) : Sym.Name); SymCtx.Kind = Sym.Kind; SymCtx.FA = FA; SymCtx.Inlined = R->isInlined(); diff --git a/clang/test/InstallAPI/diagnostics-cpp.test b/clang/test/InstallAPI/diagnostics-cpp.test index 9319a7a61d48..658886537507 100644 --- a/clang/test/InstallAPI/diagnostics-cpp.test +++ b/clang/test/InstallAPI/diagnostics-cpp.test @@ -12,8 +12,8 @@ // RUN: --verify-mode=Pedantic -o %t/output.tbd --demangle 2> %t/errors.log // RUN: FileCheck -input-file %t/errors.log %s -CHECK: warning: violations found for arm64-apple-macos13 -CHECK: CPP.h:5:7: error: declaration has external linkage, but symbol has internal linkage in dynamic library 'vtable for Bar' +CHECK: warning: violations found for arm64-apple-macos13 +CHECK: CPP.h:5:7: error: declaration has external linkage, but symbol has internal linkage in dynamic library 'vtable for Bar' CHECK-NEXT: class Bar : Foo { CHECK-NEXT: ^ CHECK-NEXT: CPP.h:5:7: error: declaration has external linkage, but symbol has internal linkage in dynamic library 'typeinfo for Bar' -- GitLab From 05bde30585710a51592eee0a6cf6df8184d09c92 Mon Sep 17 00:00:00 2001 From: Jonas Paulsson Date: Wed, 20 Mar 2024 10:29:12 -0400 Subject: [PATCH 014/296] Move assertion for AdjustsStack from PEI to MachineVerifier. (#85698) Have the verifier report a missing AdjustsStack flag rather than waiting until PEI asserts. --- llvm/lib/CodeGen/MachineVerifier.cpp | 6 +++++ llvm/lib/CodeGen/PrologEpilogInserter.cpp | 2 -- .../clear-dead-implicit-def-impdef.mir | 2 ++ ...plicit-def-remat-requires-impdef-check.mir | 2 ++ ...implicit-def-with-impdef-greedy-assert.mir | 2 ++ .../CodeGen/AMDGPU/fold-restore-undef-use.mir | 2 ++ .../greedy-alloc-fail-sgpr1024-spill.mir | 1 + .../ran-out-of-sgprs-allocation-failure.mir | 1 + .../CodeGen/AMDGPU/sched-crash-dbg-value.mir | 2 ++ .../AMDGPU/sgpr-spill-wrong-stack-id.mir | 1 + .../AMDGPU/snippet-copy-bundle-regression.mir | 1 + .../virtregrewrite-undef-identity-copy.mir | 1 + ...no-register-coalescing-in-returnsTwice.mir | 2 ++ .../CodeGen/Hexagon/regalloc-bad-undef.mir | 2 +- .../SystemZ/RAbasic-invalid-LR-update.mir | 2 ++ .../SystemZ/clear-liverange-spillreg.mir | 1 + llvm/test/CodeGen/SystemZ/int-cmp-56.mir | 4 +++ .../SystemZ/regcoal-subranges-update.mir | 2 ++ llvm/test/CodeGen/X86/callbr-asm-kill.mir | 1 + llvm/test/CodeGen/X86/regalloc-copy-hints.mir | 1 + .../CodeGen/X86/statepoint-fastregalloc.mir | 4 +++ .../X86/statepoint-invoke-ra-enter-at-end.mir | 2 +- .../X86/statepoint-invoke-ra-hoist-copies.mir | 2 +- .../statepoint-invoke-ra-inline-spiller.mir | 2 +- ...tatepoint-invoke-ra-remove-back-copies.mir | 2 +- .../test/CodeGen/X86/statepoint-invoke-ra.mir | 2 +- .../CodeGen/X86/statepoint-vreg-folding.mir | 2 +- .../DebugInfo/MIR/InstrRef/phi-coalescing.mir | 1 + .../Mips/livedebugvars-stop-trimming-loc.mir | 2 ++ .../MachineVerifier/test_adjustsstack.mir | 26 +++++++++++++++++++ 30 files changed, 74 insertions(+), 9 deletions(-) create mode 100644 llvm/test/MachineVerifier/test_adjustsstack.mir diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index c69d36fc7fdd..005efe48ac0c 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -3697,6 +3697,9 @@ void MachineVerifier::verifyStackFrame() { if (I.getOpcode() == FrameSetupOpcode) { if (BBState.ExitIsSetup) report("FrameSetup is after another FrameSetup", &I); + if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack()) + report("AdjustsStack not set in presence of a frame pseudo " + "instruction.", &I); BBState.ExitValue -= TII->getFrameTotalSize(I); BBState.ExitIsSetup = true; } @@ -3712,6 +3715,9 @@ void MachineVerifier::verifyStackFrame() { errs() << "FrameDestroy <" << Size << "> is after FrameSetup <" << AbsSPAdj << ">.\n"; } + if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack()) + report("AdjustsStack not set in presence of a frame pseudo " + "instruction.", &I); BBState.ExitValue += Size; BBState.ExitIsSetup = false; } diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp index eaf96ec5cbde..c942b8a3e268 100644 --- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp +++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp @@ -372,8 +372,6 @@ void PEI::calculateCallFrameInfo(MachineFunction &MF) { MFI.computeMaxCallFrameSize(MF, &FrameSDOps); assert(MFI.getMaxCallFrameSize() <= MaxCFSIn && "Recomputing MaxCFS gave a larger value."); - assert((FrameSDOps.empty() || MF.getFrameInfo().adjustsStack()) && - "AdjustsStack not set in presence of a frame pseudo instruction."); if (TFI->canSimplifyCallFramePseudos(MF)) { // If call frames are not being included as part of the stack frame, and diff --git a/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir b/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir index 9040937d027d..2532c76b1336 100644 --- a/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir +++ b/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir @@ -2,6 +2,8 @@ # RUN: llc -mtriple=arm64-apple-macosx -mcpu=apple-m1 -verify-regalloc -run-pass=greedy -o - %s | FileCheck %s --- name: func +frameInfo: + adjustsStack: true tracksRegLiveness: true body: | bb.0: diff --git a/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir b/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir index aa94a03786f5..47aa34e3c011 100644 --- a/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir +++ b/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir @@ -22,6 +22,7 @@ name: inst_stores_to_dead_spill_implicit_def_impdef tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true body: | bb.0: @@ -59,6 +60,7 @@ body: | name: inst_stores_to_dead_spill_movimm_impdef tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true body: | bb.0: diff --git a/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir b/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir index e5395b20afd4..d55cf71cead6 100644 --- a/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir +++ b/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir @@ -3,6 +3,8 @@ --- name: widget +frameInfo: + adjustsStack: true tracksRegLiveness: true jumpTable: kind: label-difference32 diff --git a/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir b/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir index 3616d617f84a..054eeec9e33f 100644 --- a/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir +++ b/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir @@ -7,6 +7,8 @@ --- name: restore_undef_copy_use +frameInfo: + adjustsStack: true tracksRegLiveness: true machineFunctionInfo: maxKernArgAlign: 1 diff --git a/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir b/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir index bdd89a907790..dde84af57ed2 100644 --- a/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir +++ b/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir @@ -13,6 +13,7 @@ name: greedy_fail_alloc_sgpr1024_spill tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true machineFunctionInfo: explicitKernArgSize: 16 diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir index 2ccc24152a9f..fdfc9b043cc9 100644 --- a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir +++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir @@ -24,6 +24,7 @@ registers: - { id: 10, class: sreg_64_xexec, preferred-register: '$vcc' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: maxKernArgAlign: 1 diff --git a/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir b/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir index c0d199920bd9..158874e7c827 100644 --- a/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir +++ b/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir @@ -180,6 +180,8 @@ exposesReturnsTwice: false legalized: false regBankSelected: false selected: false +frameInfo: + adjustsStack: true tracksRegLiveness: true liveins: - { reg: '$vgpr0', virtual-reg: '%0' } diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir b/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir index efbdbca9da6b..c6ccbd99bf89 100644 --- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir +++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir @@ -78,6 +78,7 @@ name: sgpr_spill_wrong_stack_id tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true machineFunctionInfo: scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3 diff --git a/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir b/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir index 355829825146..f8ec6bb5d943 100644 --- a/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir +++ b/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir @@ -21,6 +21,7 @@ name: kernel tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true machineFunctionInfo: isEntryFunction: true diff --git a/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir b/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir index 3d9db687ffa1..6659e9532376 100644 --- a/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir +++ b/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir @@ -20,6 +20,7 @@ name: undef_identity_copy tracksRegLiveness: true frameInfo: maxAlignment: 4 + adjustsStack: true hasCalls: true machineFunctionInfo: isEntryFunction: true diff --git a/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir b/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir index 5c59566247d8..b4bbb9be8ae4 100644 --- a/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir +++ b/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir @@ -86,6 +86,8 @@ --- name: main exposesReturnsTwice: true +frameInfo: + adjustsStack: true stack: - { id: 0, name: P0, size: 80, alignment: 8, local-offset: -80 } - { id: 1, name: jb1, size: 160, alignment: 8, local-offset: -240 } diff --git a/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir b/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir index 67f4dd72ea0b..9468b18bf8e4 100644 --- a/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir +++ b/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir @@ -135,7 +135,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 0 - adjustsStack: false + adjustsStack: true hasCalls: true maxCallFrameSize: 0 hasOpaqueSPAdjustment: false diff --git a/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir b/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir index 3b308ce3d0d2..fbe2b687e850 100644 --- a/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir +++ b/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir @@ -24,6 +24,8 @@ --- name: autogen_SD21418 alignment: 4 +frameInfo: + adjustsStack: true tracksRegLiveness: true registers: - { id: 0, class: vr128bit } diff --git a/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir b/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir index 7ff7d9b8b709..197c3d8551fc 100644 --- a/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir +++ b/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir @@ -157,6 +157,7 @@ registers: - { id: 129, class: grx32bit } - { id: 130, class: fp64bit } frameInfo: + adjustsStack: true hasCalls: true body: | bb.0: diff --git a/llvm/test/CodeGen/SystemZ/int-cmp-56.mir b/llvm/test/CodeGen/SystemZ/int-cmp-56.mir index e52fd44ae47d..3e00b6065eb9 100644 --- a/llvm/test/CodeGen/SystemZ/int-cmp-56.mir +++ b/llvm/test/CodeGen/SystemZ/int-cmp-56.mir @@ -48,6 +48,7 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | @@ -125,6 +126,7 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | @@ -202,6 +204,7 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | @@ -279,6 +282,7 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir b/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir index f709b70ff1b7..d3ef9b0b9abf 100644 --- a/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir +++ b/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir @@ -48,6 +48,8 @@ body: | # represented for the value carried by %7. --- name: segfault +frameInfo: + adjustsStack: true tracksRegLiveness: true liveins: [] body: | diff --git a/llvm/test/CodeGen/X86/callbr-asm-kill.mir b/llvm/test/CodeGen/X86/callbr-asm-kill.mir index 86c58c4715ed..0dded37c97af 100644 --- a/llvm/test/CodeGen/X86/callbr-asm-kill.mir +++ b/llvm/test/CodeGen/X86/callbr-asm-kill.mir @@ -45,6 +45,7 @@ liveins: - { reg: '$rsi', virtual-reg: '%3' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/CodeGen/X86/regalloc-copy-hints.mir b/llvm/test/CodeGen/X86/regalloc-copy-hints.mir index 13b5a541fa22..d09bcd6a6b40 100644 --- a/llvm/test/CodeGen/X86/regalloc-copy-hints.mir +++ b/llvm/test/CodeGen/X86/regalloc-copy-hints.mir @@ -103,6 +103,7 @@ registers: - { id: 82, class: gr32 } frameInfo: maxAlignment: 4 + adjustsStack: true hasCalls: true fixedStack: - { id: 0, size: 4, alignment: 4, stack-id: default, isImmutable: true } diff --git a/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir b/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir index 02c931067300..87ffdd7c4e6b 100644 --- a/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir +++ b/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir @@ -5,6 +5,8 @@ # Tied def/use must be assigned to the same register. --- name: test_relocate +frameInfo: + adjustsStack: true tracksRegLiveness: true body: | bb.0.entry: @@ -24,6 +26,8 @@ body: | # These regmasks have no real meaning and chosen to allow only single register to be assignable ($rbp) --- name: test_relocate_multi_regmasks +frameInfo: + adjustsStack: true tracksRegLiveness: true body: | bb.0.entry: diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir index 11968f17c70a..5f05270729fd 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir @@ -231,7 +231,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 1 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir index aae2f3870138..cf9128260f19 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir @@ -398,7 +398,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 1 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir index 87f5f0f96c50..fcebc69d9b2e 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir @@ -175,7 +175,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 4 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir index 49253968fcca..8bb39a03f7e3 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir @@ -226,7 +226,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 4 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir index 858ff3f1888b..da651039ce21 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir @@ -172,7 +172,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 4 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir b/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir index e24d5e8af1f5..d40a9a06d162 100644 --- a/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir +++ b/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir @@ -114,7 +114,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 8 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir index bc1c7ebac6ce..6460263c6025 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir @@ -106,6 +106,7 @@ liveins: - { reg: '$rsi', virtual-reg: '%5' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir b/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir index 35ab906efc90..ac67b9671f53 100644 --- a/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir +++ b/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir @@ -71,6 +71,8 @@ --- name: fn2 alignment: 4 +frameInfo: + adjustsStack: true tracksRegLiveness: true registers: - { id: 0, class: gpr32, preferred-register: '' } diff --git a/llvm/test/MachineVerifier/test_adjustsstack.mir b/llvm/test/MachineVerifier/test_adjustsstack.mir new file mode 100644 index 000000000000..d333737e000c --- /dev/null +++ b/llvm/test/MachineVerifier/test_adjustsstack.mir @@ -0,0 +1,26 @@ +# RUN: not --crash llc -o - -start-before=twoaddressinstruction -verify-machineinstrs %s 2>&1 \ +# RUN: | FileCheck %s +# REQUIRES: aarch64-registered-target +--- | + target triple = "aarch64-unknown-linux" + declare i32 @bar(i32) nounwind + define i32 @foo() nounwind { + call i32 @bar(i32 0) + ret i32 0 + } +... +--- +name: foo +registers: + - { id: 0, class: gpr32 } +body: | + bb.0 (%ir-block.0): + ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp + %0 = COPY $wzr + $w0 = COPY %0 + BL @bar, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0 + ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp + $w0 = COPY killed %0 + RET_ReallyLR implicit $w0 +... +# CHECK-LABEL: Bad machine code: AdjustsStack not set in presence of a frame pseudo instruction. -- GitLab From 5fb2797f23343999cace3afe24480e8711df4566 Mon Sep 17 00:00:00 2001 From: Neumann Hon Date: Wed, 20 Mar 2024 10:30:30 -0400 Subject: [PATCH 015/296] [GOFF][z/OS] Change PrivateGlobalPrefix and PrivateLabelPrefix to be L# (#85730) The current values for PrivateGlobalPrefix and PrivateLabelPrefix (@@ and @ respectively) are, in hindsight, poor choices for multiple reasons: First, there exist externally visible routines from the language environment that begin with @@. These functions are certainly not local/private by any means and they should not share a prefix with private globals. Secondly, both private globals and private labels should be handled the same way by GOFF, so it doesn't make much sense for them to have separate prefixes. GOFF remains the only file format where these are different and there is no reason for that to be the case --- clang/test/CodeGen/SystemZ/systemz-ppa2.c | 4 ++-- llvm/include/llvm/IR/DataLayout.h | 2 +- llvm/lib/MC/MCAsmInfoGOFF.cpp | 4 ++-- llvm/test/CodeGen/SystemZ/call-zos-01.ll | 14 +++++++------- llvm/test/CodeGen/SystemZ/call-zos-i128.ll | 4 ++-- llvm/test/CodeGen/SystemZ/call-zos-vararg.ll | 6 +++--- llvm/test/CodeGen/SystemZ/zos-ada-relocations.ll | 6 +++--- llvm/test/CodeGen/SystemZ/zos-landingpad.ll | 2 +- llvm/test/CodeGen/SystemZ/zos-ppa2.ll | 14 +++++++------- llvm/test/CodeGen/SystemZ/zos-prologue-epilog.ll | 14 +++++++------- llvm/test/MC/GOFF/ppa1.ll | 10 +++++----- llvm/unittests/IR/ManglerTest.cpp | 2 +- 12 files changed, 41 insertions(+), 41 deletions(-) diff --git a/clang/test/CodeGen/SystemZ/systemz-ppa2.c b/clang/test/CodeGen/SystemZ/systemz-ppa2.c index 0ff4cba5edfb..26b068ff03d5 100644 --- a/clang/test/CodeGen/SystemZ/systemz-ppa2.c +++ b/clang/test/CodeGen/SystemZ/systemz-ppa2.c @@ -13,14 +13,14 @@ // REQUIRES: systemz-registered-target // RUN: %clang_cc1 -triple s390x-ibm-zos -xc -S -o - %s | FileCheck %s --check-prefix CHECK-C -// CHECK-C: [[PPA2:(.L)|(@@)PPA2]]: +// CHECK-C: [[PPA2:(.L)|(L#)PPA2]]: // CHECK-C-NEXT: .byte 3{{[[:space:]]*}}.byte 0 // CHECK-C-NEXT: .byte 34{{$}} // CHECK-C-NEXT: .byte {{4}} // CHECK-C-NEXT: .long {{(CELQSTRT)}}-[[PPA2]] // RUN: %clang_cc1 -triple s390x-ibm-zos -xc++ -S -o - %s | FileCheck %s --check-prefix CHECK-CXX -// CHECK-CXX: [[PPA2:(.L)|(@@)PPA2]]: +// CHECK-CXX: [[PPA2:(.L)|(L#)PPA2]]: // CHECK-CXX-NEXT: .byte 3{{[[:space:]]*}}.byte 1 // CHECK-CXX-NEXT: .byte 34{{$}} // CHECK-CXX-NEXT: .byte {{4}} diff --git a/llvm/include/llvm/IR/DataLayout.h b/llvm/include/llvm/IR/DataLayout.h index 71f7f51d8ee4..d14adfe1590b 100644 --- a/llvm/include/llvm/IR/DataLayout.h +++ b/llvm/include/llvm/IR/DataLayout.h @@ -337,7 +337,7 @@ public: case MM_WinCOFF: return ".L"; case MM_GOFF: - return "@"; + return "L#"; case MM_Mips: return "$"; case MM_MachO: diff --git a/llvm/lib/MC/MCAsmInfoGOFF.cpp b/llvm/lib/MC/MCAsmInfoGOFF.cpp index 81704ffe4b24..3c81a466e82e 100644 --- a/llvm/lib/MC/MCAsmInfoGOFF.cpp +++ b/llvm/lib/MC/MCAsmInfoGOFF.cpp @@ -21,7 +21,7 @@ void MCAsmInfoGOFF::anchor() {} MCAsmInfoGOFF::MCAsmInfoGOFF() { Data64bitsDirective = "\t.quad\t"; HasDotTypeDotSizeDirective = false; - PrivateGlobalPrefix = "@@"; - PrivateLabelPrefix = "@"; + PrivateGlobalPrefix = "L#"; + PrivateLabelPrefix = "L#"; ZeroDirective = "\t.space\t"; } diff --git a/llvm/test/CodeGen/SystemZ/call-zos-01.ll b/llvm/test/CodeGen/SystemZ/call-zos-01.ll index 77776861186b..fc7a85caa37e 100644 --- a/llvm/test/CodeGen/SystemZ/call-zos-01.ll +++ b/llvm/test/CodeGen/SystemZ/call-zos-01.ll @@ -104,7 +104,7 @@ entry: } ; CHECK-LABEL: call_double: -; CHECK: larl [[GENREG:[0-9]+]], @{{CPI[0-9]+_[0-9]+}} +; CHECK: larl [[GENREG:[0-9]+]], L#{{CPI[0-9]+_[0-9]+}} ; CHECK-NEXT: ld 0, 0([[GENREG]]) define double @call_double() { entry: @@ -113,7 +113,7 @@ entry: } ; CHECK-LABEL: call_longdouble: -; CHECK: larl [[GENREG:[0-9]+]], @{{CPI[0-9]+_[0-9]+}} +; CHECK: larl [[GENREG:[0-9]+]], L#{{CPI[0-9]+_[0-9]+}} ; CHECK-NEXT: ld 0, 0([[GENREG]]) ; CHECK-NEXT: ld 2, 8([[GENREG]]) define fp128 @call_longdouble() { @@ -123,7 +123,7 @@ entry: } ; CHECK-LABEL: call_floats0 -; CHECK: larl [[GENREG:[0-9]+]], @{{CPI[0-9]+_[0-9]+}} +; CHECK: larl [[GENREG:[0-9]+]], L#{{CPI[0-9]+_[0-9]+}} ; CHECK-NEXT: ld 1, 0([[GENREG]]) ; CHECK-NEXT: ld 3, 8([[GENREG]]) ; CHECK: lxr 5, 0 @@ -146,7 +146,7 @@ entry: } ; CHECK-LABEL: pass_float: -; CHECK: larl 1, @{{CPI[0-9]+_[0-9]+}} +; CHECK: larl 1, L#{{CPI[0-9]+_[0-9]+}} ; CHECK: aeb 0, 0(1) define float @pass_float(float %arg) { entry: @@ -155,7 +155,7 @@ entry: } ; CHECK-LABEL: pass_double: -; CHECK: larl 1, @{{CPI[0-9]+_[0-9]+}} +; CHECK: larl 1, L#{{CPI[0-9]+_[0-9]+}} ; CHECK: adb 0, 0(1) define double @pass_double(double %arg) { entry: @@ -164,7 +164,7 @@ entry: } ; CHECK-LABEL: pass_longdouble -; CHECK: larl 1, @{{CPI[0-9]+_[0-9]+}} +; CHECK: larl 1, L#{{CPI[0-9]+_[0-9]+}} ; CHECK: lxdb 1, 0(1) ; CHECK: axbr 0, 1 define fp128 @pass_longdouble(fp128 %arg) { @@ -174,7 +174,7 @@ entry: } ; CHECK-LABEL: pass_floats0 -; CHECK: larl 1, @{{CPI[0-9]+_[0-9]+}} +; CHECK: larl 1, L#{{CPI[0-9]+_[0-9]+}} ; CHECK: axbr 0, 4 ; CHECK: axbr 1, 0 ; CHECK: cxbr 1, 5 diff --git a/llvm/test/CodeGen/SystemZ/call-zos-i128.ll b/llvm/test/CodeGen/SystemZ/call-zos-i128.ll index ccdac161d08a..775483374a32 100644 --- a/llvm/test/CodeGen/SystemZ/call-zos-i128.ll +++ b/llvm/test/CodeGen/SystemZ/call-zos-i128.ll @@ -3,10 +3,10 @@ ; RUN: llc < %s -mtriple=s390x-ibm-zos -mcpu=z13 | FileCheck %s ; CHECK-LABEL: call_i128: -; CHECK-DAG: larl 1, @CPI0_0 +; CHECK-DAG: larl 1, L#CPI0_0 ; CHECK-DAG: vl 0, 0(1), 3 ; CHECK-DAG: vst 0, 2256(4), 3 -; CHECK-DAG: larl 1, @CPI0_1 +; CHECK-DAG: larl 1, L#CPI0_1 ; CHECK-DAG: vl 0, 0(1), 3 ; CHECK-DAG: vst 0, 2272(4), 3 ; CHECK-DAG: la 1, 2288(4) diff --git a/llvm/test/CodeGen/SystemZ/call-zos-vararg.ll b/llvm/test/CodeGen/SystemZ/call-zos-vararg.ll index bde59a6be782..8290dbfe2310 100644 --- a/llvm/test/CodeGen/SystemZ/call-zos-vararg.ll +++ b/llvm/test/CodeGen/SystemZ/call-zos-vararg.ll @@ -108,7 +108,7 @@ define i64 @call_vararg_both0(i64 %arg0, double %arg1) { ; CHECK-LABEL: call_vararg_long_double0: ; CHECK: stmg 6, 7, 1872(4) ; CHECK-NEXT: aghi 4, -192 -; CHECK-NEXT: larl 1, @CPI5_0 +; CHECK-NEXT: larl 1, L#CPI5_0 ; CHECK-NEXT: ld 0, 0(1) ; CHECK-NEXT: ld 2, 8(1) ; CHECK-NEXT: lg 6, 8(5) @@ -202,7 +202,7 @@ define void @call_vec_vararg_test0(<2 x double> %v) { } ; ARCH12-LABEL: call_vec_vararg_test1 -; ARCH12: larl 1, @CPI10_0 +; ARCH12: larl 1, L#CPI10_0 ; ARCH12: vl 0, 0(1), 3 ; ARCH12: vlgvg 3, 24, 0 ; ARCH12: vrepg 2, 0, 1 @@ -294,7 +294,7 @@ entry: ; CHECK-NEXT: aghi 4, -192 ; CHECK-NEXT: lg 6, 72(5) ; CHECK-NEXT: lg 5, 64(5) -; CHECK-NEXT: larl 1, @CPI17_0 +; CHECK-NEXT: larl 1, L#CPI17_0 ; CHECK-NEXT: le 0, 0(1) ; CHECK-NEXT: llihf 0, 1073692672 ; CHECK-NEXT: llihh 2, 16384 diff --git a/llvm/test/CodeGen/SystemZ/zos-ada-relocations.ll b/llvm/test/CodeGen/SystemZ/zos-ada-relocations.ll index e25246917ec0..db67ac578186 100644 --- a/llvm/test/CodeGen/SystemZ/zos-ada-relocations.ll +++ b/llvm/test/CodeGen/SystemZ/zos-ada-relocations.ll @@ -56,9 +56,9 @@ entry: declare signext i32 @callout(i32 signext) ; CHECK: .section ".ada" -; CHECK: .set @@DoFunc@indirect0, DoFunc -; CHECK: .indirect_symbol @@DoFunc@indirect0 -; CHECK: .quad V(@@DoFunc@indirect0) * Offset 0 pointer to function descriptor DoFunc +; CHECK: .set L#DoFunc@indirect0, DoFunc +; CHECK: .indirect_symbol L#DoFunc@indirect0 +; CHECK: .quad V(L#DoFunc@indirect0) * Offset 0 pointer to function descriptor DoFunc ; CHECK: .quad R(Caller) * Offset 8 function descriptor of Caller ; CHECK: .quad V(Caller) ; CHECK: .quad A(i2) * Offset 24 pointer to data symbol i2 diff --git a/llvm/test/CodeGen/SystemZ/zos-landingpad.ll b/llvm/test/CodeGen/SystemZ/zos-landingpad.ll index 7f3214d57424..9db10114e979 100644 --- a/llvm/test/CodeGen/SystemZ/zos-landingpad.ll +++ b/llvm/test/CodeGen/SystemZ/zos-landingpad.ll @@ -19,7 +19,7 @@ done: lpad: %0 = landingpad { ptr, i32 } cleanup ; The Exception Pointer is %r1; the Exception Selector, %r2. -; CHECK: @BB{{[^%]*}} %lpad +; CHECK: L#BB{{[^%]*}} %lpad ; CHECK-DAG: stg 1, {{.*}} ; CHECK-DAG: st 2, {{.*}} %1 = extractvalue { ptr, i32 } %0, 0 diff --git a/llvm/test/CodeGen/SystemZ/zos-ppa2.ll b/llvm/test/CodeGen/SystemZ/zos-ppa2.ll index 60580aeb6d83..189b5a3757ee 100644 --- a/llvm/test/CodeGen/SystemZ/zos-ppa2.ll +++ b/llvm/test/CodeGen/SystemZ/zos-ppa2.ll @@ -2,24 +2,24 @@ ; REQUIRES: systemz-registered-target ; CHECK: .section ".ppa2" -; CHECK: @@PPA2: +; CHECK: L#PPA2: ; CHECK: .byte 3 ; CHECK: .byte 231 ; CHECK: .byte 34 ; CHECK: .byte 4 -; CHECK: .long CELQSTRT-@@PPA2 +; CHECK: .long CELQSTRT-L#PPA2 ; CHECK: .long 0 -; CHECK: .long @@DVS-@@PPA2 +; CHECK: .long L#DVS-L#PPA2 ; CHECK: .long 0 ; CHECK: .byte 129 ; CHECK: .byte 0 ; CHECK: .short 0 -; CHECK: @@DVS: +; CHECK: L#DVS: ; CHECK: .ascii "\361\371\367\360\360\361\360\361\360\360\360\360\360\360" ; CHECK: .short 0 -; CHECK: .quad @@PPA2-CELQSTRT * A(PPA2-CELQSTRT) -; CHECK: @@PPA1_void_test_0: -; CHECK: .long @@PPA2-@@PPA1_void_test_0 * Offset to PPA2 +; CHECK: .quad L#PPA2-CELQSTRT * A(PPA2-CELQSTRT) +; CHECK: L#PPA1_void_test_0: +; CHECK: .long L#PPA2-L#PPA1_void_test_0 * Offset to PPA2 ; CHECK: .section "B_IDRL" ; CHECK: .byte 0 ; CHECK: .byte 3 diff --git a/llvm/test/CodeGen/SystemZ/zos-prologue-epilog.ll b/llvm/test/CodeGen/SystemZ/zos-prologue-epilog.ll index 8c0411629da7..d3e5823fcb1f 100644 --- a/llvm/test/CodeGen/SystemZ/zos-prologue-epilog.ll +++ b/llvm/test/CodeGen/SystemZ/zos-prologue-epilog.ll @@ -15,7 +15,7 @@ ; CHECK64: aghi 4, 192 ; CHECK64: b 2(7) -; CHECK64: @@PPA1_func0_0: +; CHECK64: L#PPA1_func0_0: ; CHECK64: .short 0 * Length/4 of Parms define void @func0() { call i64 (i64) @fun(i64 10) @@ -31,7 +31,7 @@ define void @func0() { ; CHECK64: aghi 4, 160 ; CHECK64: b 2(7) -; CHECK64: @@PPA1_func1_0: +; CHECK64: L#PPA1_func1_0: ; CHECK64: .short 2 * Length/4 of Parms define void @func1(ptr %ptr) { %l01 = load volatile i64, ptr %ptr @@ -336,16 +336,16 @@ define void @large_stack0() { ; CHECK64: lgr 0, 3 ; CHECK64: llgt 3, 1208 ; CHECK64: cg 4, 64(3) -; CHECK64: jhe @BB7_2 +; CHECK64: jhe L#BB7_2 ; CHECK64: %bb.1: ; CHECK64: lg 3, 72(3) ; CHECK64: basr 3, 3 ; CHECK64: bcr 0, 7 -; CHECK64: @BB7_2: +; CHECK64: L#BB7_2: ; CHECK64: stmg 6, 7, 2064(4) ; CHECK64: lgr 3, 0 -; CHECK64: @@PPA1_large_stack1_0: +; CHECK64: L#PPA1_large_stack1_0: ; CHECK64: .short 6 * Length/4 of Parms define void @large_stack1(i64 %n1, i64 %n2, i64 %n3) { %arr = alloca [131072 x i64], align 8 @@ -361,12 +361,12 @@ define void @large_stack1(i64 %n1, i64 %n2, i64 %n3) { ; CHECK64: agfi 4, -1048768 ; CHECK64: llgt 3, 1208 ; CHECK64: cg 4, 64(3) -; CHECK64: jhe @BB8_2 +; CHECK64: jhe L#BB8_2 ; CHECK64: %bb.1: ; CHECK64: lg 3, 72(3) ; CHECK64: basr 3, 3 ; CHECK64: bcr 0, 7 -; CHECK64: @BB8_2: +; CHECK64: L#BB8_2: ; CHECK64: lgr 3, 0 ; CHECK64: lg 3, 2192(3) ; CHECK64: stmg 4, 12, 2048(4) diff --git a/llvm/test/MC/GOFF/ppa1.ll b/llvm/test/MC/GOFF/ppa1.ll index 40fc9e93780d..13971c7ec8e7 100644 --- a/llvm/test/MC/GOFF/ppa1.ll +++ b/llvm/test/MC/GOFF/ppa1.ll @@ -1,7 +1,7 @@ ; RUN: llc -mtriple s390x-ibm-zos < %s | FileCheck %s ; REQUIRES: systemz-registered-target -; CHECK: @@EPM_void_test_0: * @void_test +; CHECK: L#EPM_void_test_0: * @void_test ; CHECK: * XPLINK Routine Layout Entry ; CHECK: .long 12779717 * Eyecatcher 0x00C300C500C500 ; CHECK: .short 197 @@ -11,9 +11,9 @@ ; CHECK: * Entry Flags ; CHECK: * Bit 1: 1 = Leaf function ; CHECK: * Bit 2: 0 = Does not use alloca -; CHECK: @@func_end0: +; CHECK: L#func_end0: ; CHECK: .section ".ppa1" -; CHECK: @@PPA1_void_test_0: * PPA1 +; CHECK: L#PPA1_void_test_0: * PPA1 ; CHECK: .byte 2 * Version ; CHECK: .byte 206 * LE Signature X'CE' ; CHECK: .short 0 * Saved GPR Mask @@ -25,8 +25,8 @@ ; CHECK: .byte 0 * PPA1 Flags 3 ; CHECK: .byte 129 * PPA1 Flags 4 ; CHECK: .short 0 * Length/4 of Parms -; CHECK: .long @@func_end0-@@EPM_void_test_0 * Length of Code -; CHECK: .long @@EPM_void_test_0-@@PPA1_void_test_0 +; CHECK: .long L#func_end0-L#EPM_void_test_0 * Length of Code +; CHECK: .long L#EPM_void_test_0-L#PPA1_void_test_0 ; CHECK: .section ".text" ; CHECK: * -- End function define void @void_test() { diff --git a/llvm/unittests/IR/ManglerTest.cpp b/llvm/unittests/IR/ManglerTest.cpp index 8ad95a83b692..f2b78a1f9876 100644 --- a/llvm/unittests/IR/ManglerTest.cpp +++ b/llvm/unittests/IR/ManglerTest.cpp @@ -171,7 +171,7 @@ TEST(ManglerTest, GOFF) { "foo"); EXPECT_EQ(mangleFunc("foo", llvm::GlobalValue::PrivateLinkage, llvm::CallingConv::C, Mod, Mang), - "@foo"); + "L#foo"); } } // end anonymous namespace -- GitLab From 972f65a83f933b0f90cf975ef89452f4210e9b06 Mon Sep 17 00:00:00 2001 From: martinboehme Date: Wed, 20 Mar 2024 15:34:11 +0100 Subject: [PATCH 016/296] [clang][NFC] Add documentation for `CastExpr::path()`. (#85623) This didn't have any documentation, so I had to do some experimenting in godbolt when I used this in https://github.com/llvm/llvm-project/pull/84138, and my reviewer later also had some [questions](https://github.com/llvm/llvm-project/pull/84138#discussion_r1524855434) about this, so I figured it would be worth adding documentation. --- clang/include/clang/AST/Expr.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h index f9313f87be38..6e153ebe024b 100644 --- a/clang/include/clang/AST/Expr.h +++ b/clang/include/clang/AST/Expr.h @@ -3562,6 +3562,18 @@ public: path_const_iterator path_begin() const { return path_buffer(); } path_const_iterator path_end() const { return path_buffer() + path_size(); } + /// Path through the class hierarchy taken by casts between base and derived + /// classes (see implementation of `CastConsistency()` for a full list of + /// cast kinds that have a path). + /// + /// For each derived-to-base edge in the path, the path contains a + /// `CXXBaseSpecifier` for the base class of that edge; the entries are + /// ordered from derived class to base class. + /// + /// For example, given classes `Base`, `Intermediate : public Base` and + /// `Derived : public Intermediate`, the path for a cast from `Derived *` to + /// `Base *` contains two entries: One for `Intermediate`, and one for `Base`, + /// in that order. llvm::iterator_range path() { return llvm::make_range(path_begin(), path_end()); } -- GitLab From 647d75d3a883c008c19a79bce265388b3c95e742 Mon Sep 17 00:00:00 2001 From: Tina Jung Date: Wed, 20 Mar 2024 16:00:05 +0100 Subject: [PATCH 017/296] [mlir][emitc] Restrict integer and float types (#85788) Restrict which integers and floating-point types are valid in EmitC. This should cover the types which are supported in C++ and is aligned with what the emitter currently supports. The checks are implemented as functions and not fully in tablegen to allow them to be re-used by conversions to EmitC. --- mlir/include/mlir/Dialect/EmitC/IR/EmitC.h | 4 +++ mlir/include/mlir/Dialect/EmitC/IR/EmitC.td | 4 +-- .../mlir/Dialect/EmitC/IR/EmitCTypes.td | 6 ++++ mlir/lib/Dialect/EmitC/IR/EmitC.cpp | 29 +++++++++++++++++++ mlir/test/Dialect/EmitC/invalid_ops.mlir | 8 ++--- mlir/test/Dialect/EmitC/invalid_types.mlir | 16 ++++++++++ 6 files changed, 61 insertions(+), 6 deletions(-) diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.h b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.h index 1f0df3cb336b..725a1bcb4e6c 100644 --- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.h +++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.h @@ -30,6 +30,10 @@ namespace mlir { namespace emitc { void buildTerminatedBody(OpBuilder &builder, Location loc); +/// Determines whether \p type is a valid integer type in EmitC. +bool isSupportedIntegerType(mlir::Type type); +/// Determines whether \p type is a valid floating-point type in EmitC. +bool isSupportedFloatType(mlir::Type type); } // namespace emitc } // namespace mlir diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td index 78bfd561171f..d746222ff37a 100644 --- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td +++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td @@ -51,8 +51,8 @@ class EmitC_BinaryOp traits = []> : def CExpression : NativeOpTrait<"emitc::CExpression">; // Types only used in binary arithmetic operations. -def IntegerIndexOrOpaqueType : AnyTypeOf<[AnyInteger, Index, EmitC_OpaqueType]>; -def FloatIntegerIndexOrOpaqueType : AnyTypeOf<[AnyFloat, IntegerIndexOrOpaqueType]>; +def IntegerIndexOrOpaqueType : AnyTypeOf<[EmitCIntegerType, Index, EmitC_OpaqueType]>; +def FloatIntegerIndexOrOpaqueType : AnyTypeOf<[EmitCFloatType, IntegerIndexOrOpaqueType]>; def EmitC_AddOp : EmitC_BinaryOp<"add", [CExpression]> { let summary = "Addition operation"; diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitCTypes.td b/mlir/include/mlir/Dialect/EmitC/IR/EmitCTypes.td index a2ba45a1f6a1..bce5807230ce 100644 --- a/mlir/include/mlir/Dialect/EmitC/IR/EmitCTypes.td +++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitCTypes.td @@ -22,6 +22,12 @@ include "mlir/IR/BuiltinTypeInterfaces.td" // EmitC type definitions //===----------------------------------------------------------------------===// +def EmitCIntegerType : Type, + "integer type supported by EmitC">; + +def EmitCFloatType : Type, + "floating-point type supported by EmitC">; + class EmitC_Type traits = []> : TypeDef { let mnemonic = typeMnemonic; diff --git a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp index e401a83bcb42..ab5c418e844f 100644 --- a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp +++ b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp @@ -54,6 +54,35 @@ void mlir::emitc::buildTerminatedBody(OpBuilder &builder, Location loc) { builder.create(loc); } +bool mlir::emitc::isSupportedIntegerType(Type type) { + if (auto intType = llvm::dyn_cast(type)) { + switch (intType.getWidth()) { + case 1: + case 8: + case 16: + case 32: + case 64: + return true; + default: + return false; + } + } + return false; +} + +bool mlir::emitc::isSupportedFloatType(Type type) { + if (auto floatType = llvm::dyn_cast(type)) { + switch (floatType.getWidth()) { + case 32: + case 64: + return true; + default: + return false; + } + } + return false; +} + /// Check that the type of the initial value is compatible with the operations /// result type. static LogicalResult verifyInitializationAttribute(Operation *op, diff --git a/mlir/test/Dialect/EmitC/invalid_ops.mlir b/mlir/test/Dialect/EmitC/invalid_ops.mlir index 6294c853d999..22423cf61b55 100644 --- a/mlir/test/Dialect/EmitC/invalid_ops.mlir +++ b/mlir/test/Dialect/EmitC/invalid_ops.mlir @@ -170,7 +170,7 @@ func.func @add_float_pointer(%arg0: f32, %arg1: !emitc.ptr) { // ----- func.func @div_tensor(%arg0: tensor, %arg1: tensor) { - // expected-error @+1 {{'emitc.div' op operand #0 must be floating-point or integer or index or EmitC opaque type, but got 'tensor'}} + // expected-error @+1 {{'emitc.div' op operand #0 must be floating-point type supported by EmitC or integer type supported by EmitC or index or EmitC opaque type, but got 'tensor'}} %1 = "emitc.div" (%arg0, %arg1) : (tensor, tensor) -> tensor return } @@ -178,7 +178,7 @@ func.func @div_tensor(%arg0: tensor, %arg1: tensor) { // ----- func.func @mul_tensor(%arg0: tensor, %arg1: tensor) { - // expected-error @+1 {{'emitc.mul' op operand #0 must be floating-point or integer or index or EmitC opaque type, but got 'tensor'}} + // expected-error @+1 {{'emitc.mul' op operand #0 must be floating-point type supported by EmitC or integer type supported by EmitC or index or EmitC opaque type, but got 'tensor'}} %1 = "emitc.mul" (%arg0, %arg1) : (tensor, tensor) -> tensor return } @@ -186,7 +186,7 @@ func.func @mul_tensor(%arg0: tensor, %arg1: tensor) { // ----- func.func @rem_tensor(%arg0: tensor, %arg1: tensor) { - // expected-error @+1 {{'emitc.rem' op operand #0 must be integer or index or EmitC opaque type, but got 'tensor'}} + // expected-error @+1 {{'emitc.rem' op operand #0 must be integer type supported by EmitC or index or EmitC opaque type, but got 'tensor'}} %1 = "emitc.rem" (%arg0, %arg1) : (tensor, tensor) -> tensor return } @@ -194,7 +194,7 @@ func.func @rem_tensor(%arg0: tensor, %arg1: tensor) { // ----- func.func @rem_float(%arg0: f32, %arg1: f32) { - // expected-error @+1 {{'emitc.rem' op operand #0 must be integer or index or EmitC opaque type, but got 'f32'}} + // expected-error @+1 {{'emitc.rem' op operand #0 must be integer type supported by EmitC or index or EmitC opaque type, but got 'f32'}} %1 = "emitc.rem" (%arg0, %arg1) : (f32, f32) -> f32 return } diff --git a/mlir/test/Dialect/EmitC/invalid_types.mlir b/mlir/test/Dialect/EmitC/invalid_types.mlir index 079371b39b9d..f9d517bf689b 100644 --- a/mlir/test/Dialect/EmitC/invalid_types.mlir +++ b/mlir/test/Dialect/EmitC/invalid_types.mlir @@ -81,3 +81,19 @@ func.func @illegal_array_with_tensor_element_type( %arg0: !emitc.array<4xtensor<4xi32>> ) { } + +// ----- + +func.func @illegal_integer_type(%arg0: i11, %arg1: i11) -> i11 { + // expected-error @+1 {{'emitc.mul' op operand #0 must be floating-point type supported by EmitC or integer type supported by EmitC or index or EmitC opaque type, but got 'i11'}} + %mul = "emitc.mul" (%arg0, %arg1) : (i11, i11) -> i11 + return +} + +// ----- + +func.func @illegal_float_type(%arg0: f80, %arg1: f80) { + // expected-error @+1 {{'emitc.mul' op operand #0 must be floating-point type supported by EmitC or integer type supported by EmitC or index or EmitC opaque type, but got 'f80'}} + %mul = "emitc.mul" (%arg0, %arg1) : (f80, f80) -> f80 + return +} -- GitLab From a9fe23cde3ee554f4bd6118edcc2e747f3a8d8d5 Mon Sep 17 00:00:00 2001 From: chrulski-intel Date: Wed, 20 Mar 2024 08:02:43 -0700 Subject: [PATCH 018/296] [LLD] [COFF] Port -lto-sample-profile to COFF version of LLD (#85701) Following the commit of #83972 which added COFF support for SPGO, this patch ports the support of the option -lto-sample-profile that was only available in the ELF variant of LLD to the COFF variant to enable running the SPGO passes in the LTO/thinLTO pipelines. --- lld/COFF/Config.h | 3 +++ lld/COFF/Driver.cpp | 1 + lld/COFF/LTO.cpp | 1 + lld/COFF/Options.td | 1 + lld/test/COFF/Inputs/lto-sample-profile.prof | 1 + lld/test/COFF/lto-sample-profile.ll | 23 ++++++++++++++++++++ 6 files changed, 30 insertions(+) create mode 100644 lld/test/COFF/Inputs/lto-sample-profile.prof create mode 100644 lld/test/COFF/lto-sample-profile.ll diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h index 018f03b211e4..8f85929f1bea 100644 --- a/lld/COFF/Config.h +++ b/lld/COFF/Config.h @@ -263,6 +263,9 @@ struct Configuration { // Used for /lto-pgo-warn-mismatch: bool ltoPGOWarnMismatch = true; + // Used for /lto-sample-profile: + llvm::StringRef ltoSampleProfileName; + // Used for /call-graph-ordering-file: llvm::MapVector, uint64_t> diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp index 22ee2f133be9..1b075389325a 100644 --- a/lld/COFF/Driver.cpp +++ b/lld/COFF/Driver.cpp @@ -2028,6 +2028,7 @@ void LinkerDriver::linkerMain(ArrayRef argsArr) { config->ltoObjPath = args.getLastArgValue(OPT_lto_obj_path); config->ltoCSProfileGenerate = args.hasArg(OPT_lto_cs_profile_generate); config->ltoCSProfileFile = args.getLastArgValue(OPT_lto_cs_profile_file); + config->ltoSampleProfileName = args.getLastArgValue(OPT_lto_sample_profile); // Handle miscellaneous boolean flags. config->ltoPGOWarnMismatch = args.hasFlag(OPT_lto_pgo_warn_mismatch, OPT_lto_pgo_warn_mismatch_no, true); diff --git a/lld/COFF/LTO.cpp b/lld/COFF/LTO.cpp index 7df931911213..be49aa6e8bb3 100644 --- a/lld/COFF/LTO.cpp +++ b/lld/COFF/LTO.cpp @@ -86,6 +86,7 @@ lto::Config BitcodeCompiler::createConfig() { c.CSIRProfile = std::string(ctx.config.ltoCSProfileFile); c.RunCSIRInstr = ctx.config.ltoCSProfileGenerate; c.PGOWarnMismatch = ctx.config.ltoPGOWarnMismatch; + c.SampleProfile = ctx.config.ltoSampleProfileName; c.TimeTraceEnabled = ctx.config.timeTraceEnabled; c.TimeTraceGranularity = ctx.config.timeTraceGranularity; diff --git a/lld/COFF/Options.td b/lld/COFF/Options.td index 4dab4a207173..1e78a560bca8 100644 --- a/lld/COFF/Options.td +++ b/lld/COFF/Options.td @@ -79,6 +79,7 @@ def lldltocachepolicy : P<"lldltocachepolicy", "Pruning policy for the ThinLTO cache">; def lldsavetemps : F<"lldsavetemps">, HelpText<"Save intermediate LTO compilation results">; +def lto_sample_profile: P<"lto-sample-profile", "Sample profile file path">; def machine : P<"machine", "Specify target platform">; def merge : P<"merge", "Combine sections">; def mllvm : P<"mllvm", "Options to pass to LLVM">; diff --git a/lld/test/COFF/Inputs/lto-sample-profile.prof b/lld/test/COFF/Inputs/lto-sample-profile.prof new file mode 100644 index 000000000000..0ccd747bd376 --- /dev/null +++ b/lld/test/COFF/Inputs/lto-sample-profile.prof @@ -0,0 +1 @@ +f:0:0 diff --git a/lld/test/COFF/lto-sample-profile.ll b/lld/test/COFF/lto-sample-profile.ll new file mode 100644 index 000000000000..64d4c8ab6328 --- /dev/null +++ b/lld/test/COFF/lto-sample-profile.ll @@ -0,0 +1,23 @@ +; REQUIRES: x86 +; RUN: opt -module-summary %s -o %t1.o +; RUN: opt -module-summary %p/Inputs/thinlto.ll -o %t2.o + +; RUN: rm -f %t1.o.4.opt.bc +; RUN: lld-link /lto-sample-profile:%p/Inputs/lto-sample-profile.prof /lldsavetemps /entry:f /subsystem:console %t1.o %t2.o /out:%t3.exe +; RUN: opt -S %t1.o.4.opt.bc | FileCheck %s + +target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc-windows-msvc19.0.24215" + +; CHECK: ![[#]] = !{i32 1, !"ProfileSummary", ![[#]]} +declare void @g(...) + +define void @h() { + ret void +} +define void @f() { +entry: + call void (...) @g() + call void (...) @h() + ret void +} -- GitLab From caf8b1f654122342dc846ae4d9a86d5c6f93f945 Mon Sep 17 00:00:00 2001 From: Thomas Preud'homme Date: Wed, 20 Mar 2024 15:04:10 +0000 Subject: [PATCH 019/296] [MLIR] Add missing MLIRDialectUtils dep to TilingInterface (#84544) This fixes the following failure when doing a clean build (in particular no .ninja* lying around) of lib/libMLIRTilingInterface.a only: ``` In file included from mlir/include/mlir/Interfaces/TilingInterface.h:17, from mlir/lib/Interfaces/TilingInterface.cpp:13: mlir/include/mlir/Dialect/Utils/StructuredOpsUtils.h:27:10: fatal error: mlir/Dialect/Utils/DialectUtilsEnums.h.inc: No such file or directory ``` --- mlir/lib/Interfaces/CMakeLists.txt | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/mlir/lib/Interfaces/CMakeLists.txt b/mlir/lib/Interfaces/CMakeLists.txt index e7c76e70ed6b..d3b7bf65ad3e 100644 --- a/mlir/lib/Interfaces/CMakeLists.txt +++ b/mlir/lib/Interfaces/CMakeLists.txt @@ -101,7 +101,20 @@ add_mlir_library(MLIRSubsetOpInterface MLIRValueBoundsOpInterface ) -add_mlir_interface_library(TilingInterface) +add_mlir_library(MLIRTilingInterface + TilingInterface.cpp + + ADDITIONAL_HEADER_DIRS + ${MLIR_MAIN_INCLUDE_DIR}/mlir/Interfaces + + DEPENDS + MLIRTilingInterfaceIncGen + MLIRDialectUtils + + LINK_LIBS PUBLIC + MLIRIR +) + add_mlir_interface_library(VectorInterfaces) add_mlir_interface_library(ViewLikeInterface) -- GitLab From 7812fcf3d79ef7fe9ec6bcdfc8fd9143864956cb Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 20 Mar 2024 14:36:45 +0000 Subject: [PATCH 020/296] [VectorCombine] foldBitcastShuf - add support for binary shuffles (REAPPLIED) Generalise fold to "bitcast (shuf V0, V1, MaskC) --> shuf (bitcast V0), (bitcast V1), MaskC'". Reapplied with a clang codegen test fix. Further prep work for #67803 --- clang/test/CodeGen/X86/avx-shuffle-builtins.c | 9 +++-- .../Transforms/Vectorize/VectorCombine.cpp | 21 ++++++----- .../Transforms/PhaseOrdering/X86/pr67803.ll | 35 +++++++++++++++++-- 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/clang/test/CodeGen/X86/avx-shuffle-builtins.c b/clang/test/CodeGen/X86/avx-shuffle-builtins.c index 9109247e534f..82be43bc0504 100644 --- a/clang/test/CodeGen/X86/avx-shuffle-builtins.c +++ b/clang/test/CodeGen/X86/avx-shuffle-builtins.c @@ -60,7 +60,8 @@ __m256 test_mm256_permute2f128_ps(__m256 a, __m256 b) { __m256i test_mm256_permute2f128_si256(__m256i a, __m256i b) { // CHECK-LABEL: test_mm256_permute2f128_si256 - // CHECK: shufflevector{{.*}} <8 x i32> + // X64: shufflevector{{.*}} + // X86: shufflevector{{.*}} return _mm256_permute2f128_si256(a, b, 0x20); } @@ -104,7 +105,8 @@ __m256d test_mm256_insertf128_pd_0(__m256d a, __m128d b) { __m256i test_mm256_insertf128_si256_0(__m256i a, __m128i b) { // CHECK-LABEL: test_mm256_insertf128_si256_0 - // CHECK: shufflevector{{.*}} + // X64: shufflevector{{.*}} + // X86: shufflevector{{.*}} return _mm256_insertf128_si256(a, b, 0); } @@ -122,7 +124,8 @@ __m256d test_mm256_insertf128_pd_1(__m256d a, __m128d b) { __m256i test_mm256_insertf128_si256_1(__m256i a, __m128i b) { // CHECK-LABEL: test_mm256_insertf128_si256_1 - // CHECK: shufflevector{{.*}} + // X64: shufflevector{{.*}} + // X86: shufflevector{{.*}} return _mm256_insertf128_si256(a, b, 1); } diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp index 0b16a8b76769..23494314f132 100644 --- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp +++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp @@ -684,10 +684,10 @@ bool VectorCombine::foldInsExtFNeg(Instruction &I) { /// destination type followed by shuffle. This can enable further transforms by /// moving bitcasts or shuffles together. bool VectorCombine::foldBitcastShuffle(Instruction &I) { - Value *V0; + Value *V0, *V1; ArrayRef Mask; if (!match(&I, m_BitCast(m_OneUse( - m_Shuffle(m_Value(V0), m_Undef(), m_Mask(Mask)))))) + m_Shuffle(m_Value(V0), m_Value(V1), m_Mask(Mask)))))) return false; // 1) Do not fold bitcast shuffle for scalable type. First, shuffle cost for @@ -728,17 +728,21 @@ bool VectorCombine::foldBitcastShuffle(Instruction &I) { FixedVectorType::get(DestTy->getScalarType(), NumSrcElts); auto *OldShuffleTy = FixedVectorType::get(SrcTy->getScalarType(), Mask.size()); + bool IsUnary = isa(V1); + unsigned NumOps = IsUnary ? 1 : 2; // The new shuffle must not cost more than the old shuffle. TargetTransformInfo::TargetCostKind CK = TargetTransformInfo::TCK_RecipThroughput; TargetTransformInfo::ShuffleKind SK = - TargetTransformInfo::SK_PermuteSingleSrc; + IsUnary ? TargetTransformInfo::SK_PermuteSingleSrc + : TargetTransformInfo::SK_PermuteTwoSrc; InstructionCost DestCost = TTI.getShuffleCost(SK, NewShuffleTy, NewMask, CK) + - TTI.getCastInstrCost(Instruction::BitCast, NewShuffleTy, SrcTy, - TargetTransformInfo::CastContextHint::None, CK); + (NumOps * TTI.getCastInstrCost(Instruction::BitCast, NewShuffleTy, SrcTy, + TargetTransformInfo::CastContextHint::None, + CK)); InstructionCost SrcCost = TTI.getShuffleCost(SK, SrcTy, Mask, CK) + TTI.getCastInstrCost(Instruction::BitCast, DestTy, OldShuffleTy, @@ -746,10 +750,11 @@ bool VectorCombine::foldBitcastShuffle(Instruction &I) { if (DestCost > SrcCost || !DestCost.isValid()) return false; - // bitcast (shuf V0, MaskC) --> shuf (bitcast V0), MaskC' + // bitcast (shuf V0, V1, MaskC) --> shuf (bitcast V0), (bitcast V1), MaskC' ++NumShufOfBitcast; - Value *CastV = Builder.CreateBitCast(V0, NewShuffleTy); - Value *Shuf = Builder.CreateShuffleVector(CastV, NewMask); + Value *CastV0 = Builder.CreateBitCast(V0, NewShuffleTy); + Value *CastV1 = Builder.CreateBitCast(V1, NewShuffleTy); + Value *Shuf = Builder.CreateShuffleVector(CastV0, CastV1, NewMask); replaceValue(I, *Shuf); return true; } diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll index 211c90b5604e..e61b254b7a5f 100644 --- a/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll +++ b/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s -; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s -; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s +; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK +; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK +; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX512 define <4 x i64> @PR67803(<4 x i64> %x, <4 x i64> %y, <4 x i64> %a, <4 x i64> %b) { ; CHECK-LABEL: @PR67803( @@ -35,6 +35,35 @@ define <4 x i64> @PR67803(<4 x i64> %x, <4 x i64> %y, <4 x i64> %a, <4 x i64> %b ; CHECK-NEXT: [[SHUFFLE_I23:%.*]] = shufflevector <2 x i64> [[TMP12]], <2 x i64> [[TMP20]], <4 x i32> ; CHECK-NEXT: ret <4 x i64> [[SHUFFLE_I23]] ; +; AVX512-LABEL: @PR67803( +; AVX512-NEXT: entry: +; AVX512-NEXT: [[TMP0:%.*]] = bitcast <4 x i64> [[X:%.*]] to <8 x i32> +; AVX512-NEXT: [[TMP1:%.*]] = bitcast <4 x i64> [[Y:%.*]] to <8 x i32> +; AVX512-NEXT: [[TMP2:%.*]] = icmp sgt <8 x i32> [[TMP0]], [[TMP1]] +; AVX512-NEXT: [[CMP_I21:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> +; AVX512-NEXT: [[SEXT_I22:%.*]] = sext <4 x i1> [[CMP_I21]] to <4 x i32> +; AVX512-NEXT: [[CMP_I:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> +; AVX512-NEXT: [[SEXT_I:%.*]] = sext <4 x i1> [[CMP_I]] to <4 x i32> +; AVX512-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[SEXT_I22]], <4 x i32> [[SEXT_I]], <8 x i32> +; AVX512-NEXT: [[TMP4:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8> +; AVX512-NEXT: [[TMP5:%.*]] = shufflevector <32 x i8> [[TMP4]], <32 x i8> poison, <16 x i32> +; AVX512-NEXT: [[TMP6:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8> +; AVX512-NEXT: [[TMP7:%.*]] = shufflevector <32 x i8> [[TMP6]], <32 x i8> poison, <16 x i32> +; AVX512-NEXT: [[TMP8:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8> +; AVX512-NEXT: [[TMP9:%.*]] = shufflevector <32 x i8> [[TMP8]], <32 x i8> poison, <16 x i32> +; AVX512-NEXT: [[TMP10:%.*]] = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[TMP5]], <16 x i8> [[TMP7]], <16 x i8> [[TMP9]]) +; AVX512-NEXT: [[TMP11:%.*]] = bitcast <16 x i8> [[TMP10]] to <2 x i64> +; AVX512-NEXT: [[TMP12:%.*]] = bitcast <4 x i64> [[A]] to <32 x i8> +; AVX512-NEXT: [[TMP13:%.*]] = shufflevector <32 x i8> [[TMP12]], <32 x i8> poison, <16 x i32> +; AVX512-NEXT: [[TMP14:%.*]] = bitcast <4 x i64> [[B]] to <32 x i8> +; AVX512-NEXT: [[TMP15:%.*]] = shufflevector <32 x i8> [[TMP14]], <32 x i8> poison, <16 x i32> +; AVX512-NEXT: [[TMP16:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8> +; AVX512-NEXT: [[TMP17:%.*]] = shufflevector <32 x i8> [[TMP16]], <32 x i8> poison, <16 x i32> +; AVX512-NEXT: [[TMP18:%.*]] = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[TMP13]], <16 x i8> [[TMP15]], <16 x i8> [[TMP17]]) +; AVX512-NEXT: [[TMP19:%.*]] = bitcast <16 x i8> [[TMP18]] to <2 x i64> +; AVX512-NEXT: [[SHUFFLE_I23:%.*]] = shufflevector <2 x i64> [[TMP11]], <2 x i64> [[TMP19]], <4 x i32> +; AVX512-NEXT: ret <4 x i64> [[SHUFFLE_I23]] +; entry: %0 = bitcast <4 x i64> %x to <8 x i32> %extract = shufflevector <8 x i32> %0, <8 x i32> poison, <4 x i32> -- GitLab From 55c82f149c5065947f3de3fcde137e8172eee223 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Wed, 20 Mar 2024 08:07:28 -0700 Subject: [PATCH 021/296] [SLP][NFC]Add a test with arguments of functions, reduced by minbitwidth analysis. --- .../cmp-after-intrinsic-call-minbitwidth.ll | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/cmp-after-intrinsic-call-minbitwidth.ll diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cmp-after-intrinsic-call-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/cmp-after-intrinsic-call-minbitwidth.ll new file mode 100644 index 000000000000..a05d4fdd6315 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/X86/cmp-after-intrinsic-call-minbitwidth.ll @@ -0,0 +1,40 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt --passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=cascadelake < %s | FileCheck %s + +define void @test() { +; CHECK-LABEL: define void @test( +; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = call <2 x i32> @llvm.smin.v2i32(<2 x i32> zeroinitializer, <2 x i32> zeroinitializer) +; CHECK-NEXT: [[TMP1:%.*]] = select <2 x i1> zeroinitializer, <2 x i32> zeroinitializer, <2 x i32> [[TMP0]] +; CHECK-NEXT: [[TMP2:%.*]] = or <2 x i32> [[TMP1]], zeroinitializer +; CHECK-NEXT: [[ADD:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1 +; CHECK-NEXT: [[SHR:%.*]] = ashr i32 [[ADD]], 0 +; CHECK-NEXT: [[ADD45:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0 +; CHECK-NEXT: [[ADD152:%.*]] = or i32 [[ADD45]], [[ADD]] +; CHECK-NEXT: [[IDXPROM153:%.*]] = sext i32 [[ADD152]] to i64 +; CHECK-NEXT: [[ARRAYIDX154:%.*]] = getelementptr i8, ptr null, i64 [[IDXPROM153]] +; CHECK-NEXT: [[CALL155:%.*]] = tail call i32 null(ptr null, i32 0, ptr [[ARRAYIDX154]], i32 0) +; CHECK-NEXT: ret void +; +entry: + %conv = sext i16 0 to i32 + %cmp.i = icmp sgt i32 0, %conv + %cond.i = tail call i32 @llvm.smin.i32(i32 %conv, i32 0) + %cond5.i = select i1 %cmp.i, i32 0, i32 %cond.i + %conv43 = sext i16 0 to i32 + %cmp.i6193 = icmp sgt i32 0, %conv43 + %cond.i6194 = tail call i32 @llvm.smin.i32(i32 %conv43, i32 0) + %cond5.i6195 = select i1 %cmp.i6193, i32 0, i32 %cond.i6194 + %add = or i32 %cond5.i, 0 + %shr = ashr i32 %add, 0 + %add45 = or i32 %cond5.i6195, 0 + %add152 = or i32 %add45, %add + %idxprom153 = sext i32 %add152 to i64 + %arrayidx154 = getelementptr i8, ptr null, i64 %idxprom153 + %call155 = tail call i32 null(ptr null, i32 0, ptr %arrayidx154, i32 0) + ret void +} + +declare i32 @llvm.smin.i32(i32, i32) + -- GitLab From 4095a326c026a2b6dcb839a8c1c75906c6ba5b0a Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Wed, 20 Mar 2024 16:08:38 +0100 Subject: [PATCH 022/296] [MLIR][LLVM] Add extraData field to the DIDerivedType attribute (#85935) This commit extends the DIDerivedTypeAttr with the `extraData` field. For now, the type of it is limited to be a `DINodeAttr`, as extending the debug metadata handling to support arbitrary metadata nodes does not seem to be necessary so far. --- mlir/include/mlir-c/Dialect/LLVM.h | 2 +- .../mlir/Dialect/LLVMIR/LLVMAttrDefs.td | 3 ++- mlir/lib/CAPI/Dialect/LLVM.cpp | 19 +++++++++---------- mlir/lib/Target/LLVMIR/DebugImporter.cpp | 4 +++- mlir/lib/Target/LLVMIR/DebugTranslation.cpp | 2 +- mlir/test/CAPI/llvm.c | 2 +- mlir/test/Dialect/LLVMIR/debuginfo.mlir | 5 +++-- mlir/test/Target/LLVMIR/Import/debug-info.ll | 4 ++-- mlir/test/Target/LLVMIR/llvmir-debug.mlir | 5 +++-- 9 files changed, 25 insertions(+), 21 deletions(-) diff --git a/mlir/include/mlir-c/Dialect/LLVM.h b/mlir/include/mlir-c/Dialect/LLVM.h index b3d7a788ccbb..4f1d646f5bc8 100644 --- a/mlir/include/mlir-c/Dialect/LLVM.h +++ b/mlir/include/mlir-c/Dialect/LLVM.h @@ -238,7 +238,7 @@ MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDICompositeTypeAttrGet( MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMDIDerivedTypeAttrGet( MlirContext ctx, unsigned int tag, MlirAttribute name, MlirAttribute baseType, uint64_t sizeInBits, uint32_t alignInBits, - uint64_t offsetInBits); + uint64_t offsetInBits, MlirAttribute extraData); /// Gets the base type from a LLVM DIDerivedType attribute. MLIR_CAPI_EXPORTED MlirAttribute diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td index d4d56ae0f762..1b1824a28e99 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td @@ -405,7 +405,8 @@ def LLVM_DIDerivedTypeAttr : LLVM_Attr<"DIDerivedType", "di_derived_type", OptionalParameter<"DITypeAttr">:$baseType, OptionalParameter<"uint64_t">:$sizeInBits, OptionalParameter<"uint32_t">:$alignInBits, - OptionalParameter<"uint64_t">:$offsetInBits + OptionalParameter<"uint64_t">:$offsetInBits, + OptionalParameter<"DINodeAttr">:$extraData ); let assemblyFormat = "`<` struct(params) `>`"; } diff --git a/mlir/lib/CAPI/Dialect/LLVM.cpp b/mlir/lib/CAPI/Dialect/LLVM.cpp index d0fd5ceecfff..71f2b73dd73b 100644 --- a/mlir/lib/CAPI/Dialect/LLVM.cpp +++ b/mlir/lib/CAPI/Dialect/LLVM.cpp @@ -168,16 +168,15 @@ MlirAttribute mlirLLVMDICompositeTypeAttrGet( [](Attribute a) { return a.cast(); }))); } -MlirAttribute mlirLLVMDIDerivedTypeAttrGet(MlirContext ctx, unsigned int tag, - MlirAttribute name, - MlirAttribute baseType, - uint64_t sizeInBits, - uint32_t alignInBits, - uint64_t offsetInBits) { - return wrap(DIDerivedTypeAttr::get(unwrap(ctx), tag, - cast(unwrap(name)), - cast(unwrap(baseType)), - sizeInBits, alignInBits, offsetInBits)); +MlirAttribute +mlirLLVMDIDerivedTypeAttrGet(MlirContext ctx, unsigned int tag, + MlirAttribute name, MlirAttribute baseType, + uint64_t sizeInBits, uint32_t alignInBits, + uint64_t offsetInBits, MlirAttribute extraData) { + return wrap(DIDerivedTypeAttr::get( + unwrap(ctx), tag, cast(unwrap(name)), + cast(unwrap(baseType)), sizeInBits, alignInBits, offsetInBits, + cast(unwrap(extraData)))); } MlirAttribute diff --git a/mlir/lib/Target/LLVMIR/DebugImporter.cpp b/mlir/lib/Target/LLVMIR/DebugImporter.cpp index 5ba90bba18b1..4bdc03a3e282 100644 --- a/mlir/lib/Target/LLVMIR/DebugImporter.cpp +++ b/mlir/lib/Target/LLVMIR/DebugImporter.cpp @@ -85,10 +85,12 @@ DIDerivedTypeAttr DebugImporter::translateImpl(llvm::DIDerivedType *node) { DITypeAttr baseType = translate(node->getBaseType()); if (node->getBaseType() && !baseType) return nullptr; + DINodeAttr extraData = + translate(dyn_cast_or_null(node->getExtraData())); return DIDerivedTypeAttr::get( context, node->getTag(), getStringAttrOrNull(node->getRawName()), baseType, node->getSizeInBits(), node->getAlignInBits(), - node->getOffsetInBits()); + node->getOffsetInBits(), extraData); } DIFileAttr DebugImporter::translateImpl(llvm::DIFile *node) { diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp index 126a671a58e8..642359a23756 100644 --- a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp @@ -163,7 +163,7 @@ llvm::DIDerivedType *DebugTranslation::translateImpl(DIDerivedTypeAttr attr) { /*Scope=*/nullptr, translate(attr.getBaseType()), attr.getSizeInBits(), attr.getAlignInBits(), attr.getOffsetInBits(), /*DWARFAddressSpace=*/std::nullopt, /*PtrAuthData=*/std::nullopt, - /*Flags=*/llvm::DINode::FlagZero); + /*Flags=*/llvm::DINode::FlagZero, translate(attr.getExtraData())); } llvm::DIFile *DebugTranslation::translateImpl(DIFileAttr attr) { diff --git a/mlir/test/CAPI/llvm.c b/mlir/test/CAPI/llvm.c index 48b887ee4a95..9c3c7da46c4c 100644 --- a/mlir/test/CAPI/llvm.c +++ b/mlir/test/CAPI/llvm.c @@ -297,7 +297,7 @@ static void testDebugInfoAttributes(MlirContext ctx) { 1, 0, 8, di_type)); // CHECK: #llvm.di_derived_type<{{.*}}> mlirAttributeDump( - mlirLLVMDIDerivedTypeAttrGet(ctx, 0, bar, di_type, 64, 8, 0)); + mlirLLVMDIDerivedTypeAttrGet(ctx, 0, bar, di_type, 64, 8, 0, di_type)); // CHECK: #llvm.di_composite_type<{{.*}}> mlirAttributeDump(mlirLLVMDICompositeTypeAttrGet( diff --git a/mlir/test/Dialect/LLVMIR/debuginfo.mlir b/mlir/test/Dialect/LLVMIR/debuginfo.mlir index cef2ced391d6..4c2de0aa4c22 100644 --- a/mlir/test/Dialect/LLVMIR/debuginfo.mlir +++ b/mlir/test/Dialect/LLVMIR/debuginfo.mlir @@ -24,10 +24,11 @@ sizeInBits = 32, encoding = DW_ATE_signed > -// CHECK-DAG: #[[PTR0:.*]] = #llvm.di_derived_type +// CHECK-DAG: #[[PTR0:.*]] = #llvm.di_derived_type #ptr0 = #llvm.di_derived_type< tag = DW_TAG_pointer_type, baseType = #int0, - sizeInBits = 64, alignInBits = 32, offsetInBits = 4 + sizeInBits = 64, alignInBits = 32, offsetInBits = 4, + extraData = #int1 > // CHECK-DAG: #[[PTR1:.*]] = #llvm.di_derived_type diff --git a/mlir/test/Target/LLVMIR/Import/debug-info.ll b/mlir/test/Target/LLVMIR/Import/debug-info.ll index 9af40d8c8d3e..a7947eb0d444 100644 --- a/mlir/test/Target/LLVMIR/Import/debug-info.ll +++ b/mlir/test/Target/LLVMIR/Import/debug-info.ll @@ -136,7 +136,7 @@ define void @basic_type() !dbg !3 { ; CHECK: #[[INT:.+]] = #llvm.di_basic_type ; CHECK: #[[PTR1:.+]] = #llvm.di_derived_type -; CHECK: #[[PTR2:.+]] = #llvm.di_derived_type +; CHECK: #[[PTR2:.+]] = #llvm.di_derived_type ; CHECK: #llvm.di_subroutine_type define void @derived_type() !dbg !3 { @@ -153,7 +153,7 @@ define void @derived_type() !dbg !3 { !5 = !{!7, !8} !6 = !DIBasicType(name: "int") !7 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !6) -!8 = !DIDerivedType(name: "mypointer", tag: DW_TAG_pointer_type, baseType: !6, size: 64, align: 32, offset: 4) +!8 = !DIDerivedType(name: "mypointer", tag: DW_TAG_pointer_type, baseType: !6, size: 64, align: 32, offset: 4, extraData: !6) ; // ----- diff --git a/mlir/test/Target/LLVMIR/llvmir-debug.mlir b/mlir/test/Target/LLVMIR/llvmir-debug.mlir index c042628334d4..c34f9187d4df 100644 --- a/mlir/test/Target/LLVMIR/llvmir-debug.mlir +++ b/mlir/test/Target/LLVMIR/llvmir-debug.mlir @@ -28,7 +28,8 @@ llvm.func @func_no_debug() { > #ptr = #llvm.di_derived_type< tag = DW_TAG_pointer_type, baseType = #si32, - sizeInBits = 64, alignInBits = 32, offsetInBits = 8 + sizeInBits = 64, alignInBits = 32, offsetInBits = 8, + extraData = #si32 > #named = #llvm.di_derived_type< // Specify the name parameter. @@ -135,7 +136,7 @@ llvm.func @empty_types() { // CHECK: ![[FUNC_TYPE]] = !DISubroutineType(cc: DW_CC_normal, types: ![[FUNC_ARGS:.*]]) // CHECK: ![[FUNC_ARGS]] = !{null, ![[ARG_TYPE:.*]], ![[PTR_TYPE:.*]], ![[NAMED_TYPE:.*]], ![[COMPOSITE_TYPE:.*]], ![[VECTOR_TYPE:.*]]} // CHECK: ![[ARG_TYPE]] = !DIBasicType(name: "si64") -// CHECK: ![[PTR_TYPE]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[BASE_TYPE:.*]], size: 64, align: 32, offset: 8) +// CHECK: ![[PTR_TYPE]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[BASE_TYPE:.*]], size: 64, align: 32, offset: 8, extraData: ![[BASE_TYPE]]) // CHECK: ![[BASE_TYPE]] = !DIBasicType(name: "si32", size: 32, encoding: DW_ATE_signed) // CHECK: ![[NAMED_TYPE]] = !DIDerivedType(tag: DW_TAG_pointer_type, name: "named", baseType: ![[BASE_TYPE:.*]]) // CHECK: ![[COMPOSITE_TYPE]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "composite", file: ![[CU_FILE_LOC]], line: 42, size: 64, align: 32, elements: ![[COMPOSITE_ELEMENTS:.*]]) -- GitLab From aa8cffb9583afb583b7a329a56cbd1c7b743f6a3 Mon Sep 17 00:00:00 2001 From: ChiaHungDuan Date: Wed, 20 Mar 2024 08:15:47 -0700 Subject: [PATCH 023/296] [scudo] Fix type mismatch on DefaultMaxEntrySize (#85897) --- .../lib/scudo/standalone/allocator_config.def | 2 +- .../scudo/standalone/allocator_config_wrapper.h | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/compiler-rt/lib/scudo/standalone/allocator_config.def b/compiler-rt/lib/scudo/standalone/allocator_config.def index 92f4e39872d4..c50aadad2d63 100644 --- a/compiler-rt/lib/scudo/standalone/allocator_config.def +++ b/compiler-rt/lib/scudo/standalone/allocator_config.def @@ -110,7 +110,7 @@ SECONDARY_REQUIRED_TEMPLATE_TYPE(CacheT) SECONDARY_CACHE_OPTIONAL(const u32, EntriesArraySize, 0) SECONDARY_CACHE_OPTIONAL(const u32, QuarantineSize, 0) SECONDARY_CACHE_OPTIONAL(const u32, DefaultMaxEntriesCount, 0) -SECONDARY_CACHE_OPTIONAL(const u32, DefaultMaxEntrySize, 0) +SECONDARY_CACHE_OPTIONAL(const uptr, DefaultMaxEntrySize, 0) SECONDARY_CACHE_OPTIONAL(const s32, MinReleaseToOsIntervalMs, INT32_MIN) SECONDARY_CACHE_OPTIONAL(const s32, MaxReleaseToOsIntervalMs, INT32_MAX) diff --git a/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h b/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h index a51d770b4664..5477236ac1f3 100644 --- a/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h +++ b/compiler-rt/lib/scudo/standalone/allocator_config_wrapper.h @@ -27,6 +27,19 @@ template struct voidAdaptor { using type = void; }; +// This is used for detecting the case that defines the flag with wrong type and +// it'll be viewed as undefined optional flag. +template struct assertSameType { + template struct isSame { + static constexpr bool value = false; + }; + template struct isSame { + static constexpr bool value = true; + }; + static_assert(isSame::value, "Flag type mismatches"); + using type = R; +}; + } // namespace namespace scudo { @@ -36,7 +49,8 @@ namespace scudo { static constexpr removeConst::type getValue() { return DEFAULT; } \ }; \ template \ - struct NAME##State { \ + struct NAME##State< \ + Config, typename assertSameType::type> { \ static constexpr removeConst::type getValue() { \ return Config::MEMBER; \ } \ -- GitLab From 48a1a9b26048d2a88eaf5b09de0525ab557615de Mon Sep 17 00:00:00 2001 From: Chris B Date: Wed, 20 Mar 2024 10:16:17 -0500 Subject: [PATCH 024/296] [HLSL][docs] Document hlsl.h in the HLSL docs (#84081) This adds a brief blurb about hlsl.h in the HLSLSupport documentation where a high level view of the architecture is explained. --- clang/docs/HLSL/HLSLSupport.rst | 50 ++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/clang/docs/HLSL/HLSLSupport.rst b/clang/docs/HLSL/HLSLSupport.rst index b091c17fd2a7..2d309ddff2f3 100644 --- a/clang/docs/HLSL/HLSLSupport.rst +++ b/clang/docs/HLSL/HLSLSupport.rst @@ -91,7 +91,7 @@ performance win for HLSL. If precompiled headers are used when compiling HLSL, the ``ExternalSemaSource`` will be a ``MultiplexExternalSemaSource`` which includes both the ``ASTReader`` -and ``HLSLExternalSemaSource``. For Built-in declarations that are already +and -. For Built-in declarations that are already completed in the serialized AST, the ``HLSLExternalSemaSource`` will reuse the existing declarations and not introduce new declarations. If the built-in types are not completed in the serialized AST, the ``HLSLExternalSemaSource`` will @@ -114,6 +114,54 @@ not re-targetable, we want to share the Clang CodeGen implementation for HLSL with other GPU graphics targets like SPIR-V and possibly other GPU and even CPU targets. +hlsl.h +------ + +HLSL has a library of standalone functions. This is similar to OpenCL and CUDA, +and is analogous to C's standard library. The implementation approach for the +HLSL library functionality draws from patterns in use by OpenCL and other Clang +resource headers. All of the clang resource headers are part of the +``ClangHeaders`` component found in the source tree under +`clang/lib/Headers `_. + +.. note:: + + HLSL's complex data types are not defined in HLSL's header because many of + the semantics of those data types cannot be expressed in HLSL due to missing + language features. Data types that can't be expressed in HLSL are defined in + code in the ``HLSLExternalSemaSource``. + +Similar to OpenCL, the HLSL library functionality is implicitly declared in +translation units without needing to include a header to provide declarations. +In Clang this is handled by making ``hlsl.h`` an implicitly included header +distributed as part of the Clang resource directory. + +Similar to OpenCL, HLSL's implicit header will explicitly declare all overloads, +and each overload will map to a corresponding ``__builtin*`` compiler intrinsic +that is handled in ClangCodeGen. CUDA uses a similar pattern although many CUDA +functions have full definitions in the included headers which in turn call +corresponding ``__builtin*`` compiler intrinsics. By not having bodies HLSL +avoids the need for the inliner to clean up and inline large numbers of small +library functions. + +HLSL's implicit headers also define some of HLSL's typedefs. This is consistent +with how the AVX vector header is implemented. + +Concerns have been expressed that this approach may result in slower compile +times than the approach DXC uses where library functions are treated more like +Clang ``__builtin*`` intrinsics. No real world use cases have been identified +where parsing is a significant compile-time overhead, but the HLSL implicit +headers can be compiled into a module for performance if needed. + +Further, by treating these as functions rather than ``__builtin*`` compiler +intrinsics, the language behaviors are more consistent and aligned with user +expectation because normal overload resolution rules and implicit conversions +apply as expected. + +It is a feature of this design that clangd-powered "go to declaration" for +library functions will jump to a valid header declaration and all overloads will +be user readable. + HLSL Language ============= -- GitLab From 927308a52bc51ae786db1bd645ad5ef5889fdb2a Mon Sep 17 00:00:00 2001 From: Gheorghe-Teodor Bercea Date: Wed, 20 Mar 2024 11:22:01 -0400 Subject: [PATCH 025/296] [libomptarget][nextgen-plugin] Use SCRELEASE/SCACQUIRE in packet headers (#85678) This patch updates the construction of packet headers to replace the usage of ACQUIRE/RELEASE with SCACQUIRE/SCRELEASE which is now recommended. The patch also ensures consistency across kernel dispatches. --- libc/utils/gpu/loader/amdgpu/Loader.cpp | 3 ++- .../plugins-nextgen/amdgpu/src/rtl.cpp | 23 ++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/libc/utils/gpu/loader/amdgpu/Loader.cpp b/libc/utils/gpu/loader/amdgpu/Loader.cpp index e3911eda2bd8..7fd45acfd47e 100644 --- a/libc/utils/gpu/loader/amdgpu/Loader.cpp +++ b/libc/utils/gpu/loader/amdgpu/Loader.cpp @@ -276,7 +276,8 @@ hsa_status_t launch_kernel(hsa_agent_t dev_agent, hsa_executable_t executable, (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_SCACQUIRE_FENCE_SCOPE) | (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_SCRELEASE_FENCE_SCOPE); uint32_t header_word = header | (setup << 16u); - __atomic_store_n((uint32_t *)&packet->header, header_word, __ATOMIC_RELEASE); + __atomic_store_n(reinterpret_cast(packet), header_word, + __ATOMIC_RELEASE); hsa_signal_store_relaxed(queue->doorbell_signal, packet_id); // Wait until the kernel has completed execution on the device. Periodically diff --git a/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp index fce7454bf280..c147cefe58e9 100644 --- a/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp +++ b/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp @@ -826,15 +826,15 @@ private: /// Assumes the queue lock is acquired. void publishKernelPacket(uint64_t PacketId, uint16_t Setup, hsa_kernel_dispatch_packet_t *Packet) { - uint32_t *PacketPtr = reinterpret_cast(Packet); - - uint16_t Header = HSA_PACKET_TYPE_KERNEL_DISPATCH << HSA_PACKET_HEADER_TYPE; - Header |= HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE; - Header |= HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE; + uint16_t Header = + (HSA_PACKET_TYPE_KERNEL_DISPATCH << HSA_PACKET_HEADER_TYPE) | + (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_SCACQUIRE_FENCE_SCOPE) | + (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_SCRELEASE_FENCE_SCOPE); // Publish the packet. Do not modify the package after this point. uint32_t HeaderWord = Header | (Setup << 16u); - __atomic_store_n(PacketPtr, HeaderWord, __ATOMIC_RELEASE); + __atomic_store_n(reinterpret_cast(Packet), HeaderWord, + __ATOMIC_RELEASE); // Signal the doorbell about the published packet. hsa_signal_store_relaxed(Queue->doorbell_signal, PacketId); @@ -845,15 +845,16 @@ private: /// barrier dependencies (signals) are satisfied. Assumes the queue is locked void publishBarrierPacket(uint64_t PacketId, hsa_barrier_and_packet_t *Packet) { - uint32_t *PacketPtr = reinterpret_cast(Packet); uint16_t Setup = 0; - uint16_t Header = HSA_PACKET_TYPE_BARRIER_AND << HSA_PACKET_HEADER_TYPE; - Header |= HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE; - Header |= HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE; + uint16_t Header = + (HSA_PACKET_TYPE_BARRIER_AND << HSA_PACKET_HEADER_TYPE) | + (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_SCACQUIRE_FENCE_SCOPE) | + (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_SCRELEASE_FENCE_SCOPE); // Publish the packet. Do not modify the package after this point. uint32_t HeaderWord = Header | (Setup << 16u); - __atomic_store_n(PacketPtr, HeaderWord, __ATOMIC_RELEASE); + __atomic_store_n(reinterpret_cast(Packet), HeaderWord, + __ATOMIC_RELEASE); // Signal the doorbell about the published packet. hsa_signal_store_relaxed(Queue->doorbell_signal, PacketId); -- GitLab From 04f7cd7f4545f3368ccda625cadbe3265c3566c9 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Wed, 20 Mar 2024 08:28:40 -0700 Subject: [PATCH 026/296] [SLP][NFC]Make findBestRootPair() member function constant. --- llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 5d59f35f3081..a52064e5417b 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -2239,7 +2239,7 @@ public: /// of the cost, considered to be good enough score. std::optional findBestRootPair(ArrayRef> Candidates, - int Limit = LookAheadHeuristics::ScoreFail) { + int Limit = LookAheadHeuristics::ScoreFail) const { LookAheadHeuristics LookAhead(*TLI, *DL, *SE, *this, /*NumLanes=*/2, RootLookAheadMaxDepth); int BestScore = Limit; -- GitLab From c25e77436ea44b4c980f4974dee8984298d13a08 Mon Sep 17 00:00:00 2001 From: Gheorghe-Teodor Bercea Date: Wed, 20 Mar 2024 11:40:12 -0400 Subject: [PATCH 027/296] Revert "[libomptarget][nextgen-plugin] Use SCRELEASE/SCACQUIRE in packet headers" (#85950) Reverts llvm/llvm-project#85678 --- libc/utils/gpu/loader/amdgpu/Loader.cpp | 3 +-- .../plugins-nextgen/amdgpu/src/rtl.cpp | 23 +++++++++---------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/libc/utils/gpu/loader/amdgpu/Loader.cpp b/libc/utils/gpu/loader/amdgpu/Loader.cpp index 7fd45acfd47e..e3911eda2bd8 100644 --- a/libc/utils/gpu/loader/amdgpu/Loader.cpp +++ b/libc/utils/gpu/loader/amdgpu/Loader.cpp @@ -276,8 +276,7 @@ hsa_status_t launch_kernel(hsa_agent_t dev_agent, hsa_executable_t executable, (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_SCACQUIRE_FENCE_SCOPE) | (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_SCRELEASE_FENCE_SCOPE); uint32_t header_word = header | (setup << 16u); - __atomic_store_n(reinterpret_cast(packet), header_word, - __ATOMIC_RELEASE); + __atomic_store_n((uint32_t *)&packet->header, header_word, __ATOMIC_RELEASE); hsa_signal_store_relaxed(queue->doorbell_signal, packet_id); // Wait until the kernel has completed execution on the device. Periodically diff --git a/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp index c147cefe58e9..fce7454bf280 100644 --- a/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp +++ b/openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp @@ -826,15 +826,15 @@ private: /// Assumes the queue lock is acquired. void publishKernelPacket(uint64_t PacketId, uint16_t Setup, hsa_kernel_dispatch_packet_t *Packet) { - uint16_t Header = - (HSA_PACKET_TYPE_KERNEL_DISPATCH << HSA_PACKET_HEADER_TYPE) | - (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_SCACQUIRE_FENCE_SCOPE) | - (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_SCRELEASE_FENCE_SCOPE); + uint32_t *PacketPtr = reinterpret_cast(Packet); + + uint16_t Header = HSA_PACKET_TYPE_KERNEL_DISPATCH << HSA_PACKET_HEADER_TYPE; + Header |= HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE; + Header |= HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE; // Publish the packet. Do not modify the package after this point. uint32_t HeaderWord = Header | (Setup << 16u); - __atomic_store_n(reinterpret_cast(Packet), HeaderWord, - __ATOMIC_RELEASE); + __atomic_store_n(PacketPtr, HeaderWord, __ATOMIC_RELEASE); // Signal the doorbell about the published packet. hsa_signal_store_relaxed(Queue->doorbell_signal, PacketId); @@ -845,16 +845,15 @@ private: /// barrier dependencies (signals) are satisfied. Assumes the queue is locked void publishBarrierPacket(uint64_t PacketId, hsa_barrier_and_packet_t *Packet) { + uint32_t *PacketPtr = reinterpret_cast(Packet); uint16_t Setup = 0; - uint16_t Header = - (HSA_PACKET_TYPE_BARRIER_AND << HSA_PACKET_HEADER_TYPE) | - (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_SCACQUIRE_FENCE_SCOPE) | - (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_SCRELEASE_FENCE_SCOPE); + uint16_t Header = HSA_PACKET_TYPE_BARRIER_AND << HSA_PACKET_HEADER_TYPE; + Header |= HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE; + Header |= HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE; // Publish the packet. Do not modify the package after this point. uint32_t HeaderWord = Header | (Setup << 16u); - __atomic_store_n(reinterpret_cast(Packet), HeaderWord, - __ATOMIC_RELEASE); + __atomic_store_n(PacketPtr, HeaderWord, __ATOMIC_RELEASE); // Signal the doorbell about the published packet. hsa_signal_store_relaxed(Queue->doorbell_signal, PacketId); -- GitLab From 767e0c8bcef9cfcc57e76e66e23489ba60042762 Mon Sep 17 00:00:00 2001 From: Thomas Lively Date: Wed, 20 Mar 2024 08:42:42 -0700 Subject: [PATCH 028/296] [WebAssembly] Select BUILD_VECTOR with large unsigned lane values (#85880) Previously we expected lane constants to be in the range of signed values for each lane size, but the included test case produced large unsigned values that fall outside that range. Allow instruction selection to proceed in this case rather than failing. Fixes #63817. --- .../Target/WebAssembly/WebAssemblyInstrSIMD.td | 6 ++++-- llvm/test/CodeGen/WebAssembly/pr63817.ll | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 llvm/test/CodeGen/WebAssembly/pr63817.ll diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td index 8cd41d7017a0..af95dfa25a18 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td @@ -46,10 +46,12 @@ defm "" : ARGUMENT; defm "" : ARGUMENT; defm "" : ARGUMENT; -// Constrained immediate argument types +// Constrained immediate argument types. Allow any value from the minimum signed +// value to the maximum unsigned value for the lane size. foreach SIZE = [8, 16] in def ImmI#SIZE : ImmLeaf; foreach SIZE = [2, 4, 8, 16, 32] in def LaneIdx#SIZE : ImmLeaf; diff --git a/llvm/test/CodeGen/WebAssembly/pr63817.ll b/llvm/test/CodeGen/WebAssembly/pr63817.ll new file mode 100644 index 000000000000..252768d43f18 --- /dev/null +++ b/llvm/test/CodeGen/WebAssembly/pr63817.ll @@ -0,0 +1,15 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=wasm32 -mattr=+simd128 | FileCheck %s + +;; Regression test for a bug in which BUILD_VECTOR nodes with large unsigned +;; lane constants were not properly selected. +define <4 x i8> @test(<4 x i8> %0) { +; CHECK-LABEL: test: +; CHECK: .functype test (v128) -> (v128) +; CHECK-NEXT: # %bb.0: +; CHECK-NEXT: v128.const 255, 17, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +; CHECK-NEXT: # fallthrough-return + %V1 = or <4 x i8> , %0 + %V2 = insertelement <4 x i8> %V1, i8 17, i32 1 + ret <4 x i8> %V2 +} -- GitLab From 576d81baa5cf1801bae0fd05892be34acde33c6a Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 20 Mar 2024 08:44:24 -0700 Subject: [PATCH 029/296] [RISCV] Use REG_SEQUENCE/EXTRACT_SUBREG to move between individual GPRs and GPRPair. (#85887) Previously we used memory like we do to move between GPRs and FPR64 with the D extension on RV32. We can instead use REG_SEQUENCE/EXTRACT_SUBREG to inform register allocation how to do the copy without memory. --- llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp | 37 ++ llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 17 +- llvm/lib/Target/RISCV/RISCVInstrInfoD.td | 14 - .../test/CodeGen/RISCV/double-arith-strict.ll | 174 -------- llvm/test/CodeGen/RISCV/double-arith.ll | 358 --------------- .../RISCV/double-bitmanip-dagcombines.ll | 14 - llvm/test/CodeGen/RISCV/double-br-fcmp.ll | 210 ++------- .../test/CodeGen/RISCV/double-calling-conv.ll | 45 +- .../CodeGen/RISCV/double-convert-strict.ll | 78 ---- llvm/test/CodeGen/RISCV/double-convert.ll | 243 ++--------- llvm/test/CodeGen/RISCV/double-fcmp-strict.ll | 400 +++-------------- llvm/test/CodeGen/RISCV/double-fcmp.ll | 140 ------ llvm/test/CodeGen/RISCV/double-imm.ll | 32 +- .../CodeGen/RISCV/double-intrinsics-strict.ll | 72 +--- llvm/test/CodeGen/RISCV/double-intrinsics.ll | 127 +----- llvm/test/CodeGen/RISCV/double-isnan.ll | 12 - .../CodeGen/RISCV/double-maximum-minimum.ll | 136 ++---- llvm/test/CodeGen/RISCV/double-mem.ll | 86 +--- .../CodeGen/RISCV/double-previous-failure.ll | 4 - .../CodeGen/RISCV/double-round-conv-sat.ll | 408 ++++++------------ llvm/test/CodeGen/RISCV/double-round-conv.ll | 210 --------- llvm/test/CodeGen/RISCV/double-select-fcmp.ll | 237 ---------- llvm/test/CodeGen/RISCV/double-select-icmp.ll | 224 +++------- .../RISCV/double-stack-spill-restore.ll | 40 +- .../CodeGen/RISCV/fastcc-without-f-reg.ll | 32 +- .../test/CodeGen/RISCV/half-convert-strict.ll | 24 -- llvm/test/CodeGen/RISCV/half-convert.ll | 50 +-- llvm/test/CodeGen/RISCV/pr64645.ll | 26 -- .../CodeGen/RISCV/zdinx-asm-constraint.ll | 16 +- .../CodeGen/RISCV/zdinx-boundary-check.ll | 38 +- 30 files changed, 469 insertions(+), 3035 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp index 1b8c1434c9f2..55ba4949b3ea 100644 --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -1007,7 +1007,44 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) { ReplaceNode(Node, Res); return; } + case RISCVISD::BuildPairF64: { + if (!Subtarget->hasStdExtZdinx()) + break; + + assert(!Subtarget->is64Bit() && "Unexpected subtarget"); + + SDValue Ops[] = { + CurDAG->getTargetConstant(RISCV::GPRPairRegClassID, DL, MVT::i32), + Node->getOperand(0), + CurDAG->getTargetConstant(RISCV::sub_gpr_even, DL, MVT::i32), + Node->getOperand(1), + CurDAG->getTargetConstant(RISCV::sub_gpr_odd, DL, MVT::i32)}; + + SDNode *N = + CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::f64, Ops); + ReplaceNode(Node, N); + return; + } case RISCVISD::SplitF64: { + if (Subtarget->hasStdExtZdinx()) { + assert(!Subtarget->is64Bit() && "Unexpected subtarget"); + + if (!SDValue(Node, 0).use_empty()) { + SDValue Lo = CurDAG->getTargetExtractSubreg(RISCV::sub_gpr_even, DL, VT, + Node->getOperand(0)); + ReplaceUses(SDValue(Node, 0), Lo); + } + + if (!SDValue(Node, 1).use_empty()) { + SDValue Hi = CurDAG->getTargetExtractSubreg(RISCV::sub_gpr_odd, DL, VT, + Node->getOperand(0)); + ReplaceUses(SDValue(Node, 1), Hi); + } + + CurDAG->RemoveDeadNode(Node); + return; + } + if (!Subtarget->hasStdExtZfa()) break; assert(Subtarget->hasStdExtD() && !Subtarget->is64Bit() && diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 4bfd4d0386a8..25f035e6dd9d 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -17142,9 +17142,7 @@ static MachineBasicBlock *emitReadCounterWidePseudo(MachineInstr &MI, static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget) { - assert((MI.getOpcode() == RISCV::SplitF64Pseudo || - MI.getOpcode() == RISCV::SplitF64Pseudo_INX) && - "Unexpected instruction"); + assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction"); MachineFunction &MF = *BB->getParent(); DebugLoc DL = MI.getDebugLoc(); @@ -17154,9 +17152,7 @@ static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI, Register HiReg = MI.getOperand(1).getReg(); Register SrcReg = MI.getOperand(2).getReg(); - const TargetRegisterClass *SrcRC = MI.getOpcode() == RISCV::SplitF64Pseudo_INX - ? &RISCV::GPRPairRegClass - : &RISCV::FPR64RegClass; + const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass; int FI = MF.getInfo()->getMoveF64FrameIndex(MF); TII.storeRegToStackSlot(*BB, MI, SrcReg, MI.getOperand(2).isKill(), FI, SrcRC, @@ -17181,8 +17177,7 @@ static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI, static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget) { - assert((MI.getOpcode() == RISCV::BuildPairF64Pseudo || - MI.getOpcode() == RISCV::BuildPairF64Pseudo_INX) && + assert(MI.getOpcode() == RISCV::BuildPairF64Pseudo && "Unexpected instruction"); MachineFunction &MF = *BB->getParent(); @@ -17193,9 +17188,7 @@ static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI, Register LoReg = MI.getOperand(1).getReg(); Register HiReg = MI.getOperand(2).getReg(); - const TargetRegisterClass *DstRC = - MI.getOpcode() == RISCV::BuildPairF64Pseudo_INX ? &RISCV::GPRPairRegClass - : &RISCV::FPR64RegClass; + const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass; int FI = MF.getInfo()->getMoveF64FrameIndex(MF); MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI); @@ -17716,10 +17709,8 @@ RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, case RISCV::Select_FPR64IN32X_Using_CC_GPR: return emitSelectPseudo(MI, BB, Subtarget); case RISCV::BuildPairF64Pseudo: - case RISCV::BuildPairF64Pseudo_INX: return emitBuildPairF64Pseudo(MI, BB, Subtarget); case RISCV::SplitF64Pseudo: - case RISCV::SplitF64Pseudo_INX: return emitSplitF64Pseudo(MI, BB, Subtarget); case RISCV::PseudoQuietFLE_H: return emitQuietFCMP(MI, BB, RISCV::FLE_H, RISCV::FEQ_H, Subtarget); diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td index 9b4f93d55e33..8efefee383a6 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td @@ -524,20 +524,6 @@ let isCall = 0, mayLoad = 0, mayStore = 1, Size = 8, isCodeGenOnly = 1 in def PseudoRV32ZdinxSD : Pseudo<(outs), (ins GPRPair:$rs2, GPRNoX0:$rs1, simm12:$imm12), []>; def : Pat<(store (f64 GPRPair:$rs2), (AddrRegImmINX (XLenVT GPR:$rs1), simm12:$imm12)), (PseudoRV32ZdinxSD GPRPair:$rs2, GPR:$rs1, simm12:$imm12)>; - -/// Pseudo-instructions needed for the soft-float ABI with RV32D - -// Moves two GPRs to an FPR. -let usesCustomInserter = 1 in -def BuildPairF64Pseudo_INX - : Pseudo<(outs FPR64IN32X:$dst), (ins GPR:$src1, GPR:$src2), - [(set FPR64IN32X:$dst, (RISCVBuildPairF64 GPR:$src1, GPR:$src2))]>; - -// Moves an FPR to two GPRs. -let usesCustomInserter = 1 in -def SplitF64Pseudo_INX - : Pseudo<(outs GPR:$dst1, GPR:$dst2), (ins FPR64IN32X:$src), - [(set GPR:$dst1, GPR:$dst2, (RISCVSplitF64 FPR64IN32X:$src))]>; } // Predicates = [HasStdExtZdinx, IsRV32] let Predicates = [HasStdExtD] in { diff --git a/llvm/test/CodeGen/RISCV/double-arith-strict.ll b/llvm/test/CodeGen/RISCV/double-arith-strict.ll index 186175537772..23336933abff 100644 --- a/llvm/test/CodeGen/RISCV/double-arith-strict.ll +++ b/llvm/test/CodeGen/RISCV/double-arith-strict.ll @@ -24,21 +24,7 @@ define double @fadd_d(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fadd_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fadd_d: @@ -76,21 +62,7 @@ define double @fsub_d(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fsub_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fsub.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fsub_d: @@ -128,21 +100,7 @@ define double @fmul_d(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fmul_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fmul.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmul_d: @@ -180,21 +138,7 @@ define double @fdiv_d(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fdiv_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fdiv.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fdiv_d: @@ -232,17 +176,7 @@ define double @fsqrt_d(double %a) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fsqrt_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fsqrt.d a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fsqrt_d: @@ -398,25 +332,7 @@ define double @fmadd_d(double %a, double %b, double %c) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fmadd_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fmadd.d a0, a0, a2, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmadd_d: @@ -463,27 +379,9 @@ define double @fmsub_d(double %a, double %b, double %c) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fmsub_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.d.w a6, zero ; RV32IZFINXZDINX-NEXT: fadd.d a4, a4, a6 ; RV32IZFINXZDINX-NEXT: fmsub.d a0, a0, a2, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmsub_d: @@ -572,28 +470,10 @@ define double @fnmadd_d(double %a, double %b, double %c) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fnmadd_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.d.w a6, zero ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a6 ; RV32IZFINXZDINX-NEXT: fadd.d a4, a4, a6 ; RV32IZFINXZDINX-NEXT: fnmadd.d a0, a0, a2, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fnmadd_d: @@ -701,28 +581,10 @@ define double @fnmadd_d_2(double %a, double %b, double %c) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fnmadd_d_2: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.d.w a6, zero ; RV32IZFINXZDINX-NEXT: fadd.d a2, a2, a6 ; RV32IZFINXZDINX-NEXT: fadd.d a4, a4, a6 ; RV32IZFINXZDINX-NEXT: fnmadd.d a0, a2, a0, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fnmadd_d_2: @@ -829,27 +691,9 @@ define double @fnmsub_d(double %a, double %b, double %c) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fnmsub_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.d.w a6, zero ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a6 ; RV32IZFINXZDINX-NEXT: fnmsub.d a0, a0, a2, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fnmsub_d: @@ -932,27 +776,9 @@ define double @fnmsub_d_2(double %a, double %b, double %c) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fnmsub_d_2: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.d.w a6, zero ; RV32IZFINXZDINX-NEXT: fadd.d a2, a2, a6 ; RV32IZFINXZDINX-NEXT: fnmsub.d a0, a2, a0, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fnmsub_d_2: diff --git a/llvm/test/CodeGen/RISCV/double-arith.ll b/llvm/test/CodeGen/RISCV/double-arith.ll index 82ddf06187d3..a2093f5b5e43 100644 --- a/llvm/test/CodeGen/RISCV/double-arith.ll +++ b/llvm/test/CodeGen/RISCV/double-arith.ll @@ -25,21 +25,7 @@ define double @fadd_d(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fadd_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fadd_d: @@ -76,21 +62,7 @@ define double @fsub_d(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fsub_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fsub.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fsub_d: @@ -127,21 +99,7 @@ define double @fmul_d(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fmul_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fmul.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmul_d: @@ -178,21 +136,7 @@ define double @fdiv_d(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fdiv_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fdiv.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fdiv_d: @@ -231,17 +175,7 @@ define double @fsqrt_d(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fsqrt_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fsqrt.d a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fsqrt_d: @@ -280,21 +214,7 @@ define double @fsgnj_d(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fsgnj_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fsgnj.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fsgnj_d: @@ -335,15 +255,9 @@ define i32 @fneg_d(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fneg_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: fneg.d a2, a0 ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fneg_d: @@ -401,21 +315,7 @@ define double @fsgnjn_d(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fsgnjn_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fsgnjn.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fsgnjn_d: @@ -464,23 +364,9 @@ define double @fabs_d(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fabs_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: fabs.d a2, a0 ; RV32IZFINXZDINX-NEXT: fadd.d a0, a2, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fabs_d: @@ -532,21 +418,7 @@ define double @fmin_d(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fmin_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fmin.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmin_d: @@ -585,21 +457,7 @@ define double @fmax_d(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fmax_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fmax.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmax_d: @@ -638,25 +496,7 @@ define double @fmadd_d(double %a, double %b, double %c) nounwind { ; ; RV32IZFINXZDINX-LABEL: fmadd_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fmadd.d a0, a0, a2, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmadd_d: @@ -702,27 +542,9 @@ define double @fmsub_d(double %a, double %b, double %c) nounwind { ; ; RV32IZFINXZDINX-LABEL: fmsub_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.d.w a6, zero ; RV32IZFINXZDINX-NEXT: fadd.d a4, a4, a6 ; RV32IZFINXZDINX-NEXT: fmsub.d a0, a0, a2, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmsub_d: @@ -811,28 +633,10 @@ define double @fnmadd_d(double %a, double %b, double %c) nounwind { ; ; RV32IZFINXZDINX-LABEL: fnmadd_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.d.w a6, zero ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a6 ; RV32IZFINXZDINX-NEXT: fadd.d a4, a4, a6 ; RV32IZFINXZDINX-NEXT: fnmadd.d a0, a0, a2, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fnmadd_d: @@ -940,28 +744,10 @@ define double @fnmadd_d_2(double %a, double %b, double %c) nounwind { ; ; RV32IZFINXZDINX-LABEL: fnmadd_d_2: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.d.w a6, zero ; RV32IZFINXZDINX-NEXT: fadd.d a2, a2, a6 ; RV32IZFINXZDINX-NEXT: fadd.d a4, a4, a6 ; RV32IZFINXZDINX-NEXT: fnmadd.d a0, a2, a0, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fnmadd_d_2: @@ -1060,27 +846,9 @@ define double @fnmadd_d_3(double %a, double %b, double %c) nounwind { ; ; RV32IZFINXZDINX-LABEL: fnmadd_d_3: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fmadd.d a0, a0, a2, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) ; RV32IZFINXZDINX-NEXT: lui a2, 524288 ; RV32IZFINXZDINX-NEXT: xor a1, a1, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fnmadd_d_3: @@ -1127,27 +895,9 @@ define double @fnmadd_nsz(double %a, double %b, double %c) nounwind { ; ; RV32IZFINXZDINX-LABEL: fnmadd_nsz: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fmadd.d a0, a0, a2, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) ; RV32IZFINXZDINX-NEXT: lui a2, 524288 ; RV32IZFINXZDINX-NEXT: xor a1, a1, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fnmadd_nsz: @@ -1202,27 +952,9 @@ define double @fnmsub_d(double %a, double %b, double %c) nounwind { ; ; RV32IZFINXZDINX-LABEL: fnmsub_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.d.w a6, zero ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a6 ; RV32IZFINXZDINX-NEXT: fnmsub.d a0, a0, a2, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fnmsub_d: @@ -1305,27 +1037,9 @@ define double @fnmsub_d_2(double %a, double %b, double %c) nounwind { ; ; RV32IZFINXZDINX-LABEL: fnmsub_d_2: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.d.w a6, zero ; RV32IZFINXZDINX-NEXT: fadd.d a2, a2, a6 ; RV32IZFINXZDINX-NEXT: fnmsub.d a0, a2, a0, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fnmsub_d_2: @@ -1403,25 +1117,7 @@ define double @fmadd_d_contract(double %a, double %b, double %c) nounwind { ; ; RV32IZFINXZDINX-LABEL: fmadd_d_contract: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fmadd.d a0, a0, a2, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmadd_d_contract: @@ -1482,27 +1178,9 @@ define double @fmsub_d_contract(double %a, double %b, double %c) nounwind { ; ; RV32IZFINXZDINX-LABEL: fmsub_d_contract: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.d.w a6, zero ; RV32IZFINXZDINX-NEXT: fadd.d a4, a4, a6 ; RV32IZFINXZDINX-NEXT: fmsub.d a0, a0, a2, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmsub_d_contract: @@ -1601,29 +1279,11 @@ define double @fnmadd_d_contract(double %a, double %b, double %c) nounwind { ; ; RV32IZFINXZDINX-LABEL: fnmadd_d_contract: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.d.w a6, zero ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a6 ; RV32IZFINXZDINX-NEXT: fadd.d a2, a2, a6 ; RV32IZFINXZDINX-NEXT: fadd.d a4, a4, a6 ; RV32IZFINXZDINX-NEXT: fnmadd.d a0, a0, a2, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fnmadd_d_contract: @@ -1749,28 +1409,10 @@ define double @fnmsub_d_contract(double %a, double %b, double %c) nounwind { ; ; RV32IZFINXZDINX-LABEL: fnmsub_d_contract: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.d.w a6, zero ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a6 ; RV32IZFINXZDINX-NEXT: fadd.d a2, a2, a6 ; RV32IZFINXZDINX-NEXT: fnmsub.d a0, a0, a2, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fnmsub_d_contract: diff --git a/llvm/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll b/llvm/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll index 55bf95a126ac..99835ff59493 100644 --- a/llvm/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll +++ b/llvm/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll @@ -141,21 +141,7 @@ define double @fcopysign_fneg(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcopysign_fneg: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fsgnjn.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64I-LABEL: fcopysign_fneg: diff --git a/llvm/test/CodeGen/RISCV/double-br-fcmp.ll b/llvm/test/CodeGen/RISCV/double-br-fcmp.ll index 2c5505edb1fa..035228e73c70 100644 --- a/llvm/test/CodeGen/RISCV/double-br-fcmp.ll +++ b/llvm/test/CodeGen/RISCV/double-br-fcmp.ll @@ -89,23 +89,13 @@ define void @br_fcmp_oeq(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: br_fcmp_oeq: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: bnez a0, .LBB1_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: # %if.else -; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; RV32IZFINXZDINX-NEXT: .LBB1_2: # %if.then +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call abort ; ; RV64IZFINXZDINX-LABEL: br_fcmp_oeq: @@ -155,23 +145,13 @@ define void @br_fcmp_oeq_alt(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: br_fcmp_oeq_alt: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: bnez a0, .LBB2_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: # %if.else -; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; RV32IZFINXZDINX-NEXT: .LBB2_2: # %if.then +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call abort ; ; RV64IZFINXZDINX-LABEL: br_fcmp_oeq_alt: @@ -218,23 +198,13 @@ define void @br_fcmp_ogt(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: br_fcmp_ogt: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 4(sp) ; RV32IZFINXZDINX-NEXT: flt.d a0, a2, a0 ; RV32IZFINXZDINX-NEXT: bnez a0, .LBB3_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: # %if.else -; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; RV32IZFINXZDINX-NEXT: .LBB3_2: # %if.then +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call abort ; ; RV64IZFINXZDINX-LABEL: br_fcmp_ogt: @@ -281,23 +251,13 @@ define void @br_fcmp_oge(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: br_fcmp_oge: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 4(sp) ; RV32IZFINXZDINX-NEXT: fle.d a0, a2, a0 ; RV32IZFINXZDINX-NEXT: bnez a0, .LBB4_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: # %if.else -; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; RV32IZFINXZDINX-NEXT: .LBB4_2: # %if.then +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call abort ; ; RV64IZFINXZDINX-LABEL: br_fcmp_oge: @@ -344,23 +304,13 @@ define void @br_fcmp_olt(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: br_fcmp_olt: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) ; RV32IZFINXZDINX-NEXT: flt.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: bnez a0, .LBB5_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: # %if.else -; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; RV32IZFINXZDINX-NEXT: .LBB5_2: # %if.then +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call abort ; ; RV64IZFINXZDINX-LABEL: br_fcmp_olt: @@ -407,23 +357,13 @@ define void @br_fcmp_ole(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: br_fcmp_ole: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) ; RV32IZFINXZDINX-NEXT: fle.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: bnez a0, .LBB6_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: # %if.else -; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; RV32IZFINXZDINX-NEXT: .LBB6_2: # %if.then +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call abort ; ; RV64IZFINXZDINX-LABEL: br_fcmp_ole: @@ -474,25 +414,15 @@ define void @br_fcmp_one(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: br_fcmp_one: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) ; RV32IZFINXZDINX-NEXT: flt.d a4, a0, a2 ; RV32IZFINXZDINX-NEXT: flt.d a0, a2, a0 ; RV32IZFINXZDINX-NEXT: or a0, a0, a4 ; RV32IZFINXZDINX-NEXT: bnez a0, .LBB7_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: # %if.else -; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; RV32IZFINXZDINX-NEXT: .LBB7_2: # %if.then +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call abort ; ; RV64IZFINXZDINX-LABEL: br_fcmp_one: @@ -545,25 +475,15 @@ define void @br_fcmp_ord(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: br_fcmp_ord: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 4(sp) ; RV32IZFINXZDINX-NEXT: feq.d a2, a2, a2 ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 ; RV32IZFINXZDINX-NEXT: bnez a0, .LBB8_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: # %if.else -; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; RV32IZFINXZDINX-NEXT: .LBB8_2: # %if.then +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call abort ; ; RV64IZFINXZDINX-LABEL: br_fcmp_ord: @@ -616,25 +536,15 @@ define void @br_fcmp_ueq(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: br_fcmp_ueq: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) ; RV32IZFINXZDINX-NEXT: flt.d a4, a0, a2 ; RV32IZFINXZDINX-NEXT: flt.d a0, a2, a0 ; RV32IZFINXZDINX-NEXT: or a0, a0, a4 ; RV32IZFINXZDINX-NEXT: beqz a0, .LBB9_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: # %if.else -; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; RV32IZFINXZDINX-NEXT: .LBB9_2: # %if.then +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call abort ; ; RV64IZFINXZDINX-LABEL: br_fcmp_ueq: @@ -683,23 +593,13 @@ define void @br_fcmp_ugt(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: br_fcmp_ugt: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) ; RV32IZFINXZDINX-NEXT: fle.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: beqz a0, .LBB10_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: # %if.else -; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; RV32IZFINXZDINX-NEXT: .LBB10_2: # %if.then +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call abort ; ; RV64IZFINXZDINX-LABEL: br_fcmp_ugt: @@ -746,23 +646,13 @@ define void @br_fcmp_uge(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: br_fcmp_uge: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) ; RV32IZFINXZDINX-NEXT: flt.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: beqz a0, .LBB11_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: # %if.else -; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; RV32IZFINXZDINX-NEXT: .LBB11_2: # %if.then +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call abort ; ; RV64IZFINXZDINX-LABEL: br_fcmp_uge: @@ -809,23 +699,13 @@ define void @br_fcmp_ult(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: br_fcmp_ult: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 4(sp) ; RV32IZFINXZDINX-NEXT: fle.d a0, a2, a0 ; RV32IZFINXZDINX-NEXT: beqz a0, .LBB12_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: # %if.else -; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; RV32IZFINXZDINX-NEXT: .LBB12_2: # %if.then +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call abort ; ; RV64IZFINXZDINX-LABEL: br_fcmp_ult: @@ -872,23 +752,13 @@ define void @br_fcmp_ule(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: br_fcmp_ule: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 4(sp) ; RV32IZFINXZDINX-NEXT: flt.d a0, a2, a0 ; RV32IZFINXZDINX-NEXT: beqz a0, .LBB13_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: # %if.else -; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; RV32IZFINXZDINX-NEXT: .LBB13_2: # %if.then +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call abort ; ; RV64IZFINXZDINX-LABEL: br_fcmp_ule: @@ -935,23 +805,13 @@ define void @br_fcmp_une(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: br_fcmp_une: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: beqz a0, .LBB14_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: # %if.else -; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; RV32IZFINXZDINX-NEXT: .LBB14_2: # %if.then +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call abort ; ; RV64IZFINXZDINX-LABEL: br_fcmp_une: @@ -1002,25 +862,15 @@ define void @br_fcmp_uno(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: br_fcmp_uno: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 4(sp) ; RV32IZFINXZDINX-NEXT: feq.d a2, a2, a2 ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 ; RV32IZFINXZDINX-NEXT: beqz a0, .LBB15_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: # %if.else -; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; RV32IZFINXZDINX-NEXT: .LBB15_2: # %if.then +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call abort ; ; RV64IZFINXZDINX-LABEL: br_fcmp_uno: diff --git a/llvm/test/CodeGen/RISCV/double-calling-conv.ll b/llvm/test/CodeGen/RISCV/double-calling-conv.ll index d46256b12052..57aaa4c9f74e 100644 --- a/llvm/test/CodeGen/RISCV/double-calling-conv.ll +++ b/llvm/test/CodeGen/RISCV/double-calling-conv.ll @@ -28,21 +28,7 @@ define double @callee_double_inreg(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: callee_double_inreg: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret %1 = fadd double %a, %b ret double %1 @@ -106,22 +92,11 @@ define double @callee_double_split_reg_stack(i32 %a, i64 %b, i64 %c, double %d, ; ; RV32IZFINXZDINX-LABEL: callee_double_split_reg_stack: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: lw a0, 16(sp) -; RV32IZFINXZDINX-NEXT: sw a7, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a6, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) +; RV32IZFINXZDINX-NEXT: mv a0, a7 +; RV32IZFINXZDINX-NEXT: lw a1, 0(sp) +; RV32IZFINXZDINX-NEXT: mv a3, a6 +; RV32IZFINXZDINX-NEXT: mv a2, a5 ; RV32IZFINXZDINX-NEXT: fadd.d a0, a2, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret %1 = fadd double %d, %e ret double %1 @@ -190,17 +165,11 @@ define double @callee_double_stack(i64 %a, i64 %b, i64 %c, i64 %d, double %e, do ; ; RV32IZFINXZDINX-LABEL: callee_double_stack: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: lw a0, 24(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 28(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 16(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 20(sp) -; RV32IZFINXZDINX-NEXT: fadd.d a0, a2, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) ; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: lw a2, 0(sp) +; RV32IZFINXZDINX-NEXT: lw a3, 4(sp) +; RV32IZFINXZDINX-NEXT: fadd.d a0, a2, a0 ; RV32IZFINXZDINX-NEXT: ret %1 = fadd double %e, %f ret double %1 diff --git a/llvm/test/CodeGen/RISCV/double-convert-strict.ll b/llvm/test/CodeGen/RISCV/double-convert-strict.ll index 967b119581af..13bcafb5ebd1 100644 --- a/llvm/test/CodeGen/RISCV/double-convert-strict.ll +++ b/llvm/test/CodeGen/RISCV/double-convert-strict.ll @@ -28,13 +28,7 @@ define float @fcvt_s_d(double %a) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcvt_s_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.s.d a0, a0 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_s_d: @@ -72,13 +66,7 @@ define double @fcvt_d_s(float %a) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcvt_d_s: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: fcvt.d.s a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_d_s: @@ -116,13 +104,7 @@ define i32 @fcvt_w_d(double %a) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcvt_w_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rtz -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_w_d: @@ -162,13 +144,7 @@ define i32 @fcvt_wu_d(double %a) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcvt_wu_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rtz -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_wu_d: @@ -210,15 +186,9 @@ define i32 @fcvt_wu_d_multiple_use(double %x, ptr %y) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcvt_wu_d_multiple_use: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rtz ; RV32IZFINXZDINX-NEXT: seqz a1, a0 ; RV32IZFINXZDINX-NEXT: add a0, a0, a1 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_wu_d_multiple_use: @@ -263,13 +233,7 @@ define double @fcvt_d_w(i32 %a) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcvt_d_w: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: fcvt.d.w a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_d_w: @@ -309,14 +273,8 @@ define double @fcvt_d_w_load(ptr %p) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcvt_d_w_load: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: lw a0, 0(a0) ; RV32IZFINXZDINX-NEXT: fcvt.d.w a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_d_w_load: @@ -357,13 +315,7 @@ define double @fcvt_d_wu(i32 %a) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcvt_d_wu: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: fcvt.d.wu a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_d_wu: @@ -409,14 +361,8 @@ define double @fcvt_d_wu_load(ptr %p) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcvt_d_wu_load: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: lw a0, 0(a0) ; RV32IZFINXZDINX-NEXT: fcvt.d.wu a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_d_wu_load: @@ -661,13 +607,7 @@ define double @fcvt_d_w_i8(i8 signext %a) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcvt_d_w_i8: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: fcvt.d.w a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_d_w_i8: @@ -705,13 +645,7 @@ define double @fcvt_d_wu_i8(i8 zeroext %a) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcvt_d_wu_i8: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: fcvt.d.wu a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_d_wu_i8: @@ -749,13 +683,7 @@ define double @fcvt_d_w_i16(i16 signext %a) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcvt_d_w_i16: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: fcvt.d.w a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_d_w_i16: @@ -793,13 +721,7 @@ define double @fcvt_d_wu_i16(i16 zeroext %a) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcvt_d_wu_i16: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: fcvt.d.wu a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_d_wu_i16: diff --git a/llvm/test/CodeGen/RISCV/double-convert.ll b/llvm/test/CodeGen/RISCV/double-convert.ll index 3700a18bafc6..7a9439e5b322 100644 --- a/llvm/test/CodeGen/RISCV/double-convert.ll +++ b/llvm/test/CodeGen/RISCV/double-convert.ll @@ -20,13 +20,7 @@ define float @fcvt_s_d(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_s_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.s.d a0, a0 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_s_d: @@ -63,13 +57,7 @@ define double @fcvt_d_s(float %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_d_s: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: fcvt.d.s a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_d_s: @@ -106,13 +94,7 @@ define i32 @fcvt_w_d(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_w_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rtz -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_w_d: @@ -153,17 +135,11 @@ define i32 @fcvt_w_d_sat(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_w_d_sat: ; RV32IZFINXZDINX: # %bb.0: # %start -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a2, a0, rtz ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: seqz a0, a0 ; RV32IZFINXZDINX-NEXT: addi a0, a0, -1 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_w_d_sat: @@ -287,13 +263,7 @@ define i32 @fcvt_wu_d(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_wu_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rtz -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_wu_d: @@ -334,15 +304,9 @@ define i32 @fcvt_wu_d_multiple_use(double %x, ptr %y) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_wu_d_multiple_use: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rtz ; RV32IZFINXZDINX-NEXT: seqz a1, a0 ; RV32IZFINXZDINX-NEXT: add a0, a0, a1 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_wu_d_multiple_use: @@ -402,17 +366,11 @@ define i32 @fcvt_wu_d_sat(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_wu_d_sat: ; RV32IZFINXZDINX: # %bb.0: # %start -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a2, a0, rtz ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: seqz a0, a0 ; RV32IZFINXZDINX-NEXT: addi a0, a0, -1 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_wu_d_sat: @@ -512,13 +470,7 @@ define double @fcvt_d_w(i32 %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_d_w: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: fcvt.d.w a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_d_w: @@ -557,14 +509,8 @@ define double @fcvt_d_w_load(ptr %p) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_d_w_load: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: lw a0, 0(a0) ; RV32IZFINXZDINX-NEXT: fcvt.d.w a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_d_w_load: @@ -605,13 +551,7 @@ define double @fcvt_d_wu(i32 %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_d_wu: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: fcvt.d.wu a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_d_wu: @@ -656,14 +596,8 @@ define double @fcvt_d_wu_load(ptr %p) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_d_wu_load: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: lw a0, 0(a0) ; RV32IZFINXZDINX-NEXT: fcvt.d.wu a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_d_wu_load: @@ -809,13 +743,11 @@ define i64 @fcvt_l_d_sat(double %a) nounwind { ; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s3, 12(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw s0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI12_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI12_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI12_0)(a2) +; RV32IZFINXZDINX-NEXT: mv s1, a1 +; RV32IZFINXZDINX-NEXT: mv s0, a0 ; RV32IZFINXZDINX-NEXT: fle.d s2, a2, s0 ; RV32IZFINXZDINX-NEXT: neg s3, s2 ; RV32IZFINXZDINX-NEXT: call __fixdfdi @@ -1057,18 +989,17 @@ define i64 @fcvt_lu_d_sat(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_lu_d_sat: ; RV32IZFINXZDINX: # %bb.0: # %start -; RV32IZFINXZDINX-NEXT: addi sp, sp, -32 -; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw s1, 12(sp) +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s2, 0(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: mv s1, a1 ; RV32IZFINXZDINX-NEXT: fcvt.d.w a2, zero -; RV32IZFINXZDINX-NEXT: fle.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg s2, a2 +; RV32IZFINXZDINX-NEXT: mv s0, a0 +; RV32IZFINXZDINX-NEXT: fle.d a0, a2, s0 +; RV32IZFINXZDINX-NEXT: neg s2, a0 +; RV32IZFINXZDINX-NEXT: mv a0, s0 ; RV32IZFINXZDINX-NEXT: call __fixunsdfdi ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI14_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI14_0+4)(a2) @@ -1079,11 +1010,11 @@ define i64 @fcvt_lu_d_sat(double %a) nounwind { ; RV32IZFINXZDINX-NEXT: or a0, a2, a0 ; RV32IZFINXZDINX-NEXT: and a1, s2, a1 ; RV32IZFINXZDINX-NEXT: or a1, a2, a1 -; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 +; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s2, 0(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_lu_d_sat: @@ -1186,14 +1117,6 @@ define i64 @fmv_x_d(double %a, double %b) nounwind { ; RV32IZFINXZDINX-LABEL: fmv_x_d: ; RV32IZFINXZDINX: # %bb.0: ; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) ; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) @@ -1353,21 +1276,17 @@ define double @fmv_d_x(i64 %a, i64 %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fmv_d_x: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -32 -; RV32IZFINXZDINX-NEXT: sw a3, 20(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 16(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 28(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 24(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 16(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 20(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 24(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 28(sp) -; RV32IZFINXZDINX-NEXT: fadd.d a0, a2, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) +; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) ; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 +; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) +; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) +; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) +; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) +; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) +; RV32IZFINXZDINX-NEXT: fadd.d a0, a2, a0 +; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmv_d_x: @@ -1406,13 +1325,7 @@ define double @fcvt_d_w_i8(i8 signext %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_d_w_i8: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: fcvt.d.w a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_d_w_i8: @@ -1449,13 +1362,7 @@ define double @fcvt_d_wu_i8(i8 zeroext %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_d_wu_i8: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: fcvt.d.wu a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_d_wu_i8: @@ -1492,13 +1399,7 @@ define double @fcvt_d_w_i16(i16 signext %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_d_w_i16: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: fcvt.d.w a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_d_w_i16: @@ -1535,13 +1436,7 @@ define double @fcvt_d_wu_i16(i16 zeroext %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_d_wu_i16: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: fcvt.d.wu a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_d_wu_i16: @@ -1731,13 +1626,7 @@ define signext i16 @fcvt_w_s_i16(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_w_s_i16: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rtz -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_w_s_i16: @@ -1797,24 +1686,18 @@ define signext i16 @fcvt_w_s_sat_i16(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_w_s_sat_i16: ; RV32IZFINXZDINX: # %bb.0: # %start -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI26_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI26_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI26_0)(a2) ; RV32IZFINXZDINX-NEXT: lui a4, %hi(.LCPI26_1) ; RV32IZFINXZDINX-NEXT: lw a5, %lo(.LCPI26_1+4)(a4) ; RV32IZFINXZDINX-NEXT: lw a4, %lo(.LCPI26_1)(a4) -; RV32IZFINXZDINX-NEXT: fmax.d a2, a0, a2 -; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 -; RV32IZFINXZDINX-NEXT: neg a0, a0 -; RV32IZFINXZDINX-NEXT: fmin.d a2, a2, a4 -; RV32IZFINXZDINX-NEXT: fcvt.w.d a1, a2, rtz -; RV32IZFINXZDINX-NEXT: and a0, a0, a1 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: feq.d a6, a0, a0 +; RV32IZFINXZDINX-NEXT: neg a6, a6 +; RV32IZFINXZDINX-NEXT: fmax.d a0, a0, a2 +; RV32IZFINXZDINX-NEXT: fmin.d a0, a0, a4 +; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rtz +; RV32IZFINXZDINX-NEXT: and a0, a6, a0 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_w_s_sat_i16: @@ -1948,13 +1831,7 @@ define zeroext i16 @fcvt_wu_s_i16(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_wu_s_i16: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rtz -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_wu_s_i16: @@ -2006,11 +1883,6 @@ define zeroext i16 @fcvt_wu_s_sat_i16(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_wu_s_sat_i16: ; RV32IZFINXZDINX: # %bb.0: # %start -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI28_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI28_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI28_0)(a2) @@ -2018,7 +1890,6 @@ define zeroext i16 @fcvt_wu_s_sat_i16(double %a) nounwind { ; RV32IZFINXZDINX-NEXT: fmax.d a0, a0, a4 ; RV32IZFINXZDINX-NEXT: fmin.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rtz -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_wu_s_sat_i16: @@ -2130,13 +2001,7 @@ define signext i8 @fcvt_w_s_i8(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_w_s_i8: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rtz -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_w_s_i8: @@ -2196,24 +2061,18 @@ define signext i8 @fcvt_w_s_sat_i8(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_w_s_sat_i8: ; RV32IZFINXZDINX: # %bb.0: # %start -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI30_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI30_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI30_0)(a2) ; RV32IZFINXZDINX-NEXT: lui a4, %hi(.LCPI30_1) ; RV32IZFINXZDINX-NEXT: lw a5, %lo(.LCPI30_1+4)(a4) ; RV32IZFINXZDINX-NEXT: lw a4, %lo(.LCPI30_1)(a4) -; RV32IZFINXZDINX-NEXT: fmax.d a2, a0, a2 -; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 -; RV32IZFINXZDINX-NEXT: neg a0, a0 -; RV32IZFINXZDINX-NEXT: fmin.d a2, a2, a4 -; RV32IZFINXZDINX-NEXT: fcvt.w.d a1, a2, rtz -; RV32IZFINXZDINX-NEXT: and a0, a0, a1 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: feq.d a6, a0, a0 +; RV32IZFINXZDINX-NEXT: neg a6, a6 +; RV32IZFINXZDINX-NEXT: fmax.d a0, a0, a2 +; RV32IZFINXZDINX-NEXT: fmin.d a0, a0, a4 +; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rtz +; RV32IZFINXZDINX-NEXT: and a0, a6, a0 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_w_s_sat_i8: @@ -2344,13 +2203,7 @@ define zeroext i8 @fcvt_wu_s_i8(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_wu_s_i8: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rtz -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_wu_s_i8: @@ -2404,11 +2257,6 @@ define zeroext i8 @fcvt_wu_s_sat_i8(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_wu_s_sat_i8: ; RV32IZFINXZDINX: # %bb.0: # %start -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI32_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI32_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI32_0)(a2) @@ -2416,7 +2264,6 @@ define zeroext i8 @fcvt_wu_s_sat_i8(double %a) nounwind { ; RV32IZFINXZDINX-NEXT: fmax.d a0, a0, a4 ; RV32IZFINXZDINX-NEXT: fmin.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rtz -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_wu_s_sat_i8: @@ -2532,17 +2379,11 @@ define zeroext i32 @fcvt_wu_d_sat_zext(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_wu_d_sat_zext: ; RV32IZFINXZDINX: # %bb.0: # %start -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a2, a0, rtz ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: seqz a0, a0 ; RV32IZFINXZDINX-NEXT: addi a0, a0, -1 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_wu_d_sat_zext: @@ -2647,17 +2488,11 @@ define signext i32 @fcvt_w_d_sat_sext(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fcvt_w_d_sat_sext: ; RV32IZFINXZDINX: # %bb.0: # %start -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a2, a0, rtz ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: seqz a0, a0 ; RV32IZFINXZDINX-NEXT: addi a0, a0, -1 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcvt_w_d_sat_sext: diff --git a/llvm/test/CodeGen/RISCV/double-fcmp-strict.ll b/llvm/test/CodeGen/RISCV/double-fcmp-strict.ll index 3ae2e997019c..e864d8fb0edd 100644 --- a/llvm/test/CodeGen/RISCV/double-fcmp-strict.ll +++ b/llvm/test/CodeGen/RISCV/double-fcmp-strict.ll @@ -24,17 +24,7 @@ define i32 @fcmp_oeq(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmp_oeq: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmp_oeq: @@ -78,20 +68,11 @@ define i32 @fcmp_ogt(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmp_ogt: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: csrr a1, fflags -; RV32IZFINXZDINX-NEXT: flt.d a0, a2, a4 -; RV32IZFINXZDINX-NEXT: csrw fflags, a1 -; RV32IZFINXZDINX-NEXT: feq.d zero, a2, a4 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: csrr a5, fflags +; RV32IZFINXZDINX-NEXT: flt.d a4, a2, a0 +; RV32IZFINXZDINX-NEXT: csrw fflags, a5 +; RV32IZFINXZDINX-NEXT: feq.d zero, a2, a0 +; RV32IZFINXZDINX-NEXT: mv a0, a4 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmp_ogt: @@ -138,20 +119,11 @@ define i32 @fcmp_oge(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmp_oge: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: csrr a1, fflags -; RV32IZFINXZDINX-NEXT: fle.d a0, a2, a4 -; RV32IZFINXZDINX-NEXT: csrw fflags, a1 -; RV32IZFINXZDINX-NEXT: feq.d zero, a2, a4 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: csrr a5, fflags +; RV32IZFINXZDINX-NEXT: fle.d a4, a2, a0 +; RV32IZFINXZDINX-NEXT: csrw fflags, a5 +; RV32IZFINXZDINX-NEXT: feq.d zero, a2, a0 +; RV32IZFINXZDINX-NEXT: mv a0, a4 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmp_oge: @@ -200,20 +172,11 @@ define i32 @fcmp_olt(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmp_olt: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: csrr a1, fflags -; RV32IZFINXZDINX-NEXT: flt.d a0, a4, a2 -; RV32IZFINXZDINX-NEXT: csrw fflags, a1 -; RV32IZFINXZDINX-NEXT: feq.d zero, a4, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: csrr a5, fflags +; RV32IZFINXZDINX-NEXT: flt.d a4, a0, a2 +; RV32IZFINXZDINX-NEXT: csrw fflags, a5 +; RV32IZFINXZDINX-NEXT: feq.d zero, a0, a2 +; RV32IZFINXZDINX-NEXT: mv a0, a4 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmp_olt: @@ -260,20 +223,11 @@ define i32 @fcmp_ole(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmp_ole: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: csrr a1, fflags -; RV32IZFINXZDINX-NEXT: fle.d a0, a4, a2 -; RV32IZFINXZDINX-NEXT: csrw fflags, a1 -; RV32IZFINXZDINX-NEXT: feq.d zero, a4, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: csrr a5, fflags +; RV32IZFINXZDINX-NEXT: fle.d a4, a0, a2 +; RV32IZFINXZDINX-NEXT: csrw fflags, a5 +; RV32IZFINXZDINX-NEXT: feq.d zero, a0, a2 +; RV32IZFINXZDINX-NEXT: mv a0, a4 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmp_ole: @@ -327,25 +281,16 @@ define i32 @fcmp_one(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmp_one: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: csrr a0, fflags -; RV32IZFINXZDINX-NEXT: flt.d a1, a4, a2 -; RV32IZFINXZDINX-NEXT: csrw fflags, a0 -; RV32IZFINXZDINX-NEXT: feq.d zero, a4, a2 -; RV32IZFINXZDINX-NEXT: csrr a0, fflags -; RV32IZFINXZDINX-NEXT: flt.d a6, a2, a4 -; RV32IZFINXZDINX-NEXT: csrw fflags, a0 -; RV32IZFINXZDINX-NEXT: or a0, a6, a1 -; RV32IZFINXZDINX-NEXT: feq.d zero, a2, a4 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: csrr a4, fflags +; RV32IZFINXZDINX-NEXT: flt.d a5, a0, a2 +; RV32IZFINXZDINX-NEXT: csrw fflags, a4 +; RV32IZFINXZDINX-NEXT: feq.d zero, a0, a2 +; RV32IZFINXZDINX-NEXT: csrr a4, fflags +; RV32IZFINXZDINX-NEXT: flt.d a6, a2, a0 +; RV32IZFINXZDINX-NEXT: csrw fflags, a4 +; RV32IZFINXZDINX-NEXT: or a4, a6, a5 +; RV32IZFINXZDINX-NEXT: feq.d zero, a2, a0 +; RV32IZFINXZDINX-NEXT: mv a0, a4 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmp_one: @@ -430,19 +375,9 @@ define i32 @fcmp_ord(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmp_ord: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; RV32IZFINXZDINX-NEXT: feq.d a2, a2, a2 ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmp_ord: @@ -495,26 +430,17 @@ define i32 @fcmp_ueq(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmp_ueq: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: csrr a0, fflags -; RV32IZFINXZDINX-NEXT: flt.d a1, a4, a2 -; RV32IZFINXZDINX-NEXT: csrw fflags, a0 -; RV32IZFINXZDINX-NEXT: feq.d zero, a4, a2 -; RV32IZFINXZDINX-NEXT: csrr a0, fflags -; RV32IZFINXZDINX-NEXT: flt.d a6, a2, a4 -; RV32IZFINXZDINX-NEXT: csrw fflags, a0 -; RV32IZFINXZDINX-NEXT: or a0, a6, a1 -; RV32IZFINXZDINX-NEXT: xori a0, a0, 1 -; RV32IZFINXZDINX-NEXT: feq.d zero, a2, a4 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: csrr a4, fflags +; RV32IZFINXZDINX-NEXT: flt.d a5, a0, a2 +; RV32IZFINXZDINX-NEXT: csrw fflags, a4 +; RV32IZFINXZDINX-NEXT: feq.d zero, a0, a2 +; RV32IZFINXZDINX-NEXT: csrr a4, fflags +; RV32IZFINXZDINX-NEXT: flt.d a6, a2, a0 +; RV32IZFINXZDINX-NEXT: csrw fflags, a4 +; RV32IZFINXZDINX-NEXT: or a4, a6, a5 +; RV32IZFINXZDINX-NEXT: xori a4, a4, 1 +; RV32IZFINXZDINX-NEXT: feq.d zero, a2, a0 +; RV32IZFINXZDINX-NEXT: mv a0, a4 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmp_ueq: @@ -602,21 +528,12 @@ define i32 @fcmp_ugt(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmp_ugt: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: csrr a0, fflags -; RV32IZFINXZDINX-NEXT: fle.d a1, a4, a2 -; RV32IZFINXZDINX-NEXT: csrw fflags, a0 -; RV32IZFINXZDINX-NEXT: xori a0, a1, 1 -; RV32IZFINXZDINX-NEXT: feq.d zero, a4, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: csrr a4, fflags +; RV32IZFINXZDINX-NEXT: fle.d a5, a0, a2 +; RV32IZFINXZDINX-NEXT: csrw fflags, a4 +; RV32IZFINXZDINX-NEXT: xori a4, a5, 1 +; RV32IZFINXZDINX-NEXT: feq.d zero, a0, a2 +; RV32IZFINXZDINX-NEXT: mv a0, a4 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmp_ugt: @@ -665,21 +582,12 @@ define i32 @fcmp_uge(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmp_uge: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: csrr a0, fflags -; RV32IZFINXZDINX-NEXT: flt.d a1, a4, a2 -; RV32IZFINXZDINX-NEXT: csrw fflags, a0 -; RV32IZFINXZDINX-NEXT: xori a0, a1, 1 -; RV32IZFINXZDINX-NEXT: feq.d zero, a4, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: csrr a4, fflags +; RV32IZFINXZDINX-NEXT: flt.d a5, a0, a2 +; RV32IZFINXZDINX-NEXT: csrw fflags, a4 +; RV32IZFINXZDINX-NEXT: xori a4, a5, 1 +; RV32IZFINXZDINX-NEXT: feq.d zero, a0, a2 +; RV32IZFINXZDINX-NEXT: mv a0, a4 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmp_uge: @@ -730,21 +638,12 @@ define i32 @fcmp_ult(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmp_ult: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: csrr a0, fflags -; RV32IZFINXZDINX-NEXT: fle.d a1, a2, a4 -; RV32IZFINXZDINX-NEXT: csrw fflags, a0 -; RV32IZFINXZDINX-NEXT: xori a0, a1, 1 -; RV32IZFINXZDINX-NEXT: feq.d zero, a2, a4 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: csrr a4, fflags +; RV32IZFINXZDINX-NEXT: fle.d a5, a2, a0 +; RV32IZFINXZDINX-NEXT: csrw fflags, a4 +; RV32IZFINXZDINX-NEXT: xori a4, a5, 1 +; RV32IZFINXZDINX-NEXT: feq.d zero, a2, a0 +; RV32IZFINXZDINX-NEXT: mv a0, a4 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmp_ult: @@ -793,21 +692,12 @@ define i32 @fcmp_ule(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmp_ule: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: csrr a0, fflags -; RV32IZFINXZDINX-NEXT: flt.d a1, a2, a4 -; RV32IZFINXZDINX-NEXT: csrw fflags, a0 -; RV32IZFINXZDINX-NEXT: xori a0, a1, 1 -; RV32IZFINXZDINX-NEXT: feq.d zero, a2, a4 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: csrr a4, fflags +; RV32IZFINXZDINX-NEXT: flt.d a5, a2, a0 +; RV32IZFINXZDINX-NEXT: csrw fflags, a4 +; RV32IZFINXZDINX-NEXT: xori a4, a5, 1 +; RV32IZFINXZDINX-NEXT: feq.d zero, a2, a0 +; RV32IZFINXZDINX-NEXT: mv a0, a4 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmp_ule: @@ -853,18 +743,8 @@ define i32 @fcmp_une(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmp_une: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: xori a0, a0, 1 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmp_une: @@ -908,20 +788,10 @@ define i32 @fcmp_uno(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmp_uno: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; RV32IZFINXZDINX-NEXT: feq.d a2, a2, a2 ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 ; RV32IZFINXZDINX-NEXT: xori a0, a0, 1 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmp_uno: @@ -966,19 +836,9 @@ define i32 @fcmps_oeq(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmps_oeq: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; RV32IZFINXZDINX-NEXT: fle.d a4, a2, a0 ; RV32IZFINXZDINX-NEXT: fle.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: and a0, a0, a4 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmps_oeq: @@ -1021,17 +881,7 @@ define i32 @fcmps_ogt(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmps_ogt: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; RV32IZFINXZDINX-NEXT: flt.d a0, a2, a0 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmps_ogt: @@ -1071,17 +921,7 @@ define i32 @fcmps_oge(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmps_oge: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; RV32IZFINXZDINX-NEXT: fle.d a0, a2, a0 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmps_oge: @@ -1123,17 +963,7 @@ define i32 @fcmps_olt(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmps_olt: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: flt.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmps_olt: @@ -1173,17 +1003,7 @@ define i32 @fcmps_ole(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmps_ole: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fle.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmps_ole: @@ -1225,19 +1045,9 @@ define i32 @fcmps_one(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmps_one: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: flt.d a4, a0, a2 ; RV32IZFINXZDINX-NEXT: flt.d a0, a2, a0 ; RV32IZFINXZDINX-NEXT: or a0, a0, a4 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmps_one: @@ -1315,19 +1125,9 @@ define i32 @fcmps_ord(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmps_ord: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; RV32IZFINXZDINX-NEXT: fle.d a2, a2, a2 ; RV32IZFINXZDINX-NEXT: fle.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmps_ord: @@ -1372,20 +1172,10 @@ define i32 @fcmps_ueq(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmps_ueq: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: flt.d a4, a0, a2 ; RV32IZFINXZDINX-NEXT: flt.d a0, a2, a0 ; RV32IZFINXZDINX-NEXT: or a0, a0, a4 ; RV32IZFINXZDINX-NEXT: xori a0, a0, 1 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmps_ueq: @@ -1463,18 +1253,8 @@ define i32 @fcmps_ugt(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmps_ugt: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fle.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: xori a0, a0, 1 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmps_ugt: @@ -1516,18 +1296,8 @@ define i32 @fcmps_uge(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmps_uge: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: flt.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: xori a0, a0, 1 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmps_uge: @@ -1571,18 +1341,8 @@ define i32 @fcmps_ult(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmps_ult: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; RV32IZFINXZDINX-NEXT: fle.d a0, a2, a0 ; RV32IZFINXZDINX-NEXT: xori a0, a0, 1 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmps_ult: @@ -1624,18 +1384,8 @@ define i32 @fcmps_ule(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmps_ule: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; RV32IZFINXZDINX-NEXT: flt.d a0, a2, a0 ; RV32IZFINXZDINX-NEXT: xori a0, a0, 1 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmps_ule: @@ -1679,20 +1429,10 @@ define i32 @fcmps_une(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmps_une: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; RV32IZFINXZDINX-NEXT: fle.d a4, a2, a0 ; RV32IZFINXZDINX-NEXT: fle.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: and a0, a0, a4 ; RV32IZFINXZDINX-NEXT: xori a0, a0, 1 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmps_une: @@ -1738,20 +1478,10 @@ define i32 @fcmps_uno(double %a, double %b) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fcmps_uno: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; RV32IZFINXZDINX-NEXT: fle.d a2, a2, a2 ; RV32IZFINXZDINX-NEXT: fle.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 ; RV32IZFINXZDINX-NEXT: xori a0, a0, 1 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fcmps_uno: diff --git a/llvm/test/CodeGen/RISCV/double-fcmp.ll b/llvm/test/CodeGen/RISCV/double-fcmp.ll index 64a154f450f1..1e609f8081eb 100644 --- a/llvm/test/CodeGen/RISCV/double-fcmp.ll +++ b/llvm/test/CodeGen/RISCV/double-fcmp.ll @@ -45,17 +45,7 @@ define i32 @fcmp_oeq(double %a, double %b) nounwind { ; ; CHECKRV32IZFINXZDINX-LABEL: fcmp_oeq: ; CHECKRV32IZFINXZDINX: # %bb.0: -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32IZFINXZDINX-NEXT: feq.d a0, a0, a2 -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32IZFINXZDINX-NEXT: ret ; ; CHECKRV64IZFINXZDINX-LABEL: fcmp_oeq: @@ -95,17 +85,7 @@ define i32 @fcmp_ogt(double %a, double %b) nounwind { ; ; CHECKRV32IZFINXZDINX-LABEL: fcmp_ogt: ; CHECKRV32IZFINXZDINX: # %bb.0: -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; CHECKRV32IZFINXZDINX-NEXT: flt.d a0, a2, a0 -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32IZFINXZDINX-NEXT: ret ; ; CHECKRV64IZFINXZDINX-LABEL: fcmp_ogt: @@ -145,17 +125,7 @@ define i32 @fcmp_oge(double %a, double %b) nounwind { ; ; CHECKRV32IZFINXZDINX-LABEL: fcmp_oge: ; CHECKRV32IZFINXZDINX: # %bb.0: -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; CHECKRV32IZFINXZDINX-NEXT: fle.d a0, a2, a0 -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32IZFINXZDINX-NEXT: ret ; ; CHECKRV64IZFINXZDINX-LABEL: fcmp_oge: @@ -197,17 +167,7 @@ define i32 @fcmp_olt(double %a, double %b) nounwind { ; ; CHECKRV32IZFINXZDINX-LABEL: fcmp_olt: ; CHECKRV32IZFINXZDINX: # %bb.0: -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32IZFINXZDINX-NEXT: flt.d a0, a0, a2 -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32IZFINXZDINX-NEXT: ret ; ; CHECKRV64IZFINXZDINX-LABEL: fcmp_olt: @@ -247,17 +207,7 @@ define i32 @fcmp_ole(double %a, double %b) nounwind { ; ; CHECKRV32IZFINXZDINX-LABEL: fcmp_ole: ; CHECKRV32IZFINXZDINX: # %bb.0: -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32IZFINXZDINX-NEXT: fle.d a0, a0, a2 -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32IZFINXZDINX-NEXT: ret ; ; CHECKRV64IZFINXZDINX-LABEL: fcmp_ole: @@ -299,19 +249,9 @@ define i32 @fcmp_one(double %a, double %b) nounwind { ; ; CHECKRV32IZFINXZDINX-LABEL: fcmp_one: ; CHECKRV32IZFINXZDINX: # %bb.0: -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32IZFINXZDINX-NEXT: flt.d a4, a0, a2 ; CHECKRV32IZFINXZDINX-NEXT: flt.d a0, a2, a0 ; CHECKRV32IZFINXZDINX-NEXT: or a0, a0, a4 -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32IZFINXZDINX-NEXT: ret ; ; CHECKRV64IZFINXZDINX-LABEL: fcmp_one: @@ -389,19 +329,9 @@ define i32 @fcmp_ord(double %a, double %b) nounwind { ; ; CHECKRV32IZFINXZDINX-LABEL: fcmp_ord: ; CHECKRV32IZFINXZDINX: # %bb.0: -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; CHECKRV32IZFINXZDINX-NEXT: feq.d a2, a2, a2 ; CHECKRV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; CHECKRV32IZFINXZDINX-NEXT: and a0, a0, a2 -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32IZFINXZDINX-NEXT: ret ; ; CHECKRV64IZFINXZDINX-LABEL: fcmp_ord: @@ -446,20 +376,10 @@ define i32 @fcmp_ueq(double %a, double %b) nounwind { ; ; CHECKRV32IZFINXZDINX-LABEL: fcmp_ueq: ; CHECKRV32IZFINXZDINX: # %bb.0: -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32IZFINXZDINX-NEXT: flt.d a4, a0, a2 ; CHECKRV32IZFINXZDINX-NEXT: flt.d a0, a2, a0 ; CHECKRV32IZFINXZDINX-NEXT: or a0, a0, a4 ; CHECKRV32IZFINXZDINX-NEXT: xori a0, a0, 1 -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32IZFINXZDINX-NEXT: ret ; ; CHECKRV64IZFINXZDINX-LABEL: fcmp_ueq: @@ -537,18 +457,8 @@ define i32 @fcmp_ugt(double %a, double %b) nounwind { ; ; CHECKRV32IZFINXZDINX-LABEL: fcmp_ugt: ; CHECKRV32IZFINXZDINX: # %bb.0: -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32IZFINXZDINX-NEXT: fle.d a0, a0, a2 ; CHECKRV32IZFINXZDINX-NEXT: xori a0, a0, 1 -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32IZFINXZDINX-NEXT: ret ; ; CHECKRV64IZFINXZDINX-LABEL: fcmp_ugt: @@ -590,18 +500,8 @@ define i32 @fcmp_uge(double %a, double %b) nounwind { ; ; CHECKRV32IZFINXZDINX-LABEL: fcmp_uge: ; CHECKRV32IZFINXZDINX: # %bb.0: -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32IZFINXZDINX-NEXT: flt.d a0, a0, a2 ; CHECKRV32IZFINXZDINX-NEXT: xori a0, a0, 1 -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32IZFINXZDINX-NEXT: ret ; ; CHECKRV64IZFINXZDINX-LABEL: fcmp_uge: @@ -645,18 +545,8 @@ define i32 @fcmp_ult(double %a, double %b) nounwind { ; ; CHECKRV32IZFINXZDINX-LABEL: fcmp_ult: ; CHECKRV32IZFINXZDINX: # %bb.0: -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; CHECKRV32IZFINXZDINX-NEXT: fle.d a0, a2, a0 ; CHECKRV32IZFINXZDINX-NEXT: xori a0, a0, 1 -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32IZFINXZDINX-NEXT: ret ; ; CHECKRV64IZFINXZDINX-LABEL: fcmp_ult: @@ -698,18 +588,8 @@ define i32 @fcmp_ule(double %a, double %b) nounwind { ; ; CHECKRV32IZFINXZDINX-LABEL: fcmp_ule: ; CHECKRV32IZFINXZDINX: # %bb.0: -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; CHECKRV32IZFINXZDINX-NEXT: flt.d a0, a2, a0 ; CHECKRV32IZFINXZDINX-NEXT: xori a0, a0, 1 -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32IZFINXZDINX-NEXT: ret ; ; CHECKRV64IZFINXZDINX-LABEL: fcmp_ule: @@ -751,18 +631,8 @@ define i32 @fcmp_une(double %a, double %b) nounwind { ; ; CHECKRV32IZFINXZDINX-LABEL: fcmp_une: ; CHECKRV32IZFINXZDINX: # %bb.0: -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32IZFINXZDINX-NEXT: feq.d a0, a0, a2 ; CHECKRV32IZFINXZDINX-NEXT: xori a0, a0, 1 -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32IZFINXZDINX-NEXT: ret ; ; CHECKRV64IZFINXZDINX-LABEL: fcmp_une: @@ -806,20 +676,10 @@ define i32 @fcmp_uno(double %a, double %b) nounwind { ; ; CHECKRV32IZFINXZDINX-LABEL: fcmp_uno: ; CHECKRV32IZFINXZDINX: # %bb.0: -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32IZFINXZDINX-NEXT: lw a3, 12(sp) ; CHECKRV32IZFINXZDINX-NEXT: feq.d a2, a2, a2 ; CHECKRV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; CHECKRV32IZFINXZDINX-NEXT: and a0, a0, a2 ; CHECKRV32IZFINXZDINX-NEXT: xori a0, a0, 1 -; CHECKRV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32IZFINXZDINX-NEXT: ret ; ; CHECKRV64IZFINXZDINX-LABEL: fcmp_uno: diff --git a/llvm/test/CodeGen/RISCV/double-imm.ll b/llvm/test/CodeGen/RISCV/double-imm.ll index 9254369baf19..74d4acc4f23f 100644 --- a/llvm/test/CodeGen/RISCV/double-imm.ll +++ b/llvm/test/CodeGen/RISCV/double-imm.ll @@ -54,20 +54,10 @@ define double @double_imm_op(double %a) nounwind { ; ; CHECKRV32ZDINX-LABEL: double_imm_op: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32ZDINX-NEXT: lui a2, %hi(.LCPI1_0) ; CHECKRV32ZDINX-NEXT: lw a3, %lo(.LCPI1_0+4)(a2) ; CHECKRV32ZDINX-NEXT: lw a2, %lo(.LCPI1_0)(a2) ; CHECKRV32ZDINX-NEXT: fadd.d a0, a0, a2 -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: double_imm_op: @@ -153,24 +143,18 @@ define dso_local double @negzero_sel(i16 noundef %a, double noundef %d) nounwind ; ; CHECKRV32ZDINX-LABEL: negzero_sel: ; CHECKRV32ZDINX: # %bb.0: # %entry -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a1, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a2, 12(sp) -; CHECKRV32ZDINX-NEXT: slli a2, a0, 16 -; CHECKRV32ZDINX-NEXT: fcvt.d.w a0, zero -; CHECKRV32ZDINX-NEXT: beqz a2, .LBB4_2 +; CHECKRV32ZDINX-NEXT: slli a0, a0, 16 +; CHECKRV32ZDINX-NEXT: fcvt.d.w a4, zero +; CHECKRV32ZDINX-NEXT: beqz a0, .LBB4_2 ; CHECKRV32ZDINX-NEXT: # %bb.1: # %entry -; CHECKRV32ZDINX-NEXT: fneg.d a0, a0 +; CHECKRV32ZDINX-NEXT: fneg.d a2, a4 ; CHECKRV32ZDINX-NEXT: j .LBB4_3 ; CHECKRV32ZDINX-NEXT: .LBB4_2: -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) +; CHECKRV32ZDINX-NEXT: mv a3, a2 +; CHECKRV32ZDINX-NEXT: mv a2, a1 ; CHECKRV32ZDINX-NEXT: .LBB4_3: # %entry -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 +; CHECKRV32ZDINX-NEXT: mv a0, a2 +; CHECKRV32ZDINX-NEXT: mv a1, a3 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: negzero_sel: diff --git a/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll b/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll index c574f64150a2..38215860193e 100644 --- a/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll +++ b/llvm/test/CodeGen/RISCV/double-intrinsics-strict.ll @@ -28,17 +28,7 @@ define double @sqrt_f64(double %a) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: sqrt_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fsqrt.d a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: sqrt_f64: @@ -299,22 +289,12 @@ define double @sincos_f64(double %a) nounwind strictfp { ; RV32IZFINXZDINX-NEXT: mv s0, a1 ; RV32IZFINXZDINX-NEXT: mv s1, a0 ; RV32IZFINXZDINX-NEXT: call sin -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw s2, 0(sp) -; RV32IZFINXZDINX-NEXT: lw s3, 4(sp) +; RV32IZFINXZDINX-NEXT: mv s2, a0 +; RV32IZFINXZDINX-NEXT: mv s3, a1 ; RV32IZFINXZDINX-NEXT: mv a0, s1 ; RV32IZFINXZDINX-NEXT: mv a1, s0 ; RV32IZFINXZDINX-NEXT: call cos -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) ; RV32IZFINXZDINX-NEXT: fadd.d a0, s2, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) ; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload @@ -765,25 +745,7 @@ define double @fma_f64(double %a, double %b, double %c) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fma_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fmadd.d a0, a0, a2, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fma_f64: @@ -822,25 +784,7 @@ define double @fmuladd_f64(double %a, double %b, double %c) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: fmuladd_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fmadd.d a0, a0, a2, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmuladd_f64: @@ -1455,13 +1399,7 @@ define iXLen @lrint_f64(double %a) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: lrint_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: lrint_f64: @@ -1505,13 +1443,7 @@ define iXLen @lround_f64(double %a) nounwind strictfp { ; ; RV32IZFINXZDINX-LABEL: lround_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rmm -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: lround_f64: diff --git a/llvm/test/CodeGen/RISCV/double-intrinsics.ll b/llvm/test/CodeGen/RISCV/double-intrinsics.ll index f290cf0f7736..52c49cfbfb30 100644 --- a/llvm/test/CodeGen/RISCV/double-intrinsics.ll +++ b/llvm/test/CodeGen/RISCV/double-intrinsics.ll @@ -26,17 +26,7 @@ define double @sqrt_f64(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: sqrt_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fsqrt.d a0, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: sqrt_f64: @@ -254,22 +244,12 @@ define double @sincos_f64(double %a) nounwind { ; RV32IZFINXZDINX-NEXT: mv s0, a1 ; RV32IZFINXZDINX-NEXT: mv s1, a0 ; RV32IZFINXZDINX-NEXT: call sin -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw s2, 0(sp) -; RV32IZFINXZDINX-NEXT: lw s3, 4(sp) +; RV32IZFINXZDINX-NEXT: mv s2, a0 +; RV32IZFINXZDINX-NEXT: mv s3, a1 ; RV32IZFINXZDINX-NEXT: mv a0, s1 ; RV32IZFINXZDINX-NEXT: mv a1, s0 ; RV32IZFINXZDINX-NEXT: call cos -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) ; RV32IZFINXZDINX-NEXT: fadd.d a0, s2, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) ; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload @@ -606,25 +586,7 @@ define double @fma_f64(double %a, double %b, double %c) nounwind { ; ; RV32IZFINXZDINX-LABEL: fma_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fmadd.d a0, a0, a2, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fma_f64: @@ -663,25 +625,7 @@ define double @fmuladd_f64(double %a, double %b, double %c) nounwind { ; ; RV32IZFINXZDINX-LABEL: fmuladd_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fmadd.d a0, a0, a2, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmuladd_f64: @@ -769,21 +713,7 @@ define double @minnum_f64(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: minnum_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fmin.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: minnum_f64: @@ -822,21 +752,7 @@ define double @maxnum_f64(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: maxnum_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fmax.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: maxnum_f64: @@ -892,21 +808,7 @@ define double @copysign_f64(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: copysign_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fsgnj.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: copysign_f64: @@ -1381,13 +1283,7 @@ define iXLen @lrint_f64(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: lrint_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: lrint_f64: @@ -1432,13 +1328,7 @@ define iXLen @lround_f64(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: lround_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rmm -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: lround_f64: @@ -1475,13 +1365,7 @@ define i32 @lround_i32_f64(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: lround_i32_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rmm -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: lround_i32_f64: @@ -1625,16 +1509,9 @@ define i1 @isnan_d_fpclass(double %x) { ; ; RV32IZFINXZDINX-LABEL: isnan_d_fpclass: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fclass.d a0, a0 ; RV32IZFINXZDINX-NEXT: andi a0, a0, 768 ; RV32IZFINXZDINX-NEXT: snez a0, a0 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: isnan_d_fpclass: diff --git a/llvm/test/CodeGen/RISCV/double-isnan.ll b/llvm/test/CodeGen/RISCV/double-isnan.ll index 4d0b8151f3c4..6a3779dc2d36 100644 --- a/llvm/test/CodeGen/RISCV/double-isnan.ll +++ b/llvm/test/CodeGen/RISCV/double-isnan.ll @@ -17,14 +17,8 @@ define zeroext i1 @double_is_nan(double %a) nounwind { ; ; CHECKRV32ZDINX-LABEL: double_is_nan: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32ZDINX-NEXT: feq.d a0, a0, a0 ; CHECKRV32ZDINX-NEXT: xori a0, a0, 1 -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: double_is_nan: @@ -44,13 +38,7 @@ define zeroext i1 @double_not_nan(double %a) nounwind { ; ; CHECKRV32ZDINX-LABEL: double_not_nan: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32ZDINX-NEXT: feq.d a0, a0, a0 -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: double_not_nan: diff --git a/llvm/test/CodeGen/RISCV/double-maximum-minimum.ll b/llvm/test/CodeGen/RISCV/double-maximum-minimum.ll index 0ca20783591a..5229117caa2c 100644 --- a/llvm/test/CodeGen/RISCV/double-maximum-minimum.ll +++ b/llvm/test/CodeGen/RISCV/double-maximum-minimum.ll @@ -36,35 +36,25 @@ define double @fminimum_f64(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fminimum_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: feq.d a6, a0, a0 ; RV32IZFINXZDINX-NEXT: mv a4, a2 ; RV32IZFINXZDINX-NEXT: mv a5, a3 -; RV32IZFINXZDINX-NEXT: bnez a6, .LBB0_2 +; RV32IZFINXZDINX-NEXT: beqz a6, .LBB0_3 ; RV32IZFINXZDINX-NEXT: # %bb.1: +; RV32IZFINXZDINX-NEXT: feq.d a6, a2, a2 +; RV32IZFINXZDINX-NEXT: beqz a6, .LBB0_4 +; RV32IZFINXZDINX-NEXT: .LBB0_2: +; RV32IZFINXZDINX-NEXT: fmin.d a0, a0, a4 +; RV32IZFINXZDINX-NEXT: ret +; RV32IZFINXZDINX-NEXT: .LBB0_3: ; RV32IZFINXZDINX-NEXT: mv a4, a0 ; RV32IZFINXZDINX-NEXT: mv a5, a1 -; RV32IZFINXZDINX-NEXT: .LBB0_2: ; RV32IZFINXZDINX-NEXT: feq.d a6, a2, a2 -; RV32IZFINXZDINX-NEXT: bnez a6, .LBB0_4 -; RV32IZFINXZDINX-NEXT: # %bb.3: +; RV32IZFINXZDINX-NEXT: bnez a6, .LBB0_2 +; RV32IZFINXZDINX-NEXT: .LBB0_4: ; RV32IZFINXZDINX-NEXT: mv a0, a2 ; RV32IZFINXZDINX-NEXT: mv a1, a3 -; RV32IZFINXZDINX-NEXT: .LBB0_4: ; RV32IZFINXZDINX-NEXT: fmin.d a0, a0, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fminimum_f64: @@ -113,35 +103,25 @@ define double @fmaximum_f64(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fmaximum_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: feq.d a6, a0, a0 ; RV32IZFINXZDINX-NEXT: mv a4, a2 ; RV32IZFINXZDINX-NEXT: mv a5, a3 -; RV32IZFINXZDINX-NEXT: bnez a6, .LBB1_2 +; RV32IZFINXZDINX-NEXT: beqz a6, .LBB1_3 ; RV32IZFINXZDINX-NEXT: # %bb.1: +; RV32IZFINXZDINX-NEXT: feq.d a6, a2, a2 +; RV32IZFINXZDINX-NEXT: beqz a6, .LBB1_4 +; RV32IZFINXZDINX-NEXT: .LBB1_2: +; RV32IZFINXZDINX-NEXT: fmax.d a0, a0, a4 +; RV32IZFINXZDINX-NEXT: ret +; RV32IZFINXZDINX-NEXT: .LBB1_3: ; RV32IZFINXZDINX-NEXT: mv a4, a0 ; RV32IZFINXZDINX-NEXT: mv a5, a1 -; RV32IZFINXZDINX-NEXT: .LBB1_2: ; RV32IZFINXZDINX-NEXT: feq.d a6, a2, a2 -; RV32IZFINXZDINX-NEXT: bnez a6, .LBB1_4 -; RV32IZFINXZDINX-NEXT: # %bb.3: +; RV32IZFINXZDINX-NEXT: bnez a6, .LBB1_2 +; RV32IZFINXZDINX-NEXT: .LBB1_4: ; RV32IZFINXZDINX-NEXT: mv a0, a2 ; RV32IZFINXZDINX-NEXT: mv a1, a3 -; RV32IZFINXZDINX-NEXT: .LBB1_4: ; RV32IZFINXZDINX-NEXT: fmax.d a0, a0, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmaximum_f64: @@ -174,21 +154,7 @@ define double @fminimum_nnan_f64(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fminimum_nnan_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fmin.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fminimum_nnan_f64: @@ -221,35 +187,25 @@ define double @fmaximum_nnan_f64(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fmaximum_nnan_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: feq.d a6, a0, a0 ; RV32IZFINXZDINX-NEXT: mv a4, a2 ; RV32IZFINXZDINX-NEXT: mv a5, a3 -; RV32IZFINXZDINX-NEXT: bnez a6, .LBB3_2 +; RV32IZFINXZDINX-NEXT: beqz a6, .LBB3_3 ; RV32IZFINXZDINX-NEXT: # %bb.1: +; RV32IZFINXZDINX-NEXT: feq.d a6, a2, a2 +; RV32IZFINXZDINX-NEXT: beqz a6, .LBB3_4 +; RV32IZFINXZDINX-NEXT: .LBB3_2: +; RV32IZFINXZDINX-NEXT: fmin.d a0, a0, a4 +; RV32IZFINXZDINX-NEXT: ret +; RV32IZFINXZDINX-NEXT: .LBB3_3: ; RV32IZFINXZDINX-NEXT: mv a4, a0 ; RV32IZFINXZDINX-NEXT: mv a5, a1 -; RV32IZFINXZDINX-NEXT: .LBB3_2: ; RV32IZFINXZDINX-NEXT: feq.d a6, a2, a2 -; RV32IZFINXZDINX-NEXT: bnez a6, .LBB3_4 -; RV32IZFINXZDINX-NEXT: # %bb.3: +; RV32IZFINXZDINX-NEXT: bnez a6, .LBB3_2 +; RV32IZFINXZDINX-NEXT: .LBB3_4: ; RV32IZFINXZDINX-NEXT: mv a0, a2 ; RV32IZFINXZDINX-NEXT: mv a1, a3 -; RV32IZFINXZDINX-NEXT: .LBB3_4: ; RV32IZFINXZDINX-NEXT: fmin.d a0, a0, a4 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmaximum_nnan_f64: @@ -289,30 +245,14 @@ define double @fminimum_nnan_op_f64(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fminimum_nnan_op_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: feq.d a0, a2, a2 -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: bnez a0, .LBB4_2 +; RV32IZFINXZDINX-NEXT: feq.d a4, a2, a2 +; RV32IZFINXZDINX-NEXT: bnez a4, .LBB4_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: -; RV32IZFINXZDINX-NEXT: mv a0, a2 -; RV32IZFINXZDINX-NEXT: mv a1, a3 -; RV32IZFINXZDINX-NEXT: j .LBB4_3 +; RV32IZFINXZDINX-NEXT: fmin.d a0, a2, a2 +; RV32IZFINXZDINX-NEXT: ret ; RV32IZFINXZDINX-NEXT: .LBB4_2: -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a0 -; RV32IZFINXZDINX-NEXT: .LBB4_3: ; RV32IZFINXZDINX-NEXT: fmin.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fminimum_nnan_op_f64: @@ -341,23 +281,9 @@ define double @fmaximum_nnan_op_f64(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fmaximum_nnan_op_f64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fadd.d a4, a0, a2 ; RV32IZFINXZDINX-NEXT: fsub.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: fmax.d a0, a4, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmaximum_nnan_op_f64: diff --git a/llvm/test/CodeGen/RISCV/double-mem.ll b/llvm/test/CodeGen/RISCV/double-mem.ll index 6c6f70d6e2ed..38cb52b6f4b3 100644 --- a/llvm/test/CodeGen/RISCV/double-mem.ll +++ b/llvm/test/CodeGen/RISCV/double-mem.ll @@ -18,17 +18,11 @@ define dso_local double @fld(ptr %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fld: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: lw a2, 0(a0) ; RV32IZFINXZDINX-NEXT: lw a3, 4(a0) ; RV32IZFINXZDINX-NEXT: lw a1, 28(a0) ; RV32IZFINXZDINX-NEXT: lw a0, 24(a0) ; RV32IZFINXZDINX-NEXT: fadd.d a0, a2, a0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fld: @@ -56,21 +50,15 @@ define dso_local void @fsd(ptr %a, double %b, double %c) nounwind { ; ; RV32IZFINXZDINX-LABEL: fsd: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a3, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a4, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a4, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a5, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: fadd.d a2, a2, a4 +; RV32IZFINXZDINX-NEXT: mv a5, a4 +; RV32IZFINXZDINX-NEXT: mv a7, a2 +; RV32IZFINXZDINX-NEXT: mv a4, a3 +; RV32IZFINXZDINX-NEXT: mv a6, a1 +; RV32IZFINXZDINX-NEXT: fadd.d a2, a6, a4 ; RV32IZFINXZDINX-NEXT: sw a2, 0(a0) ; RV32IZFINXZDINX-NEXT: sw a3, 4(a0) ; RV32IZFINXZDINX-NEXT: sw a2, 64(a0) ; RV32IZFINXZDINX-NEXT: sw a3, 68(a0) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fsd: @@ -105,15 +93,6 @@ define dso_local double @fld_fsd_global(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fld_fsd_global: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: lui a2, %hi(G) ; RV32IZFINXZDINX-NEXT: lw a4, %lo(G)(a2) @@ -125,11 +104,6 @@ define dso_local double @fld_fsd_global(double %a, double %b) nounwind { ; RV32IZFINXZDINX-NEXT: lw a5, 76(a3) ; RV32IZFINXZDINX-NEXT: sw a0, 72(a3) ; RV32IZFINXZDINX-NEXT: sw a1, 76(a3) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fld_fsd_global: @@ -174,22 +148,12 @@ define dso_local double @fld_fsd_constant(double %a) nounwind { ; ; RV32IZFINXZDINX-LABEL: fld_fsd_constant: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: lui a2, 912092 ; RV32IZFINXZDINX-NEXT: lw a4, -273(a2) ; RV32IZFINXZDINX-NEXT: lw a5, -269(a2) ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a4 ; RV32IZFINXZDINX-NEXT: sw a0, -273(a2) ; RV32IZFINXZDINX-NEXT: sw a1, -269(a2) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fld_fsd_constant: @@ -246,19 +210,13 @@ define dso_local double @fld_stack(double %a) nounwind { ; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw s0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) +; RV32IZFINXZDINX-NEXT: mv s1, a1 +; RV32IZFINXZDINX-NEXT: mv s0, a0 ; RV32IZFINXZDINX-NEXT: addi a0, sp, 8 ; RV32IZFINXZDINX-NEXT: call notdead ; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) ; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, s0 -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) ; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload ; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload @@ -313,23 +271,15 @@ define dso_local void @fsd_stack(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fsd_stack: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -32 -; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 16(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 20(sp) -; RV32IZFINXZDINX-NEXT: addi a0, sp, 16 +; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) +; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) +; RV32IZFINXZDINX-NEXT: mv a0, sp ; RV32IZFINXZDINX-NEXT: call notdead -; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 +; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fsd_stack: @@ -360,14 +310,10 @@ define dso_local void @fsd_trunc(ptr %a, double %b) nounwind noinline optnone { ; ; RV32IZFINXZDINX-LABEL: fsd_trunc: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a1, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) +; RV32IZFINXZDINX-NEXT: mv a3, a2 +; RV32IZFINXZDINX-NEXT: mv a2, a1 ; RV32IZFINXZDINX-NEXT: fcvt.s.d a1, a2 ; RV32IZFINXZDINX-NEXT: sw a1, 0(a0) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fsd_trunc: diff --git a/llvm/test/CodeGen/RISCV/double-previous-failure.ll b/llvm/test/CodeGen/RISCV/double-previous-failure.ll index 8b8f538886ed..c169b1099b27 100644 --- a/llvm/test/CodeGen/RISCV/double-previous-failure.ll +++ b/llvm/test/CodeGen/RISCV/double-previous-failure.ll @@ -50,10 +50,6 @@ define i32 @main() nounwind { ; RV32IZFINXZDINX-NEXT: lui a1, 262144 ; RV32IZFINXZDINX-NEXT: li a0, 0 ; RV32IZFINXZDINX-NEXT: call test -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI1_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI1_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI1_0)(a2) diff --git a/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll b/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll index 7cdf18e2fea9..29a9e5070950 100644 --- a/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll +++ b/llvm/test/CodeGen/RISCV/double-round-conv-sat.ll @@ -20,18 +20,11 @@ define signext i32 @test_floor_si32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_floor_si32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a2, a0, rdn ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: seqz a0, a0 ; RV32IZFINXZDINX-NEXT: addi a0, a0, -1 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_floor_si32: @@ -112,13 +105,11 @@ define i64 @test_floor_si64(double %x) nounwind { ; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call floor -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw s0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI1_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI1_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI1_0)(a2) +; RV32IZFINXZDINX-NEXT: mv s0, a0 +; RV32IZFINXZDINX-NEXT: mv s1, a1 ; RV32IZFINXZDINX-NEXT: fle.d s2, a2, s0 ; RV32IZFINXZDINX-NEXT: neg s3, s2 ; RV32IZFINXZDINX-NEXT: call __fixdfdi @@ -177,18 +168,11 @@ define signext i32 @test_floor_ui32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_floor_ui32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a2, a0, rdn ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: seqz a0, a0 ; RV32IZFINXZDINX-NEXT: addi a0, a0, -1 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_floor_ui32: @@ -241,38 +225,30 @@ define i64 @test_floor_ui64(double %x) nounwind { ; ; RV32IZFINXZDINX-LABEL: test_floor_ui64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -32 -; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call floor -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw s1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw s0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw s1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) +; RV32IZFINXZDINX-NEXT: mv s0, a0 +; RV32IZFINXZDINX-NEXT: mv s1, a1 +; RV32IZFINXZDINX-NEXT: call __fixunsdfdi ; RV32IZFINXZDINX-NEXT: fcvt.d.w a2, zero +; RV32IZFINXZDINX-NEXT: lui a4, %hi(.LCPI3_0) +; RV32IZFINXZDINX-NEXT: lw a5, %lo(.LCPI3_0+4)(a4) +; RV32IZFINXZDINX-NEXT: lw a4, %lo(.LCPI3_0)(a4) ; RV32IZFINXZDINX-NEXT: fle.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg s2, a2 -; RV32IZFINXZDINX-NEXT: call __fixunsdfdi -; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI3_0) -; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI3_0+4)(a2) -; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI3_0)(a2) -; RV32IZFINXZDINX-NEXT: and a0, s2, a0 -; RV32IZFINXZDINX-NEXT: flt.d a2, a2, s0 ; RV32IZFINXZDINX-NEXT: neg a2, a2 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a1, s2, a1 -; RV32IZFINXZDINX-NEXT: or a1, a2, a1 -; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a4, s0 +; RV32IZFINXZDINX-NEXT: neg a3, a3 +; RV32IZFINXZDINX-NEXT: or a0, a3, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a1 +; RV32IZFINXZDINX-NEXT: or a1, a3, a1 +; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_floor_ui64: @@ -300,18 +276,11 @@ define signext i32 @test_ceil_si32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_ceil_si32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a2, a0, rup ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: seqz a0, a0 ; RV32IZFINXZDINX-NEXT: addi a0, a0, -1 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_ceil_si32: @@ -392,13 +361,11 @@ define i64 @test_ceil_si64(double %x) nounwind { ; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call ceil -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw s0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI5_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI5_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI5_0)(a2) +; RV32IZFINXZDINX-NEXT: mv s0, a0 +; RV32IZFINXZDINX-NEXT: mv s1, a1 ; RV32IZFINXZDINX-NEXT: fle.d s2, a2, s0 ; RV32IZFINXZDINX-NEXT: neg s3, s2 ; RV32IZFINXZDINX-NEXT: call __fixdfdi @@ -457,18 +424,11 @@ define signext i32 @test_ceil_ui32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_ceil_ui32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a2, a0, rup ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: seqz a0, a0 ; RV32IZFINXZDINX-NEXT: addi a0, a0, -1 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_ceil_ui32: @@ -521,38 +481,30 @@ define i64 @test_ceil_ui64(double %x) nounwind { ; ; RV32IZFINXZDINX-LABEL: test_ceil_ui64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -32 -; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call ceil -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw s1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw s0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw s1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) +; RV32IZFINXZDINX-NEXT: mv s0, a0 +; RV32IZFINXZDINX-NEXT: mv s1, a1 +; RV32IZFINXZDINX-NEXT: call __fixunsdfdi ; RV32IZFINXZDINX-NEXT: fcvt.d.w a2, zero +; RV32IZFINXZDINX-NEXT: lui a4, %hi(.LCPI7_0) +; RV32IZFINXZDINX-NEXT: lw a5, %lo(.LCPI7_0+4)(a4) +; RV32IZFINXZDINX-NEXT: lw a4, %lo(.LCPI7_0)(a4) ; RV32IZFINXZDINX-NEXT: fle.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg s2, a2 -; RV32IZFINXZDINX-NEXT: call __fixunsdfdi -; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI7_0) -; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI7_0+4)(a2) -; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI7_0)(a2) -; RV32IZFINXZDINX-NEXT: and a0, s2, a0 -; RV32IZFINXZDINX-NEXT: flt.d a2, a2, s0 ; RV32IZFINXZDINX-NEXT: neg a2, a2 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a1, s2, a1 -; RV32IZFINXZDINX-NEXT: or a1, a2, a1 -; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a4, s0 +; RV32IZFINXZDINX-NEXT: neg a3, a3 +; RV32IZFINXZDINX-NEXT: or a0, a3, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a1 +; RV32IZFINXZDINX-NEXT: or a1, a3, a1 +; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_ceil_ui64: @@ -580,18 +532,11 @@ define signext i32 @test_trunc_si32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_trunc_si32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a2, a0, rtz ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: seqz a0, a0 ; RV32IZFINXZDINX-NEXT: addi a0, a0, -1 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_trunc_si32: @@ -672,13 +617,11 @@ define i64 @test_trunc_si64(double %x) nounwind { ; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call trunc -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw s0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI9_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI9_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI9_0)(a2) +; RV32IZFINXZDINX-NEXT: mv s0, a0 +; RV32IZFINXZDINX-NEXT: mv s1, a1 ; RV32IZFINXZDINX-NEXT: fle.d s2, a2, s0 ; RV32IZFINXZDINX-NEXT: neg s3, s2 ; RV32IZFINXZDINX-NEXT: call __fixdfdi @@ -737,18 +680,11 @@ define signext i32 @test_trunc_ui32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_trunc_ui32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a2, a0, rtz ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: seqz a0, a0 ; RV32IZFINXZDINX-NEXT: addi a0, a0, -1 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_trunc_ui32: @@ -801,38 +737,30 @@ define i64 @test_trunc_ui64(double %x) nounwind { ; ; RV32IZFINXZDINX-LABEL: test_trunc_ui64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -32 -; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call trunc -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw s1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw s0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw s1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) +; RV32IZFINXZDINX-NEXT: mv s0, a0 +; RV32IZFINXZDINX-NEXT: mv s1, a1 +; RV32IZFINXZDINX-NEXT: call __fixunsdfdi ; RV32IZFINXZDINX-NEXT: fcvt.d.w a2, zero +; RV32IZFINXZDINX-NEXT: lui a4, %hi(.LCPI11_0) +; RV32IZFINXZDINX-NEXT: lw a5, %lo(.LCPI11_0+4)(a4) +; RV32IZFINXZDINX-NEXT: lw a4, %lo(.LCPI11_0)(a4) ; RV32IZFINXZDINX-NEXT: fle.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg s2, a2 -; RV32IZFINXZDINX-NEXT: call __fixunsdfdi -; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI11_0) -; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI11_0+4)(a2) -; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI11_0)(a2) -; RV32IZFINXZDINX-NEXT: and a0, s2, a0 -; RV32IZFINXZDINX-NEXT: flt.d a2, a2, s0 ; RV32IZFINXZDINX-NEXT: neg a2, a2 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a1, s2, a1 -; RV32IZFINXZDINX-NEXT: or a1, a2, a1 -; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a4, s0 +; RV32IZFINXZDINX-NEXT: neg a3, a3 +; RV32IZFINXZDINX-NEXT: or a0, a3, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a1 +; RV32IZFINXZDINX-NEXT: or a1, a3, a1 +; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_trunc_ui64: @@ -860,18 +788,11 @@ define signext i32 @test_round_si32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_round_si32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a2, a0, rmm ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: seqz a0, a0 ; RV32IZFINXZDINX-NEXT: addi a0, a0, -1 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_round_si32: @@ -952,13 +873,11 @@ define i64 @test_round_si64(double %x) nounwind { ; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call round -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw s0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI13_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI13_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI13_0)(a2) +; RV32IZFINXZDINX-NEXT: mv s0, a0 +; RV32IZFINXZDINX-NEXT: mv s1, a1 ; RV32IZFINXZDINX-NEXT: fle.d s2, a2, s0 ; RV32IZFINXZDINX-NEXT: neg s3, s2 ; RV32IZFINXZDINX-NEXT: call __fixdfdi @@ -1017,18 +936,11 @@ define signext i32 @test_round_ui32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_round_ui32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a2, a0, rmm ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: seqz a0, a0 ; RV32IZFINXZDINX-NEXT: addi a0, a0, -1 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_round_ui32: @@ -1081,38 +993,30 @@ define i64 @test_round_ui64(double %x) nounwind { ; ; RV32IZFINXZDINX-LABEL: test_round_ui64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -32 -; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call round -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw s1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw s0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw s1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) +; RV32IZFINXZDINX-NEXT: mv s0, a0 +; RV32IZFINXZDINX-NEXT: mv s1, a1 +; RV32IZFINXZDINX-NEXT: call __fixunsdfdi ; RV32IZFINXZDINX-NEXT: fcvt.d.w a2, zero +; RV32IZFINXZDINX-NEXT: lui a4, %hi(.LCPI15_0) +; RV32IZFINXZDINX-NEXT: lw a5, %lo(.LCPI15_0+4)(a4) +; RV32IZFINXZDINX-NEXT: lw a4, %lo(.LCPI15_0)(a4) ; RV32IZFINXZDINX-NEXT: fle.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg s2, a2 -; RV32IZFINXZDINX-NEXT: call __fixunsdfdi -; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI15_0) -; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI15_0+4)(a2) -; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI15_0)(a2) -; RV32IZFINXZDINX-NEXT: and a0, s2, a0 -; RV32IZFINXZDINX-NEXT: flt.d a2, a2, s0 ; RV32IZFINXZDINX-NEXT: neg a2, a2 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a1, s2, a1 -; RV32IZFINXZDINX-NEXT: or a1, a2, a1 -; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a4, s0 +; RV32IZFINXZDINX-NEXT: neg a3, a3 +; RV32IZFINXZDINX-NEXT: or a0, a3, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a1 +; RV32IZFINXZDINX-NEXT: or a1, a3, a1 +; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_round_ui64: @@ -1140,18 +1044,11 @@ define signext i32 @test_roundeven_si32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_roundeven_si32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a2, a0, rne ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: seqz a0, a0 ; RV32IZFINXZDINX-NEXT: addi a0, a0, -1 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_roundeven_si32: @@ -1232,13 +1129,11 @@ define i64 @test_roundeven_si64(double %x) nounwind { ; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call roundeven -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw s0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI17_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI17_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI17_0)(a2) +; RV32IZFINXZDINX-NEXT: mv s0, a0 +; RV32IZFINXZDINX-NEXT: mv s1, a1 ; RV32IZFINXZDINX-NEXT: fle.d s2, a2, s0 ; RV32IZFINXZDINX-NEXT: neg s3, s2 ; RV32IZFINXZDINX-NEXT: call __fixdfdi @@ -1297,18 +1192,11 @@ define signext i32 @test_roundeven_ui32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_roundeven_ui32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a2, a0, rne ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: seqz a0, a0 ; RV32IZFINXZDINX-NEXT: addi a0, a0, -1 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_roundeven_ui32: @@ -1361,38 +1249,30 @@ define i64 @test_roundeven_ui64(double %x) nounwind { ; ; RV32IZFINXZDINX-LABEL: test_roundeven_ui64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -32 -; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call roundeven -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw s1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw s0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw s1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) +; RV32IZFINXZDINX-NEXT: mv s0, a0 +; RV32IZFINXZDINX-NEXT: mv s1, a1 +; RV32IZFINXZDINX-NEXT: call __fixunsdfdi ; RV32IZFINXZDINX-NEXT: fcvt.d.w a2, zero +; RV32IZFINXZDINX-NEXT: lui a4, %hi(.LCPI19_0) +; RV32IZFINXZDINX-NEXT: lw a5, %lo(.LCPI19_0+4)(a4) +; RV32IZFINXZDINX-NEXT: lw a4, %lo(.LCPI19_0)(a4) ; RV32IZFINXZDINX-NEXT: fle.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg s2, a2 -; RV32IZFINXZDINX-NEXT: call __fixunsdfdi -; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI19_0) -; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI19_0+4)(a2) -; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI19_0)(a2) -; RV32IZFINXZDINX-NEXT: and a0, s2, a0 -; RV32IZFINXZDINX-NEXT: flt.d a2, a2, s0 ; RV32IZFINXZDINX-NEXT: neg a2, a2 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a1, s2, a1 -; RV32IZFINXZDINX-NEXT: or a1, a2, a1 -; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a4, s0 +; RV32IZFINXZDINX-NEXT: neg a3, a3 +; RV32IZFINXZDINX-NEXT: or a0, a3, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a1 +; RV32IZFINXZDINX-NEXT: or a1, a3, a1 +; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_roundeven_ui64: @@ -1420,18 +1300,11 @@ define signext i32 @test_rint_si32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_rint_si32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a2, a0 ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: seqz a0, a0 ; RV32IZFINXZDINX-NEXT: addi a0, a0, -1 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_rint_si32: @@ -1512,13 +1385,11 @@ define i64 @test_rint_si64(double %x) nounwind { ; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: sw s3, 12(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call rint -; RV32IZFINXZDINX-NEXT: sw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw s0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) ; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI21_0) ; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI21_0+4)(a2) ; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI21_0)(a2) +; RV32IZFINXZDINX-NEXT: mv s0, a0 +; RV32IZFINXZDINX-NEXT: mv s1, a1 ; RV32IZFINXZDINX-NEXT: fle.d s2, a2, s0 ; RV32IZFINXZDINX-NEXT: neg s3, s2 ; RV32IZFINXZDINX-NEXT: call __fixdfdi @@ -1577,18 +1448,11 @@ define signext i32 @test_rint_ui32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_rint_ui32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a2, a0 ; RV32IZFINXZDINX-NEXT: feq.d a0, a0, a0 ; RV32IZFINXZDINX-NEXT: seqz a0, a0 ; RV32IZFINXZDINX-NEXT: addi a0, a0, -1 ; RV32IZFINXZDINX-NEXT: and a0, a0, a2 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_rint_ui32: @@ -1641,38 +1505,30 @@ define i64 @test_rint_ui64(double %x) nounwind { ; ; RV32IZFINXZDINX-LABEL: test_rint_ui64: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -32 -; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s2, 16(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill ; RV32IZFINXZDINX-NEXT: call rint -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw s1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw s0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw s1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) +; RV32IZFINXZDINX-NEXT: mv s0, a0 +; RV32IZFINXZDINX-NEXT: mv s1, a1 +; RV32IZFINXZDINX-NEXT: call __fixunsdfdi ; RV32IZFINXZDINX-NEXT: fcvt.d.w a2, zero +; RV32IZFINXZDINX-NEXT: lui a4, %hi(.LCPI23_0) +; RV32IZFINXZDINX-NEXT: lw a5, %lo(.LCPI23_0+4)(a4) +; RV32IZFINXZDINX-NEXT: lw a4, %lo(.LCPI23_0)(a4) ; RV32IZFINXZDINX-NEXT: fle.d a2, a2, s0 -; RV32IZFINXZDINX-NEXT: neg s2, a2 -; RV32IZFINXZDINX-NEXT: call __fixunsdfdi -; RV32IZFINXZDINX-NEXT: lui a2, %hi(.LCPI23_0) -; RV32IZFINXZDINX-NEXT: lw a3, %lo(.LCPI23_0+4)(a2) -; RV32IZFINXZDINX-NEXT: lw a2, %lo(.LCPI23_0)(a2) -; RV32IZFINXZDINX-NEXT: and a0, s2, a0 -; RV32IZFINXZDINX-NEXT: flt.d a2, a2, s0 ; RV32IZFINXZDINX-NEXT: neg a2, a2 -; RV32IZFINXZDINX-NEXT: or a0, a2, a0 -; RV32IZFINXZDINX-NEXT: and a1, s2, a1 -; RV32IZFINXZDINX-NEXT: or a1, a2, a1 -; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s2, 16(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 +; RV32IZFINXZDINX-NEXT: and a0, a2, a0 +; RV32IZFINXZDINX-NEXT: flt.d a3, a4, s0 +; RV32IZFINXZDINX-NEXT: neg a3, a3 +; RV32IZFINXZDINX-NEXT: or a0, a3, a0 +; RV32IZFINXZDINX-NEXT: and a1, a2, a1 +; RV32IZFINXZDINX-NEXT: or a1, a3, a1 +; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_rint_ui64: diff --git a/llvm/test/CodeGen/RISCV/double-round-conv.ll b/llvm/test/CodeGen/RISCV/double-round-conv.ll index 094a4105de71..d84d80a4a10e 100644 --- a/llvm/test/CodeGen/RISCV/double-round-conv.ll +++ b/llvm/test/CodeGen/RISCV/double-round-conv.ll @@ -21,14 +21,7 @@ define signext i8 @test_floor_si8(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_floor_si8: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rdn -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_floor_si8: @@ -53,14 +46,7 @@ define signext i16 @test_floor_si16(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_floor_si16: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rdn -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_floor_si16: @@ -80,14 +66,7 @@ define signext i32 @test_floor_si32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_floor_si32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rdn -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_floor_si32: @@ -151,14 +130,7 @@ define zeroext i8 @test_floor_ui8(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_floor_ui8: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rdn -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_floor_ui8: @@ -183,14 +155,7 @@ define zeroext i16 @test_floor_ui16(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_floor_ui16: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rdn -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_floor_ui16: @@ -210,14 +175,7 @@ define signext i32 @test_floor_ui32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_floor_ui32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rdn -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_floor_ui32: @@ -281,14 +239,7 @@ define signext i8 @test_ceil_si8(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_ceil_si8: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rup -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_ceil_si8: @@ -313,14 +264,7 @@ define signext i16 @test_ceil_si16(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_ceil_si16: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rup -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_ceil_si16: @@ -340,14 +284,7 @@ define signext i32 @test_ceil_si32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_ceil_si32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rup -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_ceil_si32: @@ -411,14 +348,7 @@ define zeroext i8 @test_ceil_ui8(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_ceil_ui8: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rup -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_ceil_ui8: @@ -443,14 +373,7 @@ define zeroext i16 @test_ceil_ui16(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_ceil_ui16: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rup -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_ceil_ui16: @@ -470,14 +393,7 @@ define signext i32 @test_ceil_ui32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_ceil_ui32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rup -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_ceil_ui32: @@ -541,14 +457,7 @@ define signext i8 @test_trunc_si8(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_trunc_si8: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rtz -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_trunc_si8: @@ -573,14 +482,7 @@ define signext i16 @test_trunc_si16(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_trunc_si16: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rtz -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_trunc_si16: @@ -600,14 +502,7 @@ define signext i32 @test_trunc_si32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_trunc_si32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rtz -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_trunc_si32: @@ -671,14 +566,7 @@ define zeroext i8 @test_trunc_ui8(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_trunc_ui8: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rtz -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_trunc_ui8: @@ -703,14 +591,7 @@ define zeroext i16 @test_trunc_ui16(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_trunc_ui16: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rtz -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_trunc_ui16: @@ -730,14 +611,7 @@ define signext i32 @test_trunc_ui32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_trunc_ui32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rtz -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_trunc_ui32: @@ -801,14 +675,7 @@ define signext i8 @test_round_si8(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_round_si8: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rmm -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_round_si8: @@ -833,14 +700,7 @@ define signext i16 @test_round_si16(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_round_si16: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rmm -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_round_si16: @@ -860,14 +720,7 @@ define signext i32 @test_round_si32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_round_si32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rmm -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_round_si32: @@ -931,14 +784,7 @@ define zeroext i8 @test_round_ui8(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_round_ui8: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rmm -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_round_ui8: @@ -963,14 +809,7 @@ define zeroext i16 @test_round_ui16(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_round_ui16: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rmm -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_round_ui16: @@ -990,14 +829,7 @@ define signext i32 @test_round_ui32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_round_ui32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rmm -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_round_ui32: @@ -1061,14 +893,7 @@ define signext i8 @test_roundeven_si8(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_roundeven_si8: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rne -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_roundeven_si8: @@ -1093,14 +918,7 @@ define signext i16 @test_roundeven_si16(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_roundeven_si16: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rne -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_roundeven_si16: @@ -1120,14 +938,7 @@ define signext i32 @test_roundeven_si32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_roundeven_si32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.w.d a0, a0, rne -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_roundeven_si32: @@ -1191,14 +1002,7 @@ define zeroext i8 @test_roundeven_ui8(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_roundeven_ui8: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rne -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_roundeven_ui8: @@ -1223,14 +1027,7 @@ define zeroext i16 @test_roundeven_ui16(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_roundeven_ui16: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rne -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_roundeven_ui16: @@ -1250,14 +1047,7 @@ define signext i32 @test_roundeven_ui32(double %x) { ; ; RV32IZFINXZDINX-LABEL: test_roundeven_ui32: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fcvt.wu.d a0, a0, rne -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: test_roundeven_ui32: diff --git a/llvm/test/CodeGen/RISCV/double-select-fcmp.ll b/llvm/test/CodeGen/RISCV/double-select-fcmp.ll index 766da3680ffc..654a4609caa2 100644 --- a/llvm/test/CodeGen/RISCV/double-select-fcmp.ll +++ b/llvm/test/CodeGen/RISCV/double-select-fcmp.ll @@ -41,26 +41,12 @@ define double @select_fcmp_oeq(double %a, double %b) nounwind { ; ; CHECKRV32ZDINX-LABEL: select_fcmp_oeq: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32ZDINX-NEXT: feq.d a4, a0, a2 ; CHECKRV32ZDINX-NEXT: bnez a4, .LBB1_2 ; CHECKRV32ZDINX-NEXT: # %bb.1: ; CHECKRV32ZDINX-NEXT: mv a0, a2 ; CHECKRV32ZDINX-NEXT: mv a1, a3 ; CHECKRV32ZDINX-NEXT: .LBB1_2: -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: select_fcmp_oeq: @@ -88,26 +74,12 @@ define double @select_fcmp_ogt(double %a, double %b) nounwind { ; ; CHECKRV32ZDINX-LABEL: select_fcmp_ogt: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a3, 12(sp) ; CHECKRV32ZDINX-NEXT: flt.d a4, a2, a0 ; CHECKRV32ZDINX-NEXT: bnez a4, .LBB2_2 ; CHECKRV32ZDINX-NEXT: # %bb.1: ; CHECKRV32ZDINX-NEXT: mv a0, a2 ; CHECKRV32ZDINX-NEXT: mv a1, a3 ; CHECKRV32ZDINX-NEXT: .LBB2_2: -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: select_fcmp_ogt: @@ -135,26 +107,12 @@ define double @select_fcmp_oge(double %a, double %b) nounwind { ; ; CHECKRV32ZDINX-LABEL: select_fcmp_oge: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a3, 12(sp) ; CHECKRV32ZDINX-NEXT: fle.d a4, a2, a0 ; CHECKRV32ZDINX-NEXT: bnez a4, .LBB3_2 ; CHECKRV32ZDINX-NEXT: # %bb.1: ; CHECKRV32ZDINX-NEXT: mv a0, a2 ; CHECKRV32ZDINX-NEXT: mv a1, a3 ; CHECKRV32ZDINX-NEXT: .LBB3_2: -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: select_fcmp_oge: @@ -182,26 +140,12 @@ define double @select_fcmp_olt(double %a, double %b) nounwind { ; ; CHECKRV32ZDINX-LABEL: select_fcmp_olt: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32ZDINX-NEXT: flt.d a4, a0, a2 ; CHECKRV32ZDINX-NEXT: bnez a4, .LBB4_2 ; CHECKRV32ZDINX-NEXT: # %bb.1: ; CHECKRV32ZDINX-NEXT: mv a0, a2 ; CHECKRV32ZDINX-NEXT: mv a1, a3 ; CHECKRV32ZDINX-NEXT: .LBB4_2: -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: select_fcmp_olt: @@ -229,26 +173,12 @@ define double @select_fcmp_ole(double %a, double %b) nounwind { ; ; CHECKRV32ZDINX-LABEL: select_fcmp_ole: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32ZDINX-NEXT: fle.d a4, a0, a2 ; CHECKRV32ZDINX-NEXT: bnez a4, .LBB5_2 ; CHECKRV32ZDINX-NEXT: # %bb.1: ; CHECKRV32ZDINX-NEXT: mv a0, a2 ; CHECKRV32ZDINX-NEXT: mv a1, a3 ; CHECKRV32ZDINX-NEXT: .LBB5_2: -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: select_fcmp_ole: @@ -278,15 +208,6 @@ define double @select_fcmp_one(double %a, double %b) nounwind { ; ; CHECKRV32ZDINX-LABEL: select_fcmp_one: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32ZDINX-NEXT: flt.d a4, a0, a2 ; CHECKRV32ZDINX-NEXT: flt.d a5, a2, a0 ; CHECKRV32ZDINX-NEXT: or a4, a5, a4 @@ -295,11 +216,6 @@ define double @select_fcmp_one(double %a, double %b) nounwind { ; CHECKRV32ZDINX-NEXT: mv a0, a2 ; CHECKRV32ZDINX-NEXT: mv a1, a3 ; CHECKRV32ZDINX-NEXT: .LBB6_2: -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: select_fcmp_one: @@ -331,15 +247,6 @@ define double @select_fcmp_ord(double %a, double %b) nounwind { ; ; CHECKRV32ZDINX-LABEL: select_fcmp_ord: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a3, 12(sp) ; CHECKRV32ZDINX-NEXT: feq.d a4, a2, a2 ; CHECKRV32ZDINX-NEXT: feq.d a5, a0, a0 ; CHECKRV32ZDINX-NEXT: and a4, a5, a4 @@ -348,11 +255,6 @@ define double @select_fcmp_ord(double %a, double %b) nounwind { ; CHECKRV32ZDINX-NEXT: mv a0, a2 ; CHECKRV32ZDINX-NEXT: mv a1, a3 ; CHECKRV32ZDINX-NEXT: .LBB7_2: -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: select_fcmp_ord: @@ -384,15 +286,6 @@ define double @select_fcmp_ueq(double %a, double %b) nounwind { ; ; CHECKRV32ZDINX-LABEL: select_fcmp_ueq: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32ZDINX-NEXT: flt.d a4, a0, a2 ; CHECKRV32ZDINX-NEXT: flt.d a5, a2, a0 ; CHECKRV32ZDINX-NEXT: or a4, a5, a4 @@ -401,11 +294,6 @@ define double @select_fcmp_ueq(double %a, double %b) nounwind { ; CHECKRV32ZDINX-NEXT: mv a0, a2 ; CHECKRV32ZDINX-NEXT: mv a1, a3 ; CHECKRV32ZDINX-NEXT: .LBB8_2: -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: select_fcmp_ueq: @@ -435,26 +323,12 @@ define double @select_fcmp_ugt(double %a, double %b) nounwind { ; ; CHECKRV32ZDINX-LABEL: select_fcmp_ugt: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32ZDINX-NEXT: fle.d a4, a0, a2 ; CHECKRV32ZDINX-NEXT: beqz a4, .LBB9_2 ; CHECKRV32ZDINX-NEXT: # %bb.1: ; CHECKRV32ZDINX-NEXT: mv a0, a2 ; CHECKRV32ZDINX-NEXT: mv a1, a3 ; CHECKRV32ZDINX-NEXT: .LBB9_2: -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: select_fcmp_ugt: @@ -482,26 +356,12 @@ define double @select_fcmp_uge(double %a, double %b) nounwind { ; ; CHECKRV32ZDINX-LABEL: select_fcmp_uge: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32ZDINX-NEXT: flt.d a4, a0, a2 ; CHECKRV32ZDINX-NEXT: beqz a4, .LBB10_2 ; CHECKRV32ZDINX-NEXT: # %bb.1: ; CHECKRV32ZDINX-NEXT: mv a0, a2 ; CHECKRV32ZDINX-NEXT: mv a1, a3 ; CHECKRV32ZDINX-NEXT: .LBB10_2: -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: select_fcmp_uge: @@ -529,26 +389,12 @@ define double @select_fcmp_ult(double %a, double %b) nounwind { ; ; CHECKRV32ZDINX-LABEL: select_fcmp_ult: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a3, 12(sp) ; CHECKRV32ZDINX-NEXT: fle.d a4, a2, a0 ; CHECKRV32ZDINX-NEXT: beqz a4, .LBB11_2 ; CHECKRV32ZDINX-NEXT: # %bb.1: ; CHECKRV32ZDINX-NEXT: mv a0, a2 ; CHECKRV32ZDINX-NEXT: mv a1, a3 ; CHECKRV32ZDINX-NEXT: .LBB11_2: -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: select_fcmp_ult: @@ -576,26 +422,12 @@ define double @select_fcmp_ule(double %a, double %b) nounwind { ; ; CHECKRV32ZDINX-LABEL: select_fcmp_ule: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a3, 12(sp) ; CHECKRV32ZDINX-NEXT: flt.d a4, a2, a0 ; CHECKRV32ZDINX-NEXT: beqz a4, .LBB12_2 ; CHECKRV32ZDINX-NEXT: # %bb.1: ; CHECKRV32ZDINX-NEXT: mv a0, a2 ; CHECKRV32ZDINX-NEXT: mv a1, a3 ; CHECKRV32ZDINX-NEXT: .LBB12_2: -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: select_fcmp_ule: @@ -623,26 +455,12 @@ define double @select_fcmp_une(double %a, double %b) nounwind { ; ; CHECKRV32ZDINX-LABEL: select_fcmp_une: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32ZDINX-NEXT: feq.d a4, a0, a2 ; CHECKRV32ZDINX-NEXT: beqz a4, .LBB13_2 ; CHECKRV32ZDINX-NEXT: # %bb.1: ; CHECKRV32ZDINX-NEXT: mv a0, a2 ; CHECKRV32ZDINX-NEXT: mv a1, a3 ; CHECKRV32ZDINX-NEXT: .LBB13_2: -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: select_fcmp_une: @@ -672,15 +490,6 @@ define double @select_fcmp_uno(double %a, double %b) nounwind { ; ; CHECKRV32ZDINX-LABEL: select_fcmp_uno: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a3, 12(sp) ; CHECKRV32ZDINX-NEXT: feq.d a4, a2, a2 ; CHECKRV32ZDINX-NEXT: feq.d a5, a0, a0 ; CHECKRV32ZDINX-NEXT: and a4, a5, a4 @@ -689,11 +498,6 @@ define double @select_fcmp_uno(double %a, double %b) nounwind { ; CHECKRV32ZDINX-NEXT: mv a0, a2 ; CHECKRV32ZDINX-NEXT: mv a1, a3 ; CHECKRV32ZDINX-NEXT: .LBB14_2: -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: select_fcmp_uno: @@ -741,22 +545,12 @@ define i32 @i32_select_fcmp_oeq(double %a, double %b, i32 %c, i32 %d) nounwind { ; ; CHECKRV32ZDINX-LABEL: i32_select_fcmp_oeq: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32ZDINX-NEXT: feq.d a1, a0, a2 ; CHECKRV32ZDINX-NEXT: mv a0, a4 ; CHECKRV32ZDINX-NEXT: bnez a1, .LBB16_2 ; CHECKRV32ZDINX-NEXT: # %bb.1: ; CHECKRV32ZDINX-NEXT: mv a0, a5 ; CHECKRV32ZDINX-NEXT: .LBB16_2: -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: i32_select_fcmp_oeq: @@ -783,20 +577,9 @@ define i32 @select_fcmp_oeq_1_2(double %a, double %b) { ; ; CHECKRV32ZDINX-LABEL: select_fcmp_oeq_1_2: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: .cfi_def_cfa_offset 16 -; CHECKRV32ZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32ZDINX-NEXT: feq.d a0, a0, a2 ; CHECKRV32ZDINX-NEXT: li a1, 2 ; CHECKRV32ZDINX-NEXT: sub a0, a1, a0 -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: select_fcmp_oeq_1_2: @@ -819,18 +602,8 @@ define signext i32 @select_fcmp_uge_negone_zero(double %a, double %b) nounwind { ; ; CHECKRV32ZDINX-LABEL: select_fcmp_uge_negone_zero: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32ZDINX-NEXT: fle.d a0, a0, a2 ; CHECKRV32ZDINX-NEXT: addi a0, a0, -1 -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: select_fcmp_uge_negone_zero: @@ -852,18 +625,8 @@ define signext i32 @select_fcmp_uge_1_2(double %a, double %b) nounwind { ; ; CHECKRV32ZDINX-LABEL: select_fcmp_uge_1_2: ; CHECKRV32ZDINX: # %bb.0: -; CHECKRV32ZDINX-NEXT: addi sp, sp, -16 -; CHECKRV32ZDINX-NEXT: sw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a2, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a3, 12(sp) -; CHECKRV32ZDINX-NEXT: sw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: sw a1, 12(sp) -; CHECKRV32ZDINX-NEXT: lw a0, 8(sp) -; CHECKRV32ZDINX-NEXT: lw a1, 12(sp) ; CHECKRV32ZDINX-NEXT: fle.d a0, a0, a2 ; CHECKRV32ZDINX-NEXT: addi a0, a0, 1 -; CHECKRV32ZDINX-NEXT: addi sp, sp, 16 ; CHECKRV32ZDINX-NEXT: ret ; ; CHECKRV64ZDINX-LABEL: select_fcmp_uge_1_2: diff --git a/llvm/test/CodeGen/RISCV/double-select-icmp.ll b/llvm/test/CodeGen/RISCV/double-select-icmp.ll index d864ff51b466..929ffc578f5b 100644 --- a/llvm/test/CodeGen/RISCV/double-select-icmp.ll +++ b/llvm/test/CodeGen/RISCV/double-select-icmp.ll @@ -20,24 +20,13 @@ define double @select_icmp_eq(i32 signext %a, i32 signext %b, double %c, double ; ; RV32ZDINX-LABEL: select_icmp_eq: ; RV32ZDINX: # %bb.0: -; RV32ZDINX-NEXT: addi sp, sp, -16 -; RV32ZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) -; RV32ZDINX-NEXT: sw a2, 8(sp) -; RV32ZDINX-NEXT: sw a3, 12(sp) -; RV32ZDINX-NEXT: bne a0, a1, .LBB0_2 +; RV32ZDINX-NEXT: beq a0, a1, .LBB0_2 ; RV32ZDINX-NEXT: # %bb.1: -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) +; RV32ZDINX-NEXT: mv a2, a4 +; RV32ZDINX-NEXT: mv a3, a5 ; RV32ZDINX-NEXT: .LBB0_2: -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a0, 8(sp) -; RV32ZDINX-NEXT: lw a1, 12(sp) -; RV32ZDINX-NEXT: addi sp, sp, 16 +; RV32ZDINX-NEXT: mv a0, a2 +; RV32ZDINX-NEXT: mv a1, a3 ; RV32ZDINX-NEXT: ret ; ; RV64ZDINX-LABEL: select_icmp_eq: @@ -64,24 +53,13 @@ define double @select_icmp_ne(i32 signext %a, i32 signext %b, double %c, double ; ; RV32ZDINX-LABEL: select_icmp_ne: ; RV32ZDINX: # %bb.0: -; RV32ZDINX-NEXT: addi sp, sp, -16 -; RV32ZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) -; RV32ZDINX-NEXT: sw a2, 8(sp) -; RV32ZDINX-NEXT: sw a3, 12(sp) -; RV32ZDINX-NEXT: beq a0, a1, .LBB1_2 +; RV32ZDINX-NEXT: bne a0, a1, .LBB1_2 ; RV32ZDINX-NEXT: # %bb.1: -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) +; RV32ZDINX-NEXT: mv a2, a4 +; RV32ZDINX-NEXT: mv a3, a5 ; RV32ZDINX-NEXT: .LBB1_2: -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a0, 8(sp) -; RV32ZDINX-NEXT: lw a1, 12(sp) -; RV32ZDINX-NEXT: addi sp, sp, 16 +; RV32ZDINX-NEXT: mv a0, a2 +; RV32ZDINX-NEXT: mv a1, a3 ; RV32ZDINX-NEXT: ret ; ; RV64ZDINX-LABEL: select_icmp_ne: @@ -108,24 +86,13 @@ define double @select_icmp_ugt(i32 signext %a, i32 signext %b, double %c, double ; ; RV32ZDINX-LABEL: select_icmp_ugt: ; RV32ZDINX: # %bb.0: -; RV32ZDINX-NEXT: addi sp, sp, -16 -; RV32ZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) -; RV32ZDINX-NEXT: sw a2, 8(sp) -; RV32ZDINX-NEXT: sw a3, 12(sp) -; RV32ZDINX-NEXT: bgeu a1, a0, .LBB2_2 +; RV32ZDINX-NEXT: bltu a1, a0, .LBB2_2 ; RV32ZDINX-NEXT: # %bb.1: -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) +; RV32ZDINX-NEXT: mv a2, a4 +; RV32ZDINX-NEXT: mv a3, a5 ; RV32ZDINX-NEXT: .LBB2_2: -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a0, 8(sp) -; RV32ZDINX-NEXT: lw a1, 12(sp) -; RV32ZDINX-NEXT: addi sp, sp, 16 +; RV32ZDINX-NEXT: mv a0, a2 +; RV32ZDINX-NEXT: mv a1, a3 ; RV32ZDINX-NEXT: ret ; ; RV64ZDINX-LABEL: select_icmp_ugt: @@ -152,24 +119,13 @@ define double @select_icmp_uge(i32 signext %a, i32 signext %b, double %c, double ; ; RV32ZDINX-LABEL: select_icmp_uge: ; RV32ZDINX: # %bb.0: -; RV32ZDINX-NEXT: addi sp, sp, -16 -; RV32ZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) -; RV32ZDINX-NEXT: sw a2, 8(sp) -; RV32ZDINX-NEXT: sw a3, 12(sp) -; RV32ZDINX-NEXT: bltu a0, a1, .LBB3_2 +; RV32ZDINX-NEXT: bgeu a0, a1, .LBB3_2 ; RV32ZDINX-NEXT: # %bb.1: -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) +; RV32ZDINX-NEXT: mv a2, a4 +; RV32ZDINX-NEXT: mv a3, a5 ; RV32ZDINX-NEXT: .LBB3_2: -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a0, 8(sp) -; RV32ZDINX-NEXT: lw a1, 12(sp) -; RV32ZDINX-NEXT: addi sp, sp, 16 +; RV32ZDINX-NEXT: mv a0, a2 +; RV32ZDINX-NEXT: mv a1, a3 ; RV32ZDINX-NEXT: ret ; ; RV64ZDINX-LABEL: select_icmp_uge: @@ -196,24 +152,13 @@ define double @select_icmp_ult(i32 signext %a, i32 signext %b, double %c, double ; ; RV32ZDINX-LABEL: select_icmp_ult: ; RV32ZDINX: # %bb.0: -; RV32ZDINX-NEXT: addi sp, sp, -16 -; RV32ZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) -; RV32ZDINX-NEXT: sw a2, 8(sp) -; RV32ZDINX-NEXT: sw a3, 12(sp) -; RV32ZDINX-NEXT: bgeu a0, a1, .LBB4_2 +; RV32ZDINX-NEXT: bltu a0, a1, .LBB4_2 ; RV32ZDINX-NEXT: # %bb.1: -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) +; RV32ZDINX-NEXT: mv a2, a4 +; RV32ZDINX-NEXT: mv a3, a5 ; RV32ZDINX-NEXT: .LBB4_2: -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a0, 8(sp) -; RV32ZDINX-NEXT: lw a1, 12(sp) -; RV32ZDINX-NEXT: addi sp, sp, 16 +; RV32ZDINX-NEXT: mv a0, a2 +; RV32ZDINX-NEXT: mv a1, a3 ; RV32ZDINX-NEXT: ret ; ; RV64ZDINX-LABEL: select_icmp_ult: @@ -240,24 +185,13 @@ define double @select_icmp_ule(i32 signext %a, i32 signext %b, double %c, double ; ; RV32ZDINX-LABEL: select_icmp_ule: ; RV32ZDINX: # %bb.0: -; RV32ZDINX-NEXT: addi sp, sp, -16 -; RV32ZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) -; RV32ZDINX-NEXT: sw a2, 8(sp) -; RV32ZDINX-NEXT: sw a3, 12(sp) -; RV32ZDINX-NEXT: bltu a1, a0, .LBB5_2 +; RV32ZDINX-NEXT: bgeu a1, a0, .LBB5_2 ; RV32ZDINX-NEXT: # %bb.1: -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) +; RV32ZDINX-NEXT: mv a2, a4 +; RV32ZDINX-NEXT: mv a3, a5 ; RV32ZDINX-NEXT: .LBB5_2: -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a0, 8(sp) -; RV32ZDINX-NEXT: lw a1, 12(sp) -; RV32ZDINX-NEXT: addi sp, sp, 16 +; RV32ZDINX-NEXT: mv a0, a2 +; RV32ZDINX-NEXT: mv a1, a3 ; RV32ZDINX-NEXT: ret ; ; RV64ZDINX-LABEL: select_icmp_ule: @@ -284,24 +218,13 @@ define double @select_icmp_sgt(i32 signext %a, i32 signext %b, double %c, double ; ; RV32ZDINX-LABEL: select_icmp_sgt: ; RV32ZDINX: # %bb.0: -; RV32ZDINX-NEXT: addi sp, sp, -16 -; RV32ZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) -; RV32ZDINX-NEXT: sw a2, 8(sp) -; RV32ZDINX-NEXT: sw a3, 12(sp) -; RV32ZDINX-NEXT: bge a1, a0, .LBB6_2 +; RV32ZDINX-NEXT: blt a1, a0, .LBB6_2 ; RV32ZDINX-NEXT: # %bb.1: -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) +; RV32ZDINX-NEXT: mv a2, a4 +; RV32ZDINX-NEXT: mv a3, a5 ; RV32ZDINX-NEXT: .LBB6_2: -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a0, 8(sp) -; RV32ZDINX-NEXT: lw a1, 12(sp) -; RV32ZDINX-NEXT: addi sp, sp, 16 +; RV32ZDINX-NEXT: mv a0, a2 +; RV32ZDINX-NEXT: mv a1, a3 ; RV32ZDINX-NEXT: ret ; ; RV64ZDINX-LABEL: select_icmp_sgt: @@ -328,24 +251,13 @@ define double @select_icmp_sge(i32 signext %a, i32 signext %b, double %c, double ; ; RV32ZDINX-LABEL: select_icmp_sge: ; RV32ZDINX: # %bb.0: -; RV32ZDINX-NEXT: addi sp, sp, -16 -; RV32ZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) -; RV32ZDINX-NEXT: sw a2, 8(sp) -; RV32ZDINX-NEXT: sw a3, 12(sp) -; RV32ZDINX-NEXT: blt a0, a1, .LBB7_2 +; RV32ZDINX-NEXT: bge a0, a1, .LBB7_2 ; RV32ZDINX-NEXT: # %bb.1: -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) +; RV32ZDINX-NEXT: mv a2, a4 +; RV32ZDINX-NEXT: mv a3, a5 ; RV32ZDINX-NEXT: .LBB7_2: -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a0, 8(sp) -; RV32ZDINX-NEXT: lw a1, 12(sp) -; RV32ZDINX-NEXT: addi sp, sp, 16 +; RV32ZDINX-NEXT: mv a0, a2 +; RV32ZDINX-NEXT: mv a1, a3 ; RV32ZDINX-NEXT: ret ; ; RV64ZDINX-LABEL: select_icmp_sge: @@ -372,24 +284,13 @@ define double @select_icmp_slt(i32 signext %a, i32 signext %b, double %c, double ; ; RV32ZDINX-LABEL: select_icmp_slt: ; RV32ZDINX: # %bb.0: -; RV32ZDINX-NEXT: addi sp, sp, -16 -; RV32ZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) -; RV32ZDINX-NEXT: sw a2, 8(sp) -; RV32ZDINX-NEXT: sw a3, 12(sp) -; RV32ZDINX-NEXT: bge a0, a1, .LBB8_2 +; RV32ZDINX-NEXT: blt a0, a1, .LBB8_2 ; RV32ZDINX-NEXT: # %bb.1: -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) +; RV32ZDINX-NEXT: mv a2, a4 +; RV32ZDINX-NEXT: mv a3, a5 ; RV32ZDINX-NEXT: .LBB8_2: -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a0, 8(sp) -; RV32ZDINX-NEXT: lw a1, 12(sp) -; RV32ZDINX-NEXT: addi sp, sp, 16 +; RV32ZDINX-NEXT: mv a0, a2 +; RV32ZDINX-NEXT: mv a1, a3 ; RV32ZDINX-NEXT: ret ; ; RV64ZDINX-LABEL: select_icmp_slt: @@ -416,24 +317,13 @@ define double @select_icmp_sle(i32 signext %a, i32 signext %b, double %c, double ; ; RV32ZDINX-LABEL: select_icmp_sle: ; RV32ZDINX: # %bb.0: -; RV32ZDINX-NEXT: addi sp, sp, -16 -; RV32ZDINX-NEXT: .cfi_def_cfa_offset 16 -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) -; RV32ZDINX-NEXT: sw a2, 8(sp) -; RV32ZDINX-NEXT: sw a3, 12(sp) -; RV32ZDINX-NEXT: blt a1, a0, .LBB9_2 +; RV32ZDINX-NEXT: bge a1, a0, .LBB9_2 ; RV32ZDINX-NEXT: # %bb.1: -; RV32ZDINX-NEXT: lw a4, 8(sp) -; RV32ZDINX-NEXT: lw a5, 12(sp) +; RV32ZDINX-NEXT: mv a2, a4 +; RV32ZDINX-NEXT: mv a3, a5 ; RV32ZDINX-NEXT: .LBB9_2: -; RV32ZDINX-NEXT: sw a4, 8(sp) -; RV32ZDINX-NEXT: sw a5, 12(sp) -; RV32ZDINX-NEXT: lw a0, 8(sp) -; RV32ZDINX-NEXT: lw a1, 12(sp) -; RV32ZDINX-NEXT: addi sp, sp, 16 +; RV32ZDINX-NEXT: mv a0, a2 +; RV32ZDINX-NEXT: mv a1, a3 ; RV32ZDINX-NEXT: ret ; ; RV64ZDINX-LABEL: select_icmp_sle: @@ -458,15 +348,8 @@ define double @select_icmp_slt_one(i32 signext %a) { ; ; RV32ZDINX-LABEL: select_icmp_slt_one: ; RV32ZDINX: # %bb.0: -; RV32ZDINX-NEXT: addi sp, sp, -16 -; RV32ZDINX-NEXT: .cfi_def_cfa_offset 16 ; RV32ZDINX-NEXT: slti a0, a0, 1 ; RV32ZDINX-NEXT: fcvt.d.w a0, a0 -; RV32ZDINX-NEXT: sw a0, 8(sp) -; RV32ZDINX-NEXT: sw a1, 12(sp) -; RV32ZDINX-NEXT: lw a0, 8(sp) -; RV32ZDINX-NEXT: lw a1, 12(sp) -; RV32ZDINX-NEXT: addi sp, sp, 16 ; RV32ZDINX-NEXT: ret ; ; RV64ZDINX-LABEL: select_icmp_slt_one: @@ -488,15 +371,8 @@ define double @select_icmp_sgt_zero(i32 signext %a) { ; ; RV32ZDINX-LABEL: select_icmp_sgt_zero: ; RV32ZDINX: # %bb.0: -; RV32ZDINX-NEXT: addi sp, sp, -16 -; RV32ZDINX-NEXT: .cfi_def_cfa_offset 16 ; RV32ZDINX-NEXT: slti a0, a0, 1 ; RV32ZDINX-NEXT: fcvt.d.w a0, a0 -; RV32ZDINX-NEXT: sw a0, 8(sp) -; RV32ZDINX-NEXT: sw a1, 12(sp) -; RV32ZDINX-NEXT: lw a0, 8(sp) -; RV32ZDINX-NEXT: lw a1, 12(sp) -; RV32ZDINX-NEXT: addi sp, sp, 16 ; RV32ZDINX-NEXT: ret ; ; RV64ZDINX-LABEL: select_icmp_sgt_zero: diff --git a/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll b/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll index aa88a365431a..4ae912a34d33 100644 --- a/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll +++ b/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll @@ -62,40 +62,28 @@ define double @func(double %d, i32 %n) nounwind { ; ; RV32IZFINXZDINX-LABEL: func: ; RV32IZFINXZDINX: # %bb.0: # %entry -; RV32IZFINXZDINX-NEXT: addi sp, sp, -32 -; RV32IZFINXZDINX-NEXT: sw ra, 28(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s0, 24(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw s1, 20(sp) # 4-byte Folded Spill -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw s1, 12(sp) +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: mv s1, a1 +; RV32IZFINXZDINX-NEXT: mv s0, a0 ; RV32IZFINXZDINX-NEXT: beqz a2, .LBB0_2 ; RV32IZFINXZDINX-NEXT: # %bb.1: # %if.else ; RV32IZFINXZDINX-NEXT: addi a2, a2, -1 -; RV32IZFINXZDINX-NEXT: sw s0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw s1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) +; RV32IZFINXZDINX-NEXT: mv a0, s0 +; RV32IZFINXZDINX-NEXT: mv a1, s1 ; RV32IZFINXZDINX-NEXT: call func -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, s0 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) ; RV32IZFINXZDINX-NEXT: j .LBB0_3 ; RV32IZFINXZDINX-NEXT: .LBB0_2: # %return -; RV32IZFINXZDINX-NEXT: sw s0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw s1, 12(sp) +; RV32IZFINXZDINX-NEXT: mv a0, s0 +; RV32IZFINXZDINX-NEXT: mv a1, s1 ; RV32IZFINXZDINX-NEXT: .LBB0_3: # %return -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw ra, 28(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s0, 24(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: lw s1, 20(sp) # 4-byte Folded Reload -; RV32IZFINXZDINX-NEXT: addi sp, sp, 32 +; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: func: diff --git a/llvm/test/CodeGen/RISCV/fastcc-without-f-reg.ll b/llvm/test/CodeGen/RISCV/fastcc-without-f-reg.ll index fb0b34cf796b..a44d31dff09c 100644 --- a/llvm/test/CodeGen/RISCV/fastcc-without-f-reg.ll +++ b/llvm/test/CodeGen/RISCV/fastcc-without-f-reg.ll @@ -160,17 +160,13 @@ define double @caller_double(double %x) nounwind { ; ; ZDINX32-LABEL: caller_double: ; ZDINX32: # %bb.0: # %entry -; ZDINX32-NEXT: addi sp, sp, -32 -; ZDINX32-NEXT: sw ra, 28(sp) # 4-byte Folded Spill -; ZDINX32-NEXT: sw a0, 16(sp) -; ZDINX32-NEXT: sw a1, 20(sp) -; ZDINX32-NEXT: lw a0, 16(sp) -; ZDINX32-NEXT: lw a1, 20(sp) +; ZDINX32-NEXT: addi sp, sp, -16 +; ZDINX32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill ; ZDINX32-NEXT: sw a0, 0(sp) ; ZDINX32-NEXT: sw a1, 4(sp) ; ZDINX32-NEXT: call d -; ZDINX32-NEXT: lw ra, 28(sp) # 4-byte Folded Reload -; ZDINX32-NEXT: addi sp, sp, 32 +; ZDINX32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; ZDINX32-NEXT: addi sp, sp, 16 ; ZDINX32-NEXT: ret ; ; ZDINX64-LABEL: caller_double: @@ -200,14 +196,8 @@ define internal fastcc double @d(double %x) nounwind { ; ; ZDINX32-LABEL: d: ; ZDINX32: # %bb.0: # %entry -; ZDINX32-NEXT: addi sp, sp, -16 -; ZDINX32-NEXT: lw a0, 16(sp) -; ZDINX32-NEXT: lw a1, 20(sp) -; ZDINX32-NEXT: sw a0, 8(sp) -; ZDINX32-NEXT: sw a1, 12(sp) -; ZDINX32-NEXT: lw a0, 8(sp) -; ZDINX32-NEXT: lw a1, 12(sp) -; ZDINX32-NEXT: addi sp, sp, 16 +; ZDINX32-NEXT: lw a0, 0(sp) +; ZDINX32-NEXT: lw a1, 4(sp) ; ZDINX32-NEXT: ret ; ; ZDINX64-LABEL: d: @@ -1360,14 +1350,8 @@ define fastcc double @callee_double_32(<32 x double> %A) nounwind { ; ; ZDINX32-LABEL: callee_double_32: ; ZDINX32: # %bb.0: -; ZDINX32-NEXT: addi sp, sp, -16 -; ZDINX32-NEXT: lw a0, 16(sp) -; ZDINX32-NEXT: lw a1, 20(sp) -; ZDINX32-NEXT: sw a0, 8(sp) -; ZDINX32-NEXT: sw a1, 12(sp) -; ZDINX32-NEXT: lw a0, 8(sp) -; ZDINX32-NEXT: lw a1, 12(sp) -; ZDINX32-NEXT: addi sp, sp, 16 +; ZDINX32-NEXT: lw a0, 0(sp) +; ZDINX32-NEXT: lw a1, 4(sp) ; ZDINX32-NEXT: ret ; ; ZDINX64-LABEL: callee_double_32: diff --git a/llvm/test/CodeGen/RISCV/half-convert-strict.ll b/llvm/test/CodeGen/RISCV/half-convert-strict.ll index f03a020762bb..677aa9263ea6 100644 --- a/llvm/test/CodeGen/RISCV/half-convert-strict.ll +++ b/llvm/test/CodeGen/RISCV/half-convert-strict.ll @@ -1745,13 +1745,7 @@ define half @fcvt_h_d(double %a) nounwind strictfp { ; ; RV32IZDINXZHINX-LABEL: fcvt_h_d: ; RV32IZDINXZHINX: # %bb.0: -; RV32IZDINXZHINX-NEXT: addi sp, sp, -16 -; RV32IZDINXZHINX-NEXT: sw a0, 8(sp) -; RV32IZDINXZHINX-NEXT: sw a1, 12(sp) -; RV32IZDINXZHINX-NEXT: lw a0, 8(sp) -; RV32IZDINXZHINX-NEXT: lw a1, 12(sp) ; RV32IZDINXZHINX-NEXT: fcvt.h.d a0, a0 -; RV32IZDINXZHINX-NEXT: addi sp, sp, 16 ; RV32IZDINXZHINX-NEXT: ret ; ; RV64IZDINXZHINX-LABEL: fcvt_h_d: @@ -1807,13 +1801,7 @@ define half @fcvt_h_d(double %a) nounwind strictfp { ; ; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_d: ; CHECK32-IZDINXZHINXMIN: # %bb.0: -; CHECK32-IZDINXZHINXMIN-NEXT: addi sp, sp, -16 -; CHECK32-IZDINXZHINXMIN-NEXT: sw a0, 8(sp) -; CHECK32-IZDINXZHINXMIN-NEXT: sw a1, 12(sp) -; CHECK32-IZDINXZHINXMIN-NEXT: lw a0, 8(sp) -; CHECK32-IZDINXZHINXMIN-NEXT: lw a1, 12(sp) ; CHECK32-IZDINXZHINXMIN-NEXT: fcvt.h.d a0, a0 -; CHECK32-IZDINXZHINXMIN-NEXT: addi sp, sp, 16 ; CHECK32-IZDINXZHINXMIN-NEXT: ret ; ; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_d: @@ -1878,13 +1866,7 @@ define double @fcvt_d_h(half %a) nounwind strictfp { ; ; RV32IZDINXZHINX-LABEL: fcvt_d_h: ; RV32IZDINXZHINX: # %bb.0: -; RV32IZDINXZHINX-NEXT: addi sp, sp, -16 ; RV32IZDINXZHINX-NEXT: fcvt.d.h a0, a0 -; RV32IZDINXZHINX-NEXT: sw a0, 8(sp) -; RV32IZDINXZHINX-NEXT: sw a1, 12(sp) -; RV32IZDINXZHINX-NEXT: lw a0, 8(sp) -; RV32IZDINXZHINX-NEXT: lw a1, 12(sp) -; RV32IZDINXZHINX-NEXT: addi sp, sp, 16 ; RV32IZDINXZHINX-NEXT: ret ; ; RV64IZDINXZHINX-LABEL: fcvt_d_h: @@ -1944,13 +1926,7 @@ define double @fcvt_d_h(half %a) nounwind strictfp { ; ; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_d_h: ; CHECK32-IZDINXZHINXMIN: # %bb.0: -; CHECK32-IZDINXZHINXMIN-NEXT: addi sp, sp, -16 ; CHECK32-IZDINXZHINXMIN-NEXT: fcvt.d.h a0, a0 -; CHECK32-IZDINXZHINXMIN-NEXT: sw a0, 8(sp) -; CHECK32-IZDINXZHINXMIN-NEXT: sw a1, 12(sp) -; CHECK32-IZDINXZHINXMIN-NEXT: lw a0, 8(sp) -; CHECK32-IZDINXZHINXMIN-NEXT: lw a1, 12(sp) -; CHECK32-IZDINXZHINXMIN-NEXT: addi sp, sp, 16 ; CHECK32-IZDINXZHINXMIN-NEXT: ret ; ; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_d_h: diff --git a/llvm/test/CodeGen/RISCV/half-convert.ll b/llvm/test/CodeGen/RISCV/half-convert.ll index 28ac6e272e11..bc63b3961952 100644 --- a/llvm/test/CodeGen/RISCV/half-convert.ll +++ b/llvm/test/CodeGen/RISCV/half-convert.ll @@ -5275,21 +5275,10 @@ define half @fcvt_h_d(double %a) nounwind { ; RV64IZHINX-NEXT: addi sp, sp, 16 ; RV64IZHINX-NEXT: ret ; -; RV32IZDINXZHINX-LABEL: fcvt_h_d: -; RV32IZDINXZHINX: # %bb.0: -; RV32IZDINXZHINX-NEXT: addi sp, sp, -16 -; RV32IZDINXZHINX-NEXT: sw a0, 8(sp) -; RV32IZDINXZHINX-NEXT: sw a1, 12(sp) -; RV32IZDINXZHINX-NEXT: lw a0, 8(sp) -; RV32IZDINXZHINX-NEXT: lw a1, 12(sp) -; RV32IZDINXZHINX-NEXT: fcvt.h.d a0, a0 -; RV32IZDINXZHINX-NEXT: addi sp, sp, 16 -; RV32IZDINXZHINX-NEXT: ret -; -; RV64IZDINXZHINX-LABEL: fcvt_h_d: -; RV64IZDINXZHINX: # %bb.0: -; RV64IZDINXZHINX-NEXT: fcvt.h.d a0, a0 -; RV64IZDINXZHINX-NEXT: ret +; CHECKIZDINXZHINX-LABEL: fcvt_h_d: +; CHECKIZDINXZHINX: # %bb.0: +; CHECKIZDINXZHINX-NEXT: fcvt.h.d a0, a0 +; CHECKIZDINXZHINX-NEXT: ret ; ; RV32I-LABEL: fcvt_h_d: ; RV32I: # %bb.0: @@ -5405,13 +5394,7 @@ define half @fcvt_h_d(double %a) nounwind { ; ; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_d: ; CHECK32-IZDINXZHINXMIN: # %bb.0: -; CHECK32-IZDINXZHINXMIN-NEXT: addi sp, sp, -16 -; CHECK32-IZDINXZHINXMIN-NEXT: sw a0, 8(sp) -; CHECK32-IZDINXZHINXMIN-NEXT: sw a1, 12(sp) -; CHECK32-IZDINXZHINXMIN-NEXT: lw a0, 8(sp) -; CHECK32-IZDINXZHINXMIN-NEXT: lw a1, 12(sp) ; CHECK32-IZDINXZHINXMIN-NEXT: fcvt.h.d a0, a0 -; CHECK32-IZDINXZHINXMIN-NEXT: addi sp, sp, 16 ; CHECK32-IZDINXZHINXMIN-NEXT: ret ; ; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_d: @@ -5473,21 +5456,10 @@ define double @fcvt_d_h(half %a) nounwind { ; RV64IZHINX-NEXT: addi sp, sp, 16 ; RV64IZHINX-NEXT: ret ; -; RV32IZDINXZHINX-LABEL: fcvt_d_h: -; RV32IZDINXZHINX: # %bb.0: -; RV32IZDINXZHINX-NEXT: addi sp, sp, -16 -; RV32IZDINXZHINX-NEXT: fcvt.d.h a0, a0 -; RV32IZDINXZHINX-NEXT: sw a0, 8(sp) -; RV32IZDINXZHINX-NEXT: sw a1, 12(sp) -; RV32IZDINXZHINX-NEXT: lw a0, 8(sp) -; RV32IZDINXZHINX-NEXT: lw a1, 12(sp) -; RV32IZDINXZHINX-NEXT: addi sp, sp, 16 -; RV32IZDINXZHINX-NEXT: ret -; -; RV64IZDINXZHINX-LABEL: fcvt_d_h: -; RV64IZDINXZHINX: # %bb.0: -; RV64IZDINXZHINX-NEXT: fcvt.d.h a0, a0 -; RV64IZDINXZHINX-NEXT: ret +; CHECKIZDINXZHINX-LABEL: fcvt_d_h: +; CHECKIZDINXZHINX: # %bb.0: +; CHECKIZDINXZHINX-NEXT: fcvt.d.h a0, a0 +; CHECKIZDINXZHINX-NEXT: ret ; ; RV32I-LABEL: fcvt_d_h: ; RV32I: # %bb.0: @@ -5607,13 +5579,7 @@ define double @fcvt_d_h(half %a) nounwind { ; ; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_d_h: ; CHECK32-IZDINXZHINXMIN: # %bb.0: -; CHECK32-IZDINXZHINXMIN-NEXT: addi sp, sp, -16 ; CHECK32-IZDINXZHINXMIN-NEXT: fcvt.d.h a0, a0 -; CHECK32-IZDINXZHINXMIN-NEXT: sw a0, 8(sp) -; CHECK32-IZDINXZHINXMIN-NEXT: sw a1, 12(sp) -; CHECK32-IZDINXZHINXMIN-NEXT: lw a0, 8(sp) -; CHECK32-IZDINXZHINXMIN-NEXT: lw a1, 12(sp) -; CHECK32-IZDINXZHINXMIN-NEXT: addi sp, sp, 16 ; CHECK32-IZDINXZHINXMIN-NEXT: ret ; ; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_d_h: diff --git a/llvm/test/CodeGen/RISCV/pr64645.ll b/llvm/test/CodeGen/RISCV/pr64645.ll index 44dce5aabd22..f6d46516a3c0 100644 --- a/llvm/test/CodeGen/RISCV/pr64645.ll +++ b/llvm/test/CodeGen/RISCV/pr64645.ll @@ -5,34 +5,8 @@ define <2 x double> @v2f64(<2 x double> %x, <2 x double> %y) nounwind { ; CHECK-LABEL: v2f64: ; CHECK: # %bb.0: -; CHECK-NEXT: addi sp, sp, -16 -; CHECK-NEXT: sw a4, 8(sp) -; CHECK-NEXT: sw a5, 12(sp) -; CHECK-NEXT: lw a4, 8(sp) -; CHECK-NEXT: lw a5, 12(sp) -; CHECK-NEXT: sw a0, 8(sp) -; CHECK-NEXT: sw a1, 12(sp) -; CHECK-NEXT: lw a0, 8(sp) -; CHECK-NEXT: lw a1, 12(sp) -; CHECK-NEXT: sw a6, 8(sp) -; CHECK-NEXT: sw a7, 12(sp) -; CHECK-NEXT: lw a6, 8(sp) -; CHECK-NEXT: lw a7, 12(sp) -; CHECK-NEXT: sw a2, 8(sp) -; CHECK-NEXT: sw a3, 12(sp) -; CHECK-NEXT: lw a2, 8(sp) -; CHECK-NEXT: lw a3, 12(sp) ; CHECK-NEXT: fadd.d a2, a2, a6 ; CHECK-NEXT: fadd.d a0, a0, a4 -; CHECK-NEXT: sw a0, 8(sp) -; CHECK-NEXT: sw a1, 12(sp) -; CHECK-NEXT: lw a0, 8(sp) -; CHECK-NEXT: lw a1, 12(sp) -; CHECK-NEXT: sw a2, 8(sp) -; CHECK-NEXT: sw a3, 12(sp) -; CHECK-NEXT: lw a2, 8(sp) -; CHECK-NEXT: lw a3, 12(sp) -; CHECK-NEXT: addi sp, sp, 16 ; CHECK-NEXT: ret %1 = fadd <2 x double> %x, %y ret <2 x double> %1 diff --git a/llvm/test/CodeGen/RISCV/zdinx-asm-constraint.ll b/llvm/test/CodeGen/RISCV/zdinx-asm-constraint.ll index 63c46ca4eafc..95695aa69776 100644 --- a/llvm/test/CodeGen/RISCV/zdinx-asm-constraint.ll +++ b/llvm/test/CodeGen/RISCV/zdinx-asm-constraint.ll @@ -4,21 +4,15 @@ define dso_local void @zdinx_asm(ptr nocapture noundef writeonly %a, double noundef %b, double noundef %c) nounwind { ; CHECK-LABEL: zdinx_asm: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: addi sp, sp, -16 -; CHECK-NEXT: sw a1, 8(sp) -; CHECK-NEXT: sw a2, 12(sp) -; CHECK-NEXT: lw a6, 8(sp) -; CHECK-NEXT: lw a7, 12(sp) -; CHECK-NEXT: sw a3, 8(sp) -; CHECK-NEXT: sw a4, 12(sp) -; CHECK-NEXT: lw a2, 8(sp) -; CHECK-NEXT: lw a3, 12(sp) +; CHECK-NEXT: mv a5, a4 +; CHECK-NEXT: mv a7, a2 +; CHECK-NEXT: mv a4, a3 +; CHECK-NEXT: mv a6, a1 ; CHECK-NEXT: #APP -; CHECK-NEXT: fsgnjx.d a2, a6, a2 +; CHECK-NEXT: fsgnjx.d a2, a6, a4 ; CHECK-NEXT: #NO_APP ; CHECK-NEXT: sw a2, 8(a0) ; CHECK-NEXT: sw a3, 12(a0) -; CHECK-NEXT: addi sp, sp, 16 ; CHECK-NEXT: ret entry: %arrayidx = getelementptr inbounds double, ptr %a, i32 1 diff --git a/llvm/test/CodeGen/RISCV/zdinx-boundary-check.ll b/llvm/test/CodeGen/RISCV/zdinx-boundary-check.ll index 3eeb704f80eb..f56d47716bd7 100644 --- a/llvm/test/CodeGen/RISCV/zdinx-boundary-check.ll +++ b/llvm/test/CodeGen/RISCV/zdinx-boundary-check.ll @@ -7,15 +7,11 @@ define void @foo(ptr nocapture %p, double %d) nounwind { ; RV32ZDINX-LABEL: foo: ; RV32ZDINX: # %bb.0: # %entry -; RV32ZDINX-NEXT: addi sp, sp, -16 -; RV32ZDINX-NEXT: sw a1, 8(sp) -; RV32ZDINX-NEXT: sw a2, 12(sp) -; RV32ZDINX-NEXT: lw a2, 8(sp) -; RV32ZDINX-NEXT: lw a3, 12(sp) +; RV32ZDINX-NEXT: mv a3, a2 ; RV32ZDINX-NEXT: addi a0, a0, 2047 +; RV32ZDINX-NEXT: mv a2, a1 ; RV32ZDINX-NEXT: sw a2, -3(a0) ; RV32ZDINX-NEXT: sw a3, 1(a0) -; RV32ZDINX-NEXT: addi sp, sp, 16 ; RV32ZDINX-NEXT: ret ; ; RV64ZDINX-LABEL: foo: @@ -31,16 +27,12 @@ entry: define void @foo2(ptr nocapture %p, double %d) nounwind { ; RV32ZDINX-LABEL: foo2: ; RV32ZDINX: # %bb.0: # %entry -; RV32ZDINX-NEXT: addi sp, sp, -16 -; RV32ZDINX-NEXT: sw a1, 8(sp) -; RV32ZDINX-NEXT: sw a2, 12(sp) -; RV32ZDINX-NEXT: lw a2, 8(sp) -; RV32ZDINX-NEXT: lw a3, 12(sp) +; RV32ZDINX-NEXT: mv a3, a2 +; RV32ZDINX-NEXT: mv a2, a1 ; RV32ZDINX-NEXT: fadd.d a2, a2, a2 ; RV32ZDINX-NEXT: addi a0, a0, 2047 ; RV32ZDINX-NEXT: sw a2, -3(a0) ; RV32ZDINX-NEXT: sw a3, 1(a0) -; RV32ZDINX-NEXT: addi sp, sp, 16 ; RV32ZDINX-NEXT: ret ; ; RV64ZDINX-LABEL: foo2: @@ -117,15 +109,11 @@ entry: define void @foo5(ptr nocapture %p, double %d) nounwind { ; RV32ZDINX-LABEL: foo5: ; RV32ZDINX: # %bb.0: # %entry -; RV32ZDINX-NEXT: addi sp, sp, -16 -; RV32ZDINX-NEXT: sw a1, 8(sp) -; RV32ZDINX-NEXT: sw a2, 12(sp) -; RV32ZDINX-NEXT: lw a2, 8(sp) -; RV32ZDINX-NEXT: lw a3, 12(sp) +; RV32ZDINX-NEXT: mv a3, a2 ; RV32ZDINX-NEXT: addi a0, a0, -2048 +; RV32ZDINX-NEXT: mv a2, a1 ; RV32ZDINX-NEXT: sw a2, -1(a0) ; RV32ZDINX-NEXT: sw a3, 3(a0) -; RV32ZDINX-NEXT: addi sp, sp, 16 ; RV32ZDINX-NEXT: ret ; ; RV64ZDINX-LABEL: foo5: @@ -142,19 +130,15 @@ entry: define void @foo6(ptr %p, double %d) nounwind { ; RV32ZDINX-LABEL: foo6: ; RV32ZDINX: # %bb.0: # %entry -; RV32ZDINX-NEXT: addi sp, sp, -16 -; RV32ZDINX-NEXT: sw a1, 8(sp) -; RV32ZDINX-NEXT: sw a2, 12(sp) -; RV32ZDINX-NEXT: lw a2, 8(sp) -; RV32ZDINX-NEXT: lw a3, 12(sp) -; RV32ZDINX-NEXT: lui a1, %hi(.LCPI5_0) -; RV32ZDINX-NEXT: lw a4, %lo(.LCPI5_0)(a1) -; RV32ZDINX-NEXT: lw a5, %lo(.LCPI5_0+4)(a1) +; RV32ZDINX-NEXT: lui a3, %hi(.LCPI5_0) +; RV32ZDINX-NEXT: lw a4, %lo(.LCPI5_0)(a3) +; RV32ZDINX-NEXT: lw a5, %lo(.LCPI5_0+4)(a3) +; RV32ZDINX-NEXT: mv a3, a2 +; RV32ZDINX-NEXT: mv a2, a1 ; RV32ZDINX-NEXT: fadd.d a2, a2, a4 ; RV32ZDINX-NEXT: addi a0, a0, 2047 ; RV32ZDINX-NEXT: sw a2, -3(a0) ; RV32ZDINX-NEXT: sw a3, 1(a0) -; RV32ZDINX-NEXT: addi sp, sp, 16 ; RV32ZDINX-NEXT: ret ; ; RV64ZDINX-LABEL: foo6: -- GitLab From 3deaa77f1a25f0cdfcf23c34fac0b51293f32f9c Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Wed, 20 Mar 2024 15:47:00 +0000 Subject: [PATCH 030/296] [flang][OpenMP] simplify getReductionName (#85666) Re-use fir::getTypeAsString instead of creating something new here. This spells integer names like i32 instead of i_32 so there is a lot of test churn. --- flang/lib/Lower/OpenMP/ReductionProcessor.cpp | 75 ++++++------------- flang/lib/Lower/OpenMP/ReductionProcessor.h | 8 +- .../OpenMP/FIR/wsloop-reduction-add-byref.f90 | 22 +++--- .../Lower/OpenMP/FIR/wsloop-reduction-add.f90 | 28 +++---- .../FIR/wsloop-reduction-iand-byref.f90 | 4 +- .../FIR/wsloop-reduction-ieor-byref.f90 | 4 +- .../OpenMP/FIR/wsloop-reduction-ior-byref.f90 | 4 +- .../OpenMP/FIR/wsloop-reduction-max-byref.f90 | 8 +- .../OpenMP/FIR/wsloop-reduction-min-byref.f90 | 8 +- .../Lower/OpenMP/default-clause-byref.f90 | 2 +- flang/test/Lower/OpenMP/default-clause.f90 | 2 +- .../Lower/OpenMP/parallel-reduction-array.f90 | 4 +- .../OpenMP/parallel-reduction-array2.f90 | 4 +- .../OpenMP/parallel-reduction-rename.f90 | 4 +- .../parallel-wsloop-reduction-byref.f90 | 2 +- .../OpenMP/parallel-wsloop-reduction.f90 | 2 +- .../OpenMP/wsloop-reduction-add-byref.f90 | 22 +++--- .../wsloop-reduction-add-hlfir-byref.f90 | 6 +- .../OpenMP/wsloop-reduction-add-hlfir.f90 | 4 +- .../Lower/OpenMP/wsloop-reduction-add.f90 | 32 +++----- .../Lower/OpenMP/wsloop-reduction-array.f90 | 4 +- .../Lower/OpenMP/wsloop-reduction-array2.f90 | 4 +- .../OpenMP/wsloop-reduction-iand-byref.f90 | 4 +- .../Lower/OpenMP/wsloop-reduction-iand.f90 | 4 +- .../OpenMP/wsloop-reduction-ieor-byref.f90 | 4 +- .../OpenMP/wsloop-reduction-ior-byref.f90 | 6 +- .../Lower/OpenMP/wsloop-reduction-ior.f90 | 4 +- .../OpenMP/wsloop-reduction-max-2-byref.f90 | 2 +- .../Lower/OpenMP/wsloop-reduction-max-2.f90 | 2 +- .../OpenMP/wsloop-reduction-max-byref.f90 | 10 +-- .../wsloop-reduction-max-hlfir-byref.f90 | 4 +- .../OpenMP/wsloop-reduction-max-hlfir.f90 | 4 +- .../Lower/OpenMP/wsloop-reduction-max.f90 | 10 +-- .../OpenMP/wsloop-reduction-min-byref.f90 | 10 +-- .../Lower/OpenMP/wsloop-reduction-min.f90 | 10 +-- .../Lower/OpenMP/wsloop-reduction-min2.f90 | 4 +- .../OpenMP/wsloop-reduction-mul-byref.f90 | 22 +++--- .../Lower/OpenMP/wsloop-reduction-mul.f90 | 23 +++--- 38 files changed, 163 insertions(+), 213 deletions(-) diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp index d01c585e0ddb..2477f635792a 100644 --- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp +++ b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp @@ -81,8 +81,10 @@ bool ReductionProcessor::supportedIntrinsicProcReduction( return redType; } -std::string ReductionProcessor::getReductionName(llvm::StringRef name, - mlir::Type ty, bool isByRef) { +std::string +ReductionProcessor::getReductionName(llvm::StringRef name, + const fir::KindMapping &kindMap, + mlir::Type ty, bool isByRef) { ty = fir::unwrapRefType(ty); // extra string to distinguish reduction functions for variables passed by @@ -91,47 +93,12 @@ std::string ReductionProcessor::getReductionName(llvm::StringRef name, if (isByRef) byrefAddition = "_byref"; - if (fir::isa_trivial(ty)) - return (llvm::Twine(name) + - (ty.isIntOrIndex() ? llvm::Twine("_i_") : llvm::Twine("_f_")) + - llvm::Twine(ty.getIntOrFloatBitWidth()) + byrefAddition) - .str(); - - // creates a name like reduction_i_64_box_ux4x3 - if (auto boxTy = mlir::dyn_cast_or_null(ty)) { - // TODO: support for allocatable boxes: - // !fir.box>> - fir::SequenceType seqTy = fir::unwrapRefType(boxTy.getEleTy()) - .dyn_cast_or_null(); - if (!seqTy) - return {}; - - std::string prefix = getReductionName( - name, fir::unwrapSeqOrBoxedSeqType(ty), /*isByRef=*/false); - if (prefix.empty()) - return {}; - std::stringstream tyStr; - tyStr << prefix << "_box_"; - bool first = true; - for (std::int64_t extent : seqTy.getShape()) { - if (first) - first = false; - else - tyStr << "x"; - if (extent == seqTy.getUnknownExtent()) - tyStr << 'u'; // I'm not sure that '?' is safe in symbol names - else - tyStr << extent; - } - return (tyStr.str() + byrefAddition).str(); - } - - return {}; + return fir::getTypeAsString(ty, kindMap, (name + byrefAddition).str()); } std::string ReductionProcessor::getReductionName( - omp::clause::DefinedOperator::IntrinsicOperator intrinsicOp, mlir::Type ty, - bool isByRef) { + omp::clause::DefinedOperator::IntrinsicOperator intrinsicOp, + const fir::KindMapping &kindMap, mlir::Type ty, bool isByRef) { std::string reductionName; switch (intrinsicOp) { @@ -154,7 +121,7 @@ std::string ReductionProcessor::getReductionName( break; } - return getReductionName(reductionName, ty, isByRef); + return getReductionName(reductionName, kindMap, ty, isByRef); } mlir::Value @@ -162,9 +129,9 @@ ReductionProcessor::getReductionInitValue(mlir::Location loc, mlir::Type type, ReductionIdentifier redId, fir::FirOpBuilder &builder) { type = fir::unwrapRefType(type); - assert((fir::isa_integer(type) || fir::isa_real(type) || - type.isa()) && - "only integer, logical and real types are currently supported"); + if (!fir::isa_integer(type) && !fir::isa_real(type) && + !mlir::isa(type)) + TODO(loc, "Reduction of some types is not supported"); switch (redId) { case ReductionIdentifier::MAX: { if (auto ty = type.dyn_cast()) { @@ -463,8 +430,7 @@ mlir::omp::DeclareReductionOp ReductionProcessor::createDeclareReduction( mlir::OpBuilder::InsertionGuard guard(builder); mlir::ModuleOp module = builder.getModule(); - if (reductionOpName.empty()) - TODO(loc, "Reduction of some types is not supported"); + assert(!reductionOpName.empty()); auto decl = module.lookupSymbol(reductionOpName); @@ -601,15 +567,18 @@ void ReductionProcessor::addDeclareReduction( for (mlir::Value symVal : reductionVars) { auto redType = mlir::cast(symVal.getType()); + const auto &kindMap = firOpBuilder.getKindMap(); if (redType.getEleTy().isa()) - decl = createDeclareReduction( - firOpBuilder, - getReductionName(intrinsicOp, firOpBuilder.getI1Type(), isByRef), - redId, redType, currentLocation, isByRef); + decl = createDeclareReduction(firOpBuilder, + getReductionName(intrinsicOp, kindMap, + firOpBuilder.getI1Type(), + isByRef), + redId, redType, currentLocation, isByRef); else decl = createDeclareReduction( - firOpBuilder, getReductionName(intrinsicOp, redType, isByRef), - redId, redType, currentLocation, isByRef); + firOpBuilder, + getReductionName(intrinsicOp, kindMap, redType, isByRef), redId, + redType, currentLocation, isByRef); reductionDeclSymbols.push_back(mlir::SymbolRefAttr::get( firOpBuilder.getContext(), decl.getSymName())); } @@ -631,7 +600,7 @@ void ReductionProcessor::addDeclareReduction( decl = createDeclareReduction( firOpBuilder, getReductionName(getRealName(*reductionIntrinsic).ToString(), - redType, isByRef), + firOpBuilder.getKindMap(), redType, isByRef), redId, redType, currentLocation, isByRef); reductionDeclSymbols.push_back(mlir::SymbolRefAttr::get( firOpBuilder.getContext(), decl.getSymName())); diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.h b/flang/lib/Lower/OpenMP/ReductionProcessor.h index 52b44e555bdb..ee2732547fc2 100644 --- a/flang/lib/Lower/OpenMP/ReductionProcessor.h +++ b/flang/lib/Lower/OpenMP/ReductionProcessor.h @@ -76,12 +76,14 @@ public: static bool doReductionByRef(const llvm::SmallVectorImpl &reductionVars); - static std::string getReductionName(llvm::StringRef name, mlir::Type ty, - bool isByRef); + static std::string getReductionName(llvm::StringRef name, + const fir::KindMapping &kindMap, + mlir::Type ty, bool isByRef); static std::string getReductionName(omp::clause::DefinedOperator::IntrinsicOperator intrinsicOp, - mlir::Type ty, bool isByRef); + const fir::KindMapping &kindMap, mlir::Type ty, + bool isByRef); /// This function returns the identity value of the operator \p /// reductionOpName. For example: diff --git a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-add-byref.f90 b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-add-byref.f90 index e9eadb4bc31a..08f5a0fcdbae 100644 --- a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-add-byref.f90 +++ b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-add-byref.f90 @@ -2,7 +2,7 @@ ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py -! CHECK-LABEL: omp.declare_reduction @add_reduction_f_64_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @add_reduction_byref_f64 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[C0_1:.*]] = arith.constant 0.000000e+00 : f64 @@ -19,7 +19,7 @@ ! CHECK: omp.yield(%[[ARG0]] : !fir.ref) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @add_reduction_i_64_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @add_reduction_byref_i64 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[C0_1:.*]] = arith.constant 0 : i64 @@ -36,7 +36,7 @@ ! CHECK: omp.yield(%[[ARG0]] : !fir.ref) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @add_reduction_f_32_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @add_reduction_byref_f32 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[C0_1:.*]] = arith.constant 0.000000e+00 : f32 @@ -53,7 +53,7 @@ ! CHECK: omp.yield(%[[ARG0]] : !fir.ref) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @add_reduction_i_32_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @add_reduction_byref_i32 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[C0_1:.*]] = arith.constant 0 : i32 @@ -80,7 +80,7 @@ ! CHECK: %[[VAL_4:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_5:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_6:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_byref %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { +! CHECK: omp.wsloop byref reduction(@add_reduction_byref_i32 %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { ! CHECK: fir.store %[[VAL_8]] to %[[VAL_3]] : !fir.ref ! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_7]] : !fir.ref ! CHECK: %[[VAL_10:.*]] = fir.load %[[VAL_3]] : !fir.ref @@ -116,7 +116,7 @@ end subroutine ! CHECK: %[[VAL_4:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_5:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_6:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@add_reduction_f_32_byref %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { +! CHECK: omp.wsloop byref reduction(@add_reduction_byref_f32 %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { ! CHECK: fir.store %[[VAL_8]] to %[[VAL_3]] : !fir.ref ! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_7]] : !fir.ref ! CHECK: %[[VAL_10:.*]] = fir.load %[[VAL_3]] : !fir.ref @@ -152,7 +152,7 @@ end subroutine ! CHECK: %[[VAL_4:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_5:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_6:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_byref %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { +! CHECK: omp.wsloop byref reduction(@add_reduction_byref_i32 %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { ! CHECK: fir.store %[[VAL_8]] to %[[VAL_3]] : !fir.ref ! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_3]] : !fir.ref ! CHECK: %[[VAL_10:.*]] = fir.load %[[VAL_7]] : !fir.ref @@ -187,7 +187,7 @@ end subroutine ! CHECK: %[[VAL_4:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_5:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_6:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@add_reduction_f_32_byref %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { +! CHECK: omp.wsloop byref reduction(@add_reduction_byref_f32 %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { ! CHECK: fir.store %[[VAL_8]] to %[[VAL_3]] : !fir.ref ! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_3]] : !fir.ref ! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i32) -> f32 @@ -229,7 +229,7 @@ end subroutine ! CHECK: %[[VAL_8:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_10:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_byref %[[VAL_1]] -> %[[VAL_11:.*]] : !fir.ref, @add_reduction_i_32_byref %[[VAL_2]] -> %[[VAL_12:.*]] : !fir.ref, @add_reduction_i_32_byref %[[VAL_3]] -> %[[VAL_13:.*]] : !fir.ref) for (%[[VAL_14:.*]]) : i32 = (%[[VAL_8]]) to (%[[VAL_9]]) inclusive step (%[[VAL_10]]) { +! CHECK: omp.wsloop byref reduction(@add_reduction_byref_i32 %[[VAL_1]] -> %[[VAL_11:.*]] : !fir.ref, @add_reduction_byref_i32 %[[VAL_2]] -> %[[VAL_12:.*]] : !fir.ref, @add_reduction_byref_i32 %[[VAL_3]] -> %[[VAL_13:.*]] : !fir.ref) for (%[[VAL_14:.*]]) : i32 = (%[[VAL_8]]) to (%[[VAL_9]]) inclusive step (%[[VAL_10]]) { ! CHECK: fir.store %[[VAL_14]] to %[[VAL_7]] : !fir.ref ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_11]] : !fir.ref ! CHECK: %[[VAL_16:.*]] = fir.load %[[VAL_7]] : !fir.ref @@ -282,7 +282,7 @@ end subroutine ! CHECK: %[[VAL_8:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_10:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@add_reduction_f_32_byref %[[VAL_1]] -> %[[VAL_11:.*]] : !fir.ref, @add_reduction_f_32_byref %[[VAL_2]] -> %[[VAL_12:.*]] : !fir.ref, @add_reduction_f_32_byref %[[VAL_3]] -> %[[VAL_13:.*]] : !fir.ref) for (%[[VAL_14:.*]]) : i32 = (%[[VAL_8]]) to (%[[VAL_9]]) inclusive step (%[[VAL_10]]) { +! CHECK: omp.wsloop byref reduction(@add_reduction_byref_f32 %[[VAL_1]] -> %[[VAL_11:.*]] : !fir.ref, @add_reduction_byref_f32 %[[VAL_2]] -> %[[VAL_12:.*]] : !fir.ref, @add_reduction_byref_f32 %[[VAL_3]] -> %[[VAL_13:.*]] : !fir.ref) for (%[[VAL_14:.*]]) : i32 = (%[[VAL_8]]) to (%[[VAL_9]]) inclusive step (%[[VAL_10]]) { ! CHECK: fir.store %[[VAL_14]] to %[[VAL_7]] : !fir.ref ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_11]] : !fir.ref ! CHECK: %[[VAL_16:.*]] = fir.load %[[VAL_7]] : !fir.ref @@ -341,7 +341,7 @@ end subroutine ! CHECK: %[[VAL_10:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_11:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_12:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_byref %[[VAL_2]] -> %[[VAL_13:.*]] : !fir.ref, @add_reduction_i_64_byref %[[VAL_3]] -> %[[VAL_14:.*]] : !fir.ref, @add_reduction_f_32_byref %[[VAL_4]] -> %[[VAL_15:.*]] : !fir.ref, @add_reduction_f_64_byref %[[VAL_1]] -> %[[VAL_16:.*]] : !fir.ref) for (%[[VAL_17:.*]]) : i32 = (%[[VAL_10]]) to (%[[VAL_11]]) inclusive step (%[[VAL_12]]) { +! CHECK: omp.wsloop byref reduction(@add_reduction_byref_i32 %[[VAL_2]] -> %[[VAL_13:.*]] : !fir.ref, @add_reduction_byref_i64 %[[VAL_3]] -> %[[VAL_14:.*]] : !fir.ref, @add_reduction_byref_f32 %[[VAL_4]] -> %[[VAL_15:.*]] : !fir.ref, @add_reduction_byref_f64 %[[VAL_1]] -> %[[VAL_16:.*]] : !fir.ref) for (%[[VAL_17:.*]]) : i32 = (%[[VAL_10]]) to (%[[VAL_11]]) inclusive step (%[[VAL_12]]) { ! CHECK: fir.store %[[VAL_17]] to %[[VAL_9]] : !fir.ref ! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_13]] : !fir.ref ! CHECK: %[[VAL_19:.*]] = fir.load %[[VAL_9]] : !fir.ref diff --git a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-add.f90 b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-add.f90 index b53dc715e65b..dc96b875f745 100644 --- a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-add.f90 +++ b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-add.f90 @@ -1,13 +1,7 @@ ! RUN: bbc -emit-fir -hlfir=false -fopenmp %s -o - | FileCheck %s ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fopenmp %s -o - | FileCheck %s -! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py -! The script is designed to make adding checks to -! a test case fast, it is *not* designed to be authoritative -! about what constitutes a good test! The CHECK should be -! minimized and named to reflect the test intent. - -! CHECK-LABEL: omp.declare_reduction @add_reduction_f_64 : f64 init { +! CHECK-LABEL: omp.declare_reduction @add_reduction_f64 : f64 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: f64): ! CHECK: %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f64 ! CHECK: omp.yield(%[[VAL_1]] : f64) @@ -18,7 +12,7 @@ ! CHECK: omp.yield(%[[VAL_2]] : f64) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @add_reduction_i_64 : i64 init { +! CHECK-LABEL: omp.declare_reduction @add_reduction_i64 : i64 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: i64): ! CHECK: %[[VAL_1:.*]] = arith.constant 0 : i64 ! CHECK: omp.yield(%[[VAL_1]] : i64) @@ -29,7 +23,7 @@ ! CHECK: omp.yield(%[[VAL_2]] : i64) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @add_reduction_f_32 : f32 init { +! CHECK-LABEL: omp.declare_reduction @add_reduction_f32 : f32 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: f32): ! CHECK: %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f32 ! CHECK: omp.yield(%[[VAL_1]] : f32) @@ -40,7 +34,7 @@ ! CHECK: omp.yield(%[[VAL_2]] : f32) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @add_reduction_i_32 : i32 init { +! CHECK-LABEL: omp.declare_reduction @add_reduction_i32 : i32 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: i32): ! CHECK: %[[VAL_1:.*]] = arith.constant 0 : i32 ! CHECK: omp.yield(%[[VAL_1]] : i32) @@ -61,7 +55,7 @@ ! CHECK: %[[VAL_4:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_5:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_6:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@add_reduction_i_32 %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { +! CHECK: omp.wsloop reduction(@add_reduction_i32 %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { ! CHECK: fir.store %[[VAL_8]] to %[[VAL_3]] : !fir.ref ! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_7]] : !fir.ref ! CHECK: %[[VAL_10:.*]] = fir.load %[[VAL_3]] : !fir.ref @@ -97,7 +91,7 @@ end subroutine ! CHECK: %[[VAL_4:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_5:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_6:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@add_reduction_f_32 %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { +! CHECK: omp.wsloop reduction(@add_reduction_f32 %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { ! CHECK: fir.store %[[VAL_8]] to %[[VAL_3]] : !fir.ref ! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_7]] : !fir.ref ! CHECK: %[[VAL_10:.*]] = fir.load %[[VAL_3]] : !fir.ref @@ -133,7 +127,7 @@ end subroutine ! CHECK: %[[VAL_4:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_5:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_6:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@add_reduction_i_32 %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { +! CHECK: omp.wsloop reduction(@add_reduction_i32 %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { ! CHECK: fir.store %[[VAL_8]] to %[[VAL_3]] : !fir.ref ! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_3]] : !fir.ref ! CHECK: %[[VAL_10:.*]] = fir.load %[[VAL_7]] : !fir.ref @@ -168,7 +162,7 @@ end subroutine ! CHECK: %[[VAL_4:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_5:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_6:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@add_reduction_f_32 %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { +! CHECK: omp.wsloop reduction(@add_reduction_f32 %[[VAL_1]] -> %[[VAL_7:.*]] : !fir.ref) for (%[[VAL_8:.*]]) : i32 = (%[[VAL_4]]) to (%[[VAL_5]]) inclusive step (%[[VAL_6]]) { ! CHECK: fir.store %[[VAL_8]] to %[[VAL_3]] : !fir.ref ! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_3]] : !fir.ref ! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i32) -> f32 @@ -210,7 +204,7 @@ end subroutine ! CHECK: %[[VAL_8:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_10:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@add_reduction_i_32 %[[VAL_1]] -> %[[VAL_11:.*]] : !fir.ref, @add_reduction_i_32 %[[VAL_2]] -> %[[VAL_12:.*]] : !fir.ref, @add_reduction_i_32 %[[VAL_3]] -> %[[VAL_13:.*]] : !fir.ref) for (%[[VAL_14:.*]]) : i32 = (%[[VAL_8]]) to (%[[VAL_9]]) inclusive step (%[[VAL_10]]) { +! CHECK: omp.wsloop reduction(@add_reduction_i32 %[[VAL_1]] -> %[[VAL_11:.*]] : !fir.ref, @add_reduction_i32 %[[VAL_2]] -> %[[VAL_12:.*]] : !fir.ref, @add_reduction_i32 %[[VAL_3]] -> %[[VAL_13:.*]] : !fir.ref) for (%[[VAL_14:.*]]) : i32 = (%[[VAL_8]]) to (%[[VAL_9]]) inclusive step (%[[VAL_10]]) { ! CHECK: fir.store %[[VAL_14]] to %[[VAL_7]] : !fir.ref ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_11]] : !fir.ref ! CHECK: %[[VAL_16:.*]] = fir.load %[[VAL_7]] : !fir.ref @@ -263,7 +257,7 @@ end subroutine ! CHECK: %[[VAL_8:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_10:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@add_reduction_f_32 %[[VAL_1]] -> %[[VAL_11:.*]] : !fir.ref, @add_reduction_f_32 %[[VAL_2]] -> %[[VAL_12:.*]] : !fir.ref, @add_reduction_f_32 %[[VAL_3]] -> %[[VAL_13:.*]] : !fir.ref) for (%[[VAL_14:.*]]) : i32 = (%[[VAL_8]]) to (%[[VAL_9]]) inclusive step (%[[VAL_10]]) { +! CHECK: omp.wsloop reduction(@add_reduction_f32 %[[VAL_1]] -> %[[VAL_11:.*]] : !fir.ref, @add_reduction_f32 %[[VAL_2]] -> %[[VAL_12:.*]] : !fir.ref, @add_reduction_f32 %[[VAL_3]] -> %[[VAL_13:.*]] : !fir.ref) for (%[[VAL_14:.*]]) : i32 = (%[[VAL_8]]) to (%[[VAL_9]]) inclusive step (%[[VAL_10]]) { ! CHECK: fir.store %[[VAL_14]] to %[[VAL_7]] : !fir.ref ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_11]] : !fir.ref ! CHECK: %[[VAL_16:.*]] = fir.load %[[VAL_7]] : !fir.ref @@ -322,7 +316,7 @@ end subroutine ! CHECK: %[[VAL_10:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_11:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_12:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@add_reduction_i_32 %[[VAL_2]] -> %[[VAL_13:.*]] : !fir.ref, @add_reduction_i_64 %[[VAL_3]] -> %[[VAL_14:.*]] : !fir.ref, @add_reduction_f_32 %[[VAL_4]] -> %[[VAL_15:.*]] : !fir.ref, @add_reduction_f_64 %[[VAL_1]] -> %[[VAL_16:.*]] : !fir.ref) for (%[[VAL_17:.*]]) : i32 = (%[[VAL_10]]) to (%[[VAL_11]]) inclusive step (%[[VAL_12]]) { +! CHECK: omp.wsloop reduction(@add_reduction_i32 %[[VAL_2]] -> %[[VAL_13:.*]] : !fir.ref, @add_reduction_i64 %[[VAL_3]] -> %[[VAL_14:.*]] : !fir.ref, @add_reduction_f32 %[[VAL_4]] -> %[[VAL_15:.*]] : !fir.ref, @add_reduction_f64 %[[VAL_1]] -> %[[VAL_16:.*]] : !fir.ref) for (%[[VAL_17:.*]]) : i32 = (%[[VAL_10]]) to (%[[VAL_11]]) inclusive step (%[[VAL_12]]) { ! CHECK: fir.store %[[VAL_17]] to %[[VAL_9]] : !fir.ref ! CHECK: %[[VAL_18:.*]] = fir.load %[[VAL_13]] : !fir.ref ! CHECK: %[[VAL_19:.*]] = fir.load %[[VAL_9]] : !fir.ref diff --git a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-iand-byref.f90 b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-iand-byref.f90 index 92e5e8a0abb5..6717597ff3b0 100644 --- a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-iand-byref.f90 +++ b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-iand-byref.f90 @@ -1,7 +1,7 @@ ! RUN: bbc -emit-fir -hlfir=false -fopenmp --force-byref-reduction %s -o - | FileCheck %s ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s -!CHECK-LABEL: omp.declare_reduction @iand_i_32_byref : !fir.ref +!CHECK-LABEL: omp.declare_reduction @iand_byref_i32 : !fir.ref !CHECK-SAME: init { !CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): !CHECK: %[[C0_1:.*]] = arith.constant -1 : i32 @@ -23,7 +23,7 @@ !CHECK-SAME: %[[Y_BOX:.*]]: !fir.box> !CHECK: %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_iandEx"} !CHECK: omp.parallel -!CHECK: omp.wsloop byref reduction(@iand_i_32_byref %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for +!CHECK: omp.wsloop byref reduction(@iand_byref_i32 %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for !CHECK: %[[LPRV:.+]] = fir.load %[[PRV]] : !fir.ref !CHECK: %[[Y_I_REF:.*]] = fir.coordinate_of %[[Y_BOX]] !CHECK: %[[Y_I:.*]] = fir.load %[[Y_I_REF]] : !fir.ref diff --git a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-ieor-byref.f90 b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-ieor-byref.f90 index c55778903924..1baa59a510fa 100644 --- a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-ieor-byref.f90 +++ b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-ieor-byref.f90 @@ -1,7 +1,7 @@ ! RUN: bbc -emit-fir -hlfir=false -fopenmp --force-byref-reduction %s -o - | FileCheck %s ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -mmlir --force-byref-reduction -fopenmp %s -o - | FileCheck %s -! CHECK-LABEL: omp.declare_reduction @ieor_i_32_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @ieor_byref_i32 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[C0_1:.*]] = arith.constant 0 : i32 @@ -22,7 +22,7 @@ !CHECK-SAME: %[[Y_BOX:.*]]: !fir.box> !CHECK: %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_ieorEx"} !CHECK: omp.parallel -!CHECK: omp.wsloop byref reduction(@ieor_i_32_byref %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for +!CHECK: omp.wsloop byref reduction(@ieor_byref_i32 %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for !CHECK: %[[LPRV:.+]] = fir.load %[[PRV]] : !fir.ref !CHECK: %[[Y_I_REF:.*]] = fir.coordinate_of %[[Y_BOX]] !CHECK: %[[Y_I:.*]] = fir.load %[[Y_I_REF]] : !fir.ref diff --git a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-ior-byref.f90 b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-ior-byref.f90 index 57f393b9688e..5482ef33fc8a 100644 --- a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-ior-byref.f90 +++ b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-ior-byref.f90 @@ -1,7 +1,7 @@ ! RUN: bbc -emit-fir -hlfir=false -fopenmp --force-byref-reduction %s -o - | FileCheck %s ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s -! CHECK-LABEL: omp.declare_reduction @ior_i_32_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @ior_byref_i32 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[C0_1:.*]] = arith.constant 0 : i32 @@ -22,7 +22,7 @@ !CHECK-SAME: %[[Y_BOX:.*]]: !fir.box> !CHECK: %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_iorEx"} !CHECK: omp.parallel -!CHECK: omp.wsloop byref reduction(@ior_i_32_byref %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for +!CHECK: omp.wsloop byref reduction(@ior_byref_i32 %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for !CHECK: %[[LPRV:.+]] = fir.load %[[PRV]] : !fir.ref !CHECK: %[[Y_I_REF:.*]] = fir.coordinate_of %[[Y_BOX]] !CHECK: %[[Y_I:.*]] = fir.load %[[Y_I_REF]] : !fir.ref diff --git a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-max-byref.f90 b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-max-byref.f90 index bc655b5e4295..f0979ab95f56 100644 --- a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-max-byref.f90 +++ b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-max-byref.f90 @@ -1,7 +1,7 @@ ! RUN: bbc -emit-fir -hlfir=false -fopenmp --force-byref-reduction -o - %s 2>&1 | FileCheck %s ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -fopenmp -mmlir --force-byref-reduction -o - %s 2>&1 | FileCheck %s -!CHECK: omp.declare_reduction @max_f_32_byref : !fir.ref +!CHECK: omp.declare_reduction @max_byref_f32 : !fir.ref !CHECK-SAME: init { !CHECK: %[[MINIMUM_VAL:.*]] = arith.constant -3.40282347E+38 : f32 !CHECK: %[[REF:.*]] = fir.alloca f32 @@ -15,7 +15,7 @@ !CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref !CHECK: omp.yield(%[[ARG0]] : !fir.ref) -!CHECK-LABEL: omp.declare_reduction @max_i_32_byref : !fir.ref +!CHECK-LABEL: omp.declare_reduction @max_byref_i32 : !fir.ref !CHECK-SAME: init { !CHECK: %[[MINIMUM_VAL:.*]] = arith.constant -2147483648 : i32 !CHECK: fir.store %[[MINIMUM_VAL]] to %[[REF]] : !fir.ref @@ -32,7 +32,7 @@ !CHECK-SAME: %[[Y_BOX:.*]]: !fir.box> !CHECK: %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_max_intEx"} !CHECK: omp.parallel -!CHECK: omp.wsloop byref reduction(@max_i_32_byref %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for +!CHECK: omp.wsloop byref reduction(@max_byref_i32 %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for !CHECK: %[[LPRV:.+]] = fir.load %[[PRV]] : !fir.ref !CHECK: %[[Y_I_REF:.*]] = fir.coordinate_of %[[Y_BOX]] !CHECK: %[[Y_I:.*]] = fir.load %[[Y_I_REF]] : !fir.ref @@ -45,7 +45,7 @@ !CHECK-SAME: %[[Y_BOX:.*]]: !fir.box> !CHECK: %[[X_REF:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFreduction_max_realEx"} !CHECK: omp.parallel -!CHECK: omp.wsloop byref reduction(@max_f_32_byref %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for +!CHECK: omp.wsloop byref reduction(@max_byref_f32 %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for !CHECK: %[[LPRV:.+]] = fir.load %[[PRV]] : !fir.ref !CHECK: %[[Y_I_REF:.*]] = fir.coordinate_of %[[Y_BOX]] !CHECK: %[[Y_I:.*]] = fir.load %[[Y_I_REF]] : !fir.ref diff --git a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-min-byref.f90 b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-min-byref.f90 index ce928aa131b7..24aa8e46e5bb 100644 --- a/flang/test/Lower/OpenMP/FIR/wsloop-reduction-min-byref.f90 +++ b/flang/test/Lower/OpenMP/FIR/wsloop-reduction-min-byref.f90 @@ -1,7 +1,7 @@ ! RUN: bbc -emit-fir -hlfir=false -fopenmp --force-byref-reduction -o - %s 2>&1 | FileCheck %s ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir -mmlir --force-byref-reduction -fopenmp -o - %s 2>&1 | FileCheck %s -!CHECK: omp.declare_reduction @min_f_32_byref : !fir.ref +!CHECK: omp.declare_reduction @min_byref_f32 : !fir.ref !CHECK-SAME: init { !CHECK: %[[MAXIMUM_VAL:.*]] = arith.constant 3.40282347E+38 : f32 !CHECK: %[[REF:.*]] = fir.alloca f32 @@ -15,7 +15,7 @@ !CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref !CHECK: omp.yield(%[[ARG0]] : !fir.ref) -!CHECK-LABEL: omp.declare_reduction @min_i_32_byref : !fir.ref +!CHECK-LABEL: omp.declare_reduction @min_byref_i32 : !fir.ref !CHECK-SAME: init { !CHECK: %[[MAXIMUM_VAL:.*]] = arith.constant 2147483647 : i32 !CHECK: fir.store %[[MAXIMUM_VAL]] to %[[REF]] : !fir.ref @@ -32,7 +32,7 @@ !CHECK-SAME: %[[Y_BOX:.*]]: !fir.box> !CHECK: %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFreduction_min_intEx"} !CHECK: omp.parallel -!CHECK: omp.wsloop byref reduction(@min_i_32_byref %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for +!CHECK: omp.wsloop byref reduction(@min_byref_i32 %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for !CHECK: %[[LPRV:.+]] = fir.load %[[PRV]] : !fir.ref !CHECK: %[[Y_I_REF:.*]] = fir.coordinate_of %[[Y_BOX]] !CHECK: %[[Y_I:.*]] = fir.load %[[Y_I_REF]] : !fir.ref @@ -46,7 +46,7 @@ !CHECK-SAME: %[[Y_BOX:.*]]: !fir.box> !CHECK: %[[X_REF:.*]] = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFreduction_min_realEx"} !CHECK: omp.parallel -!CHECK: omp.wsloop byref reduction(@min_f_32_byref %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for +!CHECK: omp.wsloop byref reduction(@min_byref_f32 %[[X_REF]] -> %[[PRV:.+]] : !fir.ref) for !CHECK: %[[LPRV:.+]] = fir.load %[[PRV]] : !fir.ref !CHECK: %[[Y_I_REF:.*]] = fir.coordinate_of %[[Y_BOX]] !CHECK: %[[Y_I:.*]] = fir.load %[[Y_I_REF]] : !fir.ref diff --git a/flang/test/Lower/OpenMP/default-clause-byref.f90 b/flang/test/Lower/OpenMP/default-clause-byref.f90 index 5d9538e53069..1167ba7e6ae0 100644 --- a/flang/test/Lower/OpenMP/default-clause-byref.f90 +++ b/flang/test/Lower/OpenMP/default-clause-byref.f90 @@ -352,7 +352,7 @@ subroutine skipped_default_clause_checks() type(it)::iii !CHECK: omp.parallel { -!CHECK: omp.wsloop byref reduction(@min_i_32_byref %[[VAL_Z_DECLARE]]#0 -> %[[PRV:.+]] : !fir.ref) for (%[[ARG:.*]]) {{.*}} { +!CHECK: omp.wsloop byref reduction(@min_byref_i32 %[[VAL_Z_DECLARE]]#0 -> %[[PRV:.+]] : !fir.ref) for (%[[ARG:.*]]) {{.*}} { !CHECK: omp.yield !CHECK: } !CHECK: omp.terminator diff --git a/flang/test/Lower/OpenMP/default-clause.f90 b/flang/test/Lower/OpenMP/default-clause.f90 index 0e118742689d..f86b51aef4e0 100644 --- a/flang/test/Lower/OpenMP/default-clause.f90 +++ b/flang/test/Lower/OpenMP/default-clause.f90 @@ -352,7 +352,7 @@ subroutine skipped_default_clause_checks() type(it)::iii !CHECK: omp.parallel { -!CHECK: omp.wsloop reduction(@min_i_32 %[[VAL_Z_DECLARE]]#0 -> %[[PRV:.+]] : !fir.ref) for (%[[ARG:.*]]) {{.*}} { +!CHECK: omp.wsloop reduction(@min_i32 %[[VAL_Z_DECLARE]]#0 -> %[[PRV:.+]] : !fir.ref) for (%[[ARG:.*]]) {{.*}} { !CHECK: omp.yield !CHECK: } !CHECK: omp.terminator diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array.f90 index 359dab35e8a6..735a99854308 100644 --- a/flang/test/Lower/OpenMP/parallel-reduction-array.f90 +++ b/flang/test/Lower/OpenMP/parallel-reduction-array.f90 @@ -13,7 +13,7 @@ i(3) = 3 print *,i end program -! CHECK-LABEL: omp.declare_reduction @add_reduction_i_32_box_3_byref : !fir.ref>> init { +! CHECK-LABEL: omp.declare_reduction @add_reduction_byref_box_3xi32 : !fir.ref>> init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref>>): ! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.array<3xi32> {bindc_name = ".tmp"} ! CHECK: %[[VAL_2:.*]] = arith.constant 0 : i32 @@ -53,7 +53,7 @@ end program ! CHECK: %[[VAL_4:.*]] = fir.embox %[[VAL_3]]#1(%[[VAL_2]]) : (!fir.ref>, !fir.shape<1>) -> !fir.box> ! CHECK: %[[VAL_5:.*]] = fir.alloca !fir.box> ! CHECK: fir.store %[[VAL_4]] to %[[VAL_5]] : !fir.ref>> -! CHECK: omp.parallel byref reduction(@add_reduction_i_32_box_3_byref %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref>>) { +! CHECK: omp.parallel byref reduction(@add_reduction_byref_box_3xi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref>>) { ! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEi"} : (!fir.ref>>) -> (!fir.ref>>, !fir.ref>>) ! CHECK: %[[VAL_8:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref>> diff --git a/flang/test/Lower/OpenMP/parallel-reduction-array2.f90 b/flang/test/Lower/OpenMP/parallel-reduction-array2.f90 index 5a31be8d206b..4834047a98a4 100644 --- a/flang/test/Lower/OpenMP/parallel-reduction-array2.f90 +++ b/flang/test/Lower/OpenMP/parallel-reduction-array2.f90 @@ -13,7 +13,7 @@ i(3) = i(3) + 3 print *,i end program -! CHECK-LABEL: omp.declare_reduction @add_reduction_i_32_box_3_byref : !fir.ref>> init { +! CHECK-LABEL: omp.declare_reduction @add_reduction_byref_box_3xi32 : !fir.ref>> init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref>>): ! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.array<3xi32> {bindc_name = ".tmp"} ! CHECK: %[[VAL_2:.*]] = arith.constant 0 : i32 @@ -53,7 +53,7 @@ end program ! CHECK: %[[VAL_4:.*]] = fir.embox %[[VAL_3]]#1(%[[VAL_2]]) : (!fir.ref>, !fir.shape<1>) -> !fir.box> ! CHECK: %[[VAL_5:.*]] = fir.alloca !fir.box> ! CHECK: fir.store %[[VAL_4]] to %[[VAL_5]] : !fir.ref>> -! CHECK: omp.parallel byref reduction(@add_reduction_i_32_box_3_byref %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref>>) { +! CHECK: omp.parallel byref reduction(@add_reduction_byref_box_3xi32 %[[VAL_5]] -> %[[VAL_6:.*]] : !fir.ref>>) { ! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFEi"} : (!fir.ref>>) -> (!fir.ref>>, !fir.ref>>) ! CHECK: %[[VAL_8:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref>> ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : index diff --git a/flang/test/Lower/OpenMP/parallel-reduction-rename.f90 b/flang/test/Lower/OpenMP/parallel-reduction-rename.f90 index 7845f808e6c5..c06343e997bf 100644 --- a/flang/test/Lower/OpenMP/parallel-reduction-rename.f90 +++ b/flang/test/Lower/OpenMP/parallel-reduction-rename.f90 @@ -14,7 +14,7 @@ end program main ! test that we understood that this should be a max reduction -! CHECK-LABEL: omp.declare_reduction @max_i_32 : i32 init { +! CHECK-LABEL: omp.declare_reduction @max_i32 : i32 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: i32): ! CHECK: %[[VAL_1:.*]] = arith.constant -2147483648 : i32 ! CHECK: omp.yield(%[[VAL_1]] : i32) @@ -30,7 +30,7 @@ end program main ! CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFEn"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_2:.*]] = arith.constant 0 : i32 ! CHECK: hlfir.assign %[[VAL_2]] to %[[VAL_1]]#0 : i32, !fir.ref -! CHECK: omp.parallel reduction(@max_i_32 %[[VAL_1]]#0 -> %[[VAL_3:.*]] : !fir.ref) { +! CHECK: omp.parallel reduction(@max_i32 %[[VAL_1]]#0 -> %[[VAL_3:.*]] : !fir.ref) { ! ... ! CHECK: omp.terminator diff --git a/flang/test/Lower/OpenMP/parallel-wsloop-reduction-byref.f90 b/flang/test/Lower/OpenMP/parallel-wsloop-reduction-byref.f90 index 8492a69fed58..66c80c31917b 100644 --- a/flang/test/Lower/OpenMP/parallel-wsloop-reduction-byref.f90 +++ b/flang/test/Lower/OpenMP/parallel-wsloop-reduction-byref.f90 @@ -4,7 +4,7 @@ ! RUN: flang-new -fc1 -fopenmp -mmlir --force-byref-reduction -emit-hlfir %s -o - | FileCheck %s ! CHECK: omp.parallel { -! CHECK: omp.wsloop byref reduction(@add_reduction_i_32 +! CHECK: omp.wsloop byref reduction(@add_reduction_byref_i32 subroutine sb integer :: x x = 0 diff --git a/flang/test/Lower/OpenMP/parallel-wsloop-reduction.f90 b/flang/test/Lower/OpenMP/parallel-wsloop-reduction.f90 index c13ec2bc4aec..fdedbb061607 100644 --- a/flang/test/Lower/OpenMP/parallel-wsloop-reduction.f90 +++ b/flang/test/Lower/OpenMP/parallel-wsloop-reduction.f90 @@ -4,7 +4,7 @@ ! RUN: flang-new -fc1 -fopenmp -emit-hlfir %s -o - | FileCheck %s ! CHECK: omp.parallel { -! CHECK: omp.wsloop reduction(@add_reduction_i_32 +! CHECK: omp.wsloop reduction(@add_reduction_i32 subroutine sb integer :: x x = 0 diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90 index caec65b6051c..e63db33bbe25 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-add-byref.f90 @@ -1,7 +1,7 @@ ! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction %s -o - | FileCheck %s ! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s -! CHECK-LABEL: omp.declare_reduction @add_reduction_f_64_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @add_reduction_byref_f64 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[C0_1:.*]] = arith.constant 0.000000e+00 : f64 @@ -18,7 +18,7 @@ ! CHECK: omp.yield(%[[ARG0]] : !fir.ref) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @add_reduction_i_64_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @add_reduction_byref_i64 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[C0_1:.*]] = arith.constant 0 : i64 @@ -35,7 +35,7 @@ ! CHECK: omp.yield(%[[ARG0]] : !fir.ref) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @add_reduction_f_32_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @add_reduction_byref_f32 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[C0_1:.*]] = arith.constant 0.000000e+00 : f32 @@ -52,7 +52,7 @@ ! CHECK: omp.yield(%[[ARG0]] : !fir.ref) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @add_reduction_i_32_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @add_reduction_byref_i32 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[C0_1:.*]] = arith.constant 0 : i32 @@ -82,7 +82,7 @@ ! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_byref %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: omp.wsloop byref reduction(@add_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { ! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref ! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref @@ -122,7 +122,7 @@ end subroutine ! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@add_reduction_f_32_byref %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: omp.wsloop byref reduction(@add_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { ! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref ! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref @@ -163,7 +163,7 @@ end subroutine ! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_byref %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: omp.wsloop byref reduction(@add_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { ! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref ! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reduction_switch_orderEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref @@ -202,7 +202,7 @@ end subroutine ! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@add_reduction_f_32_byref %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: omp.wsloop byref reduction(@add_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { ! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref ! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reduction_switch_orderEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref @@ -250,7 +250,7 @@ end subroutine ! CHECK: %[[VAL_13:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_14:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_15:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_byref %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @add_reduction_i_32_byref %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @add_reduction_i_32_byref %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { +! CHECK: omp.wsloop byref reduction(@add_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @add_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @add_reduction_byref_i32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { ! CHECK: fir.store %[[VAL_19]] to %[[VAL_12]]#1 : !fir.ref ! CHECK: %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_int_reductions_same_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_int_reductions_same_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) @@ -311,7 +311,7 @@ end subroutine ! CHECK: %[[VAL_13:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_14:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_15:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@add_reduction_f_32_byref %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @add_reduction_f_32_byref %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @add_reduction_f_32_byref %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { +! CHECK: omp.wsloop byref reduction(@add_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @add_reduction_byref_f32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @add_reduction_byref_f32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { ! CHECK: fir.store %[[VAL_19]] to %[[VAL_12]]#1 : !fir.ref ! CHECK: %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_real_reductions_same_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_real_reductions_same_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) @@ -379,7 +379,7 @@ end subroutine ! CHECK: %[[VAL_16:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_17:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_18:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_byref %[[VAL_5]]#0 -> %[[VAL_19:.*]] : !fir.ref, @add_reduction_i_64_byref %[[VAL_7]]#0 -> %[[VAL_20:.*]] : !fir.ref, @add_reduction_f_32_byref %[[VAL_9]]#0 -> %[[VAL_21:.*]] : !fir.ref, @add_reduction_f_64_byref %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref) for (%[[VAL_23:.*]]) : i32 = (%[[VAL_16]]) to (%[[VAL_17]]) inclusive step (%[[VAL_18]]) { +! CHECK: omp.wsloop byref reduction(@add_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_19:.*]] : !fir.ref, @add_reduction_byref_i64 %[[VAL_7]]#0 -> %[[VAL_20:.*]] : !fir.ref, @add_reduction_byref_f32 %[[VAL_9]]#0 -> %[[VAL_21:.*]] : !fir.ref, @add_reduction_byref_f64 %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref) for (%[[VAL_23:.*]]) : i32 = (%[[VAL_16]]) to (%[[VAL_17]]) inclusive step (%[[VAL_18]]) { ! CHECK: fir.store %[[VAL_23]] to %[[VAL_15]]#1 : !fir.ref ! CHECK: %[[VAL_24:.*]]:2 = hlfir.declare %[[VAL_19]] {uniq_name = "_QFmultiple_reductions_different_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_25:.*]]:2 = hlfir.declare %[[VAL_20]] {uniq_name = "_QFmultiple_reductions_different_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir-byref.f90 index 5e5df8c1365a..3b4d9666c693 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir-byref.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir-byref.f90 @@ -1,9 +1,7 @@ ! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction %s -o - | FileCheck %s ! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s -! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py - -! CHECK-LABEL: omp.declare_reduction @add_reduction_i_32_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @add_reduction_byref_i32 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[C0_1:.*]] = arith.constant 0 : i32 @@ -33,7 +31,7 @@ ! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_byref %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) +! CHECK: omp.wsloop byref reduction(@add_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) ! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref ! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir.f90 index d6006f21b3bd..7c9070592e46 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-add-hlfir.f90 @@ -3,7 +3,7 @@ ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py -! CHECK-LABEL: omp.declare_reduction @add_reduction_i_32 : i32 init { +! CHECK-LABEL: omp.declare_reduction @add_reduction_i32 : i32 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: i32): ! CHECK: %[[VAL_1:.*]] = arith.constant 0 : i32 ! CHECK: omp.yield(%[[VAL_1]] : i32) @@ -27,7 +27,7 @@ ! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@add_reduction_i_32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) +! CHECK: omp.wsloop reduction(@add_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) ! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref ! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-add.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-add.f90 index d6222a248b7f..11e1ffb79f8e 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-add.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-add.f90 @@ -1,17 +1,7 @@ ! RUN: bbc -emit-hlfir -fopenmp %s -o - | FileCheck %s ! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s - -! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py - -! The script is designed to make adding checks to -! a test case fast, it is *not* designed to be authoritative -! about what constitutes a good test! The CHECK should be -! minimized and named to reflect the test intent. - - - -! CHECK-LABEL: omp.declare_reduction @add_reduction_f_64 : f64 init { +! CHECK-LABEL: omp.declare_reduction @add_reduction_f64 : f64 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: f64): ! CHECK: %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f64 ! CHECK: omp.yield(%[[VAL_1]] : f64) @@ -22,7 +12,7 @@ ! CHECK: omp.yield(%[[VAL_2]] : f64) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @add_reduction_i_64 : i64 init { +! CHECK-LABEL: omp.declare_reduction @add_reduction_i64 : i64 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: i64): ! CHECK: %[[VAL_1:.*]] = arith.constant 0 : i64 ! CHECK: omp.yield(%[[VAL_1]] : i64) @@ -33,7 +23,7 @@ ! CHECK: omp.yield(%[[VAL_2]] : i64) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @add_reduction_f_32 : f32 init { +! CHECK-LABEL: omp.declare_reduction @add_reduction_f32 : f32 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: f32): ! CHECK: %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f32 ! CHECK: omp.yield(%[[VAL_1]] : f32) @@ -44,7 +34,7 @@ ! CHECK: omp.yield(%[[VAL_2]] : f32) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @add_reduction_i_32 : i32 init { +! CHECK-LABEL: omp.declare_reduction @add_reduction_i32 : i32 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: i32): ! CHECK: %[[VAL_1:.*]] = arith.constant 0 : i32 ! CHECK: omp.yield(%[[VAL_1]] : i32) @@ -68,7 +58,7 @@ ! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@add_reduction_i_32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: omp.wsloop reduction(@add_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { ! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref ! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref @@ -108,7 +98,7 @@ end subroutine ! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@add_reduction_f_32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: omp.wsloop reduction(@add_reduction_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { ! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref ! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref @@ -149,7 +139,7 @@ end subroutine ! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@add_reduction_i_32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: omp.wsloop reduction(@add_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { ! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref ! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reduction_switch_orderEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref @@ -188,7 +178,7 @@ end subroutine ! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@add_reduction_f_32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: omp.wsloop reduction(@add_reduction_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { ! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref ! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reduction_switch_orderEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref @@ -236,7 +226,7 @@ end subroutine ! CHECK: %[[VAL_13:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_14:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_15:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@add_reduction_i_32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @add_reduction_i_32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @add_reduction_i_32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { +! CHECK: omp.wsloop reduction(@add_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @add_reduction_i32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @add_reduction_i32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { ! CHECK: fir.store %[[VAL_19]] to %[[VAL_12]]#1 : !fir.ref ! CHECK: %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_int_reductions_same_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_int_reductions_same_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) @@ -297,7 +287,7 @@ end subroutine ! CHECK: %[[VAL_13:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_14:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_15:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@add_reduction_f_32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @add_reduction_f_32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @add_reduction_f_32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { +! CHECK: omp.wsloop reduction(@add_reduction_f32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @add_reduction_f32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @add_reduction_f32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { ! CHECK: fir.store %[[VAL_19]] to %[[VAL_12]]#1 : !fir.ref ! CHECK: %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_real_reductions_same_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_real_reductions_same_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) @@ -365,7 +355,7 @@ end subroutine ! CHECK: %[[VAL_16:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_17:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_18:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@add_reduction_i_32 %[[VAL_5]]#0 -> %[[VAL_19:.*]] : !fir.ref, @add_reduction_i_64 %[[VAL_7]]#0 -> %[[VAL_20:.*]] : !fir.ref, @add_reduction_f_32 %[[VAL_9]]#0 -> %[[VAL_21:.*]] : !fir.ref, @add_reduction_f_64 %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref) for (%[[VAL_23:.*]]) : i32 = (%[[VAL_16]]) to (%[[VAL_17]]) inclusive step (%[[VAL_18]]) { +! CHECK: omp.wsloop reduction(@add_reduction_i32 %[[VAL_5]]#0 -> %[[VAL_19:.*]] : !fir.ref, @add_reduction_i64 %[[VAL_7]]#0 -> %[[VAL_20:.*]] : !fir.ref, @add_reduction_f32 %[[VAL_9]]#0 -> %[[VAL_21:.*]] : !fir.ref, @add_reduction_f64 %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref) for (%[[VAL_23:.*]]) : i32 = (%[[VAL_16]]) to (%[[VAL_17]]) inclusive step (%[[VAL_18]]) { ! CHECK: fir.store %[[VAL_23]] to %[[VAL_15]]#1 : !fir.ref ! CHECK: %[[VAL_24:.*]]:2 = hlfir.declare %[[VAL_19]] {uniq_name = "_QFmultiple_reductions_different_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_25:.*]]:2 = hlfir.declare %[[VAL_20]] {uniq_name = "_QFmultiple_reductions_different_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array.f90 index be4d27f6ac47..a20ed1ca83ce 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-array.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-array.f90 @@ -14,7 +14,7 @@ enddo print *,r end program -! CHECK-LABEL omp.declare_reduction @add_reduction_i_32_box_2_byref : !fir.ref>> init { +! CHECK-LABEL omp.declare_reduction @add_reduction_byref_box_2xi32 : !fir.ref>> init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref>>): ! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.array<2xi32> {bindc_name = ".tmp"} ! CHECK: %[[VAL_2:.*]] = arith.constant 0 : i32 @@ -63,7 +63,7 @@ end program ! CHECK: %[[VAL_11:.*]] = fir.embox %[[VAL_5]]#1(%[[VAL_4]]) : (!fir.ref>, !fir.shape<1>) -> !fir.box> ! CHECK: %[[VAL_12:.*]] = fir.alloca !fir.box> ! CHECK: fir.store %[[VAL_11]] to %[[VAL_12]] : !fir.ref>> -! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_box_2_byref %[[VAL_12]] -> %[[VAL_13:.*]] : !fir.ref>>) for (%[[VAL_14:.*]]) : i32 = (%[[VAL_8]]) to (%[[VAL_9]]) inclusive step (%[[VAL_10]]) { +! CHECK: omp.wsloop byref reduction(@add_reduction_byref_box_2xi32 %[[VAL_12]] -> %[[VAL_13:.*]] : !fir.ref>>) for (%[[VAL_14:.*]]) : i32 = (%[[VAL_8]]) to (%[[VAL_9]]) inclusive step (%[[VAL_10]]) { ! CHECK: fir.store %[[VAL_14]] to %[[VAL_7]]#1 : !fir.ref ! CHECK: %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_13]] {uniq_name = "_QFEr"} : (!fir.ref>>) -> (!fir.ref>>, !fir.ref>>) ! CHECK: %[[VAL_16:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90 index c77bd72f916d..61599876da8e 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-array2.f90 @@ -14,7 +14,7 @@ enddo print *,r end program -! CHECK-LABEL omp.declare_reduction @add_reduction_i_32_box_2_byref : !fir.ref>> init { +! CHECK-LABEL omp.declare_reduction @add_reduction_byref_box_2xi32 : !fir.ref>> init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref>>): ! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.array<2xi32> {bindc_name = ".tmp"} ! CHECK: %[[VAL_2:.*]] = arith.constant 0 : i32 @@ -63,7 +63,7 @@ end program ! CHECK: %[[VAL_11:.*]] = fir.embox %[[VAL_5]]#1(%[[VAL_4]]) : (!fir.ref>, !fir.shape<1>) -> !fir.box> ! CHECK: %[[VAL_12:.*]] = fir.alloca !fir.box> ! CHECK: fir.store %[[VAL_11]] to %[[VAL_12]] : !fir.ref>> -! CHECK: omp.wsloop byref reduction(@add_reduction_i_32_box_2_byref %[[VAL_12]] -> %[[VAL_13:.*]] : !fir.ref>>) for (%[[VAL_14:.*]]) : i32 = (%[[VAL_8]]) to (%[[VAL_9]]) inclusive step (%[[VAL_10]]) { +! CHECK: omp.wsloop byref reduction(@add_reduction_byref_box_2xi32 %[[VAL_12]] -> %[[VAL_13:.*]] : !fir.ref>>) for (%[[VAL_14:.*]]) : i32 = (%[[VAL_8]]) to (%[[VAL_9]]) inclusive step (%[[VAL_10]]) { ! CHECK: fir.store %[[VAL_14]] to %[[VAL_7]]#1 : !fir.ref ! CHECK: %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_13]] {uniq_name = "_QFEr"} : (!fir.ref>>) -> (!fir.ref>>, !fir.ref>>) ! CHECK: %[[VAL_16:.*]] = fir.load %[[VAL_15]]#0 : !fir.ref>> diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90 index 2df3ad393ba5..e3f06a446ed4 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-iand-byref.f90 @@ -3,7 +3,7 @@ ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py -! CHECK-LABEL: omp.declare_reduction @iand_i_32_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @iand_byref_i32 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[C0_1:.*]] = arith.constant -1 : i32 @@ -35,7 +35,7 @@ ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@iand_i_32_byref %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { +! CHECK: omp.wsloop byref reduction(@iand_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { ! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref ! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_iandEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90 index 707be66def8d..746617e21062 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-iand.f90 @@ -3,7 +3,7 @@ ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py -! CHECK-LABEL: omp.declare_reduction @iand_i_32 : i32 init { +! CHECK-LABEL: omp.declare_reduction @iand_i32 : i32 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: i32): ! CHECK: %[[VAL_1:.*]] = arith.constant -1 : i32 ! CHECK: omp.yield(%[[VAL_1]] : i32) @@ -29,7 +29,7 @@ ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@iand_i_32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { +! CHECK: omp.wsloop reduction(@iand_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { ! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref ! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_iandEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90 index 72a6901878ad..7e3a283bf783 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-ieor-byref.f90 @@ -1,7 +1,7 @@ ! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction %s -o - | FileCheck %s ! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s -! CHECK-LABEL: omp.declare_reduction @ieor_i_32_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @ieor_byref_i32 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[C0_1:.*]] = arith.constant 0 : i32 @@ -28,7 +28,7 @@ !CHECK: omp.parallel !CHECK: %[[I_REF:.*]] = fir.alloca i32 {adapt.valuebyref, pinned} !CHECK: %[[I_DECL:.*]]:2 = hlfir.declare %[[I_REF]] {uniq_name = "_QFreduction_ieorEi"} : (!fir.ref) -> (!fir.ref, !fir.ref) -!CHECK: omp.wsloop byref reduction(@ieor_i_32_byref %[[X_DECL]]#0 -> %[[PRV:.+]] : !fir.ref) for +!CHECK: omp.wsloop byref reduction(@ieor_byref_i32 %[[X_DECL]]#0 -> %[[PRV:.+]] : !fir.ref) for !CHECK: fir.store %{{.*}} to %[[I_DECL]]#1 : !fir.ref !CHECK: %[[PRV_DECL:.+]]:2 = hlfir.declare %[[PRV]] {{.*}} : (!fir.ref) -> (!fir.ref, !fir.ref) !CHECK: %[[I_32:.*]] = fir.load %[[I_DECL]]#0 : !fir.ref diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90 index 662e64b8fae6..c7f8e8bdede5 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-ior-byref.f90 @@ -1,9 +1,7 @@ ! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction %s -o - | FileCheck %s ! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction %s -o - | FileCheck %s -! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py - -! CHECK-LABEL: omp.declare_reduction @ior_i_32_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @ior_byref_i32 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[C0_1:.*]] = arith.constant 0 : i32 @@ -35,7 +33,7 @@ ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@ior_i_32_byref %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) +! CHECK: omp.wsloop byref reduction(@ior_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) ! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref ! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_iorEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90 index 8f6ca7c41c42..dd0bbeb1a076 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-ior.f90 @@ -3,7 +3,7 @@ ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py -! CHECK-LABEL: omp.declare_reduction @ior_i_32 : i32 init { +! CHECK-LABEL: omp.declare_reduction @ior_i32 : i32 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: i32): ! CHECK: %[[VAL_1:.*]] = arith.constant 0 : i32 ! CHECK: omp.yield(%[[VAL_1]] : i32) @@ -29,7 +29,7 @@ ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@ior_i_32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) +! CHECK: omp.wsloop reduction(@ior_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) ! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref ! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_iorEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-2-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-2-byref.f90 index 360cd34df2d1..5358806cdcde 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-max-2-byref.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-2-byref.f90 @@ -1,7 +1,7 @@ ! RUN: bbc -emit-hlfir -fopenmp --force-byref-reduction -o - %s 2>&1 | FileCheck %s ! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --force-byref-reduction -o - %s 2>&1 | FileCheck %s -! CHECK: omp.wsloop byref reduction(@max_i_32 +! CHECK: omp.wsloop byref reduction(@max_byref_i32 ! CHECK: arith.cmpi sgt ! CHECK: arith.select diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-2.f90 index 1f4d61985689..abd7ca1ae555 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-max-2.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-2.f90 @@ -1,7 +1,7 @@ ! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s ! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s -! CHECK: omp.wsloop reduction(@max_i_32 +! CHECK: omp.wsloop reduction(@max_i32 ! CHECK: arith.cmpi sgt ! CHECK: arith.select diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90 index 058eeea5fb92..ee562bbe1586 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90 @@ -3,7 +3,7 @@ ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py -!CHECK: omp.declare_reduction @max_f_32_byref : !fir.ref +!CHECK: omp.declare_reduction @max_byref_f32 : !fir.ref !CHECK-SAME: init { !CHECK: %[[MINIMUM_VAL:.*]] = arith.constant -3.40282347E+38 : f32 !CHECK: %[[REF:.*]] = fir.alloca f32 @@ -17,7 +17,7 @@ !CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref !CHECK: omp.yield(%[[ARG0]] : !fir.ref) -!CHECK-LABEL: omp.declare_reduction @max_i_32_byref : !fir.ref +!CHECK-LABEL: omp.declare_reduction @max_byref_i32 : !fir.ref !CHECK-SAME: init { !CHECK: %[[MINIMUM_VAL:.*]] = arith.constant -2147483648 : i32 !CHECK: %[[REF:.*]] = fir.alloca i32 @@ -46,7 +46,7 @@ ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@max_i_32_byref %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { +! CHECK: omp.wsloop byref reduction(@max_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { ! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref ! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_max_intEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref @@ -75,7 +75,7 @@ ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@max_f_32_byref %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { +! CHECK: omp.wsloop byref reduction(@max_byref_f32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { ! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref ! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_max_realEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref @@ -94,7 +94,7 @@ ! CHECK: %[[VAL_32:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_33:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_34:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@max_f_32_byref %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref) for (%[[VAL_36:.*]]) : i32 = (%[[VAL_32]]) to (%[[VAL_33]]) inclusive step (%[[VAL_34]]) { +! CHECK: omp.wsloop byref reduction(@max_byref_f32 %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref) for (%[[VAL_36:.*]]) : i32 = (%[[VAL_32]]) to (%[[VAL_33]]) inclusive step (%[[VAL_34]]) { ! CHECK: fir.store %[[VAL_36]] to %[[VAL_31]]#1 : !fir.ref ! CHECK: %[[VAL_37:.*]]:2 = hlfir.declare %[[VAL_35]] {uniq_name = "_QFreduction_max_realEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_38:.*]] = fir.load %[[VAL_31]]#0 : !fir.ref diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90 index 3a07450765cf..10bba6ac4b51 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir-byref.f90 @@ -3,7 +3,7 @@ ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py -! CHECK-LABEL: omp.declare_reduction @max_i_32_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @max_byref_i32 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[MINIMUM_VAL:.*]] = arith.constant -2147483648 : i32 @@ -33,7 +33,7 @@ ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@max_i_32_byref %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { +! CHECK: omp.wsloop byref reduction(@max_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { ! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref ! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_max_intEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir.f90 index 3dfed100a02e..5ea5d6626f18 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-hlfir.f90 @@ -3,7 +3,7 @@ ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py -! CHECK-LABEL: omp.declare_reduction @max_i_32 : i32 init { +! CHECK-LABEL: omp.declare_reduction @max_i32 : i32 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: i32): ! CHECK: %[[VAL_1:.*]] = arith.constant -2147483648 : i32 ! CHECK: omp.yield(%[[VAL_1]] : i32) @@ -29,7 +29,7 @@ ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@max_i_32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { +! CHECK: omp.wsloop reduction(@max_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { ! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref ! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_max_intEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max.f90 index 895eca85fd1e..6f11f0ec96a7 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-max.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-max.f90 @@ -3,7 +3,7 @@ ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py -! CHECK-LABEL: omp.declare_reduction @max_f_32 : f32 init { +! CHECK-LABEL: omp.declare_reduction @max_f32 : f32 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: f32): ! CHECK: %[[VAL_1:.*]] = arith.constant -3.40282347E+38 : f32 ! CHECK: omp.yield(%[[VAL_1]] : f32) @@ -14,7 +14,7 @@ ! CHECK: omp.yield(%[[VAL_2]] : f32) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @max_i_32 : i32 init { +! CHECK-LABEL: omp.declare_reduction @max_i32 : i32 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: i32): ! CHECK: %[[VAL_1:.*]] = arith.constant -2147483648 : i32 ! CHECK: omp.yield(%[[VAL_1]] : i32) @@ -40,7 +40,7 @@ ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@max_i_32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { +! CHECK: omp.wsloop reduction(@max_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { ! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref ! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_max_intEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref @@ -69,7 +69,7 @@ ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@max_f_32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { +! CHECK: omp.wsloop reduction(@max_f32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { ! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref ! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_max_realEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref @@ -88,7 +88,7 @@ ! CHECK: %[[VAL_32:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_33:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_34:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@max_f_32 %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref) for (%[[VAL_36:.*]]) : i32 = (%[[VAL_32]]) to (%[[VAL_33]]) inclusive step (%[[VAL_34]]) { +! CHECK: omp.wsloop reduction(@max_f32 %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref) for (%[[VAL_36:.*]]) : i32 = (%[[VAL_32]]) to (%[[VAL_33]]) inclusive step (%[[VAL_34]]) { ! CHECK: fir.store %[[VAL_36]] to %[[VAL_31]]#1 : !fir.ref ! CHECK: %[[VAL_37:.*]]:2 = hlfir.declare %[[VAL_35]] {uniq_name = "_QFreduction_max_realEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_38:.*]] = fir.load %[[VAL_31]]#0 : !fir.ref diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90 index c44301f68b83..c0372117a03b 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90 @@ -3,7 +3,7 @@ ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py -!CHECK: omp.declare_reduction @min_f_32_byref : !fir.ref +!CHECK: omp.declare_reduction @min_byref_f32 : !fir.ref !CHECK-SAME: init { !CHECK: %[[MAXIMUM_VAL:.*]] = arith.constant 3.40282347E+38 : f32 !CHECK: %[[REF:.*]] = fir.alloca f32 @@ -17,7 +17,7 @@ !CHECK: fir.store %[[RES]] to %[[ARG0]] : !fir.ref !CHECK: omp.yield(%[[ARG0]] : !fir.ref) -!CHECK-LABEL: omp.declare_reduction @min_i_32_byref : !fir.ref +!CHECK-LABEL: omp.declare_reduction @min_byref_i32 : !fir.ref !CHECK-SAME: init { !CHECK: %[[MAXIMUM_VAL:.*]] = arith.constant 2147483647 : i32 !CHECK: %[[REF:.*]] = fir.alloca i32 @@ -46,7 +46,7 @@ ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@min_i_32_byref %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { +! CHECK: omp.wsloop byref reduction(@min_byref_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { ! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref ! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_min_intEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref @@ -75,7 +75,7 @@ ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@min_f_32_byref %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { +! CHECK: omp.wsloop byref reduction(@min_byref_f32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { ! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref ! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_min_realEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref @@ -96,7 +96,7 @@ ! CHECK: %[[VAL_32:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_33:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_34:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@min_f_32_byref %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref) for (%[[VAL_36:.*]]) : i32 = (%[[VAL_32]]) to (%[[VAL_33]]) inclusive step (%[[VAL_34]]) { +! CHECK: omp.wsloop byref reduction(@min_byref_f32 %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref) for (%[[VAL_36:.*]]) : i32 = (%[[VAL_32]]) to (%[[VAL_33]]) inclusive step (%[[VAL_34]]) { ! CHECK: fir.store %[[VAL_36]] to %[[VAL_31]]#1 : !fir.ref ! CHECK: %[[VAL_37:.*]]:2 = hlfir.declare %[[VAL_35]] {uniq_name = "_QFreduction_min_realEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_38:.*]] = fir.load %[[VAL_31]]#0 : !fir.ref diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min.f90 index 700ff78339c1..2c694f82e279 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-min.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-min.f90 @@ -3,7 +3,7 @@ ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py -! CHECK-LABEL: omp.declare_reduction @min_f_32 : f32 init { +! CHECK-LABEL: omp.declare_reduction @min_f32 : f32 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: f32): ! CHECK: %[[VAL_1:.*]] = arith.constant 3.40282347E+38 : f32 ! CHECK: omp.yield(%[[VAL_1]] : f32) @@ -14,7 +14,7 @@ ! CHECK: omp.yield(%[[VAL_2]] : f32) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @min_i_32 : i32 init { +! CHECK-LABEL: omp.declare_reduction @min_i32 : i32 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: i32): ! CHECK: %[[VAL_1:.*]] = arith.constant 2147483647 : i32 ! CHECK: omp.yield(%[[VAL_1]] : i32) @@ -40,7 +40,7 @@ ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@min_i_32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { +! CHECK: omp.wsloop reduction(@min_i32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { ! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref ! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_min_intEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref @@ -69,7 +69,7 @@ ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_10:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_11:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@min_f_32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { +! CHECK: omp.wsloop reduction(@min_f32 %[[VAL_4]]#0 -> %[[VAL_12:.*]] : !fir.ref) for (%[[VAL_13:.*]]) : i32 = (%[[VAL_9]]) to (%[[VAL_10]]) inclusive step (%[[VAL_11]]) { ! CHECK: fir.store %[[VAL_13]] to %[[VAL_8]]#1 : !fir.ref ! CHECK: %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_min_realEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref @@ -90,7 +90,7 @@ ! CHECK: %[[VAL_32:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_33:.*]] = arith.constant 100 : i32 ! CHECK: %[[VAL_34:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@min_f_32 %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref) for (%[[VAL_36:.*]]) : i32 = (%[[VAL_32]]) to (%[[VAL_33]]) inclusive step (%[[VAL_34]]) { +! CHECK: omp.wsloop reduction(@min_f32 %[[VAL_4]]#0 -> %[[VAL_35:.*]] : !fir.ref) for (%[[VAL_36:.*]]) : i32 = (%[[VAL_32]]) to (%[[VAL_33]]) inclusive step (%[[VAL_34]]) { ! CHECK: fir.store %[[VAL_36]] to %[[VAL_31]]#1 : !fir.ref ! CHECK: %[[VAL_37:.*]]:2 = hlfir.declare %[[VAL_35]] {uniq_name = "_QFreduction_min_realEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_38:.*]] = fir.load %[[VAL_31]]#0 : !fir.ref diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90 index cef1102a2fcc..0138a9578206 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-min2.f90 @@ -17,7 +17,7 @@ print *,r end program -! CHECK-LABEL: omp.declare_reduction @min_i_32 : i32 init { +! CHECK-LABEL: omp.declare_reduction @min_i32 : i32 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: i32): ! CHECK: %[[VAL_1:.*]] = arith.constant 2147483647 : i32 ! CHECK: omp.yield(%[[VAL_1]] : i32) @@ -39,7 +39,7 @@ end program ! CHECK: %[[VAL_6:.*]] = arith.constant 0 : i32 ! CHECK: %[[VAL_7:.*]] = arith.constant 10 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@min_i_32 %[[VAL_3]]#0 -> %[[VAL_9:.*]] : !fir.ref) for (%[[VAL_10:.*]]) : i32 = (%[[VAL_6]]) to (%[[VAL_7]]) inclusive step (%[[VAL_8]]) { +! CHECK: omp.wsloop reduction(@min_i32 %[[VAL_3]]#0 -> %[[VAL_9:.*]] : !fir.ref) for (%[[VAL_10:.*]]) : i32 = (%[[VAL_6]]) to (%[[VAL_7]]) inclusive step (%[[VAL_8]]) { ! CHECK: fir.store %[[VAL_10]] to %[[VAL_5]]#1 : !fir.ref ! CHECK: %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_9]] {uniq_name = "_QFEr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_12:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90 index c6aa21c4231c..a2829948d472 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-mul-byref.f90 @@ -4,7 +4,7 @@ ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py -! CHECK-LABEL: omp.declare_reduction @multiply_reduction_f_64_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @multiply_reduction_byref_f64 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[VAL_1:.*]] = arith.constant 1.000000e+00 : f64 @@ -21,7 +21,7 @@ ! CHECK: omp.yield(%[[ARG0]] : !fir.ref) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @multiply_reduction_i_64_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @multiply_reduction_byref_i64 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[VAL_1:.*]] = arith.constant 1 : i64 @@ -38,7 +38,7 @@ ! CHECK: omp.yield(%[[ARG0]] : !fir.ref) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @multiply_reduction_f_32_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @multiply_reduction_byref_f32 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[VAL_1:.*]] = arith.constant 1.000000e+00 : f32 @@ -55,7 +55,7 @@ ! CHECK: omp.yield(%[[ARG0]] : !fir.ref) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @multiply_reduction_i_32_byref : !fir.ref +! CHECK-LABEL: omp.declare_reduction @multiply_reduction_byref_i32 : !fir.ref ! CHECK-SAME: init { ! CHECK: ^bb0(%[[VAL_0:.*]]: !fir.ref): ! CHECK: %[[VAL_1:.*]] = arith.constant 1 : i32 @@ -85,7 +85,7 @@ ! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 10 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@multiply_reduction_i_32_byref %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: omp.wsloop byref reduction(@multiply_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { ! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref ! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref @@ -121,7 +121,7 @@ end subroutine ! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 10 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@multiply_reduction_f_32_byref %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: omp.wsloop byref reduction(@multiply_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { ! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref ! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref @@ -158,7 +158,7 @@ end subroutine ! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 10 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@multiply_reduction_i_32_byref %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: omp.wsloop byref reduction(@multiply_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { ! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref ! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reduction_switch_orderEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref @@ -194,7 +194,7 @@ end subroutine ! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 10 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@multiply_reduction_f_32_byref %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: omp.wsloop byref reduction(@multiply_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { ! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref ! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reduction_switch_orderEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref @@ -239,7 +239,7 @@ end subroutine ! CHECK: %[[VAL_13:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_14:.*]] = arith.constant 10 : i32 ! CHECK: %[[VAL_15:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@multiply_reduction_i_32_byref %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @multiply_reduction_i_32_byref %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @multiply_reduction_i_32_byref %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { +! CHECK: omp.wsloop byref reduction(@multiply_reduction_byref_i32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @multiply_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @multiply_reduction_byref_i32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { ! CHECK: fir.store %[[VAL_19]] to %[[VAL_12]]#1 : !fir.ref ! CHECK: %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_int_reductions_same_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_int_reductions_same_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) @@ -297,7 +297,7 @@ end subroutine ! CHECK: %[[VAL_13:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_14:.*]] = arith.constant 10 : i32 ! CHECK: %[[VAL_15:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@multiply_reduction_f_32_byref %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @multiply_reduction_f_32_byref %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @multiply_reduction_f_32_byref %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { +! CHECK: omp.wsloop byref reduction(@multiply_reduction_byref_f32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @multiply_reduction_byref_f32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @multiply_reduction_byref_f32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { ! CHECK: fir.store %[[VAL_19]] to %[[VAL_12]]#1 : !fir.ref ! CHECK: %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_real_reductions_same_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_real_reductions_same_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) @@ -362,7 +362,7 @@ end subroutine ! CHECK: %[[VAL_16:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_17:.*]] = arith.constant 10 : i32 ! CHECK: %[[VAL_18:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop byref reduction(@multiply_reduction_i_32_byref %[[VAL_5]]#0 -> %[[VAL_19:.*]] : !fir.ref, @multiply_reduction_i_64_byref %[[VAL_7]]#0 -> %[[VAL_20:.*]] : !fir.ref, @multiply_reduction_f_32_byref %[[VAL_9]]#0 -> %[[VAL_21:.*]] : !fir.ref, @multiply_reduction_f_64_byref %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref) for (%[[VAL_23:.*]]) : i32 = (%[[VAL_16]]) to (%[[VAL_17]]) inclusive step (%[[VAL_18]]) { +! CHECK: omp.wsloop byref reduction(@multiply_reduction_byref_i32 %[[VAL_5]]#0 -> %[[VAL_19:.*]] : !fir.ref, @multiply_reduction_byref_i64 %[[VAL_7]]#0 -> %[[VAL_20:.*]] : !fir.ref, @multiply_reduction_byref_f32 %[[VAL_9]]#0 -> %[[VAL_21:.*]] : !fir.ref, @multiply_reduction_byref_f64 %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref) for (%[[VAL_23:.*]]) : i32 = (%[[VAL_16]]) to (%[[VAL_17]]) inclusive step (%[[VAL_18]]) { ! CHECK: fir.store %[[VAL_23]] to %[[VAL_15]]#1 : !fir.ref ! CHECK: %[[VAL_24:.*]]:2 = hlfir.declare %[[VAL_19]] {uniq_name = "_QFmultiple_reductions_different_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_25:.*]]:2 = hlfir.declare %[[VAL_20]] {uniq_name = "_QFmultiple_reductions_different_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-mul.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-mul.f90 index ce34aafe7dc5..90d9aa5e839b 100644 --- a/flang/test/Lower/OpenMP/wsloop-reduction-mul.f90 +++ b/flang/test/Lower/OpenMP/wsloop-reduction-mul.f90 @@ -4,8 +4,7 @@ ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py -! CHECK-LABEL: omp.declare_reduction @multiply_reduction_f_64 : f64 init { -! CHECK: ^bb0(%[[VAL_0:.*]]: f64): +! CHECK-LABEL: omp.declare_reduction @multiply_reduction_f64 : f64 init { ! CHECK: %[[VAL_1:.*]] = arith.constant 1.000000e+00 : f64 ! CHECK: omp.yield(%[[VAL_1]] : f64) @@ -15,7 +14,7 @@ ! CHECK: omp.yield(%[[VAL_2]] : f64) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @multiply_reduction_i_64 : i64 init { +! CHECK-LABEL: omp.declare_reduction @multiply_reduction_i64 : i64 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: i64): ! CHECK: %[[VAL_1:.*]] = arith.constant 1 : i64 ! CHECK: omp.yield(%[[VAL_1]] : i64) @@ -26,7 +25,7 @@ ! CHECK: omp.yield(%[[VAL_2]] : i64) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @multiply_reduction_f_32 : f32 init { +! CHECK-LABEL: omp.declare_reduction @multiply_reduction_f32 : f32 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: f32): ! CHECK: %[[VAL_1:.*]] = arith.constant 1.000000e+00 : f32 ! CHECK: omp.yield(%[[VAL_1]] : f32) @@ -37,7 +36,7 @@ ! CHECK: omp.yield(%[[VAL_2]] : f32) ! CHECK: } -! CHECK-LABEL: omp.declare_reduction @multiply_reduction_i_32 : i32 init { +! CHECK-LABEL: omp.declare_reduction @multiply_reduction_i32 : i32 init { ! CHECK: ^bb0(%[[VAL_0:.*]]: i32): ! CHECK: %[[VAL_1:.*]] = arith.constant 1 : i32 ! CHECK: omp.yield(%[[VAL_1]] : i32) @@ -61,7 +60,7 @@ ! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 10 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@multiply_reduction_i_32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: omp.wsloop reduction(@multiply_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { ! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref ! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref @@ -97,7 +96,7 @@ end subroutine ! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 10 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@multiply_reduction_f_32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: omp.wsloop reduction(@multiply_reduction_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { ! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref ! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_12]]#0 : !fir.ref @@ -134,7 +133,7 @@ end subroutine ! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 10 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@multiply_reduction_i_32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: omp.wsloop reduction(@multiply_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { ! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref ! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_int_reduction_switch_orderEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref @@ -170,7 +169,7 @@ end subroutine ! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_8:.*]] = arith.constant 10 : i32 ! CHECK: %[[VAL_9:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@multiply_reduction_f_32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { +! CHECK: omp.wsloop reduction(@multiply_reduction_f32 %[[VAL_3]]#0 -> %[[VAL_10:.*]] : !fir.ref) for (%[[VAL_11:.*]]) : i32 = (%[[VAL_7]]) to (%[[VAL_8]]) inclusive step (%[[VAL_9]]) { ! CHECK: fir.store %[[VAL_11]] to %[[VAL_6]]#1 : !fir.ref ! CHECK: %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_real_reduction_switch_orderEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_13:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref @@ -215,7 +214,7 @@ end subroutine ! CHECK: %[[VAL_13:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_14:.*]] = arith.constant 10 : i32 ! CHECK: %[[VAL_15:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@multiply_reduction_i_32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @multiply_reduction_i_32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @multiply_reduction_i_32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { +! CHECK: omp.wsloop reduction(@multiply_reduction_i32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @multiply_reduction_i32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @multiply_reduction_i32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { ! CHECK: fir.store %[[VAL_19]] to %[[VAL_12]]#1 : !fir.ref ! CHECK: %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_int_reductions_same_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_int_reductions_same_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) @@ -273,7 +272,7 @@ end subroutine ! CHECK: %[[VAL_13:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_14:.*]] = arith.constant 10 : i32 ! CHECK: %[[VAL_15:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@multiply_reduction_f_32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @multiply_reduction_f_32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @multiply_reduction_f_32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { +! CHECK: omp.wsloop reduction(@multiply_reduction_f32 %[[VAL_3]]#0 -> %[[VAL_16:.*]] : !fir.ref, @multiply_reduction_f32 %[[VAL_5]]#0 -> %[[VAL_17:.*]] : !fir.ref, @multiply_reduction_f32 %[[VAL_7]]#0 -> %[[VAL_18:.*]] : !fir.ref) for (%[[VAL_19:.*]]) : i32 = (%[[VAL_13]]) to (%[[VAL_14]]) inclusive step (%[[VAL_15]]) { ! CHECK: fir.store %[[VAL_19]] to %[[VAL_12]]#1 : !fir.ref ! CHECK: %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_16]] {uniq_name = "_QFmultiple_real_reductions_same_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]] {uniq_name = "_QFmultiple_real_reductions_same_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) @@ -338,7 +337,7 @@ end subroutine ! CHECK: %[[VAL_16:.*]] = arith.constant 1 : i32 ! CHECK: %[[VAL_17:.*]] = arith.constant 10 : i32 ! CHECK: %[[VAL_18:.*]] = arith.constant 1 : i32 -! CHECK: omp.wsloop reduction(@multiply_reduction_i_32 %[[VAL_5]]#0 -> %[[VAL_19:.*]] : !fir.ref, @multiply_reduction_i_64 %[[VAL_7]]#0 -> %[[VAL_20:.*]] : !fir.ref, @multiply_reduction_f_32 %[[VAL_9]]#0 -> %[[VAL_21:.*]] : !fir.ref, @multiply_reduction_f_64 %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref) for (%[[VAL_23:.*]]) : i32 = (%[[VAL_16]]) to (%[[VAL_17]]) inclusive step (%[[VAL_18]]) { +! CHECK: omp.wsloop reduction(@multiply_reduction_i32 %[[VAL_5]]#0 -> %[[VAL_19:.*]] : !fir.ref, @multiply_reduction_i64 %[[VAL_7]]#0 -> %[[VAL_20:.*]] : !fir.ref, @multiply_reduction_f32 %[[VAL_9]]#0 -> %[[VAL_21:.*]] : !fir.ref, @multiply_reduction_f64 %[[VAL_3]]#0 -> %[[VAL_22:.*]] : !fir.ref) for (%[[VAL_23:.*]]) : i32 = (%[[VAL_16]]) to (%[[VAL_17]]) inclusive step (%[[VAL_18]]) { ! CHECK: fir.store %[[VAL_23]] to %[[VAL_15]]#1 : !fir.ref ! CHECK: %[[VAL_24:.*]]:2 = hlfir.declare %[[VAL_19]] {uniq_name = "_QFmultiple_reductions_different_typeEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_25:.*]]:2 = hlfir.declare %[[VAL_20]] {uniq_name = "_QFmultiple_reductions_different_typeEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) -- GitLab From 9ebd329ad87ca4cde3ce62e1bf5612c4fc0fcb7f Mon Sep 17 00:00:00 2001 From: Jonas Paulsson Date: Wed, 20 Mar 2024 11:46:19 -0400 Subject: [PATCH 031/296] Revert "Move assertion for AdjustsStack from PEI to MachineVerifier. (#85698)" This reverts commit 05bde30585710a51592eee0a6cf6df8184d09c92. Reverting due to verifier complaints with expensive checks on build-bot. --- llvm/lib/CodeGen/MachineVerifier.cpp | 6 ----- llvm/lib/CodeGen/PrologEpilogInserter.cpp | 2 ++ .../clear-dead-implicit-def-impdef.mir | 2 -- ...plicit-def-remat-requires-impdef-check.mir | 2 -- ...implicit-def-with-impdef-greedy-assert.mir | 2 -- .../CodeGen/AMDGPU/fold-restore-undef-use.mir | 2 -- .../greedy-alloc-fail-sgpr1024-spill.mir | 1 - .../ran-out-of-sgprs-allocation-failure.mir | 1 - .../CodeGen/AMDGPU/sched-crash-dbg-value.mir | 2 -- .../AMDGPU/sgpr-spill-wrong-stack-id.mir | 1 - .../AMDGPU/snippet-copy-bundle-regression.mir | 1 - .../virtregrewrite-undef-identity-copy.mir | 1 - ...no-register-coalescing-in-returnsTwice.mir | 2 -- .../CodeGen/Hexagon/regalloc-bad-undef.mir | 2 +- .../SystemZ/RAbasic-invalid-LR-update.mir | 2 -- .../SystemZ/clear-liverange-spillreg.mir | 1 - llvm/test/CodeGen/SystemZ/int-cmp-56.mir | 4 --- .../SystemZ/regcoal-subranges-update.mir | 2 -- llvm/test/CodeGen/X86/callbr-asm-kill.mir | 1 - llvm/test/CodeGen/X86/regalloc-copy-hints.mir | 1 - .../CodeGen/X86/statepoint-fastregalloc.mir | 4 --- .../X86/statepoint-invoke-ra-enter-at-end.mir | 2 +- .../X86/statepoint-invoke-ra-hoist-copies.mir | 2 +- .../statepoint-invoke-ra-inline-spiller.mir | 2 +- ...tatepoint-invoke-ra-remove-back-copies.mir | 2 +- .../test/CodeGen/X86/statepoint-invoke-ra.mir | 2 +- .../CodeGen/X86/statepoint-vreg-folding.mir | 2 +- .../DebugInfo/MIR/InstrRef/phi-coalescing.mir | 1 - .../Mips/livedebugvars-stop-trimming-loc.mir | 2 -- .../MachineVerifier/test_adjustsstack.mir | 26 ------------------- 30 files changed, 9 insertions(+), 74 deletions(-) delete mode 100644 llvm/test/MachineVerifier/test_adjustsstack.mir diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index 005efe48ac0c..c69d36fc7fdd 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -3697,9 +3697,6 @@ void MachineVerifier::verifyStackFrame() { if (I.getOpcode() == FrameSetupOpcode) { if (BBState.ExitIsSetup) report("FrameSetup is after another FrameSetup", &I); - if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack()) - report("AdjustsStack not set in presence of a frame pseudo " - "instruction.", &I); BBState.ExitValue -= TII->getFrameTotalSize(I); BBState.ExitIsSetup = true; } @@ -3715,9 +3712,6 @@ void MachineVerifier::verifyStackFrame() { errs() << "FrameDestroy <" << Size << "> is after FrameSetup <" << AbsSPAdj << ">.\n"; } - if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack()) - report("AdjustsStack not set in presence of a frame pseudo " - "instruction.", &I); BBState.ExitValue += Size; BBState.ExitIsSetup = false; } diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp index c942b8a3e268..eaf96ec5cbde 100644 --- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp +++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp @@ -372,6 +372,8 @@ void PEI::calculateCallFrameInfo(MachineFunction &MF) { MFI.computeMaxCallFrameSize(MF, &FrameSDOps); assert(MFI.getMaxCallFrameSize() <= MaxCFSIn && "Recomputing MaxCFS gave a larger value."); + assert((FrameSDOps.empty() || MF.getFrameInfo().adjustsStack()) && + "AdjustsStack not set in presence of a frame pseudo instruction."); if (TFI->canSimplifyCallFramePseudos(MF)) { // If call frames are not being included as part of the stack frame, and diff --git a/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir b/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir index 2532c76b1336..9040937d027d 100644 --- a/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir +++ b/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir @@ -2,8 +2,6 @@ # RUN: llc -mtriple=arm64-apple-macosx -mcpu=apple-m1 -verify-regalloc -run-pass=greedy -o - %s | FileCheck %s --- name: func -frameInfo: - adjustsStack: true tracksRegLiveness: true body: | bb.0: diff --git a/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir b/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir index 47aa34e3c011..aa94a03786f5 100644 --- a/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir +++ b/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir @@ -22,7 +22,6 @@ name: inst_stores_to_dead_spill_implicit_def_impdef tracksRegLiveness: true frameInfo: - adjustsStack: true hasCalls: true body: | bb.0: @@ -60,7 +59,6 @@ body: | name: inst_stores_to_dead_spill_movimm_impdef tracksRegLiveness: true frameInfo: - adjustsStack: true hasCalls: true body: | bb.0: diff --git a/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir b/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir index d55cf71cead6..e5395b20afd4 100644 --- a/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir +++ b/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir @@ -3,8 +3,6 @@ --- name: widget -frameInfo: - adjustsStack: true tracksRegLiveness: true jumpTable: kind: label-difference32 diff --git a/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir b/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir index 054eeec9e33f..3616d617f84a 100644 --- a/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir +++ b/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir @@ -7,8 +7,6 @@ --- name: restore_undef_copy_use -frameInfo: - adjustsStack: true tracksRegLiveness: true machineFunctionInfo: maxKernArgAlign: 1 diff --git a/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir b/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir index dde84af57ed2..bdd89a907790 100644 --- a/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir +++ b/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir @@ -13,7 +13,6 @@ name: greedy_fail_alloc_sgpr1024_spill tracksRegLiveness: true frameInfo: - adjustsStack: true hasCalls: true machineFunctionInfo: explicitKernArgSize: 16 diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir index fdfc9b043cc9..2ccc24152a9f 100644 --- a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir +++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir @@ -24,7 +24,6 @@ registers: - { id: 10, class: sreg_64_xexec, preferred-register: '$vcc' } frameInfo: maxAlignment: 1 - adjustsStack: true hasCalls: true machineFunctionInfo: maxKernArgAlign: 1 diff --git a/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir b/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir index 158874e7c827..c0d199920bd9 100644 --- a/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir +++ b/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir @@ -180,8 +180,6 @@ exposesReturnsTwice: false legalized: false regBankSelected: false selected: false -frameInfo: - adjustsStack: true tracksRegLiveness: true liveins: - { reg: '$vgpr0', virtual-reg: '%0' } diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir b/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir index c6ccbd99bf89..efbdbca9da6b 100644 --- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir +++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir @@ -78,7 +78,6 @@ name: sgpr_spill_wrong_stack_id tracksRegLiveness: true frameInfo: - adjustsStack: true hasCalls: true machineFunctionInfo: scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3 diff --git a/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir b/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir index f8ec6bb5d943..355829825146 100644 --- a/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir +++ b/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir @@ -21,7 +21,6 @@ name: kernel tracksRegLiveness: true frameInfo: - adjustsStack: true hasCalls: true machineFunctionInfo: isEntryFunction: true diff --git a/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir b/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir index 6659e9532376..3d9db687ffa1 100644 --- a/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir +++ b/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir @@ -20,7 +20,6 @@ name: undef_identity_copy tracksRegLiveness: true frameInfo: maxAlignment: 4 - adjustsStack: true hasCalls: true machineFunctionInfo: isEntryFunction: true diff --git a/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir b/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir index b4bbb9be8ae4..5c59566247d8 100644 --- a/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir +++ b/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir @@ -86,8 +86,6 @@ --- name: main exposesReturnsTwice: true -frameInfo: - adjustsStack: true stack: - { id: 0, name: P0, size: 80, alignment: 8, local-offset: -80 } - { id: 1, name: jb1, size: 160, alignment: 8, local-offset: -240 } diff --git a/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir b/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir index 9468b18bf8e4..67f4dd72ea0b 100644 --- a/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir +++ b/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir @@ -135,7 +135,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 0 - adjustsStack: true + adjustsStack: false hasCalls: true maxCallFrameSize: 0 hasOpaqueSPAdjustment: false diff --git a/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir b/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir index fbe2b687e850..3b308ce3d0d2 100644 --- a/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir +++ b/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir @@ -24,8 +24,6 @@ --- name: autogen_SD21418 alignment: 4 -frameInfo: - adjustsStack: true tracksRegLiveness: true registers: - { id: 0, class: vr128bit } diff --git a/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir b/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir index 197c3d8551fc..7ff7d9b8b709 100644 --- a/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir +++ b/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir @@ -157,7 +157,6 @@ registers: - { id: 129, class: grx32bit } - { id: 130, class: fp64bit } frameInfo: - adjustsStack: true hasCalls: true body: | bb.0: diff --git a/llvm/test/CodeGen/SystemZ/int-cmp-56.mir b/llvm/test/CodeGen/SystemZ/int-cmp-56.mir index 3e00b6065eb9..e52fd44ae47d 100644 --- a/llvm/test/CodeGen/SystemZ/int-cmp-56.mir +++ b/llvm/test/CodeGen/SystemZ/int-cmp-56.mir @@ -48,7 +48,6 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 - adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | @@ -126,7 +125,6 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 - adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | @@ -204,7 +202,6 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 - adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | @@ -282,7 +279,6 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 - adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir b/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir index d3ef9b0b9abf..f709b70ff1b7 100644 --- a/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir +++ b/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir @@ -48,8 +48,6 @@ body: | # represented for the value carried by %7. --- name: segfault -frameInfo: - adjustsStack: true tracksRegLiveness: true liveins: [] body: | diff --git a/llvm/test/CodeGen/X86/callbr-asm-kill.mir b/llvm/test/CodeGen/X86/callbr-asm-kill.mir index 0dded37c97af..86c58c4715ed 100644 --- a/llvm/test/CodeGen/X86/callbr-asm-kill.mir +++ b/llvm/test/CodeGen/X86/callbr-asm-kill.mir @@ -45,7 +45,6 @@ liveins: - { reg: '$rsi', virtual-reg: '%3' } frameInfo: maxAlignment: 1 - adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/CodeGen/X86/regalloc-copy-hints.mir b/llvm/test/CodeGen/X86/regalloc-copy-hints.mir index d09bcd6a6b40..13b5a541fa22 100644 --- a/llvm/test/CodeGen/X86/regalloc-copy-hints.mir +++ b/llvm/test/CodeGen/X86/regalloc-copy-hints.mir @@ -103,7 +103,6 @@ registers: - { id: 82, class: gr32 } frameInfo: maxAlignment: 4 - adjustsStack: true hasCalls: true fixedStack: - { id: 0, size: 4, alignment: 4, stack-id: default, isImmutable: true } diff --git a/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir b/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir index 87ffdd7c4e6b..02c931067300 100644 --- a/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir +++ b/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir @@ -5,8 +5,6 @@ # Tied def/use must be assigned to the same register. --- name: test_relocate -frameInfo: - adjustsStack: true tracksRegLiveness: true body: | bb.0.entry: @@ -26,8 +24,6 @@ body: | # These regmasks have no real meaning and chosen to allow only single register to be assignable ($rbp) --- name: test_relocate_multi_regmasks -frameInfo: - adjustsStack: true tracksRegLiveness: true body: | bb.0.entry: diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir index 5f05270729fd..11968f17c70a 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir @@ -231,7 +231,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 1 - adjustsStack: true + adjustsStack: false hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir index cf9128260f19..aae2f3870138 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir @@ -398,7 +398,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 1 - adjustsStack: true + adjustsStack: false hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir index fcebc69d9b2e..87f5f0f96c50 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir @@ -175,7 +175,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 4 - adjustsStack: true + adjustsStack: false hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir index 8bb39a03f7e3..49253968fcca 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir @@ -226,7 +226,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 4 - adjustsStack: true + adjustsStack: false hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir index da651039ce21..858ff3f1888b 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir @@ -172,7 +172,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 4 - adjustsStack: true + adjustsStack: false hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir b/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir index d40a9a06d162..e24d5e8af1f5 100644 --- a/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir +++ b/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir @@ -114,7 +114,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 8 - adjustsStack: true + adjustsStack: false hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir index 6460263c6025..bc1c7ebac6ce 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir @@ -106,7 +106,6 @@ liveins: - { reg: '$rsi', virtual-reg: '%5' } frameInfo: maxAlignment: 1 - adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir b/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir index ac67b9671f53..35ab906efc90 100644 --- a/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir +++ b/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir @@ -71,8 +71,6 @@ --- name: fn2 alignment: 4 -frameInfo: - adjustsStack: true tracksRegLiveness: true registers: - { id: 0, class: gpr32, preferred-register: '' } diff --git a/llvm/test/MachineVerifier/test_adjustsstack.mir b/llvm/test/MachineVerifier/test_adjustsstack.mir deleted file mode 100644 index d333737e000c..000000000000 --- a/llvm/test/MachineVerifier/test_adjustsstack.mir +++ /dev/null @@ -1,26 +0,0 @@ -# RUN: not --crash llc -o - -start-before=twoaddressinstruction -verify-machineinstrs %s 2>&1 \ -# RUN: | FileCheck %s -# REQUIRES: aarch64-registered-target ---- | - target triple = "aarch64-unknown-linux" - declare i32 @bar(i32) nounwind - define i32 @foo() nounwind { - call i32 @bar(i32 0) - ret i32 0 - } -... ---- -name: foo -registers: - - { id: 0, class: gpr32 } -body: | - bb.0 (%ir-block.0): - ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp - %0 = COPY $wzr - $w0 = COPY %0 - BL @bar, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0 - ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp - $w0 = COPY killed %0 - RET_ReallyLR implicit $w0 -... -# CHECK-LABEL: Bad machine code: AdjustsStack not set in presence of a frame pseudo instruction. -- GitLab From d209d1340b99d4fbd325dffb5e13b757ab8264ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Wed, 20 Mar 2024 17:48:48 +0200 Subject: [PATCH 032/296] [libcxx] [cmake] Fix cmake_path(ABSOLUTE_PATH) for empty CMAKE_INSTALL_PREFIX This should hopefully fix the issue brought up at https://github.com/llvm/llvm-project/pull/85756#issuecomment-2009852291. --- libcxx/modules/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libcxx/modules/CMakeLists.txt b/libcxx/modules/CMakeLists.txt index 8dd1b51aac2f..6c917200d6f3 100644 --- a/libcxx/modules/CMakeLists.txt +++ b/libcxx/modules/CMakeLists.txt @@ -207,10 +207,10 @@ add_custom_target(generate-cxx-modules # Use the relative path between the installation and the module in the json # file. This allows moving the entire installation to a different location. cmake_path(ABSOLUTE_PATH LIBCXX_INSTALL_LIBRARY_DIR - BASE_DIRECTORY ${CMAKE_INSTALL_PREFIX} + BASE_DIRECTORY "${CMAKE_INSTALL_PREFIX}" OUTPUT_VARIABLE ABS_LIBRARY_DIR) cmake_path(ABSOLUTE_PATH LIBCXX_INSTALL_MODULES_DIR - BASE_DIRECTORY ${CMAKE_INSTALL_PREFIX} + BASE_DIRECTORY "${CMAKE_INSTALL_PREFIX}" OUTPUT_VARIABLE ABS_MODULES_DIR) file(RELATIVE_PATH LIBCXX_MODULE_RELATIVE_PATH ${ABS_LIBRARY_DIR} -- GitLab From eb861acd49e4c6777e2fe09dd2004ac1f4731bba Mon Sep 17 00:00:00 2001 From: Steven Varoumas Date: Wed, 20 Mar 2024 15:56:22 +0000 Subject: [PATCH 033/296] [mlir][python] Enable python bindings for Index dialect (#85827) This small patch enables python bindings for the index dialect. --------- Co-authored-by: Steven Varoumas --- mlir/python/CMakeLists.txt | 9 + mlir/python/mlir/dialects/IndexOps.td | 14 ++ mlir/python/mlir/dialects/index.py | 6 + mlir/test/python/dialects/index_dialect.py | 235 +++++++++++++++++++++ 4 files changed, 264 insertions(+) create mode 100644 mlir/python/mlir/dialects/IndexOps.td create mode 100644 mlir/python/mlir/dialects/index.py create mode 100644 mlir/test/python/dialects/index_dialect.py diff --git a/mlir/python/CMakeLists.txt b/mlir/python/CMakeLists.txt index 563d035f1552..c27ee688a040 100644 --- a/mlir/python/CMakeLists.txt +++ b/mlir/python/CMakeLists.txt @@ -108,6 +108,15 @@ declare_mlir_dialect_python_bindings( dialects/complex.py DIALECT_NAME complex) +declare_mlir_dialect_python_bindings( + ADD_TO_PARENT MLIRPythonSources.Dialects + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" + TD_FILE dialects/IndexOps.td + SOURCES + dialects/index.py + DIALECT_NAME index + GEN_ENUM_BINDINGS) + declare_mlir_dialect_python_bindings( ADD_TO_PARENT MLIRPythonSources.Dialects ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir" diff --git a/mlir/python/mlir/dialects/IndexOps.td b/mlir/python/mlir/dialects/IndexOps.td new file mode 100644 index 000000000000..13b1d782c853 --- /dev/null +++ b/mlir/python/mlir/dialects/IndexOps.td @@ -0,0 +1,14 @@ +//===-- IndexOps.td - Entry point for Index bindings -----*- tablegen -*---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef PYTHON_BINDINGS_INDEX_OPS +#define PYTHON_BINDINGS_INDEX_OPS + +include "mlir/Dialect/Index/IR/IndexOps.td" + +#endif diff --git a/mlir/python/mlir/dialects/index.py b/mlir/python/mlir/dialects/index.py new file mode 100644 index 000000000000..73708c7d71a8 --- /dev/null +++ b/mlir/python/mlir/dialects/index.py @@ -0,0 +1,6 @@ +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +from ._index_ops_gen import * +from ._index_enum_gen import * diff --git a/mlir/test/python/dialects/index_dialect.py b/mlir/test/python/dialects/index_dialect.py new file mode 100644 index 000000000000..9db883469792 --- /dev/null +++ b/mlir/test/python/dialects/index_dialect.py @@ -0,0 +1,235 @@ +# RUN: %PYTHON %s | FileCheck %s + +from mlir.ir import * +from mlir.dialects import index, arith + + +def run(f): + print("\nTEST:", f.__name__) + with Context() as ctx, Location.unknown(): + module = Module.create() + with InsertionPoint(module.body): + f(ctx) + print(module) + + +# CHECK-LABEL: TEST: testConstantOp +@run +def testConstantOp(ctx): + a = index.ConstantOp(value=42) + # CHECK: %{{.*}} = index.constant 42 + + +# CHECK-LABEL: TEST: testBoolConstantOp +@run +def testBoolConstantOp(ctx): + a = index.BoolConstantOp(value=True) + # CHECK: %{{.*}} = index.bool.constant true + + +# CHECK-LABEL: TEST: testAndOp +@run +def testAndOp(ctx): + a = index.ConstantOp(value=42) + r = index.AndOp(a, a) + # CHECK: %{{.*}} = index.and %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testOrOp +@run +def testOrOp(ctx): + a = index.ConstantOp(value=42) + r = index.OrOp(a, a) + # CHECK: %{{.*}} = index.or %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testXOrOp +@run +def testXOrOp(ctx): + a = index.ConstantOp(value=42) + r = index.XOrOp(a, a) + # CHECK: %{{.*}} = index.xor %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testCastSOp +@run +def testCastSOp(ctx): + a = index.ConstantOp(value=42) + b = arith.ConstantOp(value=23, result=IntegerType.get_signless(64)) + c = index.CastSOp(input=a, output=IntegerType.get_signless(32)) + d = index.CastSOp(input=b, output=IndexType.get()) + # CHECK: %{{.*}} = index.casts %{{.*}} : index to i32 + # CHECK: %{{.*}} = index.casts %{{.*}} : i64 to index + + +# CHECK-LABEL: TEST: testCastUOp +@run +def testCastUOp(ctx): + a = index.ConstantOp(value=42) + b = arith.ConstantOp(value=23, result=IntegerType.get_signless(64)) + c = index.CastUOp(input=a, output=IntegerType.get_signless(32)) + d = index.CastUOp(input=b, output=IndexType.get()) + # CHECK: %{{.*}} = index.castu %{{.*}} : index to i32 + # CHECK: %{{.*}} = index.castu %{{.*}} : i64 to index + + +# CHECK-LABEL: TEST: testCeilDivSOp +@run +def testCeilDivSOp(ctx): + a = index.ConstantOp(value=42) + r = index.CeilDivSOp(a, a) + # CHECK: %{{.*}} = index.ceildivs %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testCeilDivUOp +@run +def testCeilDivUOp(ctx): + a = index.ConstantOp(value=42) + r = index.CeilDivUOp(a, a) + # CHECK: %{{.*}} = index.ceildivu %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testCmpOp +@run +def testCmpOp(ctx): + a = index.ConstantOp(value=42) + b = index.ConstantOp(value=23) + pred = AttrBuilder.get("IndexCmpPredicateAttr")("slt", context=ctx) + r = index.CmpOp(pred, lhs=a, rhs=b) + # CHECK: %{{.*}} = index.cmp slt(%{{.*}}, %{{.*}}) + + +# CHECK-LABEL: TEST: testAddOp +@run +def testAddOp(ctx): + a = index.ConstantOp(value=42) + r = index.AddOp(a, a) + # CHECK: %{{.*}} = index.add %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testSubOp +@run +def testSubOp(ctx): + a = index.ConstantOp(value=42) + r = index.SubOp(a, a) + # CHECK: %{{.*}} = index.sub %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testMulOp +@run +def testMulOp(ctx): + a = index.ConstantOp(value=42) + r = index.MulOp(a, a) + # CHECK: %{{.*}} = index.mul %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testDivSOp +@run +def testDivSOp(ctx): + a = index.ConstantOp(value=42) + r = index.DivSOp(a, a) + # CHECK: %{{.*}} = index.divs %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testDivUOp +@run +def testDivUOp(ctx): + a = index.ConstantOp(value=42) + r = index.DivUOp(a, a) + # CHECK: %{{.*}} = index.divu %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testFloorDivSOp +@run +def testFloorDivSOp(ctx): + a = index.ConstantOp(value=42) + r = index.FloorDivSOp(a, a) + # CHECK: %{{.*}} = index.floordivs %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testMaxSOp +@run +def testMaxSOp(ctx): + a = index.ConstantOp(value=42) + b = index.ConstantOp(value=23) + r = index.MaxSOp(a, b) + # CHECK: %{{.*}} = index.maxs %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testMaxUOp +@run +def testMaxUOp(ctx): + a = index.ConstantOp(value=42) + b = index.ConstantOp(value=23) + r = index.MaxUOp(a, b) + # CHECK: %{{.*}} = index.maxu %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testMinSOp +@run +def testMinSOp(ctx): + a = index.ConstantOp(value=42) + b = index.ConstantOp(value=23) + r = index.MinSOp(a, b) + # CHECK: %{{.*}} = index.mins %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testMinUOp +@run +def testMinUOp(ctx): + a = index.ConstantOp(value=42) + b = index.ConstantOp(value=23) + r = index.MinUOp(a, b) + # CHECK: %{{.*}} = index.minu %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testRemSOp +@run +def testRemSOp(ctx): + a = index.ConstantOp(value=42) + b = index.ConstantOp(value=23) + r = index.RemSOp(a, b) + # CHECK: %{{.*}} = index.rems %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testRemUOp +@run +def testRemUOp(ctx): + a = index.ConstantOp(value=42) + b = index.ConstantOp(value=23) + r = index.RemUOp(a, b) + # CHECK: %{{.*}} = index.remu %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testShlOp +@run +def testShlOp(ctx): + a = index.ConstantOp(value=42) + b = index.ConstantOp(value=3) + r = index.ShlOp(a, b) + # CHECK: %{{.*}} = index.shl %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testShrSOp +@run +def testShrSOp(ctx): + a = index.ConstantOp(value=42) + b = index.ConstantOp(value=3) + r = index.ShrSOp(a, b) + # CHECK: %{{.*}} = index.shrs %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testShrUOp +@run +def testShrUOp(ctx): + a = index.ConstantOp(value=42) + b = index.ConstantOp(value=3) + r = index.ShrUOp(a, b) + # CHECK: %{{.*}} = index.shru %{{.*}}, %{{.*}} + + +# CHECK-LABEL: TEST: testSizeOfOp +@run +def testSizeOfOp(ctx): + r = index.SizeOfOp() + # CHECK: %{{.*}} = index.sizeof -- GitLab From 75dfa58ea93aa93b97534906778cb3dd24ba841a Mon Sep 17 00:00:00 2001 From: Stephen Tozer Date: Wed, 20 Mar 2024 16:00:10 +0000 Subject: [PATCH 034/296] [RemoveDIs][NFC] Rename DPMarker->DbgMarker (#85931) Another trivial rename patch, the last big one for now, which renamed DPMarkers to DbgMarkers. This required the field `DbgMarker` in `Instruction` to be renamed to `DebugMarker` to avoid a clash, but otherwise was a simple string substitution of `s/DPMarker/DbgMarker` and a manual renaming of `DPM` to `DM` in the few places where that acronym was used for debug markers. --- llvm/docs/RemoveDIsDebugInfo.md | 14 +-- llvm/include/llvm/IR/BasicBlock.h | 28 ++--- .../include/llvm/IR/DebugProgramInstruction.h | 71 +++++------ llvm/include/llvm/IR/Instruction.h | 11 +- llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 2 +- llvm/lib/IR/AsmWriter.cpp | 23 ++-- llvm/lib/IR/BasicBlock.cpp | 113 +++++++++--------- llvm/lib/IR/DebugProgramInstruction.cpp | 69 +++++------ llvm/lib/IR/Instruction.cpp | 43 +++---- llvm/lib/IR/LLVMContextImpl.h | 8 +- llvm/lib/IR/Value.cpp | 2 +- llvm/lib/IR/Verifier.cpp | 10 +- llvm/lib/Transforms/Scalar/JumpThreading.cpp | 4 +- .../Transforms/Utils/LoopRotationUtils.cpp | 4 +- llvm/unittests/IR/BasicBlockDbgInfoTest.cpp | 77 ++++++------ llvm/unittests/IR/DebugInfoTest.cpp | 84 ++++++------- llvm/unittests/Transforms/Utils/LocalTest.cpp | 8 +- 17 files changed, 292 insertions(+), 279 deletions(-) diff --git a/llvm/docs/RemoveDIsDebugInfo.md b/llvm/docs/RemoveDIsDebugInfo.md index f8405767a579..a2f1e173d9d9 100644 --- a/llvm/docs/RemoveDIsDebugInfo.md +++ b/llvm/docs/RemoveDIsDebugInfo.md @@ -82,18 +82,18 @@ Like so: | | v - +------------+ - <-------+ DPMarker |<------- - / +------------+ \ - / \ - / \ - v ^ + +-------------+ + <-------+ DbgMarker |<------- + / +-------------+ \ + / \ + / \ + v ^ +-------------+ +-------------+ +-------------+ | DbgRecord +--->| DbgRecord +-->| DbgRecord | +-------------+ +-------------+ +-------------+ ``` -Each instruction has a pointer to a `DPMarker` (which will become optional), that contains a list of `DbgRecord` objects. No debugging records appear in the instruction list at all. `DbgRecord`s have a parent pointer to their owning `DPMarker`, and each `DPMarker` has a pointer back to it's owning instruction. +Each instruction has a pointer to a `DbgMarker` (which will become optional), that contains a list of `DbgRecord` objects. No debugging records appear in the instruction list at all. `DbgRecord`s have a parent pointer to their owning `DbgMarker`, and each `DbgMarker` has a pointer back to it's owning instruction. Not shown are the links from DbgRecord to other parts of the `Value`/`Metadata` hierachy: `DbgRecord` subclasses have tracking pointers to the DIMetadata that they use, and `DbgVariableRecord` has references to `Value`s that are stored in a `DebugValueUser` base class. This refers to a `ValueAsMetadata` object referring to `Value`s, via the `TrackingMetadata` facility. diff --git a/llvm/include/llvm/IR/BasicBlock.h b/llvm/include/llvm/IR/BasicBlock.h index 51444c7f8c9c..0eea4cdccca5 100644 --- a/llvm/include/llvm/IR/BasicBlock.h +++ b/llvm/include/llvm/IR/BasicBlock.h @@ -39,7 +39,7 @@ class Module; class PHINode; class ValueSymbolTable; class DbgVariableRecord; -class DPMarker; +class DbgMarker; /// LLVM Basic Block Representation /// @@ -72,18 +72,18 @@ private: Function *Parent; public: - /// Attach a DPMarker to the given instruction. Enables the storage of any + /// Attach a DbgMarker to the given instruction. Enables the storage of any /// debug-info at this position in the program. - DPMarker *createMarker(Instruction *I); - DPMarker *createMarker(InstListType::iterator It); + DbgMarker *createMarker(Instruction *I); + DbgMarker *createMarker(InstListType::iterator It); /// Convert variable location debugging information stored in dbg.value - /// intrinsics into DPMarkers / DbgRecords. Deletes all dbg.values in + /// intrinsics into DbgMarkers / DbgRecords. Deletes all dbg.values in /// the process and sets IsNewDbgInfoFormat = true. Only takes effect if /// the UseNewDbgInfoFormat LLVM command line option is given. void convertToNewDbgValues(); - /// Convert variable location debugging information stored in DPMarkers and + /// Convert variable location debugging information stored in DbgMarkers and /// DbgRecords into the dbg.value intrinsic representation. Sets /// IsNewDbgInfoFormat = false. void convertFromNewDbgValues(); @@ -97,12 +97,12 @@ public: /// instruction of this block. These are equivalent to dbg.value intrinsics /// that exist at the end of a basic block with no terminator (a transient /// state that occurs regularly). - void setTrailingDbgRecords(DPMarker *M); + void setTrailingDbgRecords(DbgMarker *M); /// Fetch the collection of DbgRecords that "trail" after the last instruction /// of this block, see \ref setTrailingDbgRecords. If there are none, returns /// nullptr. - DPMarker *getTrailingDbgRecords(); + DbgMarker *getTrailingDbgRecords(); /// Delete any trailing DbgRecords at the end of this block, see /// \ref setTrailingDbgRecords. @@ -110,15 +110,15 @@ public: void dumpDbgValues() const; - /// Return the DPMarker for the position given by \p It, so that DbgRecords + /// Return the DbgMarker for the position given by \p It, so that DbgRecords /// can be inserted there. This will either be nullptr if not present, a - /// DPMarker, or TrailingDbgRecords if It is end(). - DPMarker *getMarker(InstListType::iterator It); + /// DbgMarker, or TrailingDbgRecords if It is end(). + DbgMarker *getMarker(InstListType::iterator It); - /// Return the DPMarker for the position that comes after \p I. \see - /// BasicBlock::getMarker, this can be nullptr, a DPMarker, or + /// Return the DbgMarker for the position that comes after \p I. \see + /// BasicBlock::getMarker, this can be nullptr, a DbgMarker, or /// TrailingDbgRecords if there is no next instruction. - DPMarker *getNextMarker(Instruction *I); + DbgMarker *getNextMarker(Instruction *I); /// Insert a DbgRecord into a block at the position given by \p I. void insertDbgRecordAfter(DbgRecord *DR, Instruction *I); diff --git a/llvm/include/llvm/IR/DebugProgramInstruction.h b/llvm/include/llvm/IR/DebugProgramInstruction.h index 8bd6331a9a3e..7214d7ad65da 100644 --- a/llvm/include/llvm/IR/DebugProgramInstruction.h +++ b/llvm/include/llvm/IR/DebugProgramInstruction.h @@ -16,22 +16,22 @@ // // and all information is stored in the Value / Metadata hierachy defined // elsewhere in LLVM. In the "DbgRecord" design, each instruction /may/ have a -// connection with a DPMarker, which identifies a position immediately before -// the instruction, and each DPMarker /may/ then have connections to DbgRecords +// connection with a DbgMarker, which identifies a position immediately before +// the instruction, and each DbgMarker /may/ then have connections to DbgRecords // which record the variable assignment information. To illustrate: // // %foo = add i32 1, %0 -// ; foo->DbgMarker == nullptr +// ; foo->DebugMarker == nullptr // ;; There are no variable assignments / debug records "in front" of -// ;; the instruction for %foo, therefore it has no DbgMarker. +// ;; the instruction for %foo, therefore it has no DebugMarker. // %bar = void call @ext(%foo) -// ; bar->DbgMarker = { +// ; bar->DebugMarker = { // ; StoredDbgRecords = { // ; DbgVariableRecord(metadata i32 %foo, ...) // ; } // ; } // ;; There is a debug-info record in front of the %bar instruction, -// ;; thus it points at a DPMarker object. That DPMarker contains a +// ;; thus it points at a DbgMarker object. That DbgMarker contains a // ;; DbgVariableRecord in it's ilist, storing the equivalent information // to the // ;; dbg.value above: the Value, DILocalVariable, etc. @@ -66,7 +66,7 @@ class DbgVariableIntrinsic; class DbgInfoIntrinsic; class DbgLabelInst; class DIAssignID; -class DPMarker; +class DbgMarker; class DbgVariableRecord; class raw_ostream; @@ -120,7 +120,7 @@ public: /// Base class for non-instruction debug metadata records that have positions /// within IR. Features various methods copied across from the Instruction /// class to aid ease-of-use. DbgRecords should always be linked into a -/// DPMarker's StoredDbgRecords list. The marker connects a DbgRecord back to +/// DbgMarker's StoredDbgRecords list. The marker connects a DbgRecord back to /// it's position in the BasicBlock. /// /// We need a discriminator for dyn/isa casts. In order to avoid paying for a @@ -134,7 +134,7 @@ public: class DbgRecord : public ilist_node { public: /// Marker that this DbgRecord is linked into. - DPMarker *Marker = nullptr; + DbgMarker *Marker = nullptr; /// Subclass discriminator. enum Kind : uint8_t { ValueKind, LabelKind }; @@ -166,10 +166,10 @@ public: Kind getRecordKind() const { return RecordKind; } - void setMarker(DPMarker *M) { Marker = M; } + void setMarker(DbgMarker *M) { Marker = M; } - DPMarker *getMarker() { return Marker; } - const DPMarker *getMarker() const { return Marker; } + DbgMarker *getMarker() { return Marker; } + const DbgMarker *getMarker() const { return Marker; } BasicBlock *getBlock(); const BasicBlock *getBlock() const; @@ -539,7 +539,7 @@ filterDbgVars(iterator_range::iterator> R) { } /// Per-instruction record of debug-info. If an Instruction is the position of -/// some debugging information, it points at a DPMarker storing that info. Each +/// some debugging information, it points at a DbgMarker storing that info. Each /// marker points back at the instruction that owns it. Various utilities are /// provided for manipulating the DbgRecords contained within this marker. /// @@ -559,9 +559,9 @@ filterDbgVars(iterator_range::iterator> R) { /// which we can improve in the future. Additionally, many improvements in the /// way that debug-info is stored can be achieved in this class, at a future /// date. -class DPMarker { +class DbgMarker { public: - DPMarker() {} + DbgMarker() {} /// Link back to the Instruction that owns this marker. Can be null during /// operations that move a marker from one instruction to another. Instruction *MarkedInstr = nullptr; @@ -585,7 +585,7 @@ public: void removeFromParent(); void eraseFromParent(); - /// Implement operator<< on DPMarker. + /// Implement operator<< on DbgMarker. void print(raw_ostream &O, bool IsForDebug = false) const; void print(raw_ostream &ROS, ModuleSlotTracker &MST, bool IsForDebug) const; @@ -593,22 +593,23 @@ public: iterator_range::iterator> getDbgRecordRange(); iterator_range::const_iterator> getDbgRecordRange() const; - /// Transfer any DbgRecords from \p Src into this DPMarker. If \p InsertAtHead - /// is true, place them before existing DbgRecords, otherwise afterwards. - void absorbDebugValues(DPMarker &Src, bool InsertAtHead); - /// Transfer the DbgRecords in \p Range from \p Src into this DPMarker. If + /// Transfer any DbgRecords from \p Src into this DbgMarker. If \p + /// InsertAtHead is true, place them before existing DbgRecords, otherwise + /// afterwards. + void absorbDebugValues(DbgMarker &Src, bool InsertAtHead); + /// Transfer the DbgRecords in \p Range from \p Src into this DbgMarker. If /// \p InsertAtHead is true, place them before existing DbgRecords, otherwise // afterwards. void absorbDebugValues(iterator_range Range, - DPMarker &Src, bool InsertAtHead); - /// Insert a DbgRecord into this DPMarker, at the end of the list. If + DbgMarker &Src, bool InsertAtHead); + /// Insert a DbgRecord into this DbgMarker, at the end of the list. If /// \p InsertAtHead is true, at the start. void insertDbgRecord(DbgRecord *New, bool InsertAtHead); /// Insert a DbgRecord prior to a DbgRecord contained within this marker. void insertDbgRecord(DbgRecord *New, DbgRecord *InsertBefore); /// Insert a DbgRecord after a DbgRecord contained within this marker. void insertDbgRecordAfter(DbgRecord *New, DbgRecord *InsertAfter); - /// Clone all DPMarkers from \p From into this marker. There are numerous + /// Clone all DbgMarkers from \p From into this marker. There are numerous /// options to customise the source/destination, due to gnarliness, see class /// comment. /// \p FromHere If non-null, copy from FromHere to the end of From's @@ -617,10 +618,10 @@ public: /// StoredDbgRecords /// \returns Range over all the newly cloned DbgRecords iterator_range::iterator> - cloneDebugInfoFrom(DPMarker *From, + cloneDebugInfoFrom(DbgMarker *From, std::optional::iterator> FromHere, bool InsertAtHead = false); - /// Erase all DbgRecords in this DPMarker. + /// Erase all DbgRecords in this DbgMarker. void dropDbgRecords(); /// Erase a single DbgRecord from this marker. In an ideal future, we would /// never erase an assignment in this way, but it's the equivalent to @@ -628,34 +629,34 @@ public: void dropOneDbgRecord(DbgRecord *DR); /// We generally act like all llvm Instructions have a range of DbgRecords - /// attached to them, but in reality sometimes we don't allocate the DPMarker + /// attached to them, but in reality sometimes we don't allocate the DbgMarker /// to save time and memory, but still have to return ranges of DbgRecords. /// When we need to describe such an unallocated DbgRecord range, use this /// static markers range instead. This will bite us if someone tries to insert /// a DbgRecord in that range, but they should be using the Official (TM) API /// for that. - static DPMarker EmptyDPMarker; + static DbgMarker EmptyDbgMarker; static iterator_range::iterator> getEmptyDbgRecordRange() { - return make_range(EmptyDPMarker.StoredDbgRecords.end(), - EmptyDPMarker.StoredDbgRecords.end()); + return make_range(EmptyDbgMarker.StoredDbgRecords.end(), + EmptyDbgMarker.StoredDbgRecords.end()); } }; -inline raw_ostream &operator<<(raw_ostream &OS, const DPMarker &Marker) { +inline raw_ostream &operator<<(raw_ostream &OS, const DbgMarker &Marker) { Marker.print(OS); return OS; } /// Inline helper to return a range of DbgRecords attached to a marker. It needs /// to be inlined as it's frequently called, but also come after the declaration -/// of DPMarker. Thus: it's pre-declared by users like Instruction, then an +/// of DbgMarker. Thus: it's pre-declared by users like Instruction, then an /// inlineable body defined here. inline iterator_range::iterator> -getDbgRecordRange(DPMarker *DbgMarker) { - if (!DbgMarker) - return DPMarker::getEmptyDbgRecordRange(); - return DbgMarker->getDbgRecordRange(); +getDbgRecordRange(DbgMarker *DebugMarker) { + if (!DebugMarker) + return DbgMarker::getEmptyDbgRecordRange(); + return DebugMarker->getDbgRecordRange(); } DEFINE_ISA_CONVERSION_FUNCTIONS(DbgRecord, LLVMDbgRecordRef) diff --git a/llvm/include/llvm/IR/Instruction.h b/llvm/include/llvm/IR/Instruction.h index d6cf15577523..6e0874c5b04f 100644 --- a/llvm/include/llvm/IR/Instruction.h +++ b/llvm/include/llvm/IR/Instruction.h @@ -29,19 +29,20 @@ namespace llvm { class BasicBlock; -class DPMarker; +class DbgMarker; class FastMathFlags; class MDNode; class Module; struct AAMDNodes; -class DPMarker; +class DbgMarker; class DbgRecord; template <> struct ilist_alloc_traits { static inline void deleteNode(Instruction *V); }; -iterator_range::iterator> getDbgRecordRange(DPMarker *); +iterator_range::iterator> +getDbgRecordRange(DbgMarker *); class Instruction : public User, public ilist_node_with_parent::iterator> getDbgRecordRange() const { - return llvm::getDbgRecordRange(DbgMarker); + return llvm::getDbgRecordRange(DebugMarker); } /// Return an iterator to the position of the "Next" DbgRecord after this diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index 5addf0ac33c4..a8b69f89e7de 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -3569,7 +3569,7 @@ void ModuleBitcodeWriter::writeFunction( // Write out non-instruction debug information attached to this // instruction. Write it after the instruction so that it's easy to // re-attach to the instruction reading the records in. - for (DbgRecord &DR : I.DbgMarker->getDbgRecordRange()) { + for (DbgRecord &DR : I.DebugMarker->getDbgRecordRange()) { if (DbgLabelRecord *DLR = dyn_cast(&DR)) { Vals.push_back(VE.getMetadataID(&*DLR->getDebugLoc())); Vals.push_back(VE.getMetadataID(DLR->getLabel())); diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index 8ce6fabb30f2..38c191a2dec6 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -286,7 +286,7 @@ static const Module *getModuleFromVal(const Value *V) { return nullptr; } -static const Module *getModuleFromDPI(const DPMarker *Marker) { +static const Module *getModuleFromDPI(const DbgMarker *Marker) { const Function *M = Marker->getParent() ? Marker->getParent()->getParent() : nullptr; return M ? M->getParent() : nullptr; @@ -2717,7 +2717,7 @@ public: void printBasicBlock(const BasicBlock *BB); void printInstructionLine(const Instruction &I); void printInstruction(const Instruction &I); - void printDPMarker(const DPMarker &DPI); + void printDbgMarker(const DbgMarker &DPI); void printDbgVariableRecord(const DbgVariableRecord &DVR); void printDbgLabelRecord(const DbgLabelRecord &DLR); void printDbgRecord(const DbgRecord &DR); @@ -4604,15 +4604,15 @@ void AssemblyWriter::printInstruction(const Instruction &I) { printInfoComment(I); } -void AssemblyWriter::printDPMarker(const DPMarker &Marker) { - // There's no formal representation of a DPMarker -- print purely as a +void AssemblyWriter::printDbgMarker(const DbgMarker &Marker) { + // There's no formal representation of a DbgMarker -- print purely as a // debugging aid. for (const DbgRecord &DPR : Marker.StoredDbgRecords) { printDbgRecord(DPR); Out << "\n"; } - Out << " DPMarker -> { "; + Out << " DbgMarker -> { "; printInstruction(*Marker.MarkedInstr); Out << " }"; return; @@ -4907,7 +4907,7 @@ static bool isReferencingMDNode(const Instruction &I) { return false; } -void DPMarker::print(raw_ostream &ROS, bool IsForDebug) const { +void DbgMarker::print(raw_ostream &ROS, bool IsForDebug) const { ModuleSlotTracker MST(getModuleFromDPI(this), true); print(ROS, MST, IsForDebug); @@ -4919,8 +4919,8 @@ void DbgVariableRecord::print(raw_ostream &ROS, bool IsForDebug) const { print(ROS, MST, IsForDebug); } -void DPMarker::print(raw_ostream &ROS, ModuleSlotTracker &MST, - bool IsForDebug) const { +void DbgMarker::print(raw_ostream &ROS, ModuleSlotTracker &MST, + bool IsForDebug) const { formatted_raw_ostream OS(ROS); SlotTracker EmptySlotTable(static_cast(nullptr)); SlotTracker &SlotTable = @@ -4931,7 +4931,7 @@ void DPMarker::print(raw_ostream &ROS, ModuleSlotTracker &MST, }; incorporateFunction(getParent() ? getParent()->getParent() : nullptr); AssemblyWriter W(OS, SlotTable, getModuleFromDPI(this), nullptr, IsForDebug); - W.printDPMarker(*this); + W.printDbgMarker(*this); } void DbgLabelRecord::print(raw_ostream &ROS, bool IsForDebug) const { @@ -5220,7 +5220,10 @@ void Value::dump() const { print(dbgs(), /*IsForDebug=*/true); dbgs() << '\n'; } // Value::dump - allow easy printing of Values from the debugger. LLVM_DUMP_METHOD -void DPMarker::dump() const { print(dbgs(), /*IsForDebug=*/true); dbgs() << '\n'; } +void DbgMarker::dump() const { + print(dbgs(), /*IsForDebug=*/true); + dbgs() << '\n'; +} // Value::dump - allow easy printing of Values from the debugger. LLVM_DUMP_METHOD diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp index 2fa9b3330d18..f088c7a2cc4e 100644 --- a/llvm/lib/IR/BasicBlock.cpp +++ b/llvm/lib/IR/BasicBlock.cpp @@ -41,28 +41,28 @@ cl::opt WriteNewDbgInfoFormatToBitcode2( "write-experimental-debuginfo-iterators-to-bitcode", cl::Hidden, cl::location(WriteNewDbgInfoFormatToBitcode), cl::init(false)); -DPMarker *BasicBlock::createMarker(Instruction *I) { +DbgMarker *BasicBlock::createMarker(Instruction *I) { assert(IsNewDbgInfoFormat && "Tried to create a marker in a non new debug-info block!"); - if (I->DbgMarker) - return I->DbgMarker; - DPMarker *Marker = new DPMarker(); + if (I->DebugMarker) + return I->DebugMarker; + DbgMarker *Marker = new DbgMarker(); Marker->MarkedInstr = I; - I->DbgMarker = Marker; + I->DebugMarker = Marker; return Marker; } -DPMarker *BasicBlock::createMarker(InstListType::iterator It) { +DbgMarker *BasicBlock::createMarker(InstListType::iterator It) { assert(IsNewDbgInfoFormat && "Tried to create a marker in a non new debug-info block!"); if (It != end()) return createMarker(&*It); - DPMarker *DPM = getTrailingDbgRecords(); - if (DPM) - return DPM; - DPM = new DPMarker(); - setTrailingDbgRecords(DPM); - return DPM; + DbgMarker *DM = getTrailingDbgRecords(); + if (DM) + return DM; + DM = new DbgMarker(); + setTrailingDbgRecords(DM); + return DM; } void BasicBlock::convertToNewDbgValues() { @@ -70,10 +70,11 @@ void BasicBlock::convertToNewDbgValues() { // Iterate over all instructions in the instruction list, collecting debug // info intrinsics and converting them to DbgRecords. Once we find a "real" - // instruction, attach all those DbgRecords to a DPMarker in that instruction. + // instruction, attach all those DbgRecords to a DbgMarker in that + // instruction. SmallVector DbgVarRecs; for (Instruction &I : make_early_inc_range(InstList)) { - assert(!I.DbgMarker && "DbgMarker already set on old-format instrs?"); + assert(!I.DebugMarker && "DebugMarker already set on old-format instrs?"); if (DbgVariableIntrinsic *DVI = dyn_cast(&I)) { // Convert this dbg.value to a DbgVariableRecord. DbgVariableRecord *Value = new DbgVariableRecord(DVI); @@ -94,7 +95,7 @@ void BasicBlock::convertToNewDbgValues() { // Create a marker to store DbgRecords in. createMarker(&I); - DPMarker *Marker = I.DbgMarker; + DbgMarker *Marker = I.DebugMarker; for (DbgRecord *DVR : DbgVarRecs) Marker->insertDbgRecord(DVR, false); @@ -107,14 +108,14 @@ void BasicBlock::convertFromNewDbgValues() { invalidateOrders(); IsNewDbgInfoFormat = false; - // Iterate over the block, finding instructions annotated with DPMarkers. + // Iterate over the block, finding instructions annotated with DbgMarkers. // Convert any attached DbgRecords to debug intrinsics and insert ahead of the // instruction. for (auto &Inst : *this) { - if (!Inst.DbgMarker) + if (!Inst.DebugMarker) continue; - DPMarker &Marker = *Inst.DbgMarker; + DbgMarker &Marker = *Inst.DebugMarker; for (DbgRecord &DR : Marker.getDbgRecordRange()) InstList.insert(Inst.getIterator(), DR.createDebugIntrinsic(getModule(), nullptr)); @@ -131,11 +132,11 @@ void BasicBlock::convertFromNewDbgValues() { #ifndef NDEBUG void BasicBlock::dumpDbgValues() const { for (auto &Inst : *this) { - if (!Inst.DbgMarker) + if (!Inst.DebugMarker) continue; - dbgs() << "@ " << Inst.DbgMarker << " "; - Inst.DbgMarker->dump(); + dbgs() << "@ " << Inst.DebugMarker << " "; + Inst.DebugMarker->dump(); }; } #endif @@ -218,9 +219,9 @@ BasicBlock::~BasicBlock() { assert(getParent() == nullptr && "BasicBlock still linked into the program!"); dropAllReferences(); for (auto &Inst : *this) { - if (!Inst.DbgMarker) + if (!Inst.DebugMarker) continue; - Inst.DbgMarker->eraseFromParent(); + Inst.DebugMarker->eraseFromParent(); } InstList.clear(); } @@ -717,13 +718,13 @@ void BasicBlock::flushTerminatorDbgRecords() { return; // Are there any dangling DbgRecords? - DPMarker *TrailingDbgRecords = getTrailingDbgRecords(); + DbgMarker *TrailingDbgRecords = getTrailingDbgRecords(); if (!TrailingDbgRecords) return; // Transfer DbgRecords from the trailing position onto the terminator. createMarker(Term); - Term->DbgMarker->absorbDebugValues(*TrailingDbgRecords, false); + Term->DebugMarker->absorbDebugValues(*TrailingDbgRecords, false); TrailingDbgRecords->eraseFromParent(); deleteTrailingDbgRecords(); } @@ -760,7 +761,7 @@ void BasicBlock::spliceDebugInfoEmptyBlock(BasicBlock::iterator Dest, // occur when a block is optimised away and the terminator has been moved // somewhere else. if (Src->empty()) { - DPMarker *SrcTrailingDbgRecords = Src->getTrailingDbgRecords(); + DbgMarker *SrcTrailingDbgRecords = Src->getTrailingDbgRecords(); if (!SrcTrailingDbgRecords) return; @@ -780,7 +781,7 @@ void BasicBlock::spliceDebugInfoEmptyBlock(BasicBlock::iterator Dest, if (!First->hasDbgRecords()) return; - createMarker(Dest)->absorbDebugValues(*First->DbgMarker, InsertAtHead); + createMarker(Dest)->absorbDebugValues(*First->DebugMarker, InsertAtHead); return; } @@ -822,8 +823,8 @@ void BasicBlock::spliceDebugInfo(BasicBlock::iterator Dest, BasicBlock *Src, // If we're inserting at end(), and not in front of dangling DbgRecords, then // move the DbgRecords onto "First". They'll then be moved naturally in the // splice process. - DPMarker *MoreDanglingDbgRecords = nullptr; - DPMarker *OurTrailingDbgRecords = getTrailingDbgRecords(); + DbgMarker *MoreDanglingDbgRecords = nullptr; + DbgMarker *OurTrailingDbgRecords = getTrailingDbgRecords(); if (Dest == end() && !Dest.getHeadBit() && OurTrailingDbgRecords) { // Are the "+" DbgRecords not supposed to move? If so, detach them // temporarily. @@ -844,7 +845,7 @@ void BasicBlock::spliceDebugInfo(BasicBlock::iterator Dest, BasicBlock *Src, } else { // No current marker, create one and absorb in. (FIXME: we can avoid an // allocation in the future). - DPMarker *CurMarker = Src->createMarker(&*First); + DbgMarker *CurMarker = Src->createMarker(&*First); CurMarker->absorbDebugValues(*OurTrailingDbgRecords, false); OurTrailingDbgRecords->eraseFromParent(); } @@ -862,7 +863,7 @@ void BasicBlock::spliceDebugInfo(BasicBlock::iterator Dest, BasicBlock *Src, // FIXME: we could avoid an allocation here sometimes. (adoptDbgRecords // requires an iterator). - DPMarker *LastMarker = Src->createMarker(Last); + DbgMarker *LastMarker = Src->createMarker(Last); LastMarker->absorbDebugValues(*MoreDanglingDbgRecords, true); MoreDanglingDbgRecords->eraseFromParent(); } @@ -943,7 +944,7 @@ void BasicBlock::spliceDebugInfoImpl(BasicBlock::iterator Dest, BasicBlock *Src, // Detach the marker at Dest -- this lets us move the "====" DbgRecords // around. - DPMarker *DestMarker = nullptr; + DbgMarker *DestMarker = nullptr; if (Dest != end()) { if ((DestMarker = getMarker(Dest))) DestMarker->removeFromParent(); @@ -952,14 +953,14 @@ void BasicBlock::spliceDebugInfoImpl(BasicBlock::iterator Dest, BasicBlock *Src, // If we're moving the tail range of DbgRecords (":::"), absorb them into the // front of the DbgRecords at Dest. if (ReadFromTail && Src->getMarker(Last)) { - DPMarker *FromLast = Src->getMarker(Last); + DbgMarker *FromLast = Src->getMarker(Last); if (LastIsEnd) { Dest->adoptDbgRecords(Src, Last, true); // adoptDbgRecords will release any trailers. assert(!Src->getTrailingDbgRecords()); } else { // FIXME: can we use adoptDbgRecords here to reduce allocations? - DPMarker *OntoDest = createMarker(Dest); + DbgMarker *OntoDest = createMarker(Dest); OntoDest->absorbDebugValues(*FromLast, true); } } @@ -971,8 +972,8 @@ void BasicBlock::spliceDebugInfoImpl(BasicBlock::iterator Dest, BasicBlock *Src, if (Last != Src->end()) { Last->adoptDbgRecords(Src, First, true); } else { - DPMarker *OntoLast = Src->createMarker(Last); - DPMarker *FromFirst = Src->createMarker(First); + DbgMarker *OntoLast = Src->createMarker(Last); + DbgMarker *FromFirst = Src->createMarker(First); // Always insert at front of Last. OntoLast->absorbDebugValues(*FromFirst, true); } @@ -983,12 +984,12 @@ void BasicBlock::spliceDebugInfoImpl(BasicBlock::iterator Dest, BasicBlock *Src, if (InsertAtHead) { // Insert them at the end of the DbgRecords at Dest. The "::::" DbgRecords // might be in front of them. - DPMarker *NewDestMarker = createMarker(Dest); + DbgMarker *NewDestMarker = createMarker(Dest); NewDestMarker->absorbDebugValues(*DestMarker, false); } else { // Insert them right at the start of the range we moved, ahead of First // and the "++++" DbgRecords. - DPMarker *FirstMarker = createMarker(First); + DbgMarker *FirstMarker = createMarker(First); FirstMarker->absorbDebugValues(*DestMarker, true); } DestMarker->eraseFromParent(); @@ -997,8 +998,8 @@ void BasicBlock::spliceDebugInfoImpl(BasicBlock::iterator Dest, BasicBlock *Src, // generate the iterator with begin() / getFirstInsertionPt(), it means // any trailing debug-info at the end of the block would "normally" have // been pushed in front of "First". Move it there now. - DPMarker *FirstMarker = getMarker(First); - DPMarker *TrailingDbgRecords = getTrailingDbgRecords(); + DbgMarker *FirstMarker = getMarker(First); + DbgMarker *TrailingDbgRecords = getTrailingDbgRecords(); if (TrailingDbgRecords) { FirstMarker->absorbDebugValues(*TrailingDbgRecords, true); TrailingDbgRecords->eraseFromParent(); @@ -1040,7 +1041,7 @@ void BasicBlock::insertDbgRecordAfter(DbgRecord *DR, Instruction *I) { assert(I->getParent() == this); iterator NextIt = std::next(I->getIterator()); - DPMarker *NextMarker = createMarker(NextIt); + DbgMarker *NextMarker = createMarker(NextIt); NextMarker->insertDbgRecord(DR, true); } @@ -1048,20 +1049,20 @@ void BasicBlock::insertDbgRecordBefore(DbgRecord *DR, InstListType::iterator Where) { assert(Where == end() || Where->getParent() == this); bool InsertAtHead = Where.getHeadBit(); - DPMarker *M = createMarker(Where); + DbgMarker *M = createMarker(Where); M->insertDbgRecord(DR, InsertAtHead); } -DPMarker *BasicBlock::getNextMarker(Instruction *I) { +DbgMarker *BasicBlock::getNextMarker(Instruction *I) { return getMarker(std::next(I->getIterator())); } -DPMarker *BasicBlock::getMarker(InstListType::iterator It) { +DbgMarker *BasicBlock::getMarker(InstListType::iterator It) { if (It == end()) { - DPMarker *DPM = getTrailingDbgRecords(); - return DPM; + DbgMarker *DM = getTrailingDbgRecords(); + return DM; } - return It->DbgMarker; + return It->DebugMarker; } void BasicBlock::reinsertInstInDbgRecords( @@ -1095,27 +1096,27 @@ void BasicBlock::reinsertInstInDbgRecords( // This happens if there were no DbgRecords on I0. Are there now DbgRecords // there? if (!Pos) { - DPMarker *NextMarker = getNextMarker(I); + DbgMarker *NextMarker = getNextMarker(I); if (!NextMarker) return; if (NextMarker->StoredDbgRecords.empty()) return; - // There are DPMarkers there now -- they fell down from "I". - DPMarker *ThisMarker = createMarker(I); + // There are DbgMarkers there now -- they fell down from "I". + DbgMarker *ThisMarker = createMarker(I); ThisMarker->absorbDebugValues(*NextMarker, false); return; } // Is there even a range of DbgRecords to move? - DPMarker *DPM = (*Pos)->getMarker(); - auto Range = make_range(DPM->StoredDbgRecords.begin(), (*Pos)); + DbgMarker *DM = (*Pos)->getMarker(); + auto Range = make_range(DM->StoredDbgRecords.begin(), (*Pos)); if (Range.begin() == Range.end()) return; // Otherwise: splice. - DPMarker *ThisMarker = createMarker(I); + DbgMarker *ThisMarker = createMarker(I); assert(ThisMarker->StoredDbgRecords.empty()); - ThisMarker->absorbDebugValues(Range, *DPM, true); + ThisMarker->absorbDebugValues(Range, *DM, true); } #ifndef NDEBUG @@ -1133,11 +1134,11 @@ void BasicBlock::validateInstrOrdering() const { } #endif -void BasicBlock::setTrailingDbgRecords(DPMarker *foo) { +void BasicBlock::setTrailingDbgRecords(DbgMarker *foo) { getContext().pImpl->setTrailingDbgRecords(this, foo); } -DPMarker *BasicBlock::getTrailingDbgRecords() { +DbgMarker *BasicBlock::getTrailingDbgRecords() { return getContext().pImpl->getTrailingDbgRecords(this); } diff --git a/llvm/lib/IR/DebugProgramInstruction.cpp b/llvm/lib/IR/DebugProgramInstruction.cpp index 1c0be6f598cc..fbca7cdfcf3f 100644 --- a/llvm/lib/IR/DebugProgramInstruction.cpp +++ b/llvm/lib/IR/DebugProgramInstruction.cpp @@ -1,4 +1,4 @@ -//=====-- DebugProgramInstruction.cpp - Implement DbgRecords/DPMarkers --=====// +//=====-- DebugProgramInstruction.cpp - Implement DbgRecords/DbgMarkers --====// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -529,40 +529,40 @@ const LLVMContext &DbgRecord::getContext() const { void DbgRecord::insertBefore(DbgRecord *InsertBefore) { assert(!getMarker() && - "Cannot insert a DbgRecord that is already has a DPMarker!"); + "Cannot insert a DbgRecord that is already has a DbgMarker!"); assert(InsertBefore->getMarker() && "Cannot insert a DbgRecord before a DbgRecord that does not have a " - "DPMarker!"); + "DbgMarker!"); InsertBefore->getMarker()->insertDbgRecord(this, InsertBefore); } void DbgRecord::insertAfter(DbgRecord *InsertAfter) { assert(!getMarker() && - "Cannot insert a DbgRecord that is already has a DPMarker!"); + "Cannot insert a DbgRecord that is already has a DbgMarker!"); assert(InsertAfter->getMarker() && "Cannot insert a DbgRecord after a DbgRecord that does not have a " - "DPMarker!"); + "DbgMarker!"); InsertAfter->getMarker()->insertDbgRecordAfter(this, InsertAfter); } void DbgRecord::moveBefore(DbgRecord *MoveBefore) { assert(getMarker() && - "Canot move a DbgRecord that does not currently have a DPMarker!"); + "Canot move a DbgRecord that does not currently have a DbgMarker!"); removeFromParent(); insertBefore(MoveBefore); } void DbgRecord::moveAfter(DbgRecord *MoveAfter) { assert(getMarker() && - "Canot move a DbgRecord that does not currently have a DPMarker!"); + "Canot move a DbgRecord that does not currently have a DbgMarker!"); removeFromParent(); insertAfter(MoveAfter); } /////////////////////////////////////////////////////////////////////////////// -// An empty, global, DPMarker for the purpose of describing empty ranges of +// An empty, global, DbgMarker for the purpose of describing empty ranges of // DbgRecords. -DPMarker DPMarker::EmptyDPMarker; +DbgMarker DbgMarker::EmptyDbgMarker; -void DPMarker::dropDbgRecords() { +void DbgMarker::dropDbgRecords() { while (!StoredDbgRecords.empty()) { auto It = StoredDbgRecords.begin(); DbgRecord *DR = &*It; @@ -571,31 +571,31 @@ void DPMarker::dropDbgRecords() { } } -void DPMarker::dropOneDbgRecord(DbgRecord *DR) { +void DbgMarker::dropOneDbgRecord(DbgRecord *DR) { assert(DR->getMarker() == this); StoredDbgRecords.erase(DR->getIterator()); DR->deleteRecord(); } -const BasicBlock *DPMarker::getParent() const { +const BasicBlock *DbgMarker::getParent() const { return MarkedInstr->getParent(); } -BasicBlock *DPMarker::getParent() { return MarkedInstr->getParent(); } +BasicBlock *DbgMarker::getParent() { return MarkedInstr->getParent(); } -void DPMarker::removeMarker() { - // Are there any DbgRecords in this DPMarker? If not, nothing to preserve. +void DbgMarker::removeMarker() { + // Are there any DbgRecords in this DbgMarker? If not, nothing to preserve. Instruction *Owner = MarkedInstr; if (StoredDbgRecords.empty()) { eraseFromParent(); - Owner->DbgMarker = nullptr; + Owner->DebugMarker = nullptr; return; } // The attached DbgRecords need to be preserved; attach them to the next // instruction. If there isn't a next instruction, put them on the // "trailing" list. - DPMarker *NextMarker = Owner->getParent()->getNextMarker(Owner); + DbgMarker *NextMarker = Owner->getParent()->getNextMarker(Owner); if (NextMarker) { NextMarker->absorbDebugValues(*this, true); eraseFromParent(); @@ -608,30 +608,30 @@ void DPMarker::removeMarker() { getParent()->setTrailingDbgRecords(this); MarkedInstr = nullptr; } else { - NextIt->DbgMarker = this; + NextIt->DebugMarker = this; MarkedInstr = &*NextIt; } } - Owner->DbgMarker = nullptr; + Owner->DebugMarker = nullptr; } -void DPMarker::removeFromParent() { - MarkedInstr->DbgMarker = nullptr; +void DbgMarker::removeFromParent() { + MarkedInstr->DebugMarker = nullptr; MarkedInstr = nullptr; } -void DPMarker::eraseFromParent() { +void DbgMarker::eraseFromParent() { if (MarkedInstr) removeFromParent(); dropDbgRecords(); delete this; } -iterator_range DPMarker::getDbgRecordRange() { +iterator_range DbgMarker::getDbgRecordRange() { return make_range(StoredDbgRecords.begin(), StoredDbgRecords.end()); } iterator_range -DPMarker::getDbgRecordRange() const { +DbgMarker::getDbgRecordRange() const { return make_range(StoredDbgRecords.begin(), StoredDbgRecords.end()); } @@ -645,25 +645,25 @@ void DbgRecord::eraseFromParent() { deleteRecord(); } -void DPMarker::insertDbgRecord(DbgRecord *New, bool InsertAtHead) { +void DbgMarker::insertDbgRecord(DbgRecord *New, bool InsertAtHead) { auto It = InsertAtHead ? StoredDbgRecords.begin() : StoredDbgRecords.end(); StoredDbgRecords.insert(It, *New); New->setMarker(this); } -void DPMarker::insertDbgRecord(DbgRecord *New, DbgRecord *InsertBefore) { +void DbgMarker::insertDbgRecord(DbgRecord *New, DbgRecord *InsertBefore) { assert(InsertBefore->getMarker() == this && - "DbgRecord 'InsertBefore' must be contained in this DPMarker!"); + "DbgRecord 'InsertBefore' must be contained in this DbgMarker!"); StoredDbgRecords.insert(InsertBefore->getIterator(), *New); New->setMarker(this); } -void DPMarker::insertDbgRecordAfter(DbgRecord *New, DbgRecord *InsertAfter) { +void DbgMarker::insertDbgRecordAfter(DbgRecord *New, DbgRecord *InsertAfter) { assert(InsertAfter->getMarker() == this && - "DbgRecord 'InsertAfter' must be contained in this DPMarker!"); + "DbgRecord 'InsertAfter' must be contained in this DbgMarker!"); StoredDbgRecords.insert(++(InsertAfter->getIterator()), *New); New->setMarker(this); } -void DPMarker::absorbDebugValues(DPMarker &Src, bool InsertAtHead) { +void DbgMarker::absorbDebugValues(DbgMarker &Src, bool InsertAtHead) { auto It = InsertAtHead ? StoredDbgRecords.begin() : StoredDbgRecords.end(); for (DbgRecord &DVR : Src.StoredDbgRecords) DVR.setMarker(this); @@ -671,8 +671,9 @@ void DPMarker::absorbDebugValues(DPMarker &Src, bool InsertAtHead) { StoredDbgRecords.splice(It, Src.StoredDbgRecords); } -void DPMarker::absorbDebugValues(iterator_range Range, - DPMarker &Src, bool InsertAtHead) { +void DbgMarker::absorbDebugValues( + iterator_range Range, DbgMarker &Src, + bool InsertAtHead) { for (DbgRecord &DR : Range) DR.setMarker(this); @@ -683,8 +684,8 @@ void DPMarker::absorbDebugValues(iterator_range Range, Range.end()); } -iterator_range::iterator> DPMarker::cloneDebugInfoFrom( - DPMarker *From, std::optional::iterator> from_here, +iterator_range::iterator> DbgMarker::cloneDebugInfoFrom( + DbgMarker *From, std::optional::iterator> from_here, bool InsertAtHead) { DbgRecord *First = nullptr; // Work out what range of DbgRecords to clone: normally all the contents of diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp index 9744eb32d27a..47a7f2c9de79 100644 --- a/llvm/lib/IR/Instruction.cpp +++ b/llvm/lib/IR/Instruction.cpp @@ -93,10 +93,10 @@ void Instruction::removeFromParent() { } void Instruction::handleMarkerRemoval() { - if (!Parent->IsNewDbgInfoFormat || !DbgMarker) + if (!Parent->IsNewDbgInfoFormat || !DebugMarker) return; - DbgMarker->removeMarker(); + DebugMarker->removeMarker(); } BasicBlock::iterator Instruction::eraseFromParent() { @@ -135,7 +135,7 @@ extern cl::opt UseNewDbgInfoFormat; void Instruction::insertBefore(BasicBlock &BB, InstListType::iterator InsertPos) { - assert(!DbgMarker); + assert(!DebugMarker); BB.getInstList().insert(InsertPos, this); @@ -147,7 +147,7 @@ void Instruction::insertBefore(BasicBlock &BB, // DbgRecords should now come before "this". bool InsertAtHead = InsertPos.getHeadBit(); if (!InsertAtHead) { - DPMarker *SrcMarker = BB.getMarker(InsertPos); + DbgMarker *SrcMarker = BB.getMarker(InsertPos); if (SrcMarker && !SrcMarker->empty()) { // If this assertion fires, the calling code is about to insert a PHI // after debug-records, which would form a sequence like: @@ -214,7 +214,7 @@ void Instruction::moveBeforeImpl(BasicBlock &BB, InstListType::iterator I, // If we've been given the "Preserve" flag, then just move the DbgRecords with // the instruction, no more special handling needed. - if (BB.IsNewDbgInfoFormat && DbgMarker && !Preserve) { + if (BB.IsNewDbgInfoFormat && DebugMarker && !Preserve) { if (I != this->getIterator() || InsertAtHead) { // "this" is definitely moving in the list, or it's moving ahead of its // attached DbgVariableRecords. Detach any existing DbgRecords. @@ -227,7 +227,7 @@ void Instruction::moveBeforeImpl(BasicBlock &BB, InstListType::iterator I, BB.getInstList().splice(I, getParent()->getInstList(), getIterator()); if (BB.IsNewDbgInfoFormat && !Preserve) { - DPMarker *NextMarker = getParent()->getNextMarker(this); + DbgMarker *NextMarker = getParent()->getNextMarker(this); // If we're inserting at point I, and not in front of the DbgRecords // attached there, then we should absorb the DbgRecords attached to I. @@ -243,23 +243,24 @@ void Instruction::moveBeforeImpl(BasicBlock &BB, InstListType::iterator I, iterator_range Instruction::cloneDebugInfoFrom( const Instruction *From, std::optional FromHere, bool InsertAtHead) { - if (!From->DbgMarker) - return DPMarker::getEmptyDbgRecordRange(); + if (!From->DebugMarker) + return DbgMarker::getEmptyDbgRecordRange(); assert(getParent()->IsNewDbgInfoFormat); assert(getParent()->IsNewDbgInfoFormat == From->getParent()->IsNewDbgInfoFormat); - if (!DbgMarker) + if (!DebugMarker) getParent()->createMarker(this); - return DbgMarker->cloneDebugInfoFrom(From->DbgMarker, FromHere, InsertAtHead); + return DebugMarker->cloneDebugInfoFrom(From->DebugMarker, FromHere, + InsertAtHead); } std::optional Instruction::getDbgReinsertionPosition() { // Is there a marker on the next instruction? - DPMarker *NextMarker = getParent()->getNextMarker(this); + DbgMarker *NextMarker = getParent()->getNextMarker(this); if (!NextMarker) return std::nullopt; @@ -274,7 +275,7 @@ bool Instruction::hasDbgRecords() const { return !getDbgRecordRange().empty(); } void Instruction::adoptDbgRecords(BasicBlock *BB, BasicBlock::iterator It, bool InsertAtHead) { - DPMarker *SrcMarker = BB->getMarker(It); + DbgMarker *SrcMarker = BB->getMarker(It); auto ReleaseTrailingDbgRecords = [BB, It, SrcMarker]() { if (BB->end() == It) { SrcMarker->eraseFromParent(); @@ -287,13 +288,13 @@ void Instruction::adoptDbgRecords(BasicBlock *BB, BasicBlock::iterator It, return; } - // If we have DPMarkers attached to this instruction, we have to honour the + // If we have DbgMarkers attached to this instruction, we have to honour the // ordering of DbgRecords between this and the other marker. Fall back to just // absorbing from the source. - if (DbgMarker || It == BB->end()) { + if (DebugMarker || It == BB->end()) { // Ensure we _do_ have a marker. getParent()->createMarker(this); - DbgMarker->absorbDebugValues(*SrcMarker, InsertAtHead); + DebugMarker->absorbDebugValues(*SrcMarker, InsertAtHead); // Having transferred everything out of SrcMarker, we _could_ clean it up // and free the marker now. However, that's a lot of heap-accounting for a @@ -309,19 +310,19 @@ void Instruction::adoptDbgRecords(BasicBlock *BB, BasicBlock::iterator It, // Optimisation: we're transferring all the DbgRecords from the source // marker onto this empty location: just adopt the other instructions // marker. - DbgMarker = SrcMarker; - DbgMarker->MarkedInstr = this; - It->DbgMarker = nullptr; + DebugMarker = SrcMarker; + DebugMarker->MarkedInstr = this; + It->DebugMarker = nullptr; } } void Instruction::dropDbgRecords() { - if (DbgMarker) - DbgMarker->dropDbgRecords(); + if (DebugMarker) + DebugMarker->dropDbgRecords(); } void Instruction::dropOneDbgRecord(DbgRecord *DVR) { - DbgMarker->dropOneDbgRecord(DVR); + DebugMarker->dropOneDbgRecord(DVR); } bool Instruction::comesBefore(const Instruction *Other) const { diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h index 4542e16e59fd..58e0f21244f7 100644 --- a/llvm/lib/IR/LLVMContextImpl.h +++ b/llvm/lib/IR/LLVMContextImpl.h @@ -58,7 +58,7 @@ class AttributeSetNode; class BasicBlock; class ConstantRangeAttributeImpl; struct DiagnosticHandler; -class DPMarker; +class DbgMarker; class ElementCount; class Function; class GlobalObject; @@ -1689,15 +1689,15 @@ public: /// "trail" in such a way. These are stored in LLVMContext because typically /// LLVM only edits a small number of blocks at a time, so there's no need to /// bloat BasicBlock with such a data structure. - SmallDenseMap TrailingDbgRecords; + SmallDenseMap TrailingDbgRecords; // Set, get and delete operations for TrailingDbgRecords. - void setTrailingDbgRecords(BasicBlock *B, DPMarker *M) { + void setTrailingDbgRecords(BasicBlock *B, DbgMarker *M) { assert(!TrailingDbgRecords.count(B)); TrailingDbgRecords[B] = M; } - DPMarker *getTrailingDbgRecords(BasicBlock *B) { + DbgMarker *getTrailingDbgRecords(BasicBlock *B) { return TrailingDbgRecords.lookup(B); } diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp index 61e1c35ba4a3..8522747ccf12 100644 --- a/llvm/lib/IR/Value.cpp +++ b/llvm/lib/IR/Value.cpp @@ -581,7 +581,7 @@ static void replaceDbgUsesOutsideBlock(Value *V, Value *New, BasicBlock *BB) { DVI->replaceVariableLocationOp(V, New); } for (auto *DVR : DPUsers) { - DPMarker *Marker = DVR->getMarker(); + DbgMarker *Marker = DVR->getMarker(); if (Marker->getParent() != BB) DVR->replaceVariableLocationOp(V, New); } diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index a99b307a3536..819722566831 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -678,15 +678,15 @@ private: } while (false) void Verifier::visitDbgRecords(Instruction &I) { - if (!I.DbgMarker) + if (!I.DebugMarker) return; - CheckDI(I.DbgMarker->MarkedInstr == &I, "Instruction has invalid DbgMarker", - &I); + CheckDI(I.DebugMarker->MarkedInstr == &I, + "Instruction has invalid DebugMarker", &I); CheckDI(!isa(&I) || !I.hasDbgRecords(), "PHI Node must not have any attached DbgRecords", &I); for (DbgRecord &DR : I.getDbgRecordRange()) { - CheckDI(DR.getMarker() == I.DbgMarker, "DbgRecord had invalid DbgMarker", - &I, &DR); + CheckDI(DR.getMarker() == I.DebugMarker, + "DbgRecord had invalid DebugMarker", &I, &DR); if (auto *Loc = dyn_cast_or_null(DR.getDebugLoc().getAsMDNode())) visitMDNode(*Loc, AreDebugLocsAllowed::Yes); diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp index fd6835952570..ffcb511e6a83 100644 --- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp +++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp @@ -2113,8 +2113,8 @@ JumpThreadingPass::cloneInstructions(BasicBlock::iterator BI, // marker to marker as there isn't an instruction there. if (BE != RangeBB->end() && BE->hasDbgRecords()) { // Dump them at the end. - DPMarker *Marker = RangeBB->getMarker(BE); - DPMarker *EndMarker = NewBB->createMarker(NewBB->end()); + DbgMarker *Marker = RangeBB->getMarker(BE); + DbgMarker *EndMarker = NewBB->createMarker(NewBB->end()); auto DVRRange = EndMarker->cloneDebugInfoFrom(Marker, std::nullopt); for (DbgVariableRecord &DVR : filterDbgVars(DVRRange)) RetargetDbgVariableRecordIfPossible(&DVR); diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp index 4470c5af870a..bc6711711371 100644 --- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp @@ -601,7 +601,7 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) { // a range because it gives us a natural way of testing whether // there were DbgRecords on the next instruction before we hoisted things). iterator_range NextDbgInsts = - (I != E) ? I->getDbgRecordRange() : DPMarker::getEmptyDbgRecordRange(); + (I != E) ? I->getDbgRecordRange() : DbgMarker::getEmptyDbgRecordRange(); while (I != E) { Instruction *Inst = &*I++; @@ -659,7 +659,7 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) { RemapDbgVariableRecordRange(M, Range, ValueMap, RF_NoModuleLevelChanges | RF_IgnoreMissingLocals); - NextDbgInsts = DPMarker::getEmptyDbgRecordRange(); + NextDbgInsts = DbgMarker::getEmptyDbgRecordRange(); // Erase anything we've seen before. for (DbgVariableRecord &DVR : make_early_inc_range(filterDbgVars(Range))) diff --git a/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp b/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp index 92658b7b6895..905928819dda 100644 --- a/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp +++ b/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp @@ -147,10 +147,10 @@ TEST(BasicBlockDbgInfoTest, MarkerOperations) { // Fetch out our two markers, Instruction *Instr1 = &*BB.begin(); Instruction *Instr2 = Instr1->getNextNode(); - DPMarker *Marker1 = Instr1->DbgMarker; - DPMarker *Marker2 = Instr2->DbgMarker; + DbgMarker *Marker1 = Instr1->DebugMarker; + DbgMarker *Marker2 = Instr2->DebugMarker; // There's no TrailingDbgRecords marker allocated yet. - DPMarker *EndMarker = nullptr; + DbgMarker *EndMarker = nullptr; // Check that the "getMarker" utilities operate as expected. EXPECT_EQ(BB.getMarker(Instr1->getIterator()), Marker1); @@ -219,7 +219,7 @@ TEST(BasicBlockDbgInfoTest, MarkerOperations) { // Inserting at end(): should dislodge the DbgVariableRecords, if they were // dbg.values then they would sit "above" the new instruction. Instr1->insertBefore(BB, BB.end()); - EXPECT_EQ(Instr1->DbgMarker->StoredDbgRecords.size(), 2u); + EXPECT_EQ(Instr1->DebugMarker->StoredDbgRecords.size(), 2u); // We should de-allocate the trailing marker when something is inserted // at end(). EXPECT_EQ(BB.getTrailingDbgRecords(), nullptr); @@ -234,7 +234,7 @@ TEST(BasicBlockDbgInfoTest, MarkerOperations) { // this be the final instr in the block, and DbgVariableRecords aren't allowed // to live off the end forever. Instr2->insertBefore(BB, BB.begin()); - EXPECT_EQ(Instr2->DbgMarker->StoredDbgRecords.size(), 2u); + EXPECT_EQ(Instr2->DebugMarker->StoredDbgRecords.size(), 2u); EXPECT_EQ(BB.getTrailingDbgRecords(), nullptr); // Teardown, @@ -297,26 +297,27 @@ TEST(BasicBlockDbgInfoTest, HeadBitOperations) { Instruction *CInst = BInst->getNextNode(); Instruction *DInst = CInst->getNextNode(); // CInst should have debug-info. - ASSERT_TRUE(CInst->DbgMarker); - EXPECT_FALSE(CInst->DbgMarker->StoredDbgRecords.empty()); + ASSERT_TRUE(CInst->DebugMarker); + EXPECT_FALSE(CInst->DebugMarker->StoredDbgRecords.empty()); // If we move "c" to the start of the block, just normally, then the // DbgVariableRecords should fall down to "d". CInst->moveBefore(BB, BeginIt2); - EXPECT_TRUE(!CInst->DbgMarker || CInst->DbgMarker->StoredDbgRecords.empty()); - ASSERT_TRUE(DInst->DbgMarker); - EXPECT_FALSE(DInst->DbgMarker->StoredDbgRecords.empty()); + EXPECT_TRUE(!CInst->DebugMarker || + CInst->DebugMarker->StoredDbgRecords.empty()); + ASSERT_TRUE(DInst->DebugMarker); + EXPECT_FALSE(DInst->DebugMarker->StoredDbgRecords.empty()); // Wheras if we move D to the start of the block with moveBeforePreserving, // the DbgVariableRecords should move with it. DInst->moveBeforePreserving(BB, BB.begin()); - EXPECT_FALSE(DInst->DbgMarker->StoredDbgRecords.empty()); + EXPECT_FALSE(DInst->DebugMarker->StoredDbgRecords.empty()); EXPECT_EQ(&*BB.begin(), DInst); // Similarly, moveAfterPreserving "D" to "C" should move DbgVariableRecords // with "D". DInst->moveAfterPreserving(CInst); - EXPECT_FALSE(DInst->DbgMarker->StoredDbgRecords.empty()); + EXPECT_FALSE(DInst->DebugMarker->StoredDbgRecords.empty()); // (move back to the start...) DInst->moveBeforePreserving(BB, BB.begin()); @@ -325,8 +326,9 @@ TEST(BasicBlockDbgInfoTest, HeadBitOperations) { // If we move "C" to the beginning of the block, it should go before the // DbgVariableRecords. They'll stay on "D". CInst->moveBefore(BB, BB.begin()); - EXPECT_TRUE(!CInst->DbgMarker || CInst->DbgMarker->StoredDbgRecords.empty()); - EXPECT_FALSE(DInst->DbgMarker->StoredDbgRecords.empty()); + EXPECT_TRUE(!CInst->DebugMarker || + CInst->DebugMarker->StoredDbgRecords.empty()); + EXPECT_FALSE(DInst->DebugMarker->StoredDbgRecords.empty()); EXPECT_EQ(&*BB.begin(), CInst); EXPECT_EQ(CInst->getNextNode(), DInst); @@ -342,8 +344,9 @@ TEST(BasicBlockDbgInfoTest, HeadBitOperations) { // run of dbg.values and the next instruction. CInst->moveBefore(BB, DInst->getIterator()); // CInst gains the DbgVariableRecords. - EXPECT_TRUE(!DInst->DbgMarker || DInst->DbgMarker->StoredDbgRecords.empty()); - EXPECT_FALSE(CInst->DbgMarker->StoredDbgRecords.empty()); + EXPECT_TRUE(!DInst->DebugMarker || + DInst->DebugMarker->StoredDbgRecords.empty()); + EXPECT_FALSE(CInst->DebugMarker->StoredDbgRecords.empty()); EXPECT_EQ(&*BB.begin(), CInst); UseNewDbgInfoFormat = false; @@ -380,7 +383,7 @@ TEST(BasicBlockDbgInfoTest, InstrDbgAccess) { )"); // Check that DbgVariableRecords can be accessed from Instructions without - // digging into the depths of DPMarkers. + // digging into the depths of DbgMarkers. BasicBlock &BB = M->getFunction("f")->getEntryBlock(); // Convert the module to "new" form debug-info. M->convertToNewDbgValues(); @@ -389,18 +392,18 @@ TEST(BasicBlockDbgInfoTest, InstrDbgAccess) { Instruction *CInst = BInst->getNextNode(); Instruction *DInst = CInst->getNextNode(); - ASSERT_FALSE(BInst->DbgMarker); - ASSERT_TRUE(CInst->DbgMarker); - ASSERT_EQ(CInst->DbgMarker->StoredDbgRecords.size(), 1u); - DbgRecord *DVR1 = &*CInst->DbgMarker->StoredDbgRecords.begin(); + ASSERT_FALSE(BInst->DebugMarker); + ASSERT_TRUE(CInst->DebugMarker); + ASSERT_EQ(CInst->DebugMarker->StoredDbgRecords.size(), 1u); + DbgRecord *DVR1 = &*CInst->DebugMarker->StoredDbgRecords.begin(); ASSERT_TRUE(DVR1); EXPECT_FALSE(BInst->hasDbgRecords()); // Clone DbgVariableRecords from one inst to another. Other arguments to clone - // are tested in DPMarker test. + // are tested in DbgMarker test. auto Range1 = BInst->cloneDebugInfoFrom(CInst); - EXPECT_EQ(BInst->DbgMarker->StoredDbgRecords.size(), 1u); - DbgRecord *DVR2 = &*BInst->DbgMarker->StoredDbgRecords.begin(); + EXPECT_EQ(BInst->DebugMarker->StoredDbgRecords.size(), 1u); + DbgRecord *DVR2 = &*BInst->DebugMarker->StoredDbgRecords.begin(); EXPECT_EQ(std::distance(Range1.begin(), Range1.end()), 1u); EXPECT_EQ(&*Range1.begin(), DVR2); EXPECT_NE(DVR1, DVR2); @@ -418,12 +421,12 @@ TEST(BasicBlockDbgInfoTest, InstrDbgAccess) { // Dropping should be easy, BInst->dropDbgRecords(); EXPECT_FALSE(BInst->hasDbgRecords()); - EXPECT_EQ(BInst->DbgMarker->StoredDbgRecords.size(), 0u); + EXPECT_EQ(BInst->DebugMarker->StoredDbgRecords.size(), 0u); // And we should be able to drop individual DbgVariableRecords. CInst->dropOneDbgRecord(DVR1); EXPECT_FALSE(CInst->hasDbgRecords()); - EXPECT_EQ(CInst->DbgMarker->StoredDbgRecords.size(), 0u); + EXPECT_EQ(CInst->DebugMarker->StoredDbgRecords.size(), 0u); UseNewDbgInfoFormat = false; } @@ -533,11 +536,11 @@ protected: CInst = &*Dest; DVRA = - cast(&*BInst->DbgMarker->StoredDbgRecords.begin()); - DVRB = - cast(&*Branch->DbgMarker->StoredDbgRecords.begin()); + cast(&*BInst->DebugMarker->StoredDbgRecords.begin()); + DVRB = cast( + &*Branch->DebugMarker->StoredDbgRecords.begin()); DVRConst = - cast(&*CInst->DbgMarker->StoredDbgRecords.begin()); + cast(&*CInst->DebugMarker->StoredDbgRecords.begin()); } void TearDown() override { UseNewDbgInfoFormat = false; } @@ -546,8 +549,8 @@ protected: for (DbgRecord &D : I->getDbgRecordRange()) { if (&D == DVR) { // Confirm too that the links between the records are correct. - EXPECT_EQ(DVR->Marker, I->DbgMarker); - EXPECT_EQ(I->DbgMarker->MarkedInstr, I); + EXPECT_EQ(DVR->Marker, I->DebugMarker); + EXPECT_EQ(I->DebugMarker->MarkedInstr, I); return true; } } @@ -1173,14 +1176,14 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceTrailing) { // The trailing DbgVariableRecord should have been placed at the front of // what's been spliced in. Instruction *BInst = &*Entry.begin(); - ASSERT_TRUE(BInst->DbgMarker); - EXPECT_EQ(BInst->DbgMarker->StoredDbgRecords.size(), 1u); + ASSERT_TRUE(BInst->DebugMarker); + EXPECT_EQ(BInst->DebugMarker->StoredDbgRecords.size(), 1u); UseNewDbgInfoFormat = false; } // When we remove instructions from the program, adjacent DbgVariableRecords -// coalesce together into one DPMarker. In "old" dbg.value mode you could +// coalesce together into one DbgMarker. In "old" dbg.value mode you could // re-insert the removed instruction back into the middle of a sequence of // dbg.values. Test that this can be replicated correctly by DbgVariableRecords TEST(BasicBlockDbgInfoTest, RemoveInstAndReinsert) { @@ -1392,7 +1395,7 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceToEmpty1) { // should be in the correct order of %a, then 0. Instruction *BInst = &*Entry.begin(); ASSERT_TRUE(BInst->hasDbgRecords()); - EXPECT_EQ(BInst->DbgMarker->StoredDbgRecords.size(), 2u); + EXPECT_EQ(BInst->DebugMarker->StoredDbgRecords.size(), 2u); SmallVector DbgVariableRecords; for (DbgRecord &DVR : BInst->getDbgRecordRange()) DbgVariableRecords.push_back(cast(&DVR)); @@ -1462,7 +1465,7 @@ TEST(BasicBlockDbgInfoTest, DbgSpliceToEmpty2) { // We should now have one dbg.values on the first instruction, %a. Instruction *BInst = &*Entry.begin(); ASSERT_TRUE(BInst->hasDbgRecords()); - EXPECT_EQ(BInst->DbgMarker->StoredDbgRecords.size(), 1u); + EXPECT_EQ(BInst->DebugMarker->StoredDbgRecords.size(), 1u); SmallVector DbgVariableRecords; for (DbgRecord &DVR : BInst->getDbgRecordRange()) DbgVariableRecords.push_back(cast(&DVR)); diff --git a/llvm/unittests/IR/DebugInfoTest.cpp b/llvm/unittests/IR/DebugInfoTest.cpp index 3672f2bccfec..d06b979bf4a1 100644 --- a/llvm/unittests/IR/DebugInfoTest.cpp +++ b/llvm/unittests/IR/DebugInfoTest.cpp @@ -947,67 +947,68 @@ TEST(MetadataTest, ConvertDbgToDbgVariableRecord) { Instruction *FirstInst = &ExitBlock->front(); Instruction *RetInst = &*std::next(FirstInst->getIterator()); - // Set-up DPMarkers in this block. + // Set-up DbgMarkers in this block. ExitBlock->IsNewDbgInfoFormat = true; ExitBlock->createMarker(FirstInst); ExitBlock->createMarker(RetInst); // Insert DbgRecords into markers, order should come out DVR2, DVR1. - FirstInst->DbgMarker->insertDbgRecord(DVR1, false); - FirstInst->DbgMarker->insertDbgRecord(DVR2, true); + FirstInst->DebugMarker->insertDbgRecord(DVR1, false); + FirstInst->DebugMarker->insertDbgRecord(DVR2, true); unsigned int ItCount = 0; - for (DbgRecord &Item : FirstInst->DbgMarker->getDbgRecordRange()) { + for (DbgRecord &Item : FirstInst->DebugMarker->getDbgRecordRange()) { EXPECT_TRUE((&Item == DVR2 && ItCount == 0) || (&Item == DVR1 && ItCount == 1)); - EXPECT_EQ(Item.getMarker(), FirstInst->DbgMarker); + EXPECT_EQ(Item.getMarker(), FirstInst->DebugMarker); ++ItCount; } // Clone them onto the second marker -- should allocate new DVRs. - RetInst->DbgMarker->cloneDebugInfoFrom(FirstInst->DbgMarker, std::nullopt, false); - EXPECT_EQ(RetInst->DbgMarker->StoredDbgRecords.size(), 2u); + RetInst->DebugMarker->cloneDebugInfoFrom(FirstInst->DebugMarker, std::nullopt, + false); + EXPECT_EQ(RetInst->DebugMarker->StoredDbgRecords.size(), 2u); ItCount = 0; // Check these things store the same information; but that they're not the same // objects. for (DbgVariableRecord &Item : - filterDbgVars(RetInst->DbgMarker->getDbgRecordRange())) { + filterDbgVars(RetInst->DebugMarker->getDbgRecordRange())) { EXPECT_TRUE( (Item.getRawLocation() == DVR2->getRawLocation() && ItCount == 0) || (Item.getRawLocation() == DVR1->getRawLocation() && ItCount == 1)); - EXPECT_EQ(Item.getMarker(), RetInst->DbgMarker); + EXPECT_EQ(Item.getMarker(), RetInst->DebugMarker); EXPECT_NE(&Item, DVR1); EXPECT_NE(&Item, DVR2); ++ItCount; } - RetInst->DbgMarker->dropDbgRecords(); - EXPECT_EQ(RetInst->DbgMarker->StoredDbgRecords.size(), 0u); + RetInst->DebugMarker->dropDbgRecords(); + EXPECT_EQ(RetInst->DebugMarker->StoredDbgRecords.size(), 0u); // Try cloning one single DbgVariableRecord. - auto DIIt = std::next(FirstInst->DbgMarker->getDbgRecordRange().begin()); - RetInst->DbgMarker->cloneDebugInfoFrom(FirstInst->DbgMarker, DIIt, false); - EXPECT_EQ(RetInst->DbgMarker->StoredDbgRecords.size(), 1u); + auto DIIt = std::next(FirstInst->DebugMarker->getDbgRecordRange().begin()); + RetInst->DebugMarker->cloneDebugInfoFrom(FirstInst->DebugMarker, DIIt, false); + EXPECT_EQ(RetInst->DebugMarker->StoredDbgRecords.size(), 1u); // The second DbgVariableRecord should have been cloned; it should have the // same values as DVR1. EXPECT_EQ( - cast(RetInst->DbgMarker->StoredDbgRecords.begin()) + cast(RetInst->DebugMarker->StoredDbgRecords.begin()) ->getRawLocation(), DVR1->getRawLocation()); // We should be able to drop individual DbgRecords. - RetInst->DbgMarker->dropOneDbgRecord( - &*RetInst->DbgMarker->StoredDbgRecords.begin()); + RetInst->DebugMarker->dropOneDbgRecord( + &*RetInst->DebugMarker->StoredDbgRecords.begin()); - // "Aborb" a DPMarker: this means pretend that the instruction it's attached + // "Aborb" a DbgMarker: this means pretend that the instruction it's attached // to is disappearing so it needs to be transferred into "this" marker. - RetInst->DbgMarker->absorbDebugValues(*FirstInst->DbgMarker, true); - EXPECT_EQ(RetInst->DbgMarker->StoredDbgRecords.size(), 2u); + RetInst->DebugMarker->absorbDebugValues(*FirstInst->DebugMarker, true); + EXPECT_EQ(RetInst->DebugMarker->StoredDbgRecords.size(), 2u); // Should be the DVR1 and DVR2 objects. ItCount = 0; - for (DbgRecord &Item : RetInst->DbgMarker->getDbgRecordRange()) { + for (DbgRecord &Item : RetInst->DebugMarker->getDbgRecordRange()) { EXPECT_TRUE((&Item == DVR2 && ItCount == 0) || (&Item == DVR1 && ItCount == 1)); - EXPECT_EQ(Item.getMarker(), RetInst->DbgMarker); + EXPECT_EQ(Item.getMarker(), RetInst->DebugMarker); ++ItCount; } @@ -1015,12 +1016,12 @@ TEST(MetadataTest, ConvertDbgToDbgVariableRecord) { // evrything in the basic block, then they should sink down into the // "TrailingDbgRecords" container for dangling debug-info. Future facilities // will restore them back when a terminator is inserted. - FirstInst->DbgMarker->removeMarker(); + FirstInst->DebugMarker->removeMarker(); FirstInst->eraseFromParent(); - RetInst->DbgMarker->removeMarker(); + RetInst->DebugMarker->removeMarker(); RetInst->eraseFromParent(); - DPMarker *EndMarker = ExitBlock->getTrailingDbgRecords(); + DbgMarker *EndMarker = ExitBlock->getTrailingDbgRecords(); ASSERT_NE(EndMarker, nullptr); EXPECT_EQ(EndMarker->StoredDbgRecords.size(), 2u); // Test again that it's those two DbgVariableRecords, DVR1 and DVR2. @@ -1083,7 +1084,7 @@ TEST(MetadataTest, DbgVariableRecordConversionRoutines) { // First instruction should be a dbg.value. EXPECT_TRUE(isa(BB1->front())); EXPECT_FALSE(BB1->IsNewDbgInfoFormat); - // Validating the block for DbgVariableRecords / DPMarkers shouldn't fail -- + // Validating the block for DbgVariableRecords / DbgMarkers shouldn't fail -- // there's no data stored right now. bool BrokenDebugInfo = false; bool Error = verifyModule(*M, &errs(), &BrokenDebugInfo); @@ -1107,38 +1108,39 @@ TEST(MetadataTest, DbgVariableRecordConversionRoutines) { for (auto &I : BB) EXPECT_FALSE(isa(I)); - // There should be a DPMarker on each of the two instructions in the entry + // There should be a DbgMarker on each of the two instructions in the entry // block, each containing one DbgVariableRecord. EXPECT_EQ(BB1->size(), 2u); Instruction *FirstInst = &BB1->front(); Instruction *SecondInst = FirstInst->getNextNode(); - ASSERT_TRUE(FirstInst->DbgMarker); - ASSERT_TRUE(SecondInst->DbgMarker); - EXPECT_NE(FirstInst->DbgMarker, SecondInst->DbgMarker); - EXPECT_EQ(FirstInst, FirstInst->DbgMarker->MarkedInstr); - EXPECT_EQ(SecondInst, SecondInst->DbgMarker->MarkedInstr); + ASSERT_TRUE(FirstInst->DebugMarker); + ASSERT_TRUE(SecondInst->DebugMarker); + EXPECT_NE(FirstInst->DebugMarker, SecondInst->DebugMarker); + EXPECT_EQ(FirstInst, FirstInst->DebugMarker->MarkedInstr); + EXPECT_EQ(SecondInst, SecondInst->DebugMarker->MarkedInstr); - EXPECT_EQ(FirstInst->DbgMarker->StoredDbgRecords.size(), 1u); + EXPECT_EQ(FirstInst->DebugMarker->StoredDbgRecords.size(), 1u); DbgVariableRecord *DVR1 = cast( - &*FirstInst->DbgMarker->getDbgRecordRange().begin()); - EXPECT_EQ(DVR1->getMarker(), FirstInst->DbgMarker); + &*FirstInst->DebugMarker->getDbgRecordRange().begin()); + EXPECT_EQ(DVR1->getMarker(), FirstInst->DebugMarker); // Should point at %a, an argument. EXPECT_TRUE(isa(DVR1->getVariableLocationOp(0))); - EXPECT_EQ(SecondInst->DbgMarker->StoredDbgRecords.size(), 1u); + EXPECT_EQ(SecondInst->DebugMarker->StoredDbgRecords.size(), 1u); DbgVariableRecord *DVR2 = cast( - &*SecondInst->DbgMarker->getDbgRecordRange().begin()); - EXPECT_EQ(DVR2->getMarker(), SecondInst->DbgMarker); + &*SecondInst->DebugMarker->getDbgRecordRange().begin()); + EXPECT_EQ(DVR2->getMarker(), SecondInst->DebugMarker); // Should point at FirstInst. EXPECT_EQ(DVR2->getVariableLocationOp(0), FirstInst); - // There should be no DbgVariableRecords / DPMarkers in the second block, but + // There should be no DbgVariableRecords / DbgMarkers in the second block, but // it should be marked as being in the new format. BasicBlock *BB2 = BB1->getNextNode(); EXPECT_TRUE(BB2->IsNewDbgInfoFormat); for (auto &Inst : *BB2) // Either there should be no marker, or it should be empty. - EXPECT_TRUE(!Inst.DbgMarker || Inst.DbgMarker->StoredDbgRecords.empty()); + EXPECT_TRUE(!Inst.DebugMarker || + Inst.DebugMarker->StoredDbgRecords.empty()); // Validating the first block should continue to not be a problem, Error = verifyModule(*M, &errs(), &BrokenDebugInfo); @@ -1152,7 +1154,7 @@ TEST(MetadataTest, DbgVariableRecordConversionRoutines) { Error = verifyModule(*M, &errs(), &BrokenDebugInfo); EXPECT_FALSE(Error); EXPECT_TRUE(BrokenDebugInfo); - DVR1->setMarker(FirstInst->DbgMarker); + DVR1->setMarker(FirstInst->DebugMarker); DILocalVariable *DLV1 = DVR1->getVariable(); DIExpression *Expr1 = DVR1->getExpression(); diff --git a/llvm/unittests/Transforms/Utils/LocalTest.cpp b/llvm/unittests/Transforms/Utils/LocalTest.cpp index 4258f218794f..a86775a1366b 100644 --- a/llvm/unittests/Transforms/Utils/LocalTest.cpp +++ b/llvm/unittests/Transforms/Utils/LocalTest.cpp @@ -1324,10 +1324,10 @@ TEST(Local, ReplaceDbgVariableRecord) { Instruction *RetInst = &*It; // Convert DVI into a DbgVariableRecord. - RetInst->DbgMarker = new DPMarker(); - RetInst->DbgMarker->MarkedInstr = RetInst; + RetInst->DebugMarker = new DbgMarker(); + RetInst->DebugMarker->MarkedInstr = RetInst; DbgVariableRecord *DVR = new DbgVariableRecord(DVI); - RetInst->DbgMarker->insertDbgRecord(DVR, false); + RetInst->DebugMarker->insertDbgRecord(DVR, false); // ... and erase the dbg.value. DVI->eraseFromParent(); @@ -1341,5 +1341,5 @@ TEST(Local, ReplaceDbgVariableRecord) { EXPECT_EQ(DVR->getVariableLocationOp(0), FooInst); // Teardown. - RetInst->DbgMarker->eraseFromParent(); + RetInst->DebugMarker->eraseFromParent(); } -- GitLab From 407937036fa7640f61f225474b1ea6623a40dbdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Wed, 20 Mar 2024 18:06:07 +0200 Subject: [PATCH 035/296] Revert "[libcxx] [modules] Fix relative paths with absolute LIBCXX_INSTALL_MODULES_DIR (#85756)" This reverts commit 272d1b44efdedb68c194970a610f0ca1b7b769c5, and the follow-up fix in d209d1340b99d4fbd325dffb5e13b757ab8264ea. Even after the follow-up fix, building with an empty CMAKE_INSTALL_PREFIX errors out with errors like this: CMake Error at /b/s/w/ir/x/w/llvm-llvm-project/libcxx/modules/CMakeLists.txt:215 (file): file RELATIVE_PATH must be passed a full path to the directory: lib/x86_64-pc-windows-msvc --- libcxx/modules/CMakeLists.txt | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/libcxx/modules/CMakeLists.txt b/libcxx/modules/CMakeLists.txt index 6c917200d6f3..0dea8cfca94a 100644 --- a/libcxx/modules/CMakeLists.txt +++ b/libcxx/modules/CMakeLists.txt @@ -206,15 +206,9 @@ add_custom_target(generate-cxx-modules # Configure the modules manifest. # Use the relative path between the installation and the module in the json # file. This allows moving the entire installation to a different location. -cmake_path(ABSOLUTE_PATH LIBCXX_INSTALL_LIBRARY_DIR - BASE_DIRECTORY "${CMAKE_INSTALL_PREFIX}" - OUTPUT_VARIABLE ABS_LIBRARY_DIR) -cmake_path(ABSOLUTE_PATH LIBCXX_INSTALL_MODULES_DIR - BASE_DIRECTORY "${CMAKE_INSTALL_PREFIX}" - OUTPUT_VARIABLE ABS_MODULES_DIR) file(RELATIVE_PATH LIBCXX_MODULE_RELATIVE_PATH - ${ABS_LIBRARY_DIR} - ${ABS_MODULES_DIR}) + ${CMAKE_INSTALL_PREFIX}/${LIBCXX_INSTALL_LIBRARY_DIR} + ${CMAKE_INSTALL_PREFIX}/${LIBCXX_INSTALL_MODULES_DIR}) configure_file( "modules.json.in" "${LIBCXX_LIBRARY_DIR}/libc++.modules.json" -- GitLab From 12028cb1dab9ba1b4ac826c3d70ca19c3b379255 Mon Sep 17 00:00:00 2001 From: Will Hawkins Date: Wed, 20 Mar 2024 12:18:31 -0400 Subject: [PATCH 036/296] [DwarfGenerator] Calculate relative offset according to Dwarf Version (#84847) The relative offset for a CU in Dwarf v5 (and later) is different than the relative offset for a CU in Dwarf v4 (and before). Signed-off-by: Will Hawkins --- llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp b/llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp index c5b9f5cfc0d4..ad5e51b7efb8 100644 --- a/llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp +++ b/llvm/unittests/DebugInfo/DWARF/DwarfGenerator.cpp @@ -568,12 +568,20 @@ StringRef dwarfgen::Generator::generate() { for (auto &CU : CompileUnits) { // Set the absolute .debug_info offset for this compile unit. CU->setOffset(SecOffset); - // The DIEs contain compile unit relative offsets. - unsigned CUOffset = 11; + // The DIEs contain compile unit relative offsets and the offset depends + // on the Dwarf version. + unsigned CUOffset = 4 + // Length + 2 + // Version + 4 + // Abbreviation offset + 1; // Address size + if (Asm->getDwarfVersion() >= 5) + CUOffset += 1; // DW_UT_compile tag. + CUOffset = CU->getUnitDIE().computeSizeAndOffsets(CUOffset); // Update our absolute .debug_info offset. SecOffset += CUOffset; - CU->setLength(CUOffset - 4); + unsigned CUOffsetUnitLength = 4; + CU->setLength(CUOffset - CUOffsetUnitLength); } Abbreviations.Emit(Asm.get(), TLOF->getDwarfAbbrevSection()); -- GitLab From 1b5b4eebb6a012cf223954013d34c6e896720822 Mon Sep 17 00:00:00 2001 From: Thurston Dang Date: Wed, 20 Mar 2024 09:19:20 -0700 Subject: [PATCH 037/296] [memprof] Move allocator base to avoid conflict with high-entropy ASLR (#85834) memprof often fails when ASLR entropy is too high ('sudo sysctl vm.mmap_rnd_bits=32; ninja check-memprof'), which is the default setting for newer versions of Ubuntu (https://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/jammy/commit/?h=hwe-6.5-next--2024.03.04-1--auto&id=6b522637c6a7dabd8530026ae933fb5ff17e877f). This patch fixes the issue by moving the allocator base, analogously to ASan (https://reviews.llvm.org/D148280). Explanation from the ASan patch: when CONFIG_ARCH_MMAP_RND_BITS == 32, it will frequently conflict with memprof's allocator, because the PIE program segment base address of 0x555555555554 plus an ASLR shift of up to ((2**32) * 4K == 0x100000000000) will sometimes exceed memprof's hardcoded base address of 0x600000000000. We fix this by simply moving the allocator base to 0x500000000000, which is below the PIE program segment base address. This is cleaner than trying to move it to another location that is sandwiched between the PIE program and library segments, because if either of those grow too large, it will collide with the allocator region. Note that we will never need to change this base address again (unless we want to increase the size of the allocator), because ASLR cannot be set above 32-bits for x86-64 Linux (the PIE program segment and library segments would collide with each other; see also ARCH_MMAP_RND_BITS_MAX in https://github.com/torvalds/linux/blob/master/arch/x86/Kconfig). --- compiler-rt/lib/memprof/memprof_allocator.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compiler-rt/lib/memprof/memprof_allocator.h b/compiler-rt/lib/memprof/memprof_allocator.h index f583cee020e4..89e924f80d41 100644 --- a/compiler-rt/lib/memprof/memprof_allocator.h +++ b/compiler-rt/lib/memprof/memprof_allocator.h @@ -46,7 +46,11 @@ struct MemprofMapUnmapCallback { void OnUnmap(uptr p, uptr size) const; }; +#if SANITIZER_APPLE constexpr uptr kAllocatorSpace = 0x600000000000ULL; +#else +constexpr uptr kAllocatorSpace = 0x500000000000ULL; +#endif constexpr uptr kAllocatorSize = 0x40000000000ULL; // 4T. typedef DefaultSizeClassMap SizeClassMap; template -- GitLab From a6a6066290679f23f2bd6b27afc7a06aab07590f Mon Sep 17 00:00:00 2001 From: Eric Li Date: Wed, 20 Mar 2024 12:45:30 -0400 Subject: [PATCH 038/296] [clang][dataflow] Fix crash when analyzing a coroutine (#85957) A coroutine function body (`CoroutineBodyStmt`) may have null children, which causes `isa` to segfault. --- .../lib/Analysis/FlowSensitive/AdornedCFG.cpp | 2 +- .../Analysis/FlowSensitive/TransferTest.cpp | 54 ++++++++++++++++++- 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/clang/lib/Analysis/FlowSensitive/AdornedCFG.cpp b/clang/lib/Analysis/FlowSensitive/AdornedCFG.cpp index 3813b8c3ee8a..daa73bed1bd9 100644 --- a/clang/lib/Analysis/FlowSensitive/AdornedCFG.cpp +++ b/clang/lib/Analysis/FlowSensitive/AdornedCFG.cpp @@ -103,7 +103,7 @@ buildContainsExprConsumedInDifferentBlock( auto CheckChildExprs = [&Result, &StmtToBlock](const Stmt *S, const CFGBlock *Block) { for (const Stmt *Child : S->children()) { - if (!isa(Child)) + if (!isa_and_nonnull(Child)) continue; const CFGBlock *ChildBlock = StmtToBlock.lookup(Child); if (ChildBlock != Block) diff --git a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp index a243535d3872..1d3b268976a7 100644 --- a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp +++ b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp @@ -37,6 +37,40 @@ using ::testing::Ne; using ::testing::NotNull; using ::testing::UnorderedElementsAre; +// Declares a minimal coroutine library. +constexpr llvm::StringRef CoroutineLibrary = R"cc( +struct promise; +struct task; + +namespace std { +template +struct coroutine_traits {}; +template <> +struct coroutine_traits { + using promise_type = promise; +}; + +template +struct coroutine_handle { + static constexpr coroutine_handle from_address(void *addr) { return {}; } +}; +} // namespace std + +struct awaitable { + bool await_ready() const noexcept; + void await_suspend(std::coroutine_handle) const noexcept; + void await_resume() const noexcept; +}; +struct task {}; +struct promise { + task get_return_object(); + awaitable initial_suspend(); + awaitable final_suspend() noexcept; + void unhandled_exception(); + void return_void(); +}; +)cc"; + void runDataflow( llvm::StringRef Code, std::function< @@ -4607,7 +4641,7 @@ TEST(TransferTest, LoopCanProveInvariantForBoolean) { } TEST(TransferTest, DoesNotCrashOnUnionThisExpr) { - std::string Code = R"( + std::string Code = R"cc( union Union { int A; float B; @@ -4618,7 +4652,7 @@ TEST(TransferTest, DoesNotCrashOnUnionThisExpr) { Union B; A = B; } - )"; + )cc"; // This is a crash regression test when calling the transfer function on a // `CXXThisExpr` that refers to a union. runDataflow( @@ -4628,6 +4662,22 @@ TEST(TransferTest, DoesNotCrashOnUnionThisExpr) { LangStandard::lang_cxx17, /*ApplyBuiltinTransfer=*/true, "operator="); } +TEST(TransferTest, DoesNotCrashOnNullChildren) { + std::string Code = (CoroutineLibrary + R"cc( + task target() noexcept { + co_return; + } + )cc") + .str(); + // This is a crash regression test when calling `AdornedCFG::build` on a + // statement (in this case, the `CoroutineBodyStmt`) with null children. + runDataflow( + Code, + [](const llvm::StringMap> &, + ASTContext &) {}, + LangStandard::lang_cxx20, /*ApplyBuiltinTransfer=*/true); +} + TEST(TransferTest, StructuredBindingAssignFromStructIntMembersToRefs) { std::string Code = R"( struct A { -- GitLab From 9cb5004209323d6fa8af8c41e456818c20585984 Mon Sep 17 00:00:00 2001 From: Alexandros Lamprineas Date: Wed, 20 Mar 2024 16:49:51 +0000 Subject: [PATCH 039/296] =?UTF-8?q?Reland=20[FMV]=20Emit=20the=20resolver?= =?UTF-8?q?=20along=20with=20the=20default=20version=20definit=E2=80=A6=20?= =?UTF-8?q?(#85923)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …ion. This was reverted because the resolver didn't look as expected in one of the tests. I believe it had some interaction with #84146. I have now regenerated it using -target-feature -fp-armv8. --- clang/lib/CodeGen/CodeGenModule.cpp | 55 +- clang/lib/CodeGen/CodeGenModule.h | 5 + clang/test/CodeGen/attr-target-version.c | 546 ++++++++++++------ clang/test/CodeGenCXX/attr-target-version.cpp | 261 +++++++-- 4 files changed, 629 insertions(+), 238 deletions(-) diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index bb26bfcddaeb..cb153066b28d 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -3449,6 +3449,9 @@ bool CodeGenModule::MayBeEmittedEagerly(const ValueDecl *Global) { // Implicit template instantiations may change linkage if they are later // explicitly instantiated, so they should not be emitted eagerly. return false; + // Defer until all versions have been semantically checked. + if (FD->hasAttr() && !FD->isMultiVersion()) + return false; } if (const auto *VD = dyn_cast(Global)) { if (Context.getInlineVariableDefinitionKind(VD) == @@ -3997,10 +4000,13 @@ void CodeGenModule::EmitMultiVersionFunctionDefinition(GlobalDecl GD, EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr); // Ensure that the resolver function is also emitted. GetOrCreateMultiVersionResolver(GD); - } else if (FD->hasAttr()) { - GetOrCreateMultiVersionResolver(GD); } else EmitGlobalFunctionDefinition(GD, GV); + + // Defer the resolver emission until we can reason whether the TU + // contains a default target version implementation. + if (FD->isTargetVersionMultiVersion()) + AddDeferredMultiVersionResolverToEmit(GD); } void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD, llvm::GlobalValue *GV) { @@ -4093,10 +4099,11 @@ void CodeGenModule::emitMultiVersionFunctions() { const auto *FD = cast(GD.getDecl()); assert(FD && "Expected a FunctionDecl"); + bool EmitResolver = !FD->isTargetVersionMultiVersion(); SmallVector Options; if (FD->isTargetMultiVersion()) { getContext().forEachMultiversionedFunctionVersion( - FD, [this, &GD, &Options](const FunctionDecl *CurFD) { + FD, [this, &GD, &Options, &EmitResolver](const FunctionDecl *CurFD) { GlobalDecl CurGD{ (CurFD->isDefined() ? CurFD->getDefinition() : CurFD)}; StringRef MangledName = getMangledName(CurGD); @@ -4122,6 +4129,9 @@ void CodeGenModule::emitMultiVersionFunctions() { TA->getArchitecture(), Feats); } else { const auto *TVA = CurFD->getAttr(); + if (CurFD->isUsed() || (TVA->isDefaultVersion() && + CurFD->doesThisDeclarationHaveABody())) + EmitResolver = true; llvm::SmallVector Feats; TVA->getFeatures(Feats); Options.emplace_back(cast(Func), @@ -4177,22 +4187,27 @@ void CodeGenModule::emitMultiVersionFunctions() { continue; } + if (!EmitResolver) + continue; + llvm::Constant *ResolverConstant = GetOrCreateMultiVersionResolver(GD); if (auto *IFunc = dyn_cast(ResolverConstant)) { ResolverConstant = IFunc->getResolver(); if (FD->isTargetClonesMultiVersion() || FD->isTargetVersionMultiVersion()) { - const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD); - llvm::FunctionType *DeclTy = getTypes().GetFunctionType(FI); std::string MangledName = getMangledNameImpl( *this, GD, FD, /*OmitMultiVersionMangling=*/true); - // In prior versions of Clang, the mangling for ifuncs incorrectly - // included an .ifunc suffix. This alias is generated for backward - // compatibility. It is deprecated, and may be removed in the future. - auto *Alias = llvm::GlobalAlias::create( - DeclTy, 0, getMultiversionLinkage(*this, GD), - MangledName + ".ifunc", IFunc, &getModule()); - SetCommonAttributes(FD, Alias); + if (!GetGlobalValue(MangledName + ".ifunc")) { + const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD); + llvm::FunctionType *DeclTy = getTypes().GetFunctionType(FI); + // In prior versions of Clang, the mangling for ifuncs incorrectly + // included an .ifunc suffix. This alias is generated for backward + // compatibility. It is deprecated, and may be removed in the future. + auto *Alias = llvm::GlobalAlias::create( + DeclTy, 0, getMultiversionLinkage(*this, GD), + MangledName + ".ifunc", IFunc, &getModule()); + SetCommonAttributes(FD, Alias); + } } } llvm::Function *ResolverFunc = cast(ResolverConstant); @@ -4349,6 +4364,20 @@ void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) { } } +/// Adds a declaration to the list of multi version functions if not present. +void CodeGenModule::AddDeferredMultiVersionResolverToEmit(GlobalDecl GD) { + const auto *FD = cast(GD.getDecl()); + assert(FD && "Not a FunctionDecl?"); + + if (FD->isTargetVersionMultiVersion()) { + std::string MangledName = + getMangledNameImpl(*this, GD, FD, /*OmitMultiVersionMangling=*/true); + if (!DeferredResolversToEmit.insert(MangledName).second) + return; + } + MultiVersionFuncs.push_back(GD); +} + /// If a dispatcher for the specified mangled name is not in the module, create /// and return an llvm Function with the specified type. llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) { @@ -4388,7 +4417,7 @@ llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) { // The resolver needs to be created. For target and target_clones, defer // creation until the end of the TU. if (FD->isTargetMultiVersion() || FD->isTargetClonesMultiVersion()) - MultiVersionFuncs.push_back(GD); + AddDeferredMultiVersionResolverToEmit(GD); // For cpu_specific, don't create an ifunc yet because we don't know if the // cpu_dispatch will be emitted in this translation unit. diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h index ec34680fd3f7..1cc447765e2c 100644 --- a/clang/lib/CodeGen/CodeGenModule.h +++ b/clang/lib/CodeGen/CodeGenModule.h @@ -348,6 +348,8 @@ private: /// yet. llvm::DenseMap DeferredDecls; + llvm::StringSet DeferredResolversToEmit; + /// This is a list of deferred decls which we have seen that *are* actually /// referenced. These get code generated when the module is done. std::vector DeferredDeclsToEmit; @@ -1588,6 +1590,9 @@ private: llvm::AttributeList ExtraAttrs = llvm::AttributeList(), ForDefinition_t IsForDefinition = NotForDefinition); + // Adds a declaration to the list of multi version functions if not present. + void AddDeferredMultiVersionResolverToEmit(GlobalDecl GD); + // References to multiversion functions are resolved through an implicitly // defined resolver function. This function is responsible for creating // the resolver symbol for the provided declaration. The value returned diff --git a/clang/test/CodeGen/attr-target-version.c b/clang/test/CodeGen/attr-target-version.c index b7112c783da9..25129605e76c 100644 --- a/clang/test/CodeGen/attr-target-version.c +++ b/clang/test/CodeGen/attr-target-version.c @@ -1,5 +1,5 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --function-signature --check-attributes --check-globals --include-generated-funcs -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +ls64 -target-feature +fullfp16 -S -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature -v9.5a -target-feature -fp-armv8 -S -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature -fmv -S -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK-NOFMV int __attribute__((target_version("rng+flagm+fp16fml"))) fmv(void) { return 1; } @@ -11,15 +11,15 @@ int __attribute__((target_version("fp+aes"))) fmv(void) { return 6; } int __attribute__((target_version("crc+ls64_v"))) fmv(void) { return 7; } int __attribute__((target_version("bti"))) fmv(void) { return 8; } int __attribute__((target_version("sme2"))) fmv(void) { return 9; } -int __attribute__((target_version("default"))) fmv(void) { return 0; } +int __attribute__((target_version("default"))) fmv(void); int __attribute__((target_version("ls64+simd"))) fmv_one(void) { return 1; } int __attribute__((target_version("dpb"))) fmv_one(void) { return 2; } -int __attribute__((target_version("default"))) fmv_one(void) { return 0; } +int __attribute__((target_version("default"))) fmv_one(void); int __attribute__((target_version("fp"))) fmv_two(void) { return 1; } int __attribute__((target_version("simd"))) fmv_two(void) { return 2; } int __attribute__((target_version("dgh"))) fmv_two(void) { return 3; } int __attribute__((target_version("fp16+simd"))) fmv_two(void) { return 4; } -int __attribute__((target_version("default"))) fmv_two(void) { return 0; } +int __attribute__((target_version("default"))) fmv_two(void); int foo() { return fmv()+fmv_one()+fmv_two(); } @@ -84,9 +84,33 @@ int hoo(void) { return fp1() + fp2(); } +// This should generate one target version but no resolver. +__attribute__((target_version("default"))) int unused_with_forward_default_decl(void); +__attribute__((target_version("mops"))) int unused_with_forward_default_decl(void) { return 0; } +// This should also generate one target version but no resolver. +extern int unused_with_implicit_extern_forward_default_decl(void); +__attribute__((target_version("dotprod"))) +int unused_with_implicit_extern_forward_default_decl(void) { return 0; } +// This should also generate one target version but no resolver. +__attribute__((target_version("aes"))) int unused_with_default_decl(void) { return 0; } +__attribute__((target_version("default"))) int unused_with_default_decl(void); +// This should generate two target versions and the resolver. +__attribute__((target_version("sve"))) int unused_with_default_def(void) { return 0; } +__attribute__((target_version("default"))) int unused_with_default_def(void) { return 1; } + +// This should also generate two target versions and the resolver. +__attribute__((target_version("fp16"))) int unused_with_implicit_default_def(void) { return 0; } +int unused_with_implicit_default_def(void) { return 1; } + +// This should also generate two target versions and the resolver. +int unused_with_implicit_forward_default_def(void) { return 0; } +__attribute__((target_version("lse"))) int unused_with_implicit_forward_default_def(void) { return 1; } + +// This should generate a normal function. +__attribute__((target_version("rdm"))) int unused_without_default(void) { return 0; } //. // CHECK: @__aarch64_cpu_features = external dso_local global { i64 } @@ -97,38 +121,107 @@ int hoo(void) { // CHECK: @fmv_c.ifunc = weak_odr alias void (), ptr @fmv_c // CHECK: @fmv_inline.ifunc = weak_odr alias i32 (), ptr @fmv_inline // CHECK: @fmv_d.ifunc = internal alias i32 (), ptr @fmv_d +// CHECK: @unused_with_default_def.ifunc = weak_odr alias i32 (), ptr @unused_with_default_def +// CHECK: @unused_with_implicit_default_def.ifunc = weak_odr alias i32 (), ptr @unused_with_implicit_default_def +// CHECK: @unused_with_implicit_forward_default_def.ifunc = weak_odr alias i32 (), ptr @unused_with_implicit_forward_default_def // CHECK: @fmv = weak_odr ifunc i32 (), ptr @fmv.resolver // CHECK: @fmv_one = weak_odr ifunc i32 (), ptr @fmv_one.resolver // CHECK: @fmv_two = weak_odr ifunc i32 (), ptr @fmv_two.resolver -// CHECK: @fmv_e = weak_odr ifunc i32 (), ptr @fmv_e.resolver -// CHECK: @fmv_c = weak_odr ifunc void (), ptr @fmv_c.resolver // CHECK: @fmv_inline = weak_odr ifunc i32 (), ptr @fmv_inline.resolver +// CHECK: @fmv_e = weak_odr ifunc i32 (), ptr @fmv_e.resolver // CHECK: @fmv_d = internal ifunc i32 (), ptr @fmv_d.resolver +// CHECK: @fmv_c = weak_odr ifunc void (), ptr @fmv_c.resolver +// CHECK: @unused_with_default_def = weak_odr ifunc i32 (), ptr @unused_with_default_def.resolver +// CHECK: @unused_with_implicit_default_def = weak_odr ifunc i32 (), ptr @unused_with_implicit_default_def.resolver +// CHECK: @unused_with_implicit_forward_default_def = weak_odr ifunc i32 (), ptr @unused_with_implicit_forward_default_def.resolver //. // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._MflagmMfp16fmlMrng +// CHECK-LABEL: define {{[^@]+}}@fmv._Mflagm2Msme-i16i64 // CHECK-SAME: () #[[ATTR0:[0-9]+]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 +// CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_one._Mls64Msimd +// CHECK-LABEL: define {{[^@]+}}@fmv._MlseMsha2 // CHECK-SAME: () #[[ATTR1:[0-9]+]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 +// CHECK-NEXT: ret i32 3 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp -// CHECK-SAME: () #[[ATTR1]] { +// CHECK-LABEL: define {{[^@]+}}@fmv._MdotprodMls64_accdata +// CHECK-SAME: () #[[ATTR2:[0-9]+]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 1 +// CHECK-NEXT: ret i32 4 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv._Mfp16fmlMmemtag +// CHECK-SAME: () #[[ATTR3:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 5 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv._MaesMfp +// CHECK-SAME: () #[[ATTR4:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 6 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv._McrcMls64_v +// CHECK-SAME: () #[[ATTR5:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 7 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv._Mbti +// CHECK-SAME: () #[[ATTR6:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 8 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv._Msme2 +// CHECK-SAME: () #[[ATTR7:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 9 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_one._Mdpb +// CHECK-SAME: () #[[ATTR8:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 2 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_two._Msimd +// CHECK-SAME: () #[[ATTR4]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 2 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mdgh +// CHECK-SAME: () #[[ATTR9:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 3 +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp16Msimd +// CHECK-SAME: () #[[ATTR10:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 4 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@foo -// CHECK-SAME: () #[[ATTR2:[0-9]+]] { +// CHECK-SAME: () #[[ATTR9]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[CALL:%.*]] = call i32 @fmv() // CHECK-NEXT: [[CALL1:%.*]] = call i32 @fmv_one() @@ -158,16 +251,16 @@ int hoo(void) { // CHECK-NEXT: ret ptr @fmv._Mflagm2Msme-i16i64 // CHECK: resolver_else2: // CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 16 -// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 16 +// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 9007199254741008 +// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 9007199254741008 // CHECK-NEXT: [[TMP11:%.*]] = and i1 true, [[TMP10]] // CHECK-NEXT: br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]] // CHECK: resolver_return3: // CHECK-NEXT: ret ptr @fmv._MdotprodMls64_accdata // CHECK: resolver_else4: // CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP13:%.*]] = and i64 [[TMP12]], 1024 -// CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 1024 +// CHECK-NEXT: [[TMP13:%.*]] = and i64 [[TMP12]], 4503599627371520 +// CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 4503599627371520 // CHECK-NEXT: [[TMP15:%.*]] = and i1 true, [[TMP14]] // CHECK-NEXT: br i1 [[TMP15]], label [[RESOLVER_RETURN5:%.*]], label [[RESOLVER_ELSE6:%.*]] // CHECK: resolver_return5: @@ -182,8 +275,8 @@ int hoo(void) { // CHECK-NEXT: ret ptr @fmv._Mfp16fmlMmemtag // CHECK: resolver_else8: // CHECK-NEXT: [[TMP20:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP21:%.*]] = and i64 [[TMP20]], 16384 -// CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[TMP21]], 16384 +// CHECK-NEXT: [[TMP21:%.*]] = and i64 [[TMP20]], 16640 +// CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[TMP21]], 16640 // CHECK-NEXT: [[TMP23:%.*]] = and i1 true, [[TMP22]] // CHECK-NEXT: br i1 [[TMP23]], label [[RESOLVER_RETURN9:%.*]], label [[RESOLVER_ELSE10:%.*]] // CHECK: resolver_return9: @@ -218,43 +311,95 @@ int hoo(void) { // // CHECK-LABEL: define {{[^@]+}}@fmv_one.resolver() comdat { // CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 2251799813685760 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 2251799813685760 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: // CHECK-NEXT: ret ptr @fmv_one._Mls64Msimd +// CHECK: resolver_else: +// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 262144 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 262144 +// CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] +// CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] +// CHECK: resolver_return1: +// CHECK-NEXT: ret ptr @fmv_one._Mdpb +// CHECK: resolver_else2: +// CHECK-NEXT: ret ptr @fmv_one.default // // // CHECK-LABEL: define {{[^@]+}}@fmv_two.resolver() comdat { // CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 66048 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 66048 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: // CHECK-NEXT: ret ptr @fmv_two._Mfp16Msimd +// CHECK: resolver_else: +// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 33554432 +// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 33554432 +// CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]] +// CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]] +// CHECK: resolver_return1: +// CHECK-NEXT: ret ptr @fmv_two._Mdgh +// CHECK: resolver_else2: +// CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 512 +// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 512 +// CHECK-NEXT: [[TMP11:%.*]] = and i1 true, [[TMP10]] +// CHECK-NEXT: br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]] +// CHECK: resolver_return3: +// CHECK-NEXT: ret ptr @fmv_two._Msimd +// CHECK: resolver_else4: +// CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP13:%.*]] = and i64 [[TMP12]], 256 +// CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 256 +// CHECK-NEXT: [[TMP15:%.*]] = and i1 true, [[TMP14]] +// CHECK-NEXT: br i1 [[TMP15]], label [[RESOLVER_RETURN5:%.*]], label [[RESOLVER_ELSE6:%.*]] +// CHECK: resolver_return5: +// CHECK-NEXT: ret ptr @fmv_two._Mfp +// CHECK: resolver_else6: +// CHECK-NEXT: ret ptr @fmv_two.default // // -// CHECK-LABEL: define {{[^@]+}}@fmv_e.resolver() comdat { -// CHECK-NEXT: resolver_entry: -// CHECK-NEXT: ret ptr @fmv_e._Mls64 +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_e.default +// CHECK-SAME: () #[[ATTR9]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 20 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_default -// CHECK-SAME: () #[[ATTR2]] { +// CHECK-SAME: () #[[ATTR9]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 111 // // -// CHECK-LABEL: define {{[^@]+}}@fmv_c.resolver() comdat { -// CHECK-NEXT: resolver_entry: -// CHECK-NEXT: call void @__init_cpu_features_resolver() -// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 281474976710656 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 281474976710656 -// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] -// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] -// CHECK: resolver_return: -// CHECK-NEXT: ret ptr @fmv_c._Mssbs -// CHECK: resolver_else: -// CHECK-NEXT: ret ptr @fmv_c.default +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_c._Mssbs +// CHECK-SAME: () #[[ATTR9]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret void +// +// +// CHECK: Function Attrs: noinline nounwind optnone +// CHECK-LABEL: define {{[^@]+}}@fmv_c.default +// CHECK-SAME: () #[[ATTR9]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: ret void // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@goo -// CHECK-SAME: () #[[ATTR2]] { +// CHECK-SAME: () #[[ATTR9]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[CALL:%.*]] = call i32 @fmv_inline() // CHECK-NEXT: [[CALL1:%.*]] = call i32 @fmv_e() @@ -268,8 +413,8 @@ int hoo(void) { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() // CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 4398048608320 -// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 4398048608320 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 4398048673856 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 4398048673856 // CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] // CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] // CHECK: resolver_return: @@ -364,8 +509,8 @@ int hoo(void) { // CHECK-NEXT: ret ptr @fmv_inline._Mdpb2Mjscvt // CHECK: resolver_else22: // CHECK-NEXT: [[TMP48:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP49:%.*]] = and i64 [[TMP48]], 8 -// CHECK-NEXT: [[TMP50:%.*]] = icmp eq i64 [[TMP49]], 8 +// CHECK-NEXT: [[TMP49:%.*]] = and i64 [[TMP48]], 520 +// CHECK-NEXT: [[TMP50:%.*]] = icmp eq i64 [[TMP49]], 520 // CHECK-NEXT: [[TMP51:%.*]] = and i1 true, [[TMP50]] // CHECK-NEXT: br i1 [[TMP51]], label [[RESOLVER_RETURN23:%.*]], label [[RESOLVER_ELSE24:%.*]] // CHECK: resolver_return23: @@ -388,8 +533,8 @@ int hoo(void) { // CHECK-NEXT: ret ptr @fmv_inline._MlseMrdm // CHECK: resolver_else28: // CHECK-NEXT: [[TMP60:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 -// CHECK-NEXT: [[TMP61:%.*]] = and i64 [[TMP60]], 32 -// CHECK-NEXT: [[TMP62:%.*]] = icmp eq i64 [[TMP61]], 32 +// CHECK-NEXT: [[TMP61:%.*]] = and i64 [[TMP60]], 288 +// CHECK-NEXT: [[TMP62:%.*]] = icmp eq i64 [[TMP61]], 288 // CHECK-NEXT: [[TMP63:%.*]] = and i1 true, [[TMP62]] // CHECK-NEXT: br i1 [[TMP63]], label [[RESOLVER_RETURN29:%.*]], label [[RESOLVER_ELSE30:%.*]] // CHECK: resolver_return29: @@ -398,6 +543,20 @@ int hoo(void) { // CHECK-NEXT: ret ptr @fmv_inline.default // // +// CHECK-LABEL: define {{[^@]+}}@fmv_e.resolver() comdat { +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 2251799813685248 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 2251799813685248 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @fmv_e._Mls64 +// CHECK: resolver_else: +// CHECK-NEXT: ret ptr @fmv_e.default +// +// // CHECK-LABEL: define {{[^@]+}}@fmv_d.resolver() { // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() @@ -412,9 +571,23 @@ int hoo(void) { // CHECK-NEXT: ret ptr @fmv_d.default // // +// CHECK-LABEL: define {{[^@]+}}@fmv_c.resolver() comdat { +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 281474976710656 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 281474976710656 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @fmv_c._Mssbs +// CHECK: resolver_else: +// CHECK-NEXT: ret ptr @fmv_c.default +// +// // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@recur -// CHECK-SAME: () #[[ATTR2]] { +// CHECK-SAME: () #[[ATTR9]] { // CHECK-NEXT: entry: // CHECK-NEXT: call void @reca() // CHECK-NEXT: ret void @@ -422,7 +595,7 @@ int hoo(void) { // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@main -// CHECK-SAME: () #[[ATTR2]] { +// CHECK-SAME: () #[[ATTR9]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[RETVAL:%.*]] = alloca i32, align 4 // CHECK-NEXT: store i32 0, ptr [[RETVAL]], align 4 @@ -433,7 +606,7 @@ int hoo(void) { // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@hoo -// CHECK-SAME: () #[[ATTR2]] { +// CHECK-SAME: () #[[ATTR9]] { // CHECK-NEXT: entry: // CHECK-NEXT: [[FP1:%.*]] = alloca ptr, align 8 // CHECK-NEXT: [[FP2:%.*]] = alloca ptr, align 8 @@ -449,288 +622,274 @@ int hoo(void) { // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._Mflagm2Msme-i16i64 -// CHECK-SAME: () #[[ATTR4:[0-9]+]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 2 -// -// -// CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._MlseMsha2 -// CHECK-SAME: () #[[ATTR5:[0-9]+]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 3 -// -// -// CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._MdotprodMls64_accdata -// CHECK-SAME: () #[[ATTR6:[0-9]+]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 4 -// -// -// CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._Mfp16fmlMmemtag -// CHECK-SAME: () #[[ATTR7:[0-9]+]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 5 -// -// -// CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._MaesMfp -// CHECK-SAME: () #[[ATTR1]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_forward_default_decl._Mmops +// CHECK-SAME: () #[[ATTR12:[0-9]+]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 6 +// CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._McrcMls64_v -// CHECK-SAME: () #[[ATTR8:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_extern_forward_default_decl._Mdotprod +// CHECK-SAME: () #[[ATTR13:[0-9]+]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 7 +// CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._Mbti -// CHECK-SAME: () #[[ATTR9:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.default +// CHECK-SAME: () #[[ATTR9]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 8 +// CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv._Msme2 -// CHECK-SAME: () #[[ATTR10:[0-9]+]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def.default +// CHECK-SAME: () #[[ATTR9]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 9 +// CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv.default -// CHECK-SAME: () #[[ATTR2]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def.default +// CHECK-SAME: () #[[ATTR9]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_one._Mdpb -// CHECK-SAME: () #[[ATTR11:[0-9]+]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 2 -// -// -// CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_one.default -// CHECK-SAME: () #[[ATTR2]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def._Mlse +// CHECK-SAME: () #[[ATTR14:[0-9]+]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 0 +// CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_two._Msimd -// CHECK-SAME: () #[[ATTR1]] { +// CHECK-LABEL: define {{[^@]+}}@fmv._MflagmMfp16fmlMrng +// CHECK-SAME: () #[[ATTR15:[0-9]+]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 2 +// CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mdgh -// CHECK-SAME: () #[[ATTR2]] { +// CHECK-LABEL: define {{[^@]+}}@fmv_one._Mls64Msimd +// CHECK-SAME: () #[[ATTR4]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 3 +// CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp16Msimd -// CHECK-SAME: () #[[ATTR1]] { +// CHECK-LABEL: define {{[^@]+}}@fmv_two._Mfp +// CHECK-SAME: () #[[ATTR4]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 4 +// CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_two.default -// CHECK-SAME: () #[[ATTR2]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_default_decl._Maes +// CHECK-SAME: () #[[ATTR4]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_e.default -// CHECK-SAME: () #[[ATTR2]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def._Msve +// CHECK-SAME: () #[[ATTR16:[0-9]+]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 20 +// CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_c.default -// CHECK-SAME: () #[[ATTR2]] { +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def._Mfp16 +// CHECK-SAME: () #[[ATTR10]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret void +// CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone -// CHECK-LABEL: define {{[^@]+}}@fmv_c._Mssbs -// CHECK-SAME: () #[[ATTR2]] { +// CHECK-LABEL: define {{[^@]+}}@unused_without_default +// CHECK-SAME: () #[[ATTR17:[0-9]+]] { // CHECK-NEXT: entry: -// CHECK-NEXT: ret void +// CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf64mmMpmullMsha1 -// CHECK-SAME: () #[[ATTR12:[0-9]+]] { +// CHECK-SAME: () #[[ATTR18:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfcmaMfp16MrdmMsme -// CHECK-SAME: () #[[ATTR13:[0-9]+]] { +// CHECK-SAME: () #[[ATTR19:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 2 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf32mmMi8mmMsha3 -// CHECK-SAME: () #[[ATTR14:[0-9]+]] { +// CHECK-SAME: () #[[ATTR20:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 12 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MditMsve-ebf16 -// CHECK-SAME: () #[[ATTR15:[0-9]+]] { +// CHECK-SAME: () #[[ATTR21:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 8 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MdpbMrcpc2 -// CHECK-SAME: () #[[ATTR16:[0-9]+]] { +// CHECK-SAME: () #[[ATTR22:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 6 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mdpb2Mjscvt -// CHECK-SAME: () #[[ATTR17:[0-9]+]] { +// CHECK-SAME: () #[[ATTR23:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 7 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfrinttsMrcpc -// CHECK-SAME: () #[[ATTR18:[0-9]+]] { +// CHECK-SAME: () #[[ATTR24:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MsveMsve-bf16 -// CHECK-SAME: () #[[ATTR19:[0-9]+]] { +// CHECK-SAME: () #[[ATTR25:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 4 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2-aesMsve2-sha3 -// CHECK-SAME: () #[[ATTR20:[0-9]+]] { +// CHECK-SAME: () #[[ATTR26:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 5 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Msve2Msve2-bitpermMsve2-pmull128 -// CHECK-SAME: () #[[ATTR21:[0-9]+]] { +// CHECK-SAME: () #[[ATTR27:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 9 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag2Msve2-sm4 -// CHECK-SAME: () #[[ATTR22:[0-9]+]] { +// CHECK-SAME: () #[[ATTR28:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 10 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mmemtag3MmopsMrcpc3 -// CHECK-SAME: () #[[ATTR23:[0-9]+]] { +// CHECK-SAME: () #[[ATTR29:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 11 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MaesMdotprod -// CHECK-SAME: () #[[ATTR6]] { +// CHECK-SAME: () #[[ATTR13]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 13 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mfp16fmlMsimd -// CHECK-SAME: () #[[ATTR7]] { +// CHECK-SAME: () #[[ATTR3]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 14 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MfpMsm4 -// CHECK-SAME: () #[[ATTR24:[0-9]+]] { +// CHECK-SAME: () #[[ATTR30:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 15 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline._MlseMrdm -// CHECK-SAME: () #[[ATTR25:[0-9]+]] { +// CHECK-SAME: () #[[ATTR31:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 16 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_inline.default -// CHECK-SAME: () #[[ATTR2]] { +// CHECK-SAME: () #[[ATTR9]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 3 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_d._Msb -// CHECK-SAME: () #[[ATTR26:[0-9]+]] { +// CHECK-SAME: () #[[ATTR32:[0-9]+]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 0 // // // CHECK: Function Attrs: noinline nounwind optnone // CHECK-LABEL: define {{[^@]+}}@fmv_d.default -// CHECK-SAME: () #[[ATTR2]] { +// CHECK-SAME: () #[[ATTR9]] { // CHECK-NEXT: entry: // CHECK-NEXT: ret i32 1 // // -// CHECK-NOFMV: Function Attrs: noinline nounwind optnone -// CHECK-NOFMV-LABEL: define {{[^@]+}}@fmv -// CHECK-NOFMV-SAME: () #[[ATTR0:[0-9]+]] { -// CHECK-NOFMV-NEXT: entry: -// CHECK-NOFMV-NEXT: ret i32 0 +// CHECK-LABEL: define {{[^@]+}}@unused_with_default_def.resolver() comdat { +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1073741824 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1073741824 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @unused_with_default_def._Msve +// CHECK: resolver_else: +// CHECK-NEXT: ret ptr @unused_with_default_def.default // // -// CHECK-NOFMV: Function Attrs: noinline nounwind optnone -// CHECK-NOFMV-LABEL: define {{[^@]+}}@fmv_one -// CHECK-NOFMV-SAME: () #[[ATTR0]] { -// CHECK-NOFMV-NEXT: entry: -// CHECK-NOFMV-NEXT: ret i32 0 +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_default_def.resolver() comdat { +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 65536 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 65536 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @unused_with_implicit_default_def._Mfp16 +// CHECK: resolver_else: +// CHECK-NEXT: ret ptr @unused_with_implicit_default_def.default // // -// CHECK-NOFMV: Function Attrs: noinline nounwind optnone -// CHECK-NOFMV-LABEL: define {{[^@]+}}@fmv_two -// CHECK-NOFMV-SAME: () #[[ATTR0]] { -// CHECK-NOFMV-NEXT: entry: -// CHECK-NOFMV-NEXT: ret i32 0 +// CHECK-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def.resolver() comdat { +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 128 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 128 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @unused_with_implicit_forward_default_def._Mlse +// CHECK: resolver_else: +// CHECK-NEXT: ret ptr @unused_with_implicit_forward_default_def.default // // // CHECK-NOFMV: Function Attrs: noinline nounwind optnone // CHECK-NOFMV-LABEL: define {{[^@]+}}@foo -// CHECK-NOFMV-SAME: () #[[ATTR0]] { +// CHECK-NOFMV-SAME: () #[[ATTR0:[0-9]+]] { // CHECK-NOFMV-NEXT: entry: // CHECK-NOFMV-NEXT: [[CALL:%.*]] = call i32 @fmv() // CHECK-NOFMV-NEXT: [[CALL1:%.*]] = call i32 @fmv_one() @@ -815,34 +974,61 @@ int hoo(void) { // CHECK-NOFMV-NEXT: [[ADD:%.*]] = add nsw i32 [[CALL]], [[CALL1]] // CHECK-NOFMV-NEXT: ret i32 [[ADD]] // +// +// CHECK-NOFMV: Function Attrs: noinline nounwind optnone +// CHECK-NOFMV-LABEL: define {{[^@]+}}@unused_with_default_def +// CHECK-NOFMV-SAME: () #[[ATTR0]] { +// CHECK-NOFMV-NEXT: entry: +// CHECK-NOFMV-NEXT: ret i32 1 +// +// +// CHECK-NOFMV: Function Attrs: noinline nounwind optnone +// CHECK-NOFMV-LABEL: define {{[^@]+}}@unused_with_implicit_default_def +// CHECK-NOFMV-SAME: () #[[ATTR0]] { +// CHECK-NOFMV-NEXT: entry: +// CHECK-NOFMV-NEXT: ret i32 1 +// +// +// CHECK-NOFMV: Function Attrs: noinline nounwind optnone +// CHECK-NOFMV-LABEL: define {{[^@]+}}@unused_with_implicit_forward_default_def +// CHECK-NOFMV-SAME: () #[[ATTR0]] { +// CHECK-NOFMV-NEXT: entry: +// CHECK-NOFMV-NEXT: ret i32 0 +// //. -// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+flagm,+fp-armv8,+fp16fml,+fullfp16,+ls64,+neon,+rand" } -// CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+ls64,+neon" } -// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+ls64" } -// CHECK: attributes #[[ATTR3:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+ls64" } -// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+altnzcv,+bf16,+flagm,+fullfp16,+ls64,+sme,+sme-i16i64" } -// CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+ls64,+lse,+neon,+sha2" } -// CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+fullfp16,+ls64,+neon" } -// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fp16fml,+fullfp16,+ls64,+neon" } -// CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,+fullfp16,+ls64" } -// CHECK: attributes #[[ATTR9]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti,+fullfp16,+ls64" } -// CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fullfp16,+ls64,+sme,+sme2" } -// CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+fullfp16,+ls64" } -// CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fp-armv8,+fullfp16,+ls64,+neon,+sve" } -// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fp-armv8,+fullfp16,+ls64,+neon,+rdm,+sme" } -// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fp-armv8,+fullfp16,+i8mm,+ls64,+neon,+sha2,+sha3,+sve" } -// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fp-armv8,+fullfp16,+ls64,+neon,+sve" } -// CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+fullfp16,+ls64,+rcpc" } -// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+fp-armv8,+fullfp16,+jsconv,+ls64,+neon" } -// CHECK: attributes #[[ATTR18]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+fullfp16,+ls64,+rcpc" } -// CHECK: attributes #[[ATTR19]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+fullfp16,+ls64,+neon,+sve" } -// CHECK: attributes #[[ATTR20]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+ls64,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3" } -// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+ls64,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm" } -// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+ls64,+mte,+neon,+sve,+sve2,+sve2-sm4" } -// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+ls64,+mops,+mte,+rcpc,+rcpc3" } -// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+ls64,+neon,+sm4" } -// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+ls64,+lse,+neon,+rdm" } -// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+ls64,+sb" } +// CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+altnzcv,+bf16,+flagm,+sme,+sme-i16i64,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+sha2,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+ls64,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp16fml,+fullfp16,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bti,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme2,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR9]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR11:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+flagm,+fp16fml,+fullfp16,+neon,+rand,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+rdm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR18]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR19]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fullfp16,+neon,+rdm,+sme,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR20]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR21]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccpp,+rcpc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ccdp,+ccpp,+jsconv,+neon,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fptoint,+rcpc,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fullfp16,+neon,+sve,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-sha3,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR27]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+neon,+sve,+sve2,+sve2-aes,+sve2-bitperm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR28]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fullfp16,+mte,+neon,+sve,+sve2,+sve2-sm4,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR29]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops,+mte,+rcpc,+rcpc3,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR30]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+neon,+sm4,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR31]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse,+neon,+rdm,-fp-armv8,-v9.5a" } +// CHECK: attributes #[[ATTR32]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb,-fp-armv8,-v9.5a" } //. // CHECK-NOFMV: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" } // CHECK-NOFMV: attributes #[[ATTR1:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-fmv" } diff --git a/clang/test/CodeGenCXX/attr-target-version.cpp b/clang/test/CodeGenCXX/attr-target-version.cpp index 82a928a385e1..e06121d1a719 100644 --- a/clang/test/CodeGenCXX/attr-target-version.cpp +++ b/clang/test/CodeGenCXX/attr-target-version.cpp @@ -10,41 +10,196 @@ struct MyClass { int __attribute__((target_version("dotprod"))) goo(int); int __attribute__((target_version("crc"))) goo(int); int __attribute__((target_version("default"))) goo(int); + + // This should generate one target version but no resolver. + int __attribute__((target_version("default"))) unused_with_forward_default_decl(void); + int __attribute__((target_version("mops"))) unused_with_forward_default_decl(void); + + // This should also generate one target version but no resolver. + int unused_with_implicit_forward_default_decl(void); + int __attribute__((target_version("dotprod"))) unused_with_implicit_forward_default_decl(void); + + // This should also generate one target version but no resolver. + int __attribute__((target_version("aes"))) unused_with_default_decl(void); + int __attribute__((target_version("default"))) unused_with_default_decl(void); + + // This should generate two target versions and the resolver. + int __attribute__((target_version("sve"))) unused_with_default_def(void); + int __attribute__((target_version("default"))) unused_with_default_def(void); + + // This should also generate two target versions and the resolver. + int __attribute__((target_version("fp16"))) unused_with_implicit_default_def(void); + int unused_with_implicit_default_def(void); + + // This should also generate two target versions and the resolver. + int unused_with_implicit_forward_default_def(void); + int __attribute__((target_version("lse"))) unused_with_implicit_forward_default_def(void); + + // This should generate a normal function. + int __attribute__((target_version("rdm"))) unused_without_default(void); }; int __attribute__((target_version("default"))) MyClass::goo(int) { return 1; } int __attribute__((target_version("crc"))) MyClass::goo(int) { return 2; } int __attribute__((target_version("dotprod"))) MyClass::goo(int) { return 3; } +int __attribute__((target_version("mops"))) MyClass::unused_with_forward_default_decl(void) { return 0; } +int __attribute__((target_version("dotprod"))) MyClass::unused_with_implicit_forward_default_decl(void) { return 0; } +int __attribute__((target_version("aes"))) MyClass::unused_with_default_decl(void) { return 0; } +int __attribute__((target_version("sve"))) MyClass::unused_with_default_def(void) { return 0; } +int __attribute__((target_version("default"))) MyClass::unused_with_default_def(void) { return 1; } +int __attribute__((target_version("fp16"))) MyClass::unused_with_implicit_default_def(void) { return 0; } +int MyClass::unused_with_implicit_default_def(void) { return 1; } +int MyClass::unused_with_implicit_forward_default_def(void) { return 0; } +int __attribute__((target_version("lse"))) MyClass::unused_with_implicit_forward_default_def(void) { return 1; } +int __attribute__((target_version("rdm"))) MyClass::unused_without_default(void) { return 0; } + int bar() { MyClass m; return m.goo(1) + foo(1) + foo(); } - - //. // CHECK: @__aarch64_cpu_features = external dso_local global { i64 } -// CHECK: @_ZN7MyClass3gooEi.ifunc = weak_odr alias i32 (ptr, i32), ptr @_ZN7MyClass3gooEi // CHECK: @_Z3fooi.ifunc = weak_odr alias i32 (i32), ptr @_Z3fooi // CHECK: @_Z3foov.ifunc = weak_odr alias i32 (), ptr @_Z3foov +// CHECK: @_ZN7MyClass3gooEi.ifunc = weak_odr alias i32 (ptr, i32), ptr @_ZN7MyClass3gooEi +// CHECK: @_ZN7MyClass23unused_with_default_defEv.ifunc = weak_odr alias i32 (ptr), ptr @_ZN7MyClass23unused_with_default_defEv +// CHECK: @_ZN7MyClass32unused_with_implicit_default_defEv.ifunc = weak_odr alias i32 (ptr), ptr @_ZN7MyClass32unused_with_implicit_default_defEv +// CHECK: @_ZN7MyClass40unused_with_implicit_forward_default_defEv.ifunc = weak_odr alias i32 (ptr), ptr @_ZN7MyClass40unused_with_implicit_forward_default_defEv // CHECK: @_ZN7MyClass3gooEi = weak_odr ifunc i32 (ptr, i32), ptr @_ZN7MyClass3gooEi.resolver // CHECK: @_Z3fooi = weak_odr ifunc i32 (i32), ptr @_Z3fooi.resolver // CHECK: @_Z3foov = weak_odr ifunc i32 (), ptr @_Z3foov.resolver +// CHECK: @_ZN7MyClass23unused_with_default_defEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClass23unused_with_default_defEv.resolver +// CHECK: @_ZN7MyClass32unused_with_implicit_default_defEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClass32unused_with_implicit_default_defEv.resolver +// CHECK: @_ZN7MyClass40unused_with_implicit_forward_default_defEv = weak_odr ifunc i32 (ptr), ptr @_ZN7MyClass40unused_with_implicit_forward_default_defEv.resolver //. -// CHECK-LABEL: @_Z3fooi._Mbf16Msme-f64f64( +// CHECK-LABEL: @_Z3fooi.default( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4 +// CHECK-NEXT: ret i32 2 +// +// +// CHECK-LABEL: @_Z3foov.default( +// CHECK-NEXT: entry: +// CHECK-NEXT: ret i32 4 +// +// +// CHECK-LABEL: @_ZN7MyClass3gooEi.default( // CHECK-NEXT: entry: +// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 // CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 // CHECK-NEXT: store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4 +// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 // CHECK-NEXT: ret i32 1 // // -// CHECK-LABEL: @_Z3foov._Mebf16Msm4( +// CHECK-LABEL: @_ZN7MyClass3gooEi._Mcrc( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4 +// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: ret i32 2 +// +// +// CHECK-LABEL: @_ZN7MyClass3gooEi._Mdotprod( // CHECK-NEXT: entry: +// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4 +// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 // CHECK-NEXT: ret i32 3 // // +// CHECK-LABEL: @_ZN7MyClass32unused_with_forward_default_declEv._Mmops( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: ret i32 0 +// +// +// CHECK-LABEL: @_ZN7MyClass41unused_with_implicit_forward_default_declEv._Mdotprod( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: ret i32 0 +// +// +// CHECK-LABEL: @_ZN7MyClass24unused_with_default_declEv._Maes( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: ret i32 0 +// +// +// CHECK-LABEL: @_ZN7MyClass23unused_with_default_defEv._Msve( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: ret i32 0 +// +// +// CHECK-LABEL: @_ZN7MyClass23unused_with_default_defEv.default( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: ret i32 1 +// +// +// CHECK-LABEL: @_ZN7MyClass32unused_with_implicit_default_defEv._Mfp16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: ret i32 0 +// +// +// CHECK-LABEL: @_ZN7MyClass32unused_with_implicit_default_defEv.default( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: ret i32 1 +// +// +// CHECK-LABEL: @_ZN7MyClass40unused_with_implicit_forward_default_defEv.default( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: ret i32 0 +// +// +// CHECK-LABEL: @_ZN7MyClass40unused_with_implicit_forward_default_defEv._Mlse( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 +// CHECK-NEXT: ret i32 1 +// +// +// CHECK-LABEL: @_Z3barv( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[M:%.*]] = alloca [[STRUCT_MYCLASS:%.*]], align 1 +// CHECK-NEXT: [[CALL:%.*]] = call noundef i32 @_ZN7MyClass3gooEi(ptr noundef nonnull align 1 dereferenceable(1) [[M]], i32 noundef 1) +// CHECK-NEXT: [[CALL1:%.*]] = call noundef i32 @_Z3fooi(i32 noundef 1) +// CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[CALL]], [[CALL1]] +// CHECK-NEXT: [[CALL2:%.*]] = call noundef i32 @_Z3foov() +// CHECK-NEXT: [[ADD3:%.*]] = add nsw i32 [[ADD]], [[CALL2]] +// CHECK-NEXT: ret i32 [[ADD3]] +// +// // CHECK-LABEL: @_ZN7MyClass3gooEi.resolver( // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() @@ -67,17 +222,6 @@ int bar() { // CHECK-NEXT: ret ptr @_ZN7MyClass3gooEi.default // // -// CHECK-LABEL: @_Z3barv( -// CHECK-NEXT: entry: -// CHECK-NEXT: [[M:%.*]] = alloca [[STRUCT_MYCLASS:%.*]], align 1 -// CHECK-NEXT: [[CALL:%.*]] = call noundef i32 @_ZN7MyClass3gooEi(ptr noundef nonnull align 1 dereferenceable(1) [[M]], i32 noundef 1) -// CHECK-NEXT: [[CALL1:%.*]] = call noundef i32 @_Z3fooi(i32 noundef 1) -// CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[CALL]], [[CALL1]] -// CHECK-NEXT: [[CALL2:%.*]] = call noundef i32 @_Z3foov() -// CHECK-NEXT: [[ADD3:%.*]] = add nsw i32 [[ADD]], [[CALL2]] -// CHECK-NEXT: ret i32 [[ADD3]] -// -// // CHECK-LABEL: @_Z3fooi.resolver( // CHECK-NEXT: resolver_entry: // CHECK-NEXT: call void @__init_cpu_features_resolver() @@ -106,53 +250,80 @@ int bar() { // CHECK-NEXT: ret ptr @_Z3foov.default // // -// CHECK-LABEL: @_ZN7MyClass3gooEi._Mdotprod( +// CHECK-LABEL: @_Z3fooi._Mbf16Msme-f64f64( // CHECK-NEXT: entry: -// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 // CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 -// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 // CHECK-NEXT: store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4 -// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 -// CHECK-NEXT: ret i32 3 +// CHECK-NEXT: ret i32 1 // // -// CHECK-LABEL: @_ZN7MyClass3gooEi._Mcrc( +// CHECK-LABEL: @_Z3foov._Mebf16Msm4( // CHECK-NEXT: entry: -// CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 -// CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 -// CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 -// CHECK-NEXT: store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4 -// CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 -// CHECK-NEXT: ret i32 2 +// CHECK-NEXT: ret i32 3 // // -// CHECK-LABEL: @_ZN7MyClass3gooEi.default( +// CHECK-LABEL: @_ZN7MyClass22unused_without_defaultEv( // CHECK-NEXT: entry: // CHECK-NEXT: [[THIS_ADDR:%.*]] = alloca ptr, align 8 -// CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 // CHECK-NEXT: store ptr [[THIS:%.*]], ptr [[THIS_ADDR]], align 8 -// CHECK-NEXT: store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4 // CHECK-NEXT: [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8 -// CHECK-NEXT: ret i32 1 +// CHECK-NEXT: ret i32 0 // // -// CHECK-LABEL: @_Z3fooi.default( -// CHECK-NEXT: entry: -// CHECK-NEXT: [[DOTADDR:%.*]] = alloca i32, align 4 -// CHECK-NEXT: store i32 [[TMP0:%.*]], ptr [[DOTADDR]], align 4 -// CHECK-NEXT: ret i32 2 +// CHECK-LABEL: @_ZN7MyClass23unused_with_default_defEv.resolver( +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 1073741824 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 1073741824 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @_ZN7MyClass23unused_with_default_defEv._Msve +// CHECK: resolver_else: +// CHECK-NEXT: ret ptr @_ZN7MyClass23unused_with_default_defEv.default // // -// CHECK-LABEL: @_Z3foov.default( -// CHECK-NEXT: entry: -// CHECK-NEXT: ret i32 4 +// CHECK-LABEL: @_ZN7MyClass32unused_with_implicit_default_defEv.resolver( +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 65536 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 65536 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @_ZN7MyClass32unused_with_implicit_default_defEv._Mfp16 +// CHECK: resolver_else: +// CHECK-NEXT: ret ptr @_ZN7MyClass32unused_with_implicit_default_defEv.default +// +// +// CHECK-LABEL: @_ZN7MyClass40unused_with_implicit_forward_default_defEv.resolver( +// CHECK-NEXT: resolver_entry: +// CHECK-NEXT: call void @__init_cpu_features_resolver() +// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8 +// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 128 +// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 128 +// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]] +// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]] +// CHECK: resolver_return: +// CHECK-NEXT: ret ptr @_ZN7MyClass40unused_with_implicit_forward_default_defEv._Mlse +// CHECK: resolver_else: +// CHECK-NEXT: ret ptr @_ZN7MyClass40unused_with_implicit_forward_default_defEv.default // //. -// CHECK: attributes #[[ATTR0:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" } -// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+neon,+sm4" } -// CHECK: attributes #[[ATTR2:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } -// CHECK: attributes #[[ATTR3:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon" } -// CHECK: attributes #[[ATTR4:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc" } +// CHECK: attributes #[[ATTR0:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +// CHECK: attributes #[[ATTR1:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc" } +// CHECK: attributes #[[ATTR2:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+dotprod,+fp-armv8,+neon" } +// CHECK: attributes #[[ATTR3:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+mops" } +// CHECK: attributes #[[ATTR4:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" } +// CHECK: attributes #[[ATTR5:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon,+sve" } +// CHECK: attributes #[[ATTR6:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+neon" } +// CHECK: attributes #[[ATTR7:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+lse" } +// CHECK: attributes #[[ATTR8:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+sme,+sme-f64f64" } +// CHECK: attributes #[[ATTR9:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+fp-armv8,+neon,+sm4" } +// CHECK: attributes #[[ATTR10:[0-9]+]] = { mustprogress noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon,+rdm" } +// CHECK: attributes #[[ATTR11:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } //. // CHECK: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4} // CHECK: [[META1:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"} -- GitLab From 0e47dfede468a292dd8cd893d6d0179052501383 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 20 Mar 2024 09:58:56 -0700 Subject: [PATCH 040/296] [ELF] Add isStaticRelSecType to simplify SHT_REL/SHT_RELA testing. NFC and make it easier to introduce a new relocation format. https://discourse.llvm.org/t/rfc-relleb-a-compact-relocation-format-for-elf/77600 Pull Request: https://github.com/llvm/llvm-project/pull/85893 --- lld/ELF/InputFiles.cpp | 4 ++-- lld/ELF/InputSection.cpp | 2 +- lld/ELF/InputSection.h | 4 ++++ lld/ELF/LinkerScript.cpp | 7 +++---- lld/ELF/MarkLive.cpp | 3 +-- lld/ELF/OutputSections.cpp | 4 ++-- lld/ELF/Writer.cpp | 8 ++++---- 7 files changed, 17 insertions(+), 15 deletions(-) diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp index 4a6e691938cf..4c614c865d24 100644 --- a/lld/ELF/InputFiles.cpp +++ b/lld/ELF/InputFiles.cpp @@ -835,7 +835,7 @@ void ObjFile::initializeSections(bool ignoreComdats, // We have a second loop. It is used to: // 1) handle SHF_LINK_ORDER sections. - // 2) create SHT_REL[A] sections. In some cases the section header index of a + // 2) create relocation sections. In some cases the section header index of a // relocation section may be smaller than that of the relocated section. In // such cases, the relocation section would attempt to reference a target // section that has not yet been created. For simplicity, delay creation of @@ -845,7 +845,7 @@ void ObjFile::initializeSections(bool ignoreComdats, continue; const Elf_Shdr &sec = objSections[i]; - if (sec.sh_type == SHT_REL || sec.sh_type == SHT_RELA) { + if (isStaticRelSecType(sec.sh_type)) { // Find a relocation target section and associate this section with that. // Target may have been discarded if it is in a different section group // and the group is discarded, even though it's a violation of the spec. diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp index 082e840adde4..c34bf08757b1 100644 --- a/lld/ELF/InputSection.cpp +++ b/lld/ELF/InputSection.cpp @@ -348,7 +348,7 @@ template void InputSection::copyShtGroup(uint8_t *buf) { } InputSectionBase *InputSection::getRelocatedSection() const { - if (file->isInternal() || (type != SHT_RELA && type != SHT_REL)) + if (file->isInternal() || !isStaticRelSecType(type)) return nullptr; ArrayRef sections = file->getSections(); return sections[info]; diff --git a/lld/ELF/InputSection.h b/lld/ELF/InputSection.h index b8af962877b4..1fb7077ca435 100644 --- a/lld/ELF/InputSection.h +++ b/lld/ELF/InputSection.h @@ -448,6 +448,10 @@ public: } }; +inline bool isStaticRelSecType(uint32_t type) { + return type == llvm::ELF::SHT_RELA || type == llvm::ELF::SHT_REL; +} + inline bool isDebugSection(const InputSectionBase &sec) { return (sec.flags & llvm::ELF::SHF_ALLOC) == 0 && sec.name.starts_with(".debug"); diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp index 9e7647f63ca5..3af09a32b651 100644 --- a/lld/ELF/LinkerScript.cpp +++ b/lld/ELF/LinkerScript.cpp @@ -740,13 +740,12 @@ static OutputDesc *addInputSec(StringMap> &map, // should combine these relocation sections into single output. // We skip synthetic sections because it can be .rela.dyn/.rela.plt or any // other REL[A] sections created by linker itself. - if (!isa(isec) && - (isec->type == SHT_REL || isec->type == SHT_RELA)) { + if (!isa(isec) && isStaticRelSecType(isec->type)) { auto *sec = cast(isec); OutputSection *out = sec->getRelocatedSection()->getOutputSection(); - if (out->relocationSection) { - out->relocationSection->recordSection(sec); + if (auto *relSec = out->relocationSection) { + relSec->recordSection(sec); return nullptr; } diff --git a/lld/ELF/MarkLive.cpp b/lld/ELF/MarkLive.cpp index 93c66e81d2fa..45431e44a6c8 100644 --- a/lld/ELF/MarkLive.cpp +++ b/lld/ELF/MarkLive.cpp @@ -276,8 +276,7 @@ template void MarkLive::run() { // collection. // - Groups members are retained or discarded as a unit. if (!(sec->flags & SHF_ALLOC)) { - bool isRel = sec->type == SHT_REL || sec->type == SHT_RELA; - if (!isRel && !sec->nextInSectionGroup) { + if (!isStaticRelSecType(sec->type) && !sec->nextInSectionGroup) { sec->markLive(); for (InputSection *isec : sec->dependentSections) isec->markLive(); diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp index f986aa5f6757..eadab9d745d6 100644 --- a/lld/ELF/OutputSections.cpp +++ b/lld/ELF/OutputSections.cpp @@ -615,7 +615,7 @@ void OutputSection::finalize() { return; } - if (!config->copyRelocs || (type != SHT_RELA && type != SHT_REL)) + if (!config->copyRelocs || !isStaticRelSecType(type)) return; // Skip if 'first' is synthetic, i.e. not a section created by --emit-relocs. @@ -750,7 +750,7 @@ std::array OutputSection::getFiller() { void OutputSection::checkDynRelAddends(const uint8_t *bufStart) { assert(config->writeAddends && config->checkDynamicRelocs); - assert(type == SHT_REL || type == SHT_RELA); + assert(isStaticRelSecType(type)); SmallVector storage; ArrayRef sections = getInputSections(*this, storage); parallelFor(0, sections.size(), [&](size_t i) { diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp index d8782affe879..4eca7b22e90b 100644 --- a/lld/ELF/Writer.cpp +++ b/lld/ELF/Writer.cpp @@ -796,7 +796,7 @@ template void Writer::addSectionSymbols() { continue; for (InputSectionBase *s : isd->sections) { // Relocations are not using REL[A] section symbols. - if (s->type == SHT_REL || s->type == SHT_RELA) + if (isStaticRelSecType(s->type)) continue; // Unlike other synthetic sections, mergeable output sections contain @@ -3045,20 +3045,20 @@ template void Writer::writeSections() { // section while doing it. parallel::TaskGroup tg; for (OutputSection *sec : outputSections) - if (sec->type == SHT_REL || sec->type == SHT_RELA) + if (isStaticRelSecType(sec->type)) sec->writeTo(Out::bufferStart + sec->offset, tg); } { parallel::TaskGroup tg; for (OutputSection *sec : outputSections) - if (sec->type != SHT_REL && sec->type != SHT_RELA) + if (!isStaticRelSecType(sec->type)) sec->writeTo(Out::bufferStart + sec->offset, tg); } // Finally, check that all dynamic relocation addends were written correctly. if (config->checkDynamicRelocs && config->writeAddends) { for (OutputSection *sec : outputSections) - if (sec->type == SHT_REL || sec->type == SHT_RELA) + if (isStaticRelSecType(sec->type)) sec->checkDynRelAddends(Out::bufferStart); } } -- GitLab From 25d61be8a5e563988661709c5d01f67c06b388e2 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Wed, 20 Mar 2024 16:59:01 +0000 Subject: [PATCH 041/296] [X86] avx-shuffle-builtins.c - limit to x86 targets Attempt to fix issue with non-x86 buildbots (sorry its blind but I can't test this) --- clang/test/CodeGen/X86/avx-shuffle-builtins.c | 1 + 1 file changed, 1 insertion(+) diff --git a/clang/test/CodeGen/X86/avx-shuffle-builtins.c b/clang/test/CodeGen/X86/avx-shuffle-builtins.c index 82be43bc0504..49a56e73230d 100644 --- a/clang/test/CodeGen/X86/avx-shuffle-builtins.c +++ b/clang/test/CodeGen/X86/avx-shuffle-builtins.c @@ -1,3 +1,4 @@ +// REQUIRES: x86-registered-target // RUN: %clang_cc1 -ffreestanding %s -O3 -triple=x86_64-apple-darwin -target-feature +avx -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,X64 // RUN: %clang_cc1 -ffreestanding %s -O3 -triple=i386-apple-darwin -target-feature +avx -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,X86 // FIXME: This is testing optimized generation of shuffle instructions and should be fixed. -- GitLab From decd88ef0538504707c5d1f0fd8b9de60a5b9b4c Mon Sep 17 00:00:00 2001 From: Stephen Tozer Date: Wed, 20 Mar 2024 16:01:54 +0000 Subject: [PATCH 042/296] [RemoveDIs][NFC] Delete a now-redundant comment Submitted without review for being a trivial comment-only change, deletes a line that requests the DbgLabelRecord class be renamed to DbgLabelRecord in the future. --- llvm/include/llvm/IR/DebugProgramInstruction.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/llvm/include/llvm/IR/DebugProgramInstruction.h b/llvm/include/llvm/IR/DebugProgramInstruction.h index 7214d7ad65da..c9477131c09c 100644 --- a/llvm/include/llvm/IR/DebugProgramInstruction.h +++ b/llvm/include/llvm/IR/DebugProgramInstruction.h @@ -220,8 +220,6 @@ inline raw_ostream &operator<<(raw_ostream &OS, const DbgRecord &R) { /// Records a position in IR for a source label (DILabel). Corresponds to the /// llvm.dbg.label intrinsic. -/// FIXME: Rename DbgLabelRecord when DbgVariableRecord is renamed to -/// DbgVariableRecord. class DbgLabelRecord : public DbgRecord { DbgRecordParamRef Label; -- GitLab From b754e6f6900e8c4205567fb2a13ff3c90811f5bc Mon Sep 17 00:00:00 2001 From: Job Henandez Lara Date: Wed, 20 Mar 2024 10:13:45 -0700 Subject: [PATCH 043/296] Fix typo (#85869) --- libc/test/src/math/smoke/FMaxTest.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/test/src/math/smoke/FMaxTest.h b/libc/test/src/math/smoke/FMaxTest.h index 98fae06ee2a0..b8781a85d10f 100644 --- a/libc/test/src/math/smoke/FMaxTest.h +++ b/libc/test/src/math/smoke/FMaxTest.h @@ -1,4 +1,4 @@ -//===-- Utility class to test fmin[f|l] -------------------------*- C++ -*-===// +//===-- Utility class to test fmax[f|l] -------------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. -- GitLab From b20360abeb3a80281dc082f1e093abd13cb1ee4c Mon Sep 17 00:00:00 2001 From: AdityaK Date: Wed, 20 Mar 2024 10:15:23 -0700 Subject: [PATCH 044/296] clang driver: enable fast unaligned access for Android on RISCV64 (#85704) Android CTS test already requires fast unaligned access https://android-review.googlesource.com/c/platform/cts/+/2675633 --- clang/lib/Driver/ToolChains/Arch/RISCV.cpp | 4 ++++ clang/test/Driver/riscv-features.c | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp index 5165bccc6d7e..b1dd7c4372d4 100644 --- a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp +++ b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp @@ -167,6 +167,10 @@ void riscv::getRISCVTargetFeatures(const Driver &D, const llvm::Triple &Triple, Features.push_back("-relax"); } + // Android requires fast unaligned access on RISCV64. + if (Triple.isAndroid()) + Features.push_back("+fast-unaligned-access"); + // -mstrict-align is default, unless -mno-strict-align is specified. AddTargetFeature(Args, Features, options::OPT_mno_strict_align, options::OPT_mstrict_align, "fast-unaligned-access"); diff --git a/clang/test/Driver/riscv-features.c b/clang/test/Driver/riscv-features.c index fe74ac773ef8..052956dfa2dc 100644 --- a/clang/test/Driver/riscv-features.c +++ b/clang/test/Driver/riscv-features.c @@ -1,7 +1,9 @@ // RUN: %clang --target=riscv32-unknown-elf -### %s -fsyntax-only 2>&1 | FileCheck %s // RUN: %clang --target=riscv64-unknown-elf -### %s -fsyntax-only 2>&1 | FileCheck %s -// RUN: %clang --target=riscv64-linux-android -### %s -fsyntax-only 2>&1 | FileCheck %s -check-prefixes=ANDROID,DEFAULT -// RUN: %clang -mabi=lp64d --target=riscv64-linux-android -### %s -fsyntax-only 2>&1 | FileCheck %s -check-prefixes=ANDROID,DEFAULT +// RUN: %clang --target=riscv64-linux-android -### %s -fsyntax-only 2>&1 | FileCheck %s -check-prefixes=ANDROID,DEFAULT,FAST-UNALIGNED-ACCESS +// RUN: %clang -mabi=lp64d --target=riscv64-linux-android -### %s -fsyntax-only 2>&1 | FileCheck %s -check-prefixes=ANDROID,DEFAULT,FAST-UNALIGNED-ACCESS +// RUN: %clang -mabi=lp64d --target=riscv64-linux-android -mstrict-align -### %s -fsyntax-only 2>&1 | FileCheck %s -check-prefixes=NO-FAST-UNALIGNED-ACCESS + // CHECK: fno-signed-char -- GitLab From 721e8f366f8665468f69dd8e29f97c6da1614a8f Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 20 Mar 2024 10:26:15 -0700 Subject: [PATCH 045/296] [ELF] Improve unknown -z test --- lld/test/ELF/driver.test | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lld/test/ELF/driver.test b/lld/test/ELF/driver.test index 260be8e8f883..b8f4584a3267 100644 --- a/lld/test/ELF/driver.test +++ b/lld/test/ELF/driver.test @@ -53,12 +53,13 @@ # RUN: not ld.lld %t -output=/no/such/file 2>&1 | FileCheck -check-prefix=ERR9 %s # ERR9: cannot open output file utput=/no/such/file -# RUN: ld.lld %t -z foo -o /dev/null 2>&1 | FileCheck -check-prefix=ERR10 %s -# RUN: ld.lld %t -z foo -o /dev/null --version 2>&1 | FileCheck -check-prefix=ERR10 %s +# RUN: ld.lld %t -z foo -o /dev/null 2>&1 | FileCheck -check-prefix=ERR10 %s --implicit-check-not=warning: +# RUN: ld.lld %t -z foo -z rel -z rela -z max-page-size=1 -z common-page-size=1 -o /dev/null --version 2>&1 | \ +# RUN: FileCheck -check-prefix=ERR10 %s --implicit-check-not=warning: # ERR10: warning: unknown -z value: foo ## Check we report "unknown -z value" error even with -v. -# RUN: ld.lld %t -z foo -o /dev/null -v 2>&1 | FileCheck -check-prefix=ERR10 %s +# RUN: ld.lld %t -z foo -z rel -o /dev/null -v 2>&1 | FileCheck -check-prefix=ERR10 %s --implicit-check-not=warning: ## Note: in GNU ld, --fatal-warning still leads to a warning. # RUN: not ld.lld %t -z foo --fatal-warnings 2>&1 | FileCheck --check-prefix=ERR10-FATAL %s -- GitLab From 49c3e78961f841717c0aeec0c04b6e5401ddd6ce Mon Sep 17 00:00:00 2001 From: Augie Fackler Date: Wed, 20 Mar 2024 13:33:35 -0400 Subject: [PATCH 046/296] [bazel] add missing dependency for 2137894a6f5475e51c541b6d16e8902125a8f002 --- utils/bazel/llvm-project-overlay/libc/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index c4f6eab06221..12fbe0dd6025 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -513,6 +513,7 @@ libc_support_library( name = "__support_uint128", hdrs = ["src/__support/UInt128.h"], deps = [ + ":__support_macros_attributes", ":__support_macros_properties_types", ":__support_uint", ], -- GitLab From 66a2ed50ccb6de64fdf82957ca0d4b55ef76f3cd Mon Sep 17 00:00:00 2001 From: Alex Langford Date: Wed, 20 Mar 2024 10:43:40 -0700 Subject: [PATCH 047/296] [lldb] Remove process restart prompt from TestSourceManager (#85861) In TestSourceManager, test_artificial_source_location will give the process restart prompt if you run the test individually. The reason is that we run the process twice: first using a convenience function to run to a specific breakpoint and then again to check for a specific message emitted when you hit the breakpoint. Instead of running twice and making the test difficult to run individually, we can just check for the specific messages using other commands. --- lldb/test/API/source-manager/TestSourceManager.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lldb/test/API/source-manager/TestSourceManager.py b/lldb/test/API/source-manager/TestSourceManager.py index eab8924d1081..ad7c85aac70e 100644 --- a/lldb/test/API/source-manager/TestSourceManager.py +++ b/lldb/test/API/source-manager/TestSourceManager.py @@ -323,13 +323,12 @@ class SourceManagerTestCase(TestBase): ) self.expect( - "run", - RUN_SUCCEEDED, + "process status", substrs=[ "stop reason = breakpoint", - "%s:%d" % (src_file, 0), - "Note: this address is compiler-generated code in " "function", - "that has no source code associated " "with it.", + f"{src_file}:0", + "Note: this address is compiler-generated code in function", + "that has no source code associated with it.", ], ) -- GitLab From d42992e71c660d57c89056f6ee4a5be74fa4d1f4 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 20 Mar 2024 09:33:37 -0700 Subject: [PATCH 048/296] [RISCV] Cleanup setOperationAction for ISD::BITCAST with Zfa and D extension. NFC We only need Custom handling for i64 on RV32. This will be used by type legalization. We don't need to make it custom for f64 to get type legalization to custom split i64. If f64 and i64 are legal types, then ISD::BITCAST should be legal. --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 25f035e6dd9d..3aa28215efc2 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -562,8 +562,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, if (Subtarget.hasStdExtZfa()) { setOperationAction(FPRndMode, MVT::f64, Legal); setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal); - setOperationAction(ISD::BITCAST, MVT::i64, Custom); - setOperationAction(ISD::BITCAST, MVT::f64, Custom); + if (!Subtarget.is64Bit()) + setOperationAction(ISD::BITCAST, MVT::i64, Custom); } else { if (Subtarget.is64Bit()) setOperationAction(FPRndMode, MVT::f64, Custom); -- GitLab From 10b0e355372fab1f4d585555536525545eef8523 Mon Sep 17 00:00:00 2001 From: Alex Langford Date: Wed, 20 Mar 2024 10:46:06 -0700 Subject: [PATCH 049/296] [lldb] Invert relationship between Process and AddressableBits (#85858) AddressableBits is in the Utility module of LLDB. It currently directly refers to Process, which is from the Target LLDB module. This is a layering violation which concretely means that it is impossible to link anything that uses Utility without it also using Target as well. This is generally not an issue for LLDB (since everything is built together) but it may make it difficult to write unit tests for AddressableBits later on. --- lldb/include/lldb/Target/Process.h | 3 ++ lldb/include/lldb/Utility/AddressableBits.h | 6 ++-- .../Process/gdb-remote/ProcessGDBRemote.cpp | 4 +-- .../Process/mach-core/ProcessMachCore.cpp | 2 +- lldb/source/Target/Process.cpp | 23 +++++++++++++++ lldb/source/Utility/AddressableBits.cpp | 28 +++++++------------ 6 files changed, 43 insertions(+), 23 deletions(-) diff --git a/lldb/include/lldb/Target/Process.h b/lldb/include/lldb/Target/Process.h index e260e1b4b797..2f3a3c22422e 100644 --- a/lldb/include/lldb/Target/Process.h +++ b/lldb/include/lldb/Target/Process.h @@ -43,6 +43,7 @@ #include "lldb/Target/ThreadList.h" #include "lldb/Target/ThreadPlanStack.h" #include "lldb/Target/Trace.h" +#include "lldb/Utility/AddressableBits.h" #include "lldb/Utility/ArchSpec.h" #include "lldb/Utility/Broadcaster.h" #include "lldb/Utility/Event.h" @@ -3219,6 +3220,8 @@ protected: void LoadOperatingSystemPlugin(bool flush); + void SetAddressableBitMasks(AddressableBits bit_masks); + private: Status DestroyImpl(bool force_kill); diff --git a/lldb/include/lldb/Utility/AddressableBits.h b/lldb/include/lldb/Utility/AddressableBits.h index 75752fcf840a..0d27c3561ec2 100644 --- a/lldb/include/lldb/Utility/AddressableBits.h +++ b/lldb/include/lldb/Utility/AddressableBits.h @@ -32,11 +32,13 @@ public: void SetLowmemAddressableBits(uint32_t lowmem_addressing_bits); + uint32_t GetLowmemAddressableBits() const; + void SetHighmemAddressableBits(uint32_t highmem_addressing_bits); - static lldb::addr_t AddressableBitToMask(uint32_t addressable_bits); + uint32_t GetHighmemAddressableBits() const; - void SetProcessMasks(lldb_private::Process &process); + static lldb::addr_t AddressableBitToMask(uint32_t addressable_bits); private: uint32_t m_low_memory_addr_bits; diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp index 5b9a9d71802f..871683a60568 100644 --- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp +++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp @@ -900,7 +900,7 @@ void ProcessGDBRemote::DidLaunchOrAttach(ArchSpec &process_arch) { } AddressableBits addressable_bits = m_gdb_comm.GetAddressableBits(); - addressable_bits.SetProcessMasks(*this); + SetAddressableBitMasks(addressable_bits); if (process_arch.IsValid()) { const ArchSpec &target_arch = GetTarget().GetArchitecture(); @@ -2337,7 +2337,7 @@ StateType ProcessGDBRemote::SetThreadStopInfo(StringExtractor &stop_packet) { } } - addressable_bits.SetProcessMasks(*this); + SetAddressableBitMasks(addressable_bits); ThreadSP thread_sp = SetThreadStopInfo( tid, expedited_register_map, signo, thread_name, reason, description, diff --git a/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp b/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp index 7b9938d4f020..1da7696c9a35 100644 --- a/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp +++ b/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp @@ -574,7 +574,7 @@ Status ProcessMachCore::DoLoadCore() { CleanupMemoryRegionPermissions(); AddressableBits addressable_bits = core_objfile->GetAddressableBits(); - addressable_bits.SetProcessMasks(*this); + SetAddressableBitMasks(addressable_bits); return error; } diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp index 6d58873b54a3..f02ec37cb0f0 100644 --- a/lldb/source/Target/Process.cpp +++ b/lldb/source/Target/Process.cpp @@ -63,6 +63,7 @@ #include "lldb/Target/ThreadPlanCallFunction.h" #include "lldb/Target/ThreadPlanStack.h" #include "lldb/Target/UnixSignals.h" +#include "lldb/Utility/AddressableBits.h" #include "lldb/Utility/Event.h" #include "lldb/Utility/LLDBLog.h" #include "lldb/Utility/Log.h" @@ -6453,3 +6454,25 @@ Status Process::CalculateCoreFileSaveRanges(lldb::SaveCoreStyle core_style, return Status(); // Success! } + +void Process::SetAddressableBitMasks(AddressableBits bit_masks) { + uint32_t low_memory_addr_bits = bit_masks.GetLowmemAddressableBits(); + uint32_t high_memory_addr_bits = bit_masks.GetHighmemAddressableBits(); + + if (low_memory_addr_bits == 0 && high_memory_addr_bits == 0) + return; + + if (low_memory_addr_bits != 0) { + addr_t low_addr_mask = + AddressableBits::AddressableBitToMask(low_memory_addr_bits); + SetCodeAddressMask(low_addr_mask); + SetDataAddressMask(low_addr_mask); + } + + if (high_memory_addr_bits != 0) { + addr_t high_addr_mask = + AddressableBits::AddressableBitToMask(high_memory_addr_bits); + SetHighmemCodeAddressMask(high_addr_mask); + SetHighmemDataAddressMask(high_addr_mask); + } +} diff --git a/lldb/source/Utility/AddressableBits.cpp b/lldb/source/Utility/AddressableBits.cpp index 7f9d7ec6c134..4c98addc1f07 100644 --- a/lldb/source/Utility/AddressableBits.cpp +++ b/lldb/source/Utility/AddressableBits.cpp @@ -7,9 +7,10 @@ //===----------------------------------------------------------------------===// #include "lldb/Utility/AddressableBits.h" -#include "lldb/Target/Process.h" #include "lldb/lldb-types.h" +#include + using namespace lldb; using namespace lldb_private; @@ -28,11 +29,19 @@ void AddressableBits::SetLowmemAddressableBits( m_low_memory_addr_bits = lowmem_addressing_bits; } +uint32_t AddressableBits::GetLowmemAddressableBits() const { + return m_low_memory_addr_bits; +} + void AddressableBits::SetHighmemAddressableBits( uint32_t highmem_addressing_bits) { m_high_memory_addr_bits = highmem_addressing_bits; } +uint32_t AddressableBits::GetHighmemAddressableBits() const { + return m_high_memory_addr_bits; +} + addr_t AddressableBits::AddressableBitToMask(uint32_t addressable_bits) { assert(addressable_bits <= sizeof(addr_t) * 8); if (addressable_bits == 64) @@ -40,20 +49,3 @@ addr_t AddressableBits::AddressableBitToMask(uint32_t addressable_bits) { else return ~((1ULL << addressable_bits) - 1); } - -void AddressableBits::SetProcessMasks(Process &process) { - if (m_low_memory_addr_bits == 0 && m_high_memory_addr_bits == 0) - return; - - if (m_low_memory_addr_bits != 0) { - addr_t low_addr_mask = AddressableBitToMask(m_low_memory_addr_bits); - process.SetCodeAddressMask(low_addr_mask); - process.SetDataAddressMask(low_addr_mask); - } - - if (m_high_memory_addr_bits != 0) { - addr_t hi_addr_mask = AddressableBitToMask(m_high_memory_addr_bits); - process.SetHighmemCodeAddressMask(hi_addr_mask); - process.SetHighmemDataAddressMask(hi_addr_mask); - } -} -- GitLab From 5ea152033ee99ac7ccde791009c372a0983b4eaf Mon Sep 17 00:00:00 2001 From: Vinayak Dev <104419489+vinayakdsci@users.noreply.github.com> Date: Wed, 20 Mar 2024 23:17:35 +0530 Subject: [PATCH 050/296] [libc]: Implement strfromf() and shared utilities (#85438) Fixes #84244. Implements the function `strfromf()` introduced in C23, and adds shared utilities for implementation of other `strfrom*()` functions, including `strfromd()` and `strfroml()`. --- libc/config/linux/x86_64/entrypoints.txt | 1 + libc/spec/stdc.td | 2 + libc/src/stdlib/CMakeLists.txt | 22 ++++ libc/src/stdlib/str_from_util.h | 138 +++++++++++++++++++++++ libc/src/stdlib/strfromf.cpp | 42 +++++++ libc/src/stdlib/strfromf.h | 21 ++++ libc/test/src/stdlib/CMakeLists.txt | 11 +- libc/test/src/stdlib/strfromf_test.cpp | 107 ++++++++++++++++++ 8 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 libc/src/stdlib/str_from_util.h create mode 100644 libc/src/stdlib/strfromf.cpp create mode 100644 libc/src/stdlib/strfromf.h create mode 100644 libc/test/src/stdlib/strfromf_test.cpp diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index e8cf11266624..c216f4349627 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -180,6 +180,7 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.qsort_r libc.src.stdlib.rand libc.src.stdlib.srand + libc.src.stdlib.strfromf libc.src.stdlib.strtod libc.src.stdlib.strtof libc.src.stdlib.strtol diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index 84d28cc33503..920036adfed5 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -956,6 +956,8 @@ def StdC : StandardSpec<"stdc"> { FunctionSpec<"rand", RetValSpec, [ArgSpec]>, FunctionSpec<"srand", RetValSpec, [ArgSpec]>, + FunctionSpec<"strfromf", RetValSpec, [ArgSpec, ArgSpec, ArgSpec, ArgSpec]>, + FunctionSpec<"strtof", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"strtod", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"strtold", RetValSpec, [ArgSpec, ArgSpec]>, diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt index bd0bcffe0045..22f7f990fb08 100644 --- a/libc/src/stdlib/CMakeLists.txt +++ b/libc/src/stdlib/CMakeLists.txt @@ -52,6 +52,28 @@ add_entrypoint_object( libc.config.linux.app_h ) +add_entrypoint_object( + strfromf + SRCS + strfromf.cpp + HDRS + strfromf.h + DEPENDS + .str_from_util +) + +add_header_library( + str_from_util + HDRS + str_from_util.h + DEPENDS + libc.src.stdio.printf_core.converter + libc.src.stdio.printf_core.core_structs + libc.src.stdio.printf_core.writer + libc.src.__support.str_to_integer + libc.src.__support.CPP.type_traits +) + add_entrypoint_object( strtof SRCS diff --git a/libc/src/stdlib/str_from_util.h b/libc/src/stdlib/str_from_util.h new file mode 100644 index 000000000000..c4c1c0a0ba4e --- /dev/null +++ b/libc/src/stdlib/str_from_util.h @@ -0,0 +1,138 @@ +//===-- Implementation header for strfromx() utilitites -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// According to the C23 standard, any input character sequences except a +// precision specifier and the usual floating point formats, namely +// %{a,A,e,E,f,F,g,G}, are not allowed and any code that does otherwise results +// in undefined behaviour(including use of a '%%' conversion specifier); which +// in this case is that the buffer string is simply populated with the format +// string. The case of the input being NULL should be handled in the calling +// function (strfromf, strfromd, strfroml) itself. + +#ifndef LLVM_LIBC_SRC_STDLIB_STRFROM_UTIL_H +#define LLVM_LIBC_SRC_STDLIB_STRFROM_UTIL_H + +#include "src/__support/CPP/type_traits.h" +#include "src/__support/str_to_integer.h" +#include "src/stdio/printf_core/converter_atlas.h" +#include "src/stdio/printf_core/core_structs.h" +#include "src/stdio/printf_core/writer.h" + +#include + +namespace LIBC_NAMESPACE::internal { + +template +using storage_type = typename fputil::FPBits::StorageType; + +template +printf_core::FormatSection parse_format_string(const char *__restrict format, + T fp) { + printf_core::FormatSection section; + size_t cur_pos = 0; + + // There is no typed conversion function to convert single precision float + // to hex exponential format, and the function convert_float_hex_exp() + // requires a double or long double value to work correctly. + // To work around this, we convert fp to double if it is single precision, and + // then use that double precision value in the %{A, a} conversion specifiers. + [[maybe_unused]] double new_fp; + bool t_is_single_prec_type = cpp::is_same::value; + if (t_is_single_prec_type) + new_fp = (double)fp; + + if (format[cur_pos] == '%') { + section.has_conv = true; + ++cur_pos; + + // handle precision + section.precision = -1; + if (format[cur_pos] == '.') { + ++cur_pos; + section.precision = 0; + + // The standard does not allow the '*' (asterisk) operator for strfromx() + // functions + if (internal::isdigit(format[cur_pos])) { + auto result = internal::strtointeger(format + cur_pos, 10); + section.precision += result.value; + cur_pos += result.parsed_len; + } + } + + section.conv_name = format[cur_pos]; + switch (format[cur_pos]) { + case 'a': + case 'A': + if (t_is_single_prec_type) + section.conv_val_raw = cpp::bit_cast>(new_fp); + else + section.conv_val_raw = cpp::bit_cast>(fp); + break; + case 'e': + case 'E': + case 'f': + case 'F': + case 'g': + case 'G': + section.conv_val_raw = cpp::bit_cast>(fp); + break; + default: + section.has_conv = false; + while (format[cur_pos] != '\0') + ++cur_pos; + break; + } + + if (format[cur_pos] != '\0') + ++cur_pos; + } else { + section.has_conv = false; + // We are looking for exactly one section, so no more '%' + while (format[cur_pos] != '\0') + ++cur_pos; + } + + section.raw_string = {format, cur_pos}; + return section; +} + +template +int strfromfloat_convert(printf_core::Writer *writer, + const printf_core::FormatSection §ion) { + if (!section.has_conv) + return writer->write(section.raw_string); + + auto res = static_cast>(section.conv_val_raw); + + fputil::FPBits strfromfloat_bits(res); + if (strfromfloat_bits.is_inf_or_nan()) + return convert_inf_nan(writer, section); + + switch (section.conv_name) { + case 'f': + case 'F': + return convert_float_decimal_typed(writer, section, strfromfloat_bits); + case 'e': + case 'E': + return convert_float_dec_exp_typed(writer, section, strfromfloat_bits); + case 'a': + case 'A': + return convert_float_hex_exp(writer, section); + case 'g': + case 'G': + return convert_float_dec_auto_typed(writer, section, strfromfloat_bits); + default: + return writer->write(section.raw_string); + } + return -1; +} + +} // namespace LIBC_NAMESPACE::internal + +#endif // LLVM_LIBC_SRC_STDLIB_STRFROM_UTIL_H diff --git a/libc/src/stdlib/strfromf.cpp b/libc/src/stdlib/strfromf.cpp new file mode 100644 index 000000000000..40eff87eb454 --- /dev/null +++ b/libc/src/stdlib/strfromf.cpp @@ -0,0 +1,42 @@ +//===-- Implementation of strfromf ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdlib/strfromf.h" +#include "src/stdlib/str_from_util.h" + +#include +#include + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, strfromf, + (char *__restrict s, size_t n, const char *__restrict format, + float fp)) { + LIBC_ASSERT(s != nullptr); + + printf_core::FormatSection section = + internal::parse_format_string(format, fp); + printf_core::WriteBuffer wb(s, (n > 0 ? n - 1 : 0)); + printf_core::Writer writer(&wb); + + int result = 0; + if (section.has_conv) + result = internal::strfromfloat_convert(&writer, section); + else + result = writer.write(section.raw_string); + + if (result < 0) + return result; + + if (n > 0) + wb.buff[wb.buff_cur] = '\0'; + + return writer.get_chars_written(); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdlib/strfromf.h b/libc/src/stdlib/strfromf.h new file mode 100644 index 000000000000..b551a58af05a --- /dev/null +++ b/libc/src/stdlib/strfromf.h @@ -0,0 +1,21 @@ +//===-- Implementation header for strfromf ------------------------*- C++--===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDLIB_STRFROMF_H +#define LLVM_LIBC_SRC_STDLIB_STRFROMF_H + +#include + +namespace LIBC_NAMESPACE { + +int strfromf(char *__restrict s, size_t n, const char *__restrict format, + float fp); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDLIB_STRTOF_H diff --git a/libc/test/src/stdlib/CMakeLists.txt b/libc/test/src/stdlib/CMakeLists.txt index 5488a61c4ef1..cb42bc56f51c 100644 --- a/libc/test/src/stdlib/CMakeLists.txt +++ b/libc/test/src/stdlib/CMakeLists.txt @@ -168,6 +168,16 @@ add_libc_test( .strtol_test_support ) +add_libc_test( + strfromf_test + SUITE + libc-stdlib-tests + SRCS + strfromf_test.cpp + DEPENDS + libc.src.stdlib.strfromf +) + add_libc_test( abs_test SUITE @@ -259,7 +269,6 @@ add_libc_test( libc.src.stdlib.qsort ) - add_libc_test( qsort_r_test SUITE diff --git a/libc/test/src/stdlib/strfromf_test.cpp b/libc/test/src/stdlib/strfromf_test.cpp new file mode 100644 index 000000000000..c5489f5f3af2 --- /dev/null +++ b/libc/test/src/stdlib/strfromf_test.cpp @@ -0,0 +1,107 @@ +//===-- Unittests for strfromf --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdlib/strfromf.h" +#include "test/UnitTest/Test.h" + +TEST(LlvmLibcStrfromfTest, DecimalFloatFormat) { + char buff[100]; + int written; + + written = LIBC_NAMESPACE::strfromf(buff, 16, "%f", 1.0); + EXPECT_EQ(written, 8); + ASSERT_STREQ(buff, "1.000000"); + + written = LIBC_NAMESPACE::strfromf(buff, 20, "%f", 1234567890.0); + EXPECT_EQ(written, 17); + ASSERT_STREQ(buff, "1234567936.000000"); + + written = LIBC_NAMESPACE::strfromf(buff, 5, "%f", 1234567890.0); + EXPECT_EQ(written, 17); + ASSERT_STREQ(buff, "1234"); + + written = LIBC_NAMESPACE::strfromf(buff, 67, "%.3f", 1.0); + EXPECT_EQ(written, 5); + ASSERT_STREQ(buff, "1.000"); + + written = LIBC_NAMESPACE::strfromf(buff, 20, "%1f", 1234567890.0); + EXPECT_EQ(written, 3); + ASSERT_STREQ(buff, "%1f"); +} + +TEST(LlvmLibcStrfromfTest, HexExpFloatFormat) { + char buff[100]; + int written; + + written = LIBC_NAMESPACE::strfromf(buff, 0, "%a", 1234567890.0); + EXPECT_EQ(written, 14); + + written = LIBC_NAMESPACE::strfromf(buff, 20, "%a", 1234567890.0); + EXPECT_EQ(written, 14); + ASSERT_STREQ(buff, "0x1.26580cp+30"); + + written = LIBC_NAMESPACE::strfromf(buff, 20, "%A", 1234567890.0); + EXPECT_EQ(written, 14); + ASSERT_STREQ(buff, "0X1.26580CP+30"); +} + +TEST(LlvmLibcStrfromfTest, DecimalExpFloatFormat) { + char buff[100]; + int written; + written = LIBC_NAMESPACE::strfromf(buff, 20, "%.9e", 1234567890.0); + EXPECT_EQ(written, 15); + ASSERT_STREQ(buff, "1.234567936e+09"); + + written = LIBC_NAMESPACE::strfromf(buff, 20, "%.9E", 1234567890.0); + EXPECT_EQ(written, 15); + ASSERT_STREQ(buff, "1.234567936E+09"); +} + +TEST(LlvmLibcStrfromfTest, AutoDecimalFloatFormat) { + char buff[100]; + int written; + + written = LIBC_NAMESPACE::strfromf(buff, 20, "%.9g", 1234567890.0); + EXPECT_EQ(written, 14); + ASSERT_STREQ(buff, "1.23456794e+09"); + + written = LIBC_NAMESPACE::strfromf(buff, 20, "%.9G", 1234567890.0); + EXPECT_EQ(written, 14); + ASSERT_STREQ(buff, "1.23456794E+09"); + + written = LIBC_NAMESPACE::strfromf(buff, 0, "%G", 1.0); + EXPECT_EQ(written, 1); +} + +TEST(LlvmLibcStrfromfTest, ImproperFormatString) { + + char buff[100]; + int retval; + retval = LIBC_NAMESPACE::strfromf( + buff, 37, "A simple string with no conversions.", 1.0); + EXPECT_EQ(retval, 36); + ASSERT_STREQ(buff, "A simple string with no conversions."); + + retval = LIBC_NAMESPACE::strfromf( + buff, 37, "%A simple string with one conversion, should overwrite.", 1.0); + EXPECT_EQ(retval, 6); + ASSERT_STREQ(buff, "0X1P+0"); + + retval = LIBC_NAMESPACE::strfromf(buff, 74, + "A simple string with one conversion in %A " + "between, writes string as it is", + 1.0); + EXPECT_EQ(retval, 73); + ASSERT_STREQ(buff, "A simple string with one conversion in %A between, " + "writes string as it is"); + + retval = LIBC_NAMESPACE::strfromf(buff, 36, + "A simple string with one conversion", 1.0); + EXPECT_EQ(retval, 35); + ASSERT_STREQ(buff, "A simple string with one conversion"); +} -- GitLab From 5e6bb1fb885abec2e8bc85422bbd83fe4ece6d3b Mon Sep 17 00:00:00 2001 From: Jason Eckhardt Date: Wed, 20 Mar 2024 12:56:49 -0500 Subject: [PATCH 051/296] [TableGen][Target] Add documentation to `Constraints`. (#85951) This patch adds some basic documentation for `Constraints`, along with some "see also" pointers for backend writers to learn more. --- llvm/include/llvm/Target/Target.td | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td index 1e40cc49040d..cb1c0ed2513d 100644 --- a/llvm/include/llvm/Target/Target.td +++ b/llvm/include/llvm/Target/Target.td @@ -697,7 +697,20 @@ class Instruction : InstructionEncoding { // Scheduling information from TargetSchedule.td. list SchedRW; - string Constraints = ""; // OperandConstraint, e.g. $src = $dst. + /// Support for operand constraints. There are currently two kinds: + /// "$src = $dst" + /// Ensures that the operands are allocated to the same register. + /// + /// "@earlyclobber $rd" + /// Ensures that LLVM will not use the same register for any inputs (other + /// than an input tied to this output). + /// + /// See also: + /// - MC/MCInstrDesc.h:OperandConstraint::{TIED_TO, EARLY_CLOBBER}. + /// - CodeGen/MachineOperand.h:MachineOperand::{TiedTo, IsEarlyClobber}. + /// - The LLVM IR specification: Section `Output constraints` in the + /// discussion of inline assembly constraint strings. + string Constraints = ""; /// DisableEncoding - List of operand names (e.g. "$op1,$op2") that should not /// be encoded into the output machineinstr. -- GitLab From 4df099e447840bae24e88efca1ab4c03a5d7b21b Mon Sep 17 00:00:00 2001 From: Augie Fackler Date: Wed, 20 Mar 2024 13:59:43 -0400 Subject: [PATCH 052/296] [bazel] another BUILD fix for 2137894a6f5475e51c541b6d16e8902125a8f002 --- utils/bazel/llvm-project-overlay/libc/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 12fbe0dd6025..40cfb1f470db 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -505,7 +505,7 @@ libc_support_library( name = "__support_sign", hdrs = ["src/__support/sign.h"], deps = [ - ":__support_macros_properties_types", + ":__support_macros_attributes", ], ) -- GitLab From 62ed009ce218897d7707144de54fa1e2beec59dc Mon Sep 17 00:00:00 2001 From: Thurston Dang Date: Wed, 20 Mar 2024 11:04:52 -0700 Subject: [PATCH 053/296] [dfsan] Re-exec with no ASLR if memory layout is incompatible on Linux (#85674) DFSan's shadow mappings are incompatible with 32 bits of ASLR entropy ('sudo sysctl vm.mmap_rnd_bits=32; ninja check-dfsan') and it is difficult to fix this via increasing the size of the shadow mappings, due to the overhead of shadow memory. This patch works around the issue by detecting if the memory layout is incompatible, and if so, re-exec'ing without ASLR. DFSan and MSan share copy-pasted shadow memory code, hence this workaround is ported from MSan: - "[msan] Re-exec with no ASLR if memory layout is incompatible on Linux" (https://github.com/llvm/llvm-project/commit/58f7251820b14c93168726a24816d8a094599be5) - "[msan] Add 'MappingDesc::ALLOCATOR' type and check it is available" (https://github.com/llvm/llvm-project/commit/af2bf86a372cacf5f536bae06e2f2d3886eefb7b) (which in turn are inspired by TSan: "Re-exec TSan with no ASLR if memory layout is incompatible on Linux" (https://github.com/llvm/llvm-project/commit/0784b1eefa36d4acbb0dacd2d18796e26313b6c5 )) aeubanks had remarked in https://github.com/llvm/llvm-project/pull/85142#issuecomment-2004442883 that this issue occurs in Chromium: https://ci.chromium.org/ui/p/chromium/builders/try/linux_upload_clang/5066/overview --- compiler-rt/lib/dfsan/dfsan.cpp | 70 ++++++++++++++++++----- compiler-rt/lib/dfsan/dfsan_allocator.cpp | 3 + compiler-rt/lib/dfsan/dfsan_platform.h | 24 ++++++-- 3 files changed, 78 insertions(+), 19 deletions(-) diff --git a/compiler-rt/lib/dfsan/dfsan.cpp b/compiler-rt/lib/dfsan/dfsan.cpp index 5e85c8fda3e2..302e3c3032ac 100644 --- a/compiler-rt/lib/dfsan/dfsan.cpp +++ b/compiler-rt/lib/dfsan/dfsan.cpp @@ -33,6 +33,9 @@ #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_report_decorator.h" #include "sanitizer_common/sanitizer_stacktrace.h" +#if SANITIZER_LINUX +# include +#endif using namespace __dfsan; @@ -1127,11 +1130,12 @@ static void CheckMemoryLayoutSanity() { // TODO: CheckMemoryRangeAvailability is based on msan. // Consider refactoring these into a shared implementation. -static bool CheckMemoryRangeAvailability(uptr beg, uptr size) { +static bool CheckMemoryRangeAvailability(uptr beg, uptr size, bool verbose) { if (size > 0) { uptr end = beg + size - 1; if (!MemoryRangeIsAvailable(beg, end)) { - Printf("FATAL: Memory range %p - %p is not available.\n", beg, end); + if (verbose) + Printf("FATAL: Memory range %p - %p is not available.\n", beg, end); return false; } } @@ -1163,7 +1167,7 @@ static bool ProtectMemoryRange(uptr beg, uptr size, const char *name) { // TODO: InitShadow is based on msan. // Consider refactoring these into a shared implementation. -bool InitShadow(bool init_origins) { +bool InitShadow(bool init_origins, bool dry_run) { // Let user know mapping parameters first. VPrintf(1, "dfsan_init %p\n", (void *)&__dfsan::dfsan_init); for (unsigned i = 0; i < kMemoryLayoutSize; ++i) @@ -1173,8 +1177,9 @@ bool InitShadow(bool init_origins) { CheckMemoryLayoutSanity(); if (!MEM_IS_APP(&__dfsan::dfsan_init)) { - Printf("FATAL: Code %p is out of application range. Non-PIE build?\n", - (uptr)&__dfsan::dfsan_init); + if (!dry_run) + Printf("FATAL: Code %p is out of application range. Non-PIE build?\n", + (uptr)&__dfsan::dfsan_init); return false; } @@ -1195,20 +1200,26 @@ bool InitShadow(bool init_origins) { bool protect = type == MappingDesc::INVALID || (!init_origins && type == MappingDesc::ORIGIN); CHECK(!(map && protect)); - if (!map && !protect) - CHECK(type == MappingDesc::APP); + if (!map && !protect) { + CHECK(type == MappingDesc::APP || type == MappingDesc::ALLOCATOR); + + if (dry_run && type == MappingDesc::ALLOCATOR && + !CheckMemoryRangeAvailability(start, size, !dry_run)) + return false; + } if (map) { - if (!CheckMemoryRangeAvailability(start, size)) + if (dry_run && !CheckMemoryRangeAvailability(start, size, !dry_run)) return false; - if (!MmapFixedSuperNoReserve(start, size, kMemoryLayout[i].name)) + if (!dry_run && + !MmapFixedSuperNoReserve(start, size, kMemoryLayout[i].name)) return false; - if (common_flags()->use_madv_dontdump) + if (!dry_run && common_flags()->use_madv_dontdump) DontDumpShadowMemory(start, size); } if (protect) { - if (!CheckMemoryRangeAvailability(start, size)) + if (dry_run && !CheckMemoryRangeAvailability(start, size, !dry_run)) return false; - if (!ProtectMemoryRange(start, size, kMemoryLayout[i].name)) + if (!dry_run && !ProtectMemoryRange(start, size, kMemoryLayout[i].name)) return false; } } @@ -1216,6 +1227,35 @@ bool InitShadow(bool init_origins) { return true; } +bool InitShadowWithReExec(bool init_origins) { + // Start with dry run: check layout is ok, but don't print warnings because + // warning messages will cause tests to fail (even if we successfully re-exec + // after the warning). + bool success = InitShadow(init_origins, true); + if (!success) { +#if SANITIZER_LINUX + // Perhaps ASLR entropy is too high. If ASLR is enabled, re-exec without it. + int old_personality = personality(0xffffffff); + bool aslr_on = + (old_personality != -1) && ((old_personality & ADDR_NO_RANDOMIZE) == 0); + + if (aslr_on) { + VReport(1, + "WARNING: DataflowSanitizer: memory layout is incompatible, " + "possibly due to high-entropy ASLR.\n" + "Re-execing with fixed virtual address space.\n" + "N.B. reducing ASLR entropy is preferable.\n"); + CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1); + ReExec(); + } +#endif + } + + // The earlier dry run didn't actually map or protect anything. Run again in + // non-dry run mode. + return success && InitShadow(init_origins, false); +} + static void DFsanInit(int argc, char **argv, char **envp) { CHECK(!dfsan_init_is_running); if (dfsan_inited) @@ -1229,7 +1269,11 @@ static void DFsanInit(int argc, char **argv, char **envp) { CheckASLR(); - InitShadow(dfsan_get_track_origins()); + if (!InitShadowWithReExec(dfsan_get_track_origins())) { + Printf("FATAL: DataflowSanitizer can not mmap the shadow memory.\n"); + DumpProcessMap(); + Die(); + } initialize_interceptors(); diff --git a/compiler-rt/lib/dfsan/dfsan_allocator.cpp b/compiler-rt/lib/dfsan/dfsan_allocator.cpp index df8be2cf5ae0..63475f434cd1 100644 --- a/compiler-rt/lib/dfsan/dfsan_allocator.cpp +++ b/compiler-rt/lib/dfsan/dfsan_allocator.cpp @@ -37,6 +37,9 @@ struct DFsanMapUnmapCallback { void OnUnmap(uptr p, uptr size) const { dfsan_set_label(0, (void *)p, size); } }; +// Note: to ensure that the allocator is compatible with the application memory +// layout (especially with high-entropy ASLR), kSpaceBeg and kSpaceSize must be +// duplicated as MappingDesc::ALLOCATOR in dfsan_platform.h. #if defined(__aarch64__) const uptr kAllocatorSpace = 0xE00000000000ULL; #else diff --git a/compiler-rt/lib/dfsan/dfsan_platform.h b/compiler-rt/lib/dfsan/dfsan_platform.h index b849b4b528ad..01f0de47d960 100644 --- a/compiler-rt/lib/dfsan/dfsan_platform.h +++ b/compiler-rt/lib/dfsan/dfsan_platform.h @@ -27,10 +27,19 @@ using __sanitizer::uptr; struct MappingDesc { uptr start; uptr end; - enum Type { INVALID, APP, SHADOW, ORIGIN } type; + enum Type { + INVALID = 1, + ALLOCATOR = 2, + APP = 4, + SHADOW = 8, + ORIGIN = 16, + } type; const char *name; }; +// Note: MappingDesc::ALLOCATOR entries are only used to check for memory +// layout compatibility. The actual allocation settings are in +// dfsan_allocator.cpp, which need to be kept in sync. #if SANITIZER_LINUX && SANITIZER_WORDSIZE == 64 # if defined(__aarch64__) @@ -53,7 +62,8 @@ const MappingDesc kMemoryLayout[] = { {0X0B00000000000, 0X0C00000000000, MappingDesc::SHADOW, "shadow-10-13"}, {0X0C00000000000, 0X0D00000000000, MappingDesc::INVALID, "invalid"}, {0X0D00000000000, 0X0E00000000000, MappingDesc::ORIGIN, "origin-10-13"}, - {0X0E00000000000, 0X1000000000000, MappingDesc::APP, "app-15"}, + {0X0E00000000000, 0X0E40000000000, MappingDesc::ALLOCATOR, "allocator"}, + {0X0E40000000000, 0X1000000000000, MappingDesc::APP, "app-15"}, }; # define MEM_TO_SHADOW(mem) ((uptr)mem ^ 0xB00000000000ULL) # define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x200000000000ULL) @@ -76,7 +86,8 @@ const MappingDesc kMemoryLayout[] = { {0x510000000000ULL, 0x600000000000ULL, MappingDesc::APP, "app-2"}, {0x600000000000ULL, 0x610000000000ULL, MappingDesc::ORIGIN, "origin-1"}, {0x610000000000ULL, 0x700000000000ULL, MappingDesc::INVALID, "invalid"}, - {0x700000000000ULL, 0x800000000000ULL, MappingDesc::APP, "app-3"}}; + {0x700000000000ULL, 0x740000000000ULL, MappingDesc::ALLOCATOR, "allocator"}, + {0x740000000000ULL, 0x800000000000ULL, MappingDesc::APP, "app-3"}}; # define MEM_TO_SHADOW(mem) (((uptr)(mem)) ^ 0x500000000000ULL) # define SHADOW_TO_ORIGIN(mem) (((uptr)(mem)) + 0x100000000000ULL) # endif @@ -93,20 +104,21 @@ const uptr kMemoryLayoutSize = sizeof(kMemoryLayout) / sizeof(kMemoryLayout[0]); __attribute__((optimize("unroll-loops"))) #endif inline bool -addr_is_type(uptr addr, MappingDesc::Type mapping_type) { +addr_is_type(uptr addr, int mapping_types) { // It is critical for performance that this loop is unrolled (because then it is // simplified into just a few constant comparisons). #ifdef __clang__ # pragma unroll #endif for (unsigned i = 0; i < kMemoryLayoutSize; ++i) - if (kMemoryLayout[i].type == mapping_type && + if ((kMemoryLayout[i].type & mapping_types) && addr >= kMemoryLayout[i].start && addr < kMemoryLayout[i].end) return true; return false; } -#define MEM_IS_APP(mem) addr_is_type((uptr)(mem), MappingDesc::APP) +#define MEM_IS_APP(mem) \ + (addr_is_type((uptr)(mem), MappingDesc::APP | MappingDesc::ALLOCATOR)) #define MEM_IS_SHADOW(mem) addr_is_type((uptr)(mem), MappingDesc::SHADOW) #define MEM_IS_ORIGIN(mem) addr_is_type((uptr)(mem), MappingDesc::ORIGIN) -- GitLab From 5231005193afb8db01afe9a8a1aa308d25f60ba1 Mon Sep 17 00:00:00 2001 From: Sirraide Date: Wed, 20 Mar 2024 19:19:30 +0100 Subject: [PATCH 054/296] [Clang] Update missing varargs arg extension warnings (#84520) This updates a few warnings that were diagnosing no arguments for a `...` variadic macro parameter as a GNU extension when it actually is a C++20/C23 extension now. This fixes #84495. --- clang/docs/ReleaseNotes.rst | 3 +++ .../include/clang/Basic/DiagnosticLexKinds.td | 13 ++++++++++--- clang/lib/Lex/PPMacroExpansion.cpp | 19 ++++++++++++++----- clang/test/C/C2x/n2975.c | 4 ++-- clang/test/Lexer/gnu-flags.c | 2 -- clang/test/Preprocessor/empty_va_arg.cpp | 18 +++++++++++------- clang/test/Preprocessor/macro_fn.c | 4 ++-- 7 files changed, 42 insertions(+), 21 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index a10e942615ff..c0b0c8a8a3ea 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -263,6 +263,9 @@ Improvements to Clang's diagnostics operands, distinguishing it from potential typographical errors or unintended bitwise operations. Fixes #GH77601. +- Clang now correctly diagnoses no arguments to a variadic macro parameter as a C23/C++20 extension. + Fixes #GH84495. + Improvements to Clang's time-trace ---------------------------------- diff --git a/clang/include/clang/Basic/DiagnosticLexKinds.td b/clang/include/clang/Basic/DiagnosticLexKinds.td index d7c172e65463..ad6bacfb118d 100644 --- a/clang/include/clang/Basic/DiagnosticLexKinds.td +++ b/clang/include/clang/Basic/DiagnosticLexKinds.td @@ -465,9 +465,16 @@ def err_embedded_directive : Error< def ext_embedded_directive : Extension< "embedding a directive within macro arguments has undefined behavior">, InGroup>; -def ext_missing_varargs_arg : Extension< - "must specify at least one argument for '...' parameter of variadic macro">, - InGroup; +def ext_c_missing_varargs_arg : Extension< + "passing no argument for the '...' parameter of a variadic macro is " + "a C23 extension">, InGroup; +def ext_cxx_missing_varargs_arg : Extension< + "passing no argument for the '...' parameter of a variadic macro is " + "a C++20 extension">, InGroup; +def warn_c17_compat_missing_varargs_arg : Warning< + "passing no argument for the '...' parameter of a variadic macro is " + "incompatible with C standards before C23">, + InGroup, DefaultIgnore; def warn_cxx17_compat_missing_varargs_arg : Warning< "passing no argument for the '...' parameter of a variadic macro is " "incompatible with C++ standards before C++20">, diff --git a/clang/lib/Lex/PPMacroExpansion.cpp b/clang/lib/Lex/PPMacroExpansion.cpp index 3017461dc66e..516269c0c601 100644 --- a/clang/lib/Lex/PPMacroExpansion.cpp +++ b/clang/lib/Lex/PPMacroExpansion.cpp @@ -993,11 +993,20 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, // If the macro contains the comma pasting extension, the diagnostic // is suppressed; we know we'll get another diagnostic later. if (!MI->hasCommaPasting()) { - // C++20 allows this construct, but standards before C++20 and all C - // standards do not allow the construct (we allow it as an extension). - Diag(Tok, getLangOpts().CPlusPlus20 - ? diag::warn_cxx17_compat_missing_varargs_arg - : diag::ext_missing_varargs_arg); + // C++20 [cpp.replace]p15, C23 6.10.5p12 + // + // C++20 and C23 allow this construct, but standards before that + // do not (we allow it as an extension). + unsigned ID; + if (getLangOpts().CPlusPlus20) + ID = diag::warn_cxx17_compat_missing_varargs_arg; + else if (getLangOpts().CPlusPlus) + ID = diag::ext_cxx_missing_varargs_arg; + else if (getLangOpts().C23) + ID = diag::warn_c17_compat_missing_varargs_arg; + else + ID = diag::ext_c_missing_varargs_arg; + Diag(Tok, ID); Diag(MI->getDefinitionLoc(), diag::note_macro_here) << MacroName.getIdentifierInfo(); } diff --git a/clang/test/C/C2x/n2975.c b/clang/test/C/C2x/n2975.c index 5fc641dd66e7..2269400fe47c 100644 --- a/clang/test/C/C2x/n2975.c +++ b/clang/test/C/C2x/n2975.c @@ -11,7 +11,7 @@ void func(...) { // expected-warning {{'...' as the only parameter of a function is incompatible with C standards before C23}} // Show that va_start doesn't require the second argument in C23 mode. va_list list; - va_start(list); // FIXME: it would be nice to issue a portability warning to C17 and earlier here. + va_start(list); // expected-warning {{passing no argument for the '...' parameter of a variadic macro is incompatible with C standards before C23}} expected-note@* {{macro 'va_start' defined here}} va_end(list); // Show that va_start doesn't expand or evaluate the second argument. @@ -26,7 +26,7 @@ void func(...) { // expected-warning {{'...' as the only parameter of a function __builtin_va_start(list); // expected-error {{too few arguments to function call, expected 2, have 1}} // Verify that the return type of a call to va_start is 'void'. - _Static_assert(__builtin_types_compatible_p(__typeof__(va_start(list)), void), ""); + _Static_assert(__builtin_types_compatible_p(__typeof__(va_start(list)), void), ""); // expected-warning {{passing no argument for the '...' parameter of a variadic macro is incompatible with C standards before C23}} expected-note@* {{macro 'va_start' defined here}} _Static_assert(__builtin_types_compatible_p(__typeof__(__builtin_va_start(list, 0)), void), ""); } diff --git a/clang/test/Lexer/gnu-flags.c b/clang/test/Lexer/gnu-flags.c index 384339fc8594..4d6d216b101f 100644 --- a/clang/test/Lexer/gnu-flags.c +++ b/clang/test/Lexer/gnu-flags.c @@ -17,8 +17,6 @@ #if ALL || ZEROARGS -// expected-warning@+9 {{must specify at least one argument for '...' parameter of variadic macro}} -// expected-note@+4 {{macro 'efoo' defined here}} // expected-warning@+3 {{token pasting of ',' and __VA_ARGS__ is a GNU extension}} #endif diff --git a/clang/test/Preprocessor/empty_va_arg.cpp b/clang/test/Preprocessor/empty_va_arg.cpp index 2ee431f6bde8..7c7d49d8fb16 100644 --- a/clang/test/Preprocessor/empty_va_arg.cpp +++ b/clang/test/Preprocessor/empty_va_arg.cpp @@ -1,12 +1,16 @@ -// RUN: %clang_cc1 -Eonly -std=c++17 -pedantic -verify %s -// RUN: %clang_cc1 -Eonly -std=c17 -pedantic -verify -x c %s -// RUN: %clang_cc1 -Eonly -std=c++20 -pedantic -Wpre-c++20-compat -verify=compat %s +// RUN: %clang_cc1 -Eonly -std=c17 -pedantic -verify=c17,expected -x c %s +// RUN: %clang_cc1 -Eonly -std=c23 -pedantic -Wpre-c23-compat -verify=c23,expected -x c %s +// RUN: %clang_cc1 -Eonly -std=c++17 -pedantic -verify=cxx17,expected %s +// RUN: %clang_cc1 -Eonly -std=c++20 -pedantic -Wpre-c++20-compat -verify=cxx20,expected %s -#define FOO(x, ...) // expected-note {{macro 'FOO' defined here}} \ - // compat-note {{macro 'FOO' defined here}} +// silent-no-diagnostics + +#define FOO(x, ...) // expected-note {{macro 'FOO' defined here}} int main() { - FOO(42) // expected-warning {{must specify at least one argument for '...' parameter of variadic macro}} \ - // compat-warning {{passing no argument for the '...' parameter of a variadic macro is incompatible with C++ standards before C++20}} + FOO(42) // c17-warning {{passing no argument for the '...' parameter of a variadic macro is a C23 extension}} \ + // cxx17-warning {{passing no argument for the '...' parameter of a variadic macro is a C++20 extension}} \ + // c23-warning {{passing no argument for the '...' parameter of a variadic macro is incompatible with C standards before C23}} \ + // cxx20-warning {{passing no argument for the '...' parameter of a variadic macro is incompatible with C++ standards before C++20}} } diff --git a/clang/test/Preprocessor/macro_fn.c b/clang/test/Preprocessor/macro_fn.c index 5f4ea0e26d5d..81d836321407 100644 --- a/clang/test/Preprocessor/macro_fn.c +++ b/clang/test/Preprocessor/macro_fn.c @@ -37,8 +37,8 @@ e(x) e() zero_dot() -one_dot(x) /* empty ... argument: expected-warning {{must specify at least one argument for '...' parameter of variadic macro}} */ -one_dot() /* empty first argument, elided ...: expected-warning {{must specify at least one argument for '...' parameter of variadic macro}} */ +one_dot(x) /* empty ... argument: expected-warning {{passing no argument for the '...' parameter of a variadic macro is a C23 extension}} */ +one_dot() /* empty first argument, elided ...: expected-warning {{passing no argument for the '...' parameter of a variadic macro is a C23 extension}} */ /* Crash with function-like macro test at end of directive. */ -- GitLab From cc9186060ad75c68a052bbcf43d7d1ee93143a60 Mon Sep 17 00:00:00 2001 From: Guray Ozen Date: Wed, 20 Mar 2024 19:21:19 +0100 Subject: [PATCH 055/296] [mlir][nvgpu][nvvm] Add myself as a primary reviewer for nvgpu and nvvm dialects (#85414) --- .github/CODEOWNERS | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 561da6a588c0..c246c42b0904 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -106,6 +106,14 @@ clang/test/AST/Interp/ @tbaederr # MLIR Sparsifier. /mlir/**/*SparseTensor*/ @aartbik @PeimingLiu @yinying-lisa-li @matthias-springer +# MLIR NVGPU Dialect +/mlir/**/NVGPU*/ @grypp +/mlir/test/**/CUDA/ @grypp + +# MLIR NVVM Dialect in MLIR +/mlir/**/LLVMIR/**/BasicPtxBuilderInterface* @grypp +/mlir/**/NVVM*/ @grypp + # BOLT /bolt/ @aaupov @maksfb @rafaelauler @ayermolo @dcci -- GitLab From 949d70d5e023b34b741b7d577c61a7ef60c3316f Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Wed, 20 Mar 2024 19:23:12 +0100 Subject: [PATCH 056/296] [SPIR-V] Fix incorrect bitwise instructions applied to the bool type (#85929) This PR ensures that LLVM IR bitwise instructions result in logical SPIR-V instructions when applied to i1 type. --- llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 20 ++++++ .../CodeGen/SPIRV/instructions/bitwise-i1.ll | 69 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 llvm/test/CodeGen/SPIRV/instructions/bitwise-i1.ll diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp index e6e9131d8dc2..55b4c47c197d 100644 --- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp @@ -145,6 +145,26 @@ void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const { validatePtrTypes(STI, MRI, GR, MI, GR.getSPIRVTypeForVReg(MI.getOperand(1).getReg()), 0); break; + // ensure that LLVM IR bitwise instructions result in logical SPIR-V + // instructions when applied to bool type + case SPIRV::OpBitwiseOrS: + case SPIRV::OpBitwiseOrV: + if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(), + SPIRV::OpTypeBool)) + MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalOr)); + break; + case SPIRV::OpBitwiseAndS: + case SPIRV::OpBitwiseAndV: + if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(), + SPIRV::OpTypeBool)) + MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalAnd)); + break; + case SPIRV::OpBitwiseXorS: + case SPIRV::OpBitwiseXorV: + if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(), + SPIRV::OpTypeBool)) + MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalNotEqual)); + break; } } } diff --git a/llvm/test/CodeGen/SPIRV/instructions/bitwise-i1.ll b/llvm/test/CodeGen/SPIRV/instructions/bitwise-i1.ll new file mode 100644 index 000000000000..8d3657b36454 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/instructions/bitwise-i1.ll @@ -0,0 +1,69 @@ +; This test ensures that LLVM IR bitwise instructions result in logical SPIR-V instructions +; when applied to i1 type + +; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-DAG: %[[#Char:]] = OpTypeInt 8 0 +; CHECK-DAG: %[[#Vec2Char:]] = OpTypeVector %[[#Char]] 2 +; CHECK-DAG: %[[#Bool:]] = OpTypeBool +; CHECK-DAG: %[[#Vec2Bool:]] = OpTypeVector %[[#Bool]] 2 + +; CHECK: OpBitwiseAnd %[[#Char]] +; CHECK: OpBitwiseOr %[[#Char]] +; CHECK: OpBitwiseXor %[[#Char]] +; CHECK: OpBitwiseAnd %[[#Vec2Char]] +; CHECK: OpBitwiseOr %[[#Vec2Char]] +; CHECK: OpBitwiseXor %[[#Vec2Char]] + +; CHECK: OpLogicalAnd %[[#Bool]] + +; CHECK: OpLogicalAnd %[[#Bool]] +; CHECK: OpLogicalOr %[[#Bool]] +; CHECK: OpLogicalNotEqual %[[#Bool]] +; CHECK: OpLogicalAnd %[[#Vec2Bool]] +; CHECK: OpLogicalOr %[[#Vec2Bool]] +; CHECK: OpLogicalNotEqual %[[#Vec2Bool]] + +define void @test1(i8 noundef %arg1, i8 noundef %arg2) { + %cond1 = and i8 %arg1, %arg2 + %cond2 = or i8 %arg1, %arg2 + %cond3 = xor i8 %arg1, %arg2 + ret void +} + +define void @test1v(<2 x i8> noundef %arg1, <2 x i8> noundef %arg2) { + %cond1 = and <2 x i8> %arg1, %arg2 + %cond2 = or <2 x i8> %arg1, %arg2 + %cond3 = xor <2 x i8> %arg1, %arg2 + ret void +} + +define void @test2(float noundef %real, float noundef %imag) { +entry: + %realabs = tail call spir_func noundef float @_Z16__spirv_ocl_fabsf(float noundef %real) + %cond1 = fcmp oeq float %realabs, 1.000000e+00 + %cond2 = fcmp oeq float %imag, 0.000000e+00 + %cond3 = and i1 %cond1, %cond2 + br i1 %cond3, label %midlbl, label %cleanup +midlbl: + br label %cleanup +cleanup: + ret void +} + +define void @test3(i1 noundef %arg1, i1 noundef %arg2) { + %cond1 = and i1 %arg1, %arg2 + %cond2 = or i1 %arg1, %arg2 + %cond3 = xor i1 %arg1, %arg2 + ret void +} + +define void @test3v(<2 x i1> noundef %arg1, <2 x i1> noundef %arg2) { + %cond1 = and <2 x i1> %arg1, %arg2 + %cond2 = or <2 x i1> %arg1, %arg2 + %cond3 = xor <2 x i1> %arg1, %arg2 + ret void +} + +declare dso_local spir_func noundef float @_Z16__spirv_ocl_fabsf(float noundef) -- GitLab From c2483ed52d6f600a91663a49e35bab1dff2ed977 Mon Sep 17 00:00:00 2001 From: Vyacheslav Levytskyy Date: Wed, 20 Mar 2024 19:28:29 +0100 Subject: [PATCH 057/296] [SPIRV] Add __spirv_ builtins for existing instructions (#85654) This PR: * adds __spirv_ builtins for existing instructions; * fixes parsing of "syncscope" values in atomic instructions; * fix a special case of binary header emision. --- llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp | 30 +++---- llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp | 68 ++++++++++++--- llvm/lib/Target/SPIRV/SPIRVBuiltins.td | 24 ++++++ .../Target/SPIRV/SPIRVInstructionSelector.cpp | 85 ++++++++++++++++--- llvm/test/CodeGen/SPIRV/empty-logical.ll | 1 + llvm/test/CodeGen/SPIRV/empty-module.ll | 1 + llvm/test/CodeGen/SPIRV/empty-opencl32.ll | 1 + llvm/test/CodeGen/SPIRV/empty-opencl64.ll | 1 + llvm/test/CodeGen/SPIRV/empty.ll | 1 + llvm/test/CodeGen/SPIRV/fence.ll | 54 ++++++++++++ 10 files changed, 226 insertions(+), 40 deletions(-) create mode 100644 llvm/test/CodeGen/SPIRV/fence.ll diff --git a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp index 30c67d3fde63..4eee8062f282 100644 --- a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp @@ -103,22 +103,22 @@ void SPIRVAsmPrinter::emitEndOfAsmFile(Module &M) { if (ModuleSectionsEmitted == false) { outputModuleSections(); ModuleSectionsEmitted = true; - } else { - ST = static_cast(TM).getSubtargetImpl(); - uint32_t DecSPIRVVersion = ST->getSPIRVVersion(); - uint32_t Major = DecSPIRVVersion / 10; - uint32_t Minor = DecSPIRVVersion - Major * 10; - // TODO: calculate Bound more carefully from maximum used register number, - // accounting for generated OpLabels and other related instructions if - // needed. - unsigned Bound = 2 * (ST->getBound() + 1); - bool FlagToRestore = OutStreamer->getUseAssemblerInfoForParsing(); - OutStreamer->setUseAssemblerInfoForParsing(true); - if (MCAssembler *Asm = OutStreamer->getAssemblerPtr()) - Asm->setBuildVersion(static_cast(0), Major, Minor, - Bound, VersionTuple(Major, Minor, 0, Bound)); - OutStreamer->setUseAssemblerInfoForParsing(FlagToRestore); } + + ST = static_cast(TM).getSubtargetImpl(); + uint32_t DecSPIRVVersion = ST->getSPIRVVersion(); + uint32_t Major = DecSPIRVVersion / 10; + uint32_t Minor = DecSPIRVVersion - Major * 10; + // TODO: calculate Bound more carefully from maximum used register number, + // accounting for generated OpLabels and other related instructions if + // needed. + unsigned Bound = 2 * (ST->getBound() + 1); + bool FlagToRestore = OutStreamer->getUseAssemblerInfoForParsing(); + OutStreamer->setUseAssemblerInfoForParsing(true); + if (MCAssembler *Asm = OutStreamer->getAssemblerPtr()) + Asm->setBuildVersion(static_cast(0), Major, Minor, + Bound, VersionTuple(Major, Minor, 0, Bound)); + OutStreamer->setUseAssemblerInfoForParsing(FlagToRestore); } void SPIRVAsmPrinter::emitFunctionHeader() { diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp index 07be0b34b182..0478fc33cedc 100644 --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp @@ -53,6 +53,8 @@ struct IncomingCall { : BuiltinName(BuiltinName), Builtin(Builtin), ReturnRegister(ReturnRegister), ReturnType(ReturnType), Arguments(Arguments) {} + + bool isSpirvOp() const { return BuiltinName.rfind("__spirv_", 0) == 0; } }; struct NativeBuiltin { @@ -485,9 +487,27 @@ static Register buildMemSemanticsReg(Register SemanticsRegister, return buildConstantIntReg(Semantics, MIRBuilder, GR); } +static bool buildOpFromWrapper(MachineIRBuilder &MIRBuilder, unsigned Opcode, + const SPIRV::IncomingCall *Call, + Register TypeReg = Register(0)) { + MachineRegisterInfo *MRI = MIRBuilder.getMRI(); + auto MIB = MIRBuilder.buildInstr(Opcode); + if (TypeReg.isValid()) + MIB.addDef(Call->ReturnRegister).addUse(TypeReg); + for (Register ArgReg : Call->Arguments) { + if (!MRI->getRegClassOrNull(ArgReg)) + MRI->setRegClass(ArgReg, &SPIRV::IDRegClass); + MIB.addUse(ArgReg); + } + return true; +} + /// Helper function for translating atomic init to OpStore. static bool buildAtomicInitInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder) { + if (Call->isSpirvOp()) + return buildOpFromWrapper(MIRBuilder, SPIRV::OpStore, Call); + assert(Call->Arguments.size() == 2 && "Need 2 arguments for atomic init translation"); MIRBuilder.getMRI()->setRegClass(Call->Arguments[0], &SPIRV::IDRegClass); @@ -502,6 +522,10 @@ static bool buildAtomicInitInst(const SPIRV::IncomingCall *Call, static bool buildAtomicLoadInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) { + Register TypeReg = GR->getSPIRVTypeID(Call->ReturnType); + if (Call->isSpirvOp()) + return buildOpFromWrapper(MIRBuilder, SPIRV::OpAtomicLoad, Call, TypeReg); + Register PtrRegister = Call->Arguments[0]; MIRBuilder.getMRI()->setRegClass(PtrRegister, &SPIRV::IDRegClass); // TODO: if true insert call to __translate_ocl_memory_sccope before @@ -528,7 +552,7 @@ static bool buildAtomicLoadInst(const SPIRV::IncomingCall *Call, MIRBuilder.buildInstr(SPIRV::OpAtomicLoad) .addDef(Call->ReturnRegister) - .addUse(GR->getSPIRVTypeID(Call->ReturnType)) + .addUse(TypeReg) .addUse(PtrRegister) .addUse(ScopeRegister) .addUse(MemSemanticsReg); @@ -539,6 +563,9 @@ static bool buildAtomicLoadInst(const SPIRV::IncomingCall *Call, static bool buildAtomicStoreInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) { + if (Call->isSpirvOp()) + return buildOpFromWrapper(MIRBuilder, SPIRV::OpAtomicStore, Call); + Register ScopeRegister = buildConstantIntReg(SPIRV::Scope::Device, MIRBuilder, GR); Register PtrRegister = Call->Arguments[0]; @@ -557,12 +584,13 @@ static bool buildAtomicStoreInst(const SPIRV::IncomingCall *Call, } /// Helper function for building an atomic compare-exchange instruction. -static bool buildAtomicCompareExchangeInst(const SPIRV::IncomingCall *Call, - MachineIRBuilder &MIRBuilder, - SPIRVGlobalRegistry *GR) { - const SPIRV::DemangledBuiltin *Builtin = Call->Builtin; - unsigned Opcode = - SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode; +static bool buildAtomicCompareExchangeInst( + const SPIRV::IncomingCall *Call, const SPIRV::DemangledBuiltin *Builtin, + unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) { + if (Call->isSpirvOp()) + return buildOpFromWrapper(MIRBuilder, Opcode, Call, + GR->getSPIRVTypeID(Call->ReturnType)); + bool IsCmpxchg = Call->Builtin->Name.contains("cmpxchg"); MachineRegisterInfo *MRI = MIRBuilder.getMRI(); @@ -667,6 +695,10 @@ static bool buildAtomicCompareExchangeInst(const SPIRV::IncomingCall *Call, static bool buildAtomicRMWInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) { + if (Call->isSpirvOp()) + return buildOpFromWrapper(MIRBuilder, Opcode, Call, + GR->getSPIRVTypeID(Call->ReturnType)); + MachineRegisterInfo *MRI = MIRBuilder.getMRI(); Register ScopeRegister = Call->Arguments.size() >= 4 ? Call->Arguments[3] : Register(); @@ -731,6 +763,12 @@ static bool buildAtomicFloatingRMWInst(const SPIRV::IncomingCall *Call, static bool buildAtomicFlagInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) { + bool IsSet = Opcode == SPIRV::OpAtomicFlagTestAndSet; + Register TypeReg = GR->getSPIRVTypeID(Call->ReturnType); + if (Call->isSpirvOp()) + return buildOpFromWrapper(MIRBuilder, Opcode, Call, + IsSet ? TypeReg : Register(0)); + MachineRegisterInfo *MRI = MIRBuilder.getMRI(); Register PtrRegister = Call->Arguments[0]; unsigned Semantics = SPIRV::MemorySemantics::SequentiallyConsistent; @@ -750,9 +788,8 @@ static bool buildAtomicFlagInst(const SPIRV::IncomingCall *Call, buildScopeReg(ScopeRegister, SPIRV::Scope::Device, MIRBuilder, GR, MRI); auto MIB = MIRBuilder.buildInstr(Opcode); - if (Opcode == SPIRV::OpAtomicFlagTestAndSet) - MIB.addDef(Call->ReturnRegister) - .addUse(GR->getSPIRVTypeID(Call->ReturnType)); + if (IsSet) + MIB.addDef(Call->ReturnRegister).addUse(TypeReg); MIB.addUse(PtrRegister).addUse(ScopeRegister).addUse(MemSemanticsReg); return true; @@ -763,6 +800,9 @@ static bool buildAtomicFlagInst(const SPIRV::IncomingCall *Call, static bool buildBarrierInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) { + if (Call->isSpirvOp()) + return buildOpFromWrapper(MIRBuilder, Opcode, Call); + MachineRegisterInfo *MRI = MIRBuilder.getMRI(); unsigned MemFlags = getIConstVal(Call->Arguments[0], MRI); unsigned MemSemantics = SPIRV::MemorySemantics::None; @@ -1240,7 +1280,8 @@ static bool generateAtomicInst(const SPIRV::IncomingCall *Call, return buildAtomicStoreInst(Call, MIRBuilder, GR); case SPIRV::OpAtomicCompareExchange: case SPIRV::OpAtomicCompareExchangeWeak: - return buildAtomicCompareExchangeInst(Call, MIRBuilder, GR); + return buildAtomicCompareExchangeInst(Call, Builtin, Opcode, MIRBuilder, + GR); case SPIRV::OpAtomicIAdd: case SPIRV::OpAtomicISub: case SPIRV::OpAtomicOr: @@ -1815,14 +1856,15 @@ static bool buildEnqueueKernel(const SPIRV::IncomingCall *Call, SPIRVGlobalRegistry *GR) { MachineRegisterInfo *MRI = MIRBuilder.getMRI(); const DataLayout &DL = MIRBuilder.getDataLayout(); - bool HasEvents = Call->Builtin->Name.contains("events"); + bool IsSpirvOp = Call->isSpirvOp(); + bool HasEvents = Call->Builtin->Name.contains("events") || IsSpirvOp; const SPIRVType *Int32Ty = GR->getOrCreateSPIRVIntegerType(32, MIRBuilder); // Make vararg instructions before OpEnqueueKernel. // Local sizes arguments: Sizes of block invoke arguments. Clang generates // local size operands as an array, so we need to unpack them. SmallVector LocalSizes; - if (Call->Builtin->Name.find("_varargs") != StringRef::npos) { + if (Call->Builtin->Name.find("_varargs") != StringRef::npos || IsSpirvOp) { const unsigned LocalSizeArrayIdx = HasEvents ? 9 : 6; Register GepReg = Call->Arguments[LocalSizeArrayIdx]; MachineInstr *GepMI = MRI->getUniqueVRegDef(GepReg); diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td index eb26f70b1861..ee4f13d89c3c 100644 --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td @@ -500,27 +500,38 @@ defm : DemangledNativeBuiltin<"__spirv_All", OpenCL_std, Relational, 1, 1, OpAll defm : DemangledNativeBuiltin<"atomic_init", OpenCL_std, Atomic, 2, 2, OpStore>; defm : DemangledNativeBuiltin<"atomic_load", OpenCL_std, Atomic, 1, 1, OpAtomicLoad>; defm : DemangledNativeBuiltin<"atomic_load_explicit", OpenCL_std, Atomic, 2, 3, OpAtomicLoad>; +defm : DemangledNativeBuiltin<"__spirv_AtomicLoad", OpenCL_std, Atomic, 3, 3, OpAtomicLoad>; defm : DemangledNativeBuiltin<"atomic_store", OpenCL_std, Atomic, 2, 2, OpAtomicStore>; defm : DemangledNativeBuiltin<"atomic_store_explicit", OpenCL_std, Atomic, 2, 4, OpAtomicStore>; +defm : DemangledNativeBuiltin<"__spirv_AtomicStore", OpenCL_std, Atomic, 4, 4, OpAtomicStore>; defm : DemangledNativeBuiltin<"atomic_compare_exchange_strong", OpenCL_std, Atomic, 3, 6, OpAtomicCompareExchange>; +defm : DemangledNativeBuiltin<"__spirv_AtomicCompareExchange", OpenCL_std, Atomic, 6, 6, OpAtomicCompareExchange>; defm : DemangledNativeBuiltin<"atomic_compare_exchange_strong_explicit", OpenCL_std, Atomic, 5, 6, OpAtomicCompareExchange>; defm : DemangledNativeBuiltin<"atomic_compare_exchange_weak", OpenCL_std, Atomic, 3, 6, OpAtomicCompareExchangeWeak>; defm : DemangledNativeBuiltin<"atomic_compare_exchange_weak_explicit", OpenCL_std, Atomic, 5, 6, OpAtomicCompareExchangeWeak>; +defm : DemangledNativeBuiltin<"__spirv_AtomicCompareExchangeWeak", OpenCL_std, Atomic, 6, 6, OpAtomicCompareExchangeWeak>; defm : DemangledNativeBuiltin<"atom_cmpxchg", OpenCL_std, Atomic, 3, 6, OpAtomicCompareExchange>; defm : DemangledNativeBuiltin<"atomic_cmpxchg", OpenCL_std, Atomic, 3, 6, OpAtomicCompareExchange>; defm : DemangledNativeBuiltin<"atom_add", OpenCL_std, Atomic, 2, 4, OpAtomicIAdd>; defm : DemangledNativeBuiltin<"atomic_add", OpenCL_std, Atomic, 2, 4, OpAtomicIAdd>; +defm : DemangledNativeBuiltin<"__spirv_AtomicIAdd", OpenCL_std, Atomic, 4, 4, OpAtomicIAdd>; defm : DemangledNativeBuiltin<"atom_sub", OpenCL_std, Atomic, 2, 4, OpAtomicISub>; defm : DemangledNativeBuiltin<"atomic_sub", OpenCL_std, Atomic, 2, 4, OpAtomicISub>; +defm : DemangledNativeBuiltin<"__spirv_AtomicISub", OpenCL_std, Atomic, 4, 4, OpAtomicISub>; defm : DemangledNativeBuiltin<"atom_or", OpenCL_std, Atomic, 2, 4, OpAtomicOr>; defm : DemangledNativeBuiltin<"atomic_or", OpenCL_std, Atomic, 2, 4, OpAtomicOr>; +defm : DemangledNativeBuiltin<"__spirv_AtomicOr", OpenCL_std, Atomic, 4, 4, OpAtomicOr>; defm : DemangledNativeBuiltin<"atom_xor", OpenCL_std, Atomic, 2, 4, OpAtomicXor>; defm : DemangledNativeBuiltin<"atomic_xor", OpenCL_std, Atomic, 2, 4, OpAtomicXor>; +defm : DemangledNativeBuiltin<"__spirv_AtomicXor", OpenCL_std, Atomic, 4, 4, OpAtomicXor>; defm : DemangledNativeBuiltin<"atom_and", OpenCL_std, Atomic, 2, 4, OpAtomicAnd>; defm : DemangledNativeBuiltin<"atomic_and", OpenCL_std, Atomic, 2, 4, OpAtomicAnd>; +defm : DemangledNativeBuiltin<"__spirv_AtomicAnd", OpenCL_std, Atomic, 4, 4, OpAtomicAnd>; defm : DemangledNativeBuiltin<"atomic_exchange", OpenCL_std, Atomic, 2, 4, OpAtomicExchange>; defm : DemangledNativeBuiltin<"atomic_exchange_explicit", OpenCL_std, Atomic, 2, 4, OpAtomicExchange>; +defm : DemangledNativeBuiltin<"AtomicEx__spirv_change", OpenCL_std, Atomic, 2, 4, OpAtomicExchange>; defm : DemangledNativeBuiltin<"atomic_work_item_fence", OpenCL_std, Atomic, 1, 3, OpMemoryBarrier>; +defm : DemangledNativeBuiltin<"__spirv_MemoryBarrier", OpenCL_std, Atomic, 2, 2, OpMemoryBarrier>; defm : DemangledNativeBuiltin<"atomic_fetch_add", OpenCL_std, Atomic, 2, 4, OpAtomicIAdd>; defm : DemangledNativeBuiltin<"atomic_fetch_sub", OpenCL_std, Atomic, 2, 4, OpAtomicISub>; defm : DemangledNativeBuiltin<"atomic_fetch_or", OpenCL_std, Atomic, 2, 4, OpAtomicOr>; @@ -532,26 +543,37 @@ defm : DemangledNativeBuiltin<"atomic_fetch_or_explicit", OpenCL_std, Atomic, 4, defm : DemangledNativeBuiltin<"atomic_fetch_xor_explicit", OpenCL_std, Atomic, 4, 6, OpAtomicXor>; defm : DemangledNativeBuiltin<"atomic_fetch_and_explicit", OpenCL_std, Atomic, 4, 6, OpAtomicAnd>; defm : DemangledNativeBuiltin<"atomic_flag_test_and_set", OpenCL_std, Atomic, 1, 1, OpAtomicFlagTestAndSet>; +defm : DemangledNativeBuiltin<"__spirv_AtomicFlagTestAndSet", OpenCL_std, Atomic, 3, 3, OpAtomicFlagTestAndSet>; defm : DemangledNativeBuiltin<"atomic_flag_test_and_set_explicit", OpenCL_std, Atomic, 2, 3, OpAtomicFlagTestAndSet>; defm : DemangledNativeBuiltin<"atomic_flag_clear", OpenCL_std, Atomic, 1, 1, OpAtomicFlagClear>; +defm : DemangledNativeBuiltin<"__spirv_AtomicFlagClear", OpenCL_std, Atomic, 3, 3, OpAtomicFlagClear>; defm : DemangledNativeBuiltin<"atomic_flag_clear_explicit", OpenCL_std, Atomic, 2, 3, OpAtomicFlagClear>; // Barrier builtin records: defm : DemangledNativeBuiltin<"barrier", OpenCL_std, Barrier, 1, 3, OpControlBarrier>; defm : DemangledNativeBuiltin<"work_group_barrier", OpenCL_std, Barrier, 1, 3, OpControlBarrier>; +defm : DemangledNativeBuiltin<"__spirv_ControlBarrier", OpenCL_std, Barrier, 3, 3, OpControlBarrier>; // Kernel enqueue builtin records: defm : DemangledNativeBuiltin<"__enqueue_kernel_basic", OpenCL_std, Enqueue, 5, 5, OpEnqueueKernel>; defm : DemangledNativeBuiltin<"__enqueue_kernel_basic_events", OpenCL_std, Enqueue, 8, 8, OpEnqueueKernel>; defm : DemangledNativeBuiltin<"__enqueue_kernel_varargs", OpenCL_std, Enqueue, 7, 7, OpEnqueueKernel>; defm : DemangledNativeBuiltin<"__enqueue_kernel_events_varargs", OpenCL_std, Enqueue, 10, 10, OpEnqueueKernel>; +defm : DemangledNativeBuiltin<"__spirv_EnqueueKernel", OpenCL_std, Enqueue, 10, 0, OpEnqueueKernel>; defm : DemangledNativeBuiltin<"retain_event", OpenCL_std, Enqueue, 1, 1, OpRetainEvent>; +defm : DemangledNativeBuiltin<"__spirv_RetainEvent", OpenCL_std, Enqueue, 1, 1, OpRetainEvent>; defm : DemangledNativeBuiltin<"release_event", OpenCL_std, Enqueue, 1, 1, OpReleaseEvent>; +defm : DemangledNativeBuiltin<"__spirv_ReleaseEvent", OpenCL_std, Enqueue, 1, 1, OpReleaseEvent>; defm : DemangledNativeBuiltin<"create_user_event", OpenCL_std, Enqueue, 0, 0, OpCreateUserEvent>; +defm : DemangledNativeBuiltin<"__spirv_CreateUserEvent", OpenCL_std, Enqueue, 0, 0, OpCreateUserEvent>; defm : DemangledNativeBuiltin<"is_valid_event", OpenCL_std, Enqueue, 1, 1, OpIsValidEvent>; +defm : DemangledNativeBuiltin<"__spirv_IsValidEvent", OpenCL_std, Enqueue, 1, 1, OpIsValidEvent>; defm : DemangledNativeBuiltin<"set_user_event_status", OpenCL_std, Enqueue, 2, 2, OpSetUserEventStatus>; +defm : DemangledNativeBuiltin<"__spirv_SetUserEventStatus", OpenCL_std, Enqueue, 2, 2, OpSetUserEventStatus>; defm : DemangledNativeBuiltin<"capture_event_profiling_info", OpenCL_std, Enqueue, 3, 3, OpCaptureEventProfilingInfo>; +defm : DemangledNativeBuiltin<"__spirv_CaptureEventProfilingInfo", OpenCL_std, Enqueue, 3, 3, OpCaptureEventProfilingInfo>; defm : DemangledNativeBuiltin<"get_default_queue", OpenCL_std, Enqueue, 0, 0, OpGetDefaultQueue>; +defm : DemangledNativeBuiltin<"__spirv_GetDefaultQueue", OpenCL_std, Enqueue, 0, 0, OpGetDefaultQueue>; defm : DemangledNativeBuiltin<"ndrange_1D", OpenCL_std, Enqueue, 1, 3, OpBuildNDRange>; defm : DemangledNativeBuiltin<"ndrange_2D", OpenCL_std, Enqueue, 1, 3, OpBuildNDRange>; defm : DemangledNativeBuiltin<"ndrange_3D", OpenCL_std, Enqueue, 1, 3, OpBuildNDRange>; @@ -562,7 +584,9 @@ defm : DemangledNativeBuiltin<"__spirv_SpecConstantComposite", OpenCL_std, SpecC // Async Copy and Prefetch builtin records: defm : DemangledNativeBuiltin<"async_work_group_copy", OpenCL_std, AsyncCopy, 4, 4, OpGroupAsyncCopy>; +defm : DemangledNativeBuiltin<"__spirv_GroupAsyncCopy", OpenCL_std, AsyncCopy, 4, 4, OpGroupAsyncCopy>; defm : DemangledNativeBuiltin<"wait_group_events", OpenCL_std, AsyncCopy, 2, 2, OpGroupWaitEvents>; +defm : DemangledNativeBuiltin<"__spirv_GroupWaitEvents", OpenCL_std, AsyncCopy, 2, 2, OpGroupWaitEvents>; // Load and store builtin records: defm : DemangledNativeBuiltin<"__spirv_Load", OpenCL_std, LoadStore, 1, 3, OpLoad>; diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 0fef19c2d534..5bb8f6084f96 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -26,10 +26,33 @@ #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" #include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineModuleInfoImpls.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/IR/IntrinsicsSPIRV.h" #include "llvm/Support/Debug.h" +namespace llvm { + +class SPIRVMachineModuleInfo : public MachineModuleInfoImpl { +public: + SyncScope::ID Work_ItemSSID; + SyncScope::ID WorkGroupSSID; + SyncScope::ID DeviceSSID; + SyncScope::ID AllSVMDevicesSSID; + SyncScope::ID SubGroupSSID; + + SPIRVMachineModuleInfo(const MachineModuleInfo &MMI) { + LLVMContext &CTX = MMI.getModule()->getContext(); + Work_ItemSSID = CTX.getOrInsertSyncScopeID("work_item"); + WorkGroupSSID = CTX.getOrInsertSyncScopeID("workgroup"); + DeviceSSID = CTX.getOrInsertSyncScopeID("device"); + AllSVMDevicesSSID = CTX.getOrInsertSyncScopeID("all_svm_devices"); + SubGroupSSID = CTX.getOrInsertSyncScopeID("sub_group"); + } +}; + +} // end namespace llvm + #define DEBUG_TYPE "spirv-isel" using namespace llvm; @@ -52,6 +75,7 @@ class SPIRVInstructionSelector : public InstructionSelector { const RegisterBankInfo &RBI; SPIRVGlobalRegistry &GR; MachineRegisterInfo *MRI; + SPIRVMachineModuleInfo *MMI = nullptr; /// We need to keep track of the number we give to anonymous global values to /// generate the same name every time when this is needed. @@ -233,6 +257,7 @@ void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) { + MMI = &MF.getMMI().getObjFileInfo(); MRI = &MF.getRegInfo(); GR.setCurrentFunc(MF); InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); @@ -613,15 +638,27 @@ bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); } -static SPIRV::Scope::Scope getScope(SyncScope::ID Ord) { - switch (Ord) { - case SyncScope::SingleThread: +static SPIRV::Scope::Scope getScope(SyncScope::ID Ord, + SPIRVMachineModuleInfo *MMI) { + if (Ord == SyncScope::SingleThread || Ord == MMI->Work_ItemSSID) return SPIRV::Scope::Invocation; - case SyncScope::System: + else if (Ord == SyncScope::System || Ord == MMI->DeviceSSID) + return SPIRV::Scope::Device; + else if (Ord == MMI->WorkGroupSSID) + return SPIRV::Scope::Workgroup; + else if (Ord == MMI->AllSVMDevicesSSID) + return SPIRV::Scope::CrossDevice; + else if (Ord == MMI->SubGroupSSID) + return SPIRV::Scope::Subgroup; + else + // OpenCL approach is: "The functions that do not have memory_scope argument + // have the same semantics as the corresponding functions with the + // memory_scope argument set to memory_scope_device." See ref.: // + // https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_C.html#atomic-functions + // In our case if the scope is unknown, assuming that SPIR-V code is to be + // consumed in an OpenCL environment, we use the same approach and set the + // scope to memory_scope_device. return SPIRV::Scope::Device; - default: - llvm_unreachable("Unsupported synchronization Scope ID."); - } } static void addMemoryOperands(MachineMemOperand *MemOp, @@ -773,7 +810,8 @@ bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, unsigned NegateOpcode) const { assert(I.hasOneMemOperand()); const MachineMemOperand *MemOp = *I.memoperands_begin(); - uint32_t Scope = static_cast(getScope(MemOp->getSyncScopeID())); + uint32_t Scope = + static_cast(getScope(MemOp->getSyncScopeID(), MMI)); Register ScopeReg = buildI32Constant(Scope, I); Register Ptr = I.getOperand(1).getReg(); @@ -844,7 +882,7 @@ bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { uint32_t MemSem = static_cast(getMemSemantics(AO)); Register MemSemReg = buildI32Constant(MemSem, I); SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); - uint32_t Scope = static_cast(getScope(Ord)); + uint32_t Scope = static_cast(getScope(Ord, MMI)); Register ScopeReg = buildI32Constant(Scope, I); MachineBasicBlock &BB = *I.getParent(); return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) @@ -863,7 +901,8 @@ bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, if (!isa(I)) { assert(I.hasOneMemOperand()); const MachineMemOperand *MemOp = *I.memoperands_begin(); - unsigned Scope = static_cast(getScope(MemOp->getSyncScopeID())); + unsigned Scope = + static_cast(getScope(MemOp->getSyncScopeID(), MMI)); ScopeReg = buildI32Constant(Scope, I); unsigned ScSem = static_cast( @@ -1189,12 +1228,34 @@ bool SPIRVInstructionSelector::selectConstVector(Register ResVReg, return MIB.constrainAllUses(TII, TRI, RBI); } +static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, + const SPIRVType *ResType) { + Register OpReg = ResType->getOperand(2).getReg(); + SPIRVType *OpDef = MRI->getVRegDef(OpReg); + if (!OpDef) + return 0; + if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && + OpDef->getOperand(1).isReg()) { + if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) + OpDef = RefDef; + } + unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT + ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() + : 0; + return N; +} + bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { - if (ResType->getOpcode() != SPIRV::OpTypeVector) + unsigned N = 0; + if (ResType->getOpcode() == SPIRV::OpTypeVector) + N = GR.getScalarOrVectorComponentCount(ResType); + else if (ResType->getOpcode() == SPIRV::OpTypeArray) + N = getArrayComponentCount(MRI, ResType); + else report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); - unsigned N = GR.getScalarOrVectorComponentCount(ResType); + unsigned OpIdx = I.getNumExplicitDefs(); if (!I.getOperand(OpIdx).isReg()) report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); diff --git a/llvm/test/CodeGen/SPIRV/empty-logical.ll b/llvm/test/CodeGen/SPIRV/empty-logical.ll index a99df5f7eaaa..1c6604006e2c 100644 --- a/llvm/test/CodeGen/SPIRV/empty-logical.ll +++ b/llvm/test/CodeGen/SPIRV/empty-logical.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; Ensure the required Capabilities are listed. ; CHECK-DAG: OpCapability Shader diff --git a/llvm/test/CodeGen/SPIRV/empty-module.ll b/llvm/test/CodeGen/SPIRV/empty-module.ll index f220176a4099..b56e58ccaf8f 100644 --- a/llvm/test/CodeGen/SPIRV/empty-module.ll +++ b/llvm/test/CodeGen/SPIRV/empty-module.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK-DAG: OpCapability Addresses ; CHECK-DAG: OpCapability Linkage diff --git a/llvm/test/CodeGen/SPIRV/empty-opencl32.ll b/llvm/test/CodeGen/SPIRV/empty-opencl32.ll index a373781e290b..8e826ec35f37 100644 --- a/llvm/test/CodeGen/SPIRV/empty-opencl32.ll +++ b/llvm/test/CodeGen/SPIRV/empty-opencl32.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; FIXME: ensure Magic Number, version number, generator's magic number, "bound" and "schema" are at least present diff --git a/llvm/test/CodeGen/SPIRV/empty-opencl64.ll b/llvm/test/CodeGen/SPIRV/empty-opencl64.ll index d10196525368..4eaa2e4af0fc 100644 --- a/llvm/test/CodeGen/SPIRV/empty-opencl64.ll +++ b/llvm/test/CodeGen/SPIRV/empty-opencl64.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} ;; FIXME: ensure Magic Number, version number, generator's magic number, "bound" and "schema" are at least present diff --git a/llvm/test/CodeGen/SPIRV/empty.ll b/llvm/test/CodeGen/SPIRV/empty.ll index fdcf316b0256..390ab329aea3 100644 --- a/llvm/test/CodeGen/SPIRV/empty.ll +++ b/llvm/test/CodeGen/SPIRV/empty.ll @@ -1,4 +1,5 @@ ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} ; CHECK: OpCapability Addresses ; CHECK: "foo" diff --git a/llvm/test/CodeGen/SPIRV/fence.ll b/llvm/test/CodeGen/SPIRV/fence.ll new file mode 100644 index 000000000000..5da58667f24f --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/fence.ll @@ -0,0 +1,54 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +; CHECK-DAG: OpName %[[#GetScope:]] "_Z8getScopev" +; CHECK-DAG: %[[#Long:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#ScopeDevice:]] = OpConstant %[[#Long]] 1 +; CHECK-DAG: %[[#WrkGrpConst2:]] = OpConstant %[[#Long]] 2 +; CHECK-DAG: %[[#Const3:]] = OpConstant %[[#Long]] 3 +; CHECK-DAG: %[[#InvocationConst4:]] = OpConstant %[[#Long]] 4 +; CHECK-DAG: %[[#Const8:]] = OpConstant %[[#Long]] 8 +; CHECK-DAG: %[[#Const16:]] = OpConstant %[[#Long]] 16 +; CHECK-DAG: %[[#Const912:]] = OpConstant %[[#Long]] 912 +; CHECK: OpMemoryBarrier %[[#ScopeDevice]] %[[#WrkGrpConst2]] +; CHECK: OpMemoryBarrier %[[#ScopeDevice]] %[[#InvocationConst4]] +; CHECK: OpMemoryBarrier %[[#ScopeDevice]] %[[#Const8]] +; CHECK: OpMemoryBarrier %[[#InvocationConst4]] %[[#Const16]] +; CHECK: OpMemoryBarrier %[[#WrkGrpConst2]] %[[#InvocationConst4]] +; CHECK: OpFunctionEnd +; CHECK: %[[#ScopeId:]] = OpFunctionCall %[[#Long]] %[[#GetScope]] +; CHECK: OpControlBarrier %[[#Const3]] %[[#ScopeId:]] %[[#Const912]] + +define spir_kernel void @fence_test_kernel1(ptr addrspace(1) noalias %s.ascast) { + fence acquire + ret void +} + +define spir_kernel void @fence_test_kernel2(ptr addrspace(1) noalias %s.ascast) { + fence release + ret void +} + +define spir_kernel void @fence_test_kernel3(ptr addrspace(1) noalias %s.ascast) { + fence acq_rel + ret void +} + +define spir_kernel void @fence_test_kernel4(ptr addrspace(1) noalias %s.ascast) { + fence syncscope("singlethread") seq_cst + ret void +} + +define spir_kernel void @fence_test_kernel5(ptr addrspace(1) noalias %s.ascast) { + fence syncscope("workgroup") release + ret void +} + +define spir_func void @barrier_test1() { + %scope = call noundef i32 @_Z8getScopev() + call void @_Z22__spirv_ControlBarrieriii(i32 noundef 3, i32 noundef %scope, i32 noundef 912) + ret void +} + +declare spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef, i32 noundef, i32 noundef) +declare spir_func i32 @_Z8getScopev() -- GitLab From f676e84bba016157f5879fa66d86737ac8920c4b Mon Sep 17 00:00:00 2001 From: Jason Eckhardt Date: Wed, 20 Mar 2024 13:32:38 -0500 Subject: [PATCH 058/296] [TableGen] Fix operand constraint checking problem. (#85859) Currently operand constraint checks on "$dest = $src" are inadvertently accepting any token that contains "=". This has surprising results, e.g, "$dest != $src" is accepted as a constraint but then treated as "=". This patch ensures that only exactly the token "=" is accepted. --- llvm/test/TableGen/ConstraintChecking3.td | 2 +- llvm/test/TableGen/ConstraintChecking8.td | 34 ++++++++++++++++++++++ llvm/utils/TableGen/CodeGenInstruction.cpp | 4 ++- 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 llvm/test/TableGen/ConstraintChecking8.td diff --git a/llvm/test/TableGen/ConstraintChecking3.td b/llvm/test/TableGen/ConstraintChecking3.td index 2d5fe6b7ef96..886e6d52a039 100644 --- a/llvm/test/TableGen/ConstraintChecking3.td +++ b/llvm/test/TableGen/ConstraintChecking3.td @@ -4,5 +4,5 @@ include "ConstraintChecking.inc" // (This is illegal because the '=' has to be surrounded by whitespace) -// CHECK: [[FILE]]:[[@LINE+1]]:5: error: Illegal format for tied-to constraint in 'Foo' +// CHECK: [[FILE]]:[[@LINE+1]]:5: error: Unrecognized constraint '$dest1=$dest2' in 'Foo' def Foo : TestInstructionWithConstraints<"$dest1=$dest2">; diff --git a/llvm/test/TableGen/ConstraintChecking8.td b/llvm/test/TableGen/ConstraintChecking8.td new file mode 100644 index 000000000000..37d35150911e --- /dev/null +++ b/llvm/test/TableGen/ConstraintChecking8.td @@ -0,0 +1,34 @@ +// RUN: not llvm-tblgen -gen-asm-writer -DT0 -I %p -I %p/../../include %s 2>&1 | FileCheck %s -DFILE=%s +// RUN: not llvm-tblgen -gen-asm-writer -DT1 -I %p -I %p/../../include %s 2>&1 | FileCheck %s -DFILE=%s --check-prefix=CHECK1 +// RUN: not llvm-tblgen -gen-asm-writer -DT2 -I %p -I %p/../../include %s 2>&1 | FileCheck %s -DFILE=%s --check-prefix=CHECK2 +// RUN: not llvm-tblgen -gen-asm-writer -DT3 -I %p -I %p/../../include %s 2>&1 | FileCheck %s -DFILE=%s --check-prefix=CHECK3 +// RUN: not llvm-tblgen -gen-asm-writer -DT4 -I %p -I %p/../../include %s 2>&1 | FileCheck %s -DFILE=%s --check-prefix=CHECK4 + +include "ConstraintChecking.inc" + +// Make sure exactly the token "=" appears. + +#ifdef T0 +// CHECK: [[FILE]]:[[@LINE+1]]:5: error: Unrecognized constraint '$dest1 != $src2' in 'Foo' +def Foo : TestInstructionWithConstraints<"$dest1 != $src2">; +#endif + +#ifdef T1 +// CHECK1: [[FILE]]:[[@LINE+1]]:5: error: Unrecognized constraint '$dest1 == $src2' in 'Foo' +def Foo : TestInstructionWithConstraints<"$dest1 == $src2">; +#endif + +#ifdef T2 +// CHECK2: [[FILE]]:[[@LINE+1]]:5: error: Unrecognized constraint '= $rhs' in 'Foo' +def Foo : TestInstructionWithConstraints<"= $rhs">; +#endif + +#ifdef T3 +// CHECK3: [[FILE]]:[[@LINE+1]]:5: error: Unrecognized constraint '$lhs =' in 'Foo' +def Foo : TestInstructionWithConstraints<"$lhs =">; +#endif + +#ifdef T4 +// CHECK4: [[FILE]]:[[@LINE+1]]:5: error: Unrecognized constraint '=' in 'Foo' +def Foo : TestInstructionWithConstraints<"=">; +#endif diff --git a/llvm/utils/TableGen/CodeGenInstruction.cpp b/llvm/utils/TableGen/CodeGenInstruction.cpp index b00b95da5fc2..18a4e7b0f18b 100644 --- a/llvm/utils/TableGen/CodeGenInstruction.cpp +++ b/llvm/utils/TableGen/CodeGenInstruction.cpp @@ -325,7 +325,9 @@ static void ParseConstraint(StringRef CStr, CGIOperandList &Ops, Record *Rec) { // Only other constraint is "TIED_TO" for now. StringRef::size_type pos = CStr.find_first_of('='); - if (pos == StringRef::npos) + if (pos == StringRef::npos || pos == 0 || + CStr.find_first_of(" \t", pos) != (pos + 1) || + CStr.find_last_of(" \t", pos) != (pos - 1)) PrintFatalError(Rec->getLoc(), "Unrecognized constraint '" + CStr + "' in '" + Rec->getName() + "'"); start = CStr.find_first_not_of(" \t"); -- GitLab From 891172d9be01dd7c7e5298f2d8fdb143add448da Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 20 Mar 2024 11:35:19 -0700 Subject: [PATCH 059/296] [RISCV] Use 'riscv-isa' module flag to set ELF flags and attributes. (#85155) Walk all the ISA strings and set the subtarget bits for any extension we find in any string. This allows LTO output to have a ELF attributes from the union of all of the files used to compile it. --- llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp | 32 ++++++++++++++++--- .../CodeGen/RISCV/attributes-module-flag.ll | 17 ++++++++++ llvm/test/CodeGen/RISCV/module-elf-flags.ll | 13 ++++++++ 3 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 llvm/test/CodeGen/RISCV/attributes-module-flag.ll create mode 100644 llvm/test/CodeGen/RISCV/module-elf-flags.ll diff --git a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp index cb82e74b1efa..5bf594c0b5ea 100644 --- a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp +++ b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp @@ -100,7 +100,7 @@ public: bool emitDirectiveOptionArch(); private: - void emitAttributes(); + void emitAttributes(const MCSubtargetInfo &SubtargetInfo); void emitNTLHint(const MachineInstr *MI); @@ -385,8 +385,32 @@ void RISCVAsmPrinter::emitStartOfAsmFile(Module &M) { if (const MDString *ModuleTargetABI = dyn_cast_or_null(M.getModuleFlag("target-abi"))) RTS.setTargetABI(RISCVABI::getTargetABI(ModuleTargetABI->getString())); + + MCSubtargetInfo SubtargetInfo = *TM.getMCSubtargetInfo(); + + // Use module flag to update feature bits. + if (auto *MD = dyn_cast_or_null(M.getModuleFlag("riscv-isa"))) { + for (auto &ISA : MD->operands()) { + if (auto *ISAString = dyn_cast_or_null(ISA)) { + auto ParseResult = llvm::RISCVISAInfo::parseArchString( + ISAString->getString(), /*EnableExperimentalExtension=*/true, + /*ExperimentalExtensionVersionCheck=*/true); + if (!errorToBool(ParseResult.takeError())) { + auto &ISAInfo = *ParseResult; + for (const auto &Feature : RISCVFeatureKV) { + if (ISAInfo->hasExtension(Feature.Key) && + !SubtargetInfo.hasFeature(Feature.Value)) + SubtargetInfo.ToggleFeature(Feature.Key); + } + } + } + } + + RTS.setFlagsFromFeatures(SubtargetInfo); + } + if (TM.getTargetTriple().isOSBinFormatELF()) - emitAttributes(); + emitAttributes(SubtargetInfo); } void RISCVAsmPrinter::emitEndOfAsmFile(Module &M) { @@ -398,13 +422,13 @@ void RISCVAsmPrinter::emitEndOfAsmFile(Module &M) { EmitHwasanMemaccessSymbols(M); } -void RISCVAsmPrinter::emitAttributes() { +void RISCVAsmPrinter::emitAttributes(const MCSubtargetInfo &SubtargetInfo) { RISCVTargetStreamer &RTS = static_cast(*OutStreamer->getTargetStreamer()); // Use MCSubtargetInfo from TargetMachine. Individual functions may have // attributes that differ from other functions in the module and we have no // way to know which function is correct. - RTS.emitTargetAttributes(*TM.getMCSubtargetInfo(), /*EmitStackAlign*/ true); + RTS.emitTargetAttributes(SubtargetInfo, /*EmitStackAlign*/ true); } void RISCVAsmPrinter::emitFunctionEntryLabel() { diff --git a/llvm/test/CodeGen/RISCV/attributes-module-flag.ll b/llvm/test/CodeGen/RISCV/attributes-module-flag.ll new file mode 100644 index 000000000000..4580539fbb29 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/attributes-module-flag.ll @@ -0,0 +1,17 @@ +; RUN: llc -mtriple=riscv32 %s -o - | FileCheck %s --check-prefix=RV32 +; RUN: llc -mtriple=riscv64 %s -o - | FileCheck %s --check-prefix=RV64 + +; Test generation of ELF attribute from module metadata + +; RV32: .attribute 5, "rv32i2p1_m2p0_zba1p0" +; RV64: .attribute 5, "rv64i2p1_m2p0_zba1p0" + +define i32 @addi(i32 %a) { + %1 = add i32 %a, 1 + ret i32 %1 +} + +!llvm.module.flags = !{!0} + +!0 = !{i32 6, !"riscv-isa", !1} +!1 = !{!"rv64i2p1_m2p0", !"rv64i2p1_zba1p0"} diff --git a/llvm/test/CodeGen/RISCV/module-elf-flags.ll b/llvm/test/CodeGen/RISCV/module-elf-flags.ll new file mode 100644 index 000000000000..1b4bc9fd5466 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/module-elf-flags.ll @@ -0,0 +1,13 @@ +; RUN: llc -mtriple=riscv32 -filetype=obj < %s | llvm-readelf -h - | FileCheck -check-prefixes=FLAGS %s + +; FLAGS: Flags: 0x11, RVC, TSO + +define i32 @addi(i32 %a) { + %1 = add i32 %a, 1 + ret i32 %1 +} + +!llvm.module.flags = !{!0} + +!0 = !{i32 6, !"riscv-isa", !1} +!1 = !{!"rv64i2p1_c2p0_ztso0p1"} -- GitLab From 3f39571228fe2cf402e6ea5727cd5b32f9299356 Mon Sep 17 00:00:00 2001 From: "S. Bharadwaj Yadavalli" Date: Wed, 20 Mar 2024 14:48:16 -0400 Subject: [PATCH 060/296] [DirectX][DXIL] Distinguish return type for overload type resolution. (#85646) Return type of DXIL Ops may be different from valid overload type of the parameters, if any. Such DXIL Ops are correctly represented in DXIL.td. However, DXILEmitter assumes the return type to be the same as parameter overload type, if one exists. This results in generation in incorrect overload index value in DXILOperation.inc for the DXIL Op and incorrect DXIL operation function call in DXILOpLowering pass. This change distinguishes return types correctly from parameter overload types in DXILEmitter backend to handle such DXIL ops. Add specification for DXIL Op `isinf` and corresponding tests to verify the above change. Fixes issue #85125 --- llvm/lib/Target/DirectX/DXIL.td | 3 ++ llvm/lib/Target/DirectX/DXILOpBuilder.cpp | 43 +++++++++++----------- llvm/lib/Target/DirectX/DXILOpBuilder.h | 8 +++- llvm/lib/Target/DirectX/DXILOpLowering.cpp | 7 +--- llvm/test/CodeGen/DirectX/isinf.ll | 25 +++++++++++++ llvm/test/CodeGen/DirectX/isinf_error.ll | 13 +++++++ llvm/utils/TableGen/DXILEmitter.cpp | 14 ++++++- 7 files changed, 83 insertions(+), 30 deletions(-) create mode 100644 llvm/test/CodeGen/DirectX/isinf.ll create mode 100644 llvm/test/CodeGen/DirectX/isinf_error.ll diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index 216fa5b10c8f..36eb29d53766 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -255,6 +255,9 @@ class DXILOpMapping; def Sin : DXILOpMapping<13, unary, int_sin, "Returns sine(theta) for theta in radians.", [llvm_halforfloat_ty, LLVMMatchType<0>]>; diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp index 11b24d044923..a1eacc2d4800 100644 --- a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp +++ b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp @@ -229,13 +229,13 @@ static Type *getTypeFromParameterKind(ParameterKind Kind, Type *OverloadTy) { /// its specification in DXIL.td. /// \param OverloadTy Return type to be used to construct DXIL function type. static FunctionType *getDXILOpFunctionType(const OpCodeProperty *Prop, - Type *OverloadTy) { + Type *ReturnTy, Type *OverloadTy) { SmallVector ArgTys; auto ParamKinds = getOpCodeParameterKind(*Prop); - // Add OverloadTy as return type of the function - ArgTys.emplace_back(OverloadTy); + // Add ReturnTy as return type of the function + ArgTys.emplace_back(ReturnTy); // Add DXIL Opcode value type viz., Int32 as first argument ArgTys.emplace_back(Type::getInt32Ty(OverloadTy->getContext())); @@ -249,34 +249,33 @@ static FunctionType *getDXILOpFunctionType(const OpCodeProperty *Prop, ArgTys[0], ArrayRef(&ArgTys[1], ArgTys.size() - 1), false); } -static FunctionCallee getOrCreateDXILOpFunction(dxil::OpCode DXILOp, - Type *OverloadTy, Module &M) { - const OpCodeProperty *Prop = getOpCodeProperty(DXILOp); +namespace llvm { +namespace dxil { + +CallInst *DXILOpBuilder::createDXILOpCall(dxil::OpCode OpCode, Type *ReturnTy, + Type *OverloadTy, + llvm::iterator_range Args) { + const OpCodeProperty *Prop = getOpCodeProperty(OpCode); OverloadKind Kind = getOverloadKind(OverloadTy); if ((Prop->OverloadTys & (uint16_t)Kind) == 0) { report_fatal_error("Invalid Overload Type", /* gen_crash_diag=*/false); } - std::string FnName = constructOverloadName(Kind, OverloadTy, *Prop); - // Dependent on name to dedup. - if (auto *Fn = M.getFunction(FnName)) - return FunctionCallee(Fn); - - FunctionType *DXILOpFT = getDXILOpFunctionType(Prop, OverloadTy); - return M.getOrInsertFunction(FnName, DXILOpFT); -} - -namespace llvm { -namespace dxil { - -CallInst *DXILOpBuilder::createDXILOpCall(dxil::OpCode OpCode, Type *OverloadTy, - llvm::iterator_range Args) { - auto Fn = getOrCreateDXILOpFunction(OpCode, OverloadTy, M); + std::string DXILFnName = constructOverloadName(Kind, OverloadTy, *Prop); + FunctionCallee DXILFn; + // Get the function with name DXILFnName, if one exists + if (auto *Func = M.getFunction(DXILFnName)) { + DXILFn = FunctionCallee(Func); + } else { + // Construct and add a function with name DXILFnName + FunctionType *DXILOpFT = getDXILOpFunctionType(Prop, ReturnTy, OverloadTy); + DXILFn = M.getOrInsertFunction(DXILFnName, DXILOpFT); + } SmallVector FullArgs; FullArgs.emplace_back(B.getInt32((int32_t)OpCode)); FullArgs.append(Args.begin(), Args.end()); - return B.CreateCall(Fn, FullArgs); + return B.CreateCall(DXILFn, FullArgs); } Type *DXILOpBuilder::getOverloadTy(dxil::OpCode OpCode, FunctionType *FT) { diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.h b/llvm/lib/Target/DirectX/DXILOpBuilder.h index 1c15f109184a..f3abcc6e02a4 100644 --- a/llvm/lib/Target/DirectX/DXILOpBuilder.h +++ b/llvm/lib/Target/DirectX/DXILOpBuilder.h @@ -29,7 +29,13 @@ namespace dxil { class DXILOpBuilder { public: DXILOpBuilder(Module &M, IRBuilderBase &B) : M(M), B(B) {} - CallInst *createDXILOpCall(dxil::OpCode OpCode, Type *OverloadTy, + /// Create an instruction that calls DXIL Op with return type, specified + /// opcode, and call arguments. \param OpCode Opcode of the DXIL Op call + /// constructed \param ReturnTy Return type of the DXIL Op call constructed + /// \param OverloadTy Overload type of the DXIL Op call constructed + /// \return DXIL Op call constructed + CallInst *createDXILOpCall(dxil::OpCode OpCode, Type *ReturnTy, + Type *OverloadTy, llvm::iterator_range Args); Type *getOverloadTy(dxil::OpCode OpCode, FunctionType *FT); static const char *getOpCodeName(dxil::OpCode DXILOp); diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp index e5c2042e7d16..3e334b0ec298 100644 --- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp +++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp @@ -32,7 +32,6 @@ using namespace llvm::dxil; static void lowerIntrinsic(dxil::OpCode DXILOp, Function &F, Module &M) { IRBuilder<> B(M.getContext()); - Value *DXILOpArg = B.getInt32(static_cast(DXILOp)); DXILOpBuilder DXILB(M, B); Type *OverloadTy = DXILB.getOverloadTy(DXILOp, F.getFunctionType()); for (User *U : make_early_inc_range(F.users())) { @@ -40,11 +39,9 @@ static void lowerIntrinsic(dxil::OpCode DXILOp, Function &F, Module &M) { if (!CI) continue; - SmallVector Args; - Args.emplace_back(DXILOpArg); - Args.append(CI->arg_begin(), CI->arg_end()); B.SetInsertPoint(CI); - CallInst *DXILCI = DXILB.createDXILOpCall(DXILOp, OverloadTy, CI->args()); + CallInst *DXILCI = DXILB.createDXILOpCall(DXILOp, F.getReturnType(), + OverloadTy, CI->args()); CI->replaceAllUsesWith(DXILCI); CI->eraseFromParent(); diff --git a/llvm/test/CodeGen/DirectX/isinf.ll b/llvm/test/CodeGen/DirectX/isinf.ll new file mode 100644 index 000000000000..e2975da90bfc --- /dev/null +++ b/llvm/test/CodeGen/DirectX/isinf.ll @@ -0,0 +1,25 @@ +; RUN: opt -S -dxil-op-lower < %s | FileCheck %s + +; Make sure dxil operation function calls for isinf are generated for float and half. +; CHECK: call i1 @dx.op.isSpecialFloat.f32(i32 9, float %{{.*}}) +; CHECK: call i1 @dx.op.isSpecialFloat.f16(i32 9, half %{{.*}}) + +; Function Attrs: noinline nounwind optnone +define noundef i1 @isinf_float(float noundef %a) #0 { +entry: + %a.addr = alloca float, align 4 + store float %a, ptr %a.addr, align 4 + %0 = load float, ptr %a.addr, align 4 + %dx.isinf = call i1 @llvm.dx.isinf.f32(float %0) + ret i1 %dx.isinf +} + +; Function Attrs: noinline nounwind optnone +define noundef i1 @isinf_half(half noundef %p0) #0 { +entry: + %p0.addr = alloca half, align 2 + store half %p0, ptr %p0.addr, align 2 + %0 = load half, ptr %p0.addr, align 2 + %dx.isinf = call i1 @llvm.dx.isinf.f16(half %0) + ret i1 %dx.isinf +} diff --git a/llvm/test/CodeGen/DirectX/isinf_error.ll b/llvm/test/CodeGen/DirectX/isinf_error.ll new file mode 100644 index 000000000000..95b2d0cabcc4 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/isinf_error.ll @@ -0,0 +1,13 @@ +; RUN: not opt -S -dxil-op-lower %s 2>&1 | FileCheck %s + +; DXIL operation isinf does not support double overload type +; CHECK: LLVM ERROR: Invalid Overload Type + +define noundef i1 @isinf_double(double noundef %a) #0 { +entry: + %a.addr = alloca double, align 8 + store double %a, ptr %a.addr, align 8 + %0 = load double, ptr %a.addr, align 8 + %dx.isinf = call i1 @llvm.dx.isinf.f64(double %0) + ret i1 %dx.isinf +} diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp index 59089929837e..af1efb8aa99f 100644 --- a/llvm/utils/TableGen/DXILEmitter.cpp +++ b/llvm/utils/TableGen/DXILEmitter.cpp @@ -119,7 +119,7 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) { // Populate OpTypes with return type and parameter types // Parameter indices of overloaded parameters. - // This vector contains overload parameters in the order order used to + // This vector contains overload parameters in the order used to // resolve an LLVMMatchType in accordance with convention outlined in // the comment before the definition of class LLVMMatchType in // llvm/IR/Intrinsics.td @@ -398,10 +398,20 @@ static void emitDXILOperationTable(std::vector &Ops, OS << " static const OpCodeProperty OpCodeProps[] = {\n"; for (auto &Op : Ops) { + // Consider Op.OverloadParamIndex as the overload parameter index, by + // default + auto OLParamIdx = Op.OverloadParamIndex; + // If no overload parameter index is set, treat first parameter type as + // overload type - unless the Op has no parameters, in which case treat the + // return type - as overload parameter to emit the appropriate overload kind + // enum. + if (OLParamIdx < 0) { + OLParamIdx = (Op.OpTypes.size() > 1) ? 1 : 0; + } OS << " { dxil::OpCode::" << Op.OpName << ", " << OpStrings.get(Op.OpName) << ", OpCodeClass::" << Op.OpClass << ", " << OpClassStrings.get(Op.OpClass.data()) << ", " - << getOverloadKindStr(Op.OpTypes[0]) << ", " + << getOverloadKindStr(Op.OpTypes[OLParamIdx]) << ", " << emitDXILOperationAttr(Op.OpAttributes) << ", " << Op.OverloadParamIndex << ", " << Op.OpTypes.size() - 1 << ", " << Parameters.get(ParameterMap[Op.OpClass]) << " },\n"; -- GitLab From 2e817bfb4890249d1f5c0e50827d1d742c4f3df4 Mon Sep 17 00:00:00 2001 From: Min-Yih Hsu Date: Wed, 20 Mar 2024 12:03:01 -0700 Subject: [PATCH 061/296] [RISCV] Add missing feature predicates to some of the RVV pseudos (#85983) Some of the RVV pseudos are missing HasVInstructions. This is effectively a NFC. --- llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td index ae93bf694875..8be4c7741ca1 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -6698,6 +6698,7 @@ defm PseudoVFWREDOSUM : VPseudoVFWREDO_VS_RM; // 15. Vector Mask Instructions //===----------------------------------------------------------------------===// +let Predicates = [HasVInstructions] in { //===----------------------------------------------------------------------===// // 15.1 Vector Mask-Register Logical Instructions //===----------------------------------------------------------------------===// @@ -6718,7 +6719,6 @@ defm PseudoVMSET : VPseudoNullaryPseudoM<"VMXNOR">; //===----------------------------------------------------------------------===// // 15.2. Vector mask population count vcpop //===----------------------------------------------------------------------===// - let IsSignExtendingOpW = 1 in defm PseudoVCPOP: VPseudoVPOP_M; @@ -6753,6 +6753,7 @@ defm PseudoVIOTA_M: VPseudoVIOTA_M; // 15.9. Vector Element Index Instruction //===----------------------------------------------------------------------===// defm PseudoVID : VPseudoVID_V; +} // Predicates = [HasVInstructions] //===----------------------------------------------------------------------===// // 16. Vector Permutation Instructions @@ -6828,6 +6829,7 @@ let Predicates = [HasVInstructionsAnyF] in { //===----------------------------------------------------------------------===// // 16.4. Vector Register Gather Instructions //===----------------------------------------------------------------------===// +let Predicates = [HasVInstructions] in { defm PseudoVRGATHER : VPseudoVGTR_VV_VX_VI; defm PseudoVRGATHEREI16 : VPseudoVGTR_VV_EEW; @@ -6836,6 +6838,7 @@ defm PseudoVRGATHEREI16 : VPseudoVGTR_VV_EEW Date: Wed, 20 Mar 2024 15:07:23 -0400 Subject: [PATCH 062/296] [C11] Add test coverage for N1310 and claim conformance This is about the best I could do for testing that `signed char` does not have any padding bits. --- clang/test/C/C11/n1310.c | 31 +++++++++++++++++++++++++++++++ clang/www/c_status.html | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 clang/test/C/C11/n1310.c diff --git a/clang/test/C/C11/n1310.c b/clang/test/C/C11/n1310.c new file mode 100644 index 000000000000..7f8dd754f746 --- /dev/null +++ b/clang/test/C/C11/n1310.c @@ -0,0 +1,31 @@ +// RUN: %clang_cc1 -verify -std=c89 %s +// RUN: %clang_cc1 -verify -std=c99 %s +// RUN: %clang_cc1 -verify -std=c11 %s +// RUN: %clang_cc1 -verify -std=c17 %s +// RUN: %clang_cc1 -verify -std=c23 %s +// expected-no-diagnostics + +/* WG14 N1310: Yes + * Requiring signed char to have no padding bits + */ + +/* This is shockingly hard to test, but we're trying our best by checking that + * setting each bit of an unsigned char, then bit-casting it to signed char, + * results in a value we expect to see. If we have padding bits, then it's + * possible (but not mandatory) for the value to not be as we expect, so a + * failing assertion means the implementation is broken but a passing test does + * not *prove* there aren't padding bits. + */ +_Static_assert(__CHAR_BIT__ == 8, ""); +_Static_assert(sizeof(signed char) == 1, ""); + +#define TEST(Bit, Expected) __builtin_bit_cast(signed char, (unsigned char)(1 << Bit)) == Expected +_Static_assert(TEST(0, 1), ""); +_Static_assert(TEST(1, 2), ""); +_Static_assert(TEST(2, 4), ""); +_Static_assert(TEST(3, 8), ""); +_Static_assert(TEST(4, 16), ""); +_Static_assert(TEST(5, 32), ""); +_Static_assert(TEST(6, 64), ""); +_Static_assert(TEST(7, (signed char)128), ""); + diff --git a/clang/www/c_status.html b/clang/www/c_status.html index 9e4600b3e66a..b1f5ab4cbc4f 100644 --- a/clang/www/c_status.html +++ b/clang/www/c_status.html @@ -411,7 +411,7 @@ conformance.

Requiring signed char to have no padding bits N1310 - Unknown + Yes Initializing static or external variables -- GitLab From d9c4c312d81b4a2059d0ca2bb454c3452e52042e Mon Sep 17 00:00:00 2001 From: Zahira Ammarguellat Date: Wed, 20 Mar 2024 12:14:09 -0700 Subject: [PATCH 063/296] [CLANG] Full support of complex multiplication and division. (#81514) In clang there are two options `-f[no]cx-limited-range` and `-f[no]cx-fortran-rules` that control the range of complex multiplication and division. However, it is unclear how these options interact with one another. For instance, what should happen when the users compile with `-fcx-fortran-rules -fno-cx-limited-range` or `-fcx-limited-range -fno-cx-fortran-rules`? In this patch we are introducing a new option to solve the issue and give a greater flexibility to the user to control the behavior of the compiler when performing multiplication and division of complex floating-point values. `-fcomplex-arihmetic=[full|improved|promoted|basic]` `full`: Implementation of complex division and multiplication using a call to runtime library functions (generally the case, but the BE might sometimes replace the library call if it knows enough about the potential range of the inputs). Overflow and non-finite values are handled by the library implementation. For the case of multiplication overflow will occur in accordance with normal floating-point rules. This is the default value. `improved`: Implementation of complex division using the Smith algorithm at source precision. Smith's algorithm for complex division. See SMITH, R. L. Algorithm 116: Complex division. Commun. ACM 5, 8 (1962). This value offers improved handling for overflow in intermediate calculations, but overflow may occur. NaN and infinite values are not handled in some cases. `promoted`: Implementation of complex division using algebraic formulas at higher precision. Overflow is handled. Non-finite values are handled in some cases. If the target does not have native support for a higher precision data type, the implementation for the complex operation using the Smith algorithm will be used. Overflow may still occur in some cases. NaN and infinite values are not handled. `basic`: Implementation of complex division and multiplication using algebraic formulas at source precision. No special handling to avoid overflow. NaN and infinite values are not handled. `fcx-limited-range` will alias `-fcomplex-arithmetic=basic` `-fcx-fortran-rules` will alias `-fcomplex-arithmetic=improved` `-fno-cx-limited-range` and `-fno-cx-fortran-rules` will alias `-fcomplex-arithmetic=full` The complex division and multiplication will be implemented as follows depending on the option used. -fcomplex-arithmetic | div | mul -- | -- | -- basic | algebraic form | algebraic form improved | smith's algorithm | algebraic form full | libcall | libcall + nan processing promoted | algebraic form + higher precision | algebraic form --- clang/docs/UsersManual.rst | 45 +- .../clang/Basic/DiagnosticCommonKinds.td | 5 + clang/include/clang/Basic/LangOptions.h | 33 +- clang/include/clang/Driver/Options.td | 37 +- clang/lib/CodeGen/CGExprComplex.cpp | 83 +- clang/lib/Driver/ToolChains/Clang.cpp | 173 +- clang/lib/Parse/ParsePragma.cpp | 11 +- clang/test/CodeGen/X86/cx-complex-range.c | 1425 +++++++ clang/test/CodeGen/complex-math.c | 2 +- clang/test/CodeGen/cx-complex-range.c | 3519 ++++++++++++++++- clang/test/CodeGen/pragma-cx-limited-range.c | 224 +- clang/test/CodeGen/smiths-complex-div.c | 104 +- clang/test/Driver/range.c | 169 +- 13 files changed, 5518 insertions(+), 312 deletions(-) create mode 100644 clang/test/CodeGen/X86/cx-complex-range.c diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst index 7a63d720241a..129e75fc9a78 100644 --- a/clang/docs/UsersManual.rst +++ b/clang/docs/UsersManual.rst @@ -1847,19 +1847,50 @@ floating point semantic models: precise (the default), strict, and fast. * ``16`` - Forces ``_Float16`` operations to be emitted without using excess precision arithmetic. +.. option:: -fcomplex-arithmetic=: + + This option specifies the implementation for complex multiplication and division. + + Valid values are: ``basic``, ``improved``, ``full`` and ``promoted``. + + * ``basic`` Implementation of complex division and multiplication using + algebraic formulas at source precision. No special handling to avoid + overflow. NaN and infinite values are not handled. + * ``improved`` Implementation of complex division using the Smith algorithm + at source precision. Smith's algorithm for complex division. + See SMITH, R. L. Algorithm 116: Complex division. Commun. ACM 5, 8 (1962). + This value offers improved handling for overflow in intermediate + calculations, but overflow may occur. NaN and infinite values are not + handled in some cases. + * ``full`` Implementation of complex division and multiplication using a + call to runtime library functions (generally the case, but the BE might + sometimes replace the library call if it knows enough about the potential + range of the inputs). Overflow and non-finite values are handled by the + library implementation. For the case of multiplication overflow will occur in + accordance with normal floating-point rules. This is the default value. + * ``promoted`` Implementation of complex division using algebraic formulas at + higher precision. Overflow is handled. Non-finite values are handled in some + cases. If the target does not have native support for a higher precision + data type, the implementation for the complex operation using the Smith + algorithm will be used. Overflow may still occur in some cases. NaN and + infinite values are not handled. + .. option:: -fcx-limited-range: - This option enables the naive mathematical formulas for complex division and - multiplication with no NaN checking of results. The default is - ``-fno-cx-limited-range``, but this option is enabled by the ``-ffast-math`` + This option is aliased to ``-fcomplex-arithmetic=basic``. It enables the + naive mathematical formulas for complex division and multiplication with no + NaN checking of results. The default is ``-fno-cx-limited-range`` aliased to + ``-fcomplex-arithmetic=full``. This option is enabled by the ``-ffast-math`` option. .. option:: -fcx-fortran-rules: - This option enables the naive mathematical formulas for complex - multiplication and enables application of Smith's algorithm for complex - division. See SMITH, R. L. Algorithm 116: Complex division. Commun. - ACM 5, 8 (1962). The default is ``-fno-cx-fortran-rules``. + This option is aliased to ``-fcomplex-arithmetic=improved``. It enables the + naive mathematical formulas for complex multiplication and enables application + of Smith's algorithm for complex division. See SMITH, R. L. Algorithm 116: + Complex division. Commun. ACM 5, 8 (1962). + The default is ``-fno-cx-fortran-rules`` aliased to + ``-fcomplex-arithmetic=full``. .. _floating-point-environment: diff --git a/clang/include/clang/Basic/DiagnosticCommonKinds.td b/clang/include/clang/Basic/DiagnosticCommonKinds.td index 43e132e56658..a52bf62e2420 100644 --- a/clang/include/clang/Basic/DiagnosticCommonKinds.td +++ b/clang/include/clang/Basic/DiagnosticCommonKinds.td @@ -45,6 +45,11 @@ def note_using : Note<"using">; def note_possibility : Note<"one possibility">; def note_also_found : Note<"also found">; +def warn_next_larger_fp_type_same_size_than_fp : Warning< + "higher precision floating-point type size has the same size than " + "floating-point type size">, + InGroup>; + // Parse && Lex let CategoryName = "Lexical or Preprocessor Issue" in { diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h index 08fc706e3cbf..24b109e32cdd 100644 --- a/clang/include/clang/Basic/LangOptions.h +++ b/clang/include/clang/Basic/LangOptions.h @@ -396,7 +396,38 @@ public: IncompleteOnly = 3, }; - enum ComplexRangeKind { CX_Full, CX_Limited, CX_Fortran, CX_None }; + /// Controls the various implementations for complex multiplication and + // division. + enum ComplexRangeKind { + /// Implementation of complex division and multiplication using a call to + /// runtime library functions(generally the case, but the BE might + /// sometimes replace the library call if it knows enough about the + /// potential range of the inputs). Overflow and non-finite values are + /// handled by the library implementation. This is the default value. + CX_Full, + + /// Implementation of complex division offering an improved handling + /// for overflow in intermediate calculations with no special handling for + /// NaN and infinite values. + CX_Improved, + + /// Implementation of complex division using algebraic formulas at + /// higher precision. Overflow is handled. Non-finite values are handled in + /// some cases. If the target hardware does not have native support for a + /// higher precision data type, an implementation for the complex operation + /// will be used to provide improved guards against intermediate overflow, + /// but overflow and underflow may still occur in some cases. NaN and + /// infinite values are not handled. + CX_Promoted, + + /// Implementation of complex division and multiplication using + /// algebraic formulas at source precision. No special handling to avoid + /// overflow. NaN and infinite values are not handled. + CX_Basic, + + /// No range rule is enabled. + CX_None + }; // Define simple language options (with no accessors). #define LANGOPT(Name, Bits, Default, Description) unsigned Name : Bits; diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 9b5125ecfed8..4a954258ce40 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -1046,30 +1046,29 @@ defm offload_uniform_block : BoolFOption<"offload-uniform-block", NegFlag, BothFlags<[], [ClangOption], " that kernels are launched with uniform block sizes (default true for CUDA/HIP and false otherwise)">>; -def fcx_limited_range : Joined<["-"], "fcx-limited-range">, - Group, Visibility<[ClangOption, CC1Option]>, - HelpText<"Basic algebraic expansions of complex arithmetic operations " - "involving are enabled.">; - -def fno_cx_limited_range : Joined<["-"], "fno-cx-limited-range">, - Group, Visibility<[ClangOption, CC1Option]>, - HelpText<"Basic algebraic expansions of complex arithmetic operations " - "involving are disabled.">; - -def fcx_fortran_rules : Joined<["-"], "fcx-fortran-rules">, - Group, Visibility<[ClangOption, CC1Option]>, - HelpText<"Range reduction is enabled for complex arithmetic operations.">; - -def fno_cx_fortran_rules : Joined<["-"], "fno-cx-fortran-rules">, - Group, Visibility<[ClangOption, CC1Option]>, - HelpText<"Range reduction is disabled for complex arithmetic operations.">; +def fcomplex_arithmetic_EQ : Joined<["-"], "fcomplex-arithmetic=">, Group, + Visibility<[ClangOption, CC1Option]>, + Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">, + NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>; def complex_range_EQ : Joined<["-"], "complex-range=">, Group, Visibility<[CC1Option]>, - Values<"full,limited,fortran">, NormalizedValuesScope<"LangOptions">, - NormalizedValues<["CX_Full", "CX_Limited", "CX_Fortran"]>, + Values<"full,improved,promoted,basic">, NormalizedValuesScope<"LangOptions">, + NormalizedValues<["CX_Full", "CX_Improved", "CX_Promoted", "CX_Basic"]>, MarshallingInfoEnum, "CX_Full">; +defm cx_limited_range: BoolOptionWithoutMarshalling<"f", "cx-limited-range", + PosFlag, + NegFlag>; + +defm cx_fortran_rules: BoolOptionWithoutMarshalling<"f", "cx-fortran-rules", + PosFlag, + NegFlag>; + // OpenCL-only Options def cl_opt_disable : Flag<["-"], "cl-opt-disable">, Group, Visibility<[ClangOption, CC1Option]>, diff --git a/clang/lib/CodeGen/CGExprComplex.cpp b/clang/lib/CodeGen/CGExprComplex.cpp index 0266ba934da6..27ddaacc28f5 100644 --- a/clang/lib/CodeGen/CGExprComplex.cpp +++ b/clang/lib/CodeGen/CGExprComplex.cpp @@ -51,11 +51,12 @@ class ComplexExprEmitter CGBuilderTy &Builder; bool IgnoreReal; bool IgnoreImag; -public: - ComplexExprEmitter(CodeGenFunction &cgf, bool ir=false, bool ii=false) - : CGF(cgf), Builder(CGF.Builder), IgnoreReal(ir), IgnoreImag(ii) { - } + bool FPHasBeenPromoted; +public: + ComplexExprEmitter(CodeGenFunction &cgf, bool ir = false, bool ii = false) + : CGF(cgf), Builder(CGF.Builder), IgnoreReal(ir), IgnoreImag(ii), + FPHasBeenPromoted(false) {} //===--------------------------------------------------------------------===// // Utilities @@ -287,9 +288,54 @@ public: ComplexPairTy EmitComplexBinOpLibCall(StringRef LibCallName, const BinOpInfo &Op); - QualType getPromotionType(QualType Ty) { + QualType GetHigherPrecisionFPType(QualType ElementType) { + const auto *CurrentBT = dyn_cast(ElementType); + switch (CurrentBT->getKind()) { + case BuiltinType::Kind::Float16: + return CGF.getContext().FloatTy; + case BuiltinType::Kind::Float: + case BuiltinType::Kind::BFloat16: + return CGF.getContext().DoubleTy; + case BuiltinType::Kind::Double: + return CGF.getContext().LongDoubleTy; + default: + return ElementType; + } + } + + QualType HigherPrecisionTypeForComplexArithmetic(QualType ElementType, + bool IsDivOpCode) { + QualType HigherElementType = GetHigherPrecisionFPType(ElementType); + const llvm::fltSemantics &ElementTypeSemantics = + CGF.getContext().getFloatTypeSemantics(ElementType); + const llvm::fltSemantics &HigherElementTypeSemantics = + CGF.getContext().getFloatTypeSemantics(HigherElementType); + // Check that the promoted type can handle the intermediate values without + // overflowing. This can be interpreted as: + // (SmallerType.LargestFiniteVal * SmallerType.LargestFiniteVal) * 2 <= + // LargerType.LargestFiniteVal. + // In terms of exponent it gives this formula: + // (SmallerType.LargestFiniteVal * SmallerType.LargestFiniteVal + // doubles the exponent of SmallerType.LargestFiniteVal) + if (llvm::APFloat::semanticsMaxExponent(ElementTypeSemantics) * 2 + 1 <= + llvm::APFloat::semanticsMaxExponent(HigherElementTypeSemantics)) { + return CGF.getContext().getComplexType(HigherElementType); + } else { + FPHasBeenPromoted = true; + DiagnosticsEngine &Diags = CGF.CGM.getDiags(); + Diags.Report(diag::warn_next_larger_fp_type_same_size_than_fp); + return CGF.getContext().getComplexType(ElementType); + } + } + + QualType getPromotionType(QualType Ty, bool IsDivOpCode = false) { if (auto *CT = Ty->getAs()) { QualType ElementType = CT->getElementType(); + if (IsDivOpCode && ElementType->isFloatingType() && + CGF.getLangOpts().getComplexRange() == + LangOptions::ComplexRangeKind::CX_Promoted) + return HigherPrecisionTypeForComplexArithmetic(ElementType, + IsDivOpCode); if (ElementType.UseExcessPrecision(CGF.getContext())) return CGF.getContext().getComplexType(CGF.getContext().FloatTy); } @@ -300,11 +346,12 @@ public: #define HANDLEBINOP(OP) \ ComplexPairTy VisitBin##OP(const BinaryOperator *E) { \ - QualType promotionTy = getPromotionType(E->getType()); \ + QualType promotionTy = getPromotionType( \ + E->getType(), \ + (E->getOpcode() == BinaryOperatorKind::BO_Div) ? true : false); \ ComplexPairTy result = EmitBin##OP(EmitBinOps(E, promotionTy)); \ if (!promotionTy.isNull()) \ - result = \ - CGF.EmitUnPromotedValue(result, E->getType()); \ + result = CGF.EmitUnPromotedValue(result, E->getType()); \ return result; \ } @@ -794,8 +841,9 @@ ComplexPairTy ComplexExprEmitter::EmitBinMul(const BinOpInfo &Op) { ResR = Builder.CreateFSub(AC, BD, "mul_r"); ResI = Builder.CreateFAdd(AD, BC, "mul_i"); - if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Limited || - Op.FPFeatures.getComplexRange() == LangOptions::CX_Fortran) + if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Basic || + Op.FPFeatures.getComplexRange() == LangOptions::CX_Improved || + Op.FPFeatures.getComplexRange() == LangOptions::CX_Promoted) return ComplexPairTy(ResR, ResI); // Emit the test for the real part becoming NaN and create a branch to @@ -986,14 +1034,17 @@ ComplexPairTy ComplexExprEmitter::EmitBinDiv(const BinOpInfo &Op) { llvm::Value *OrigLHSi = LHSi; if (!LHSi) LHSi = llvm::Constant::getNullValue(RHSi->getType()); - if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Fortran) + QualType ComplexElementTy = Op.Ty->castAs()->getElementType(); + if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Improved || + (Op.FPFeatures.getComplexRange() == LangOptions::CX_Promoted && + FPHasBeenPromoted)) return EmitRangeReductionDiv(LHSr, LHSi, RHSr, RHSi); - else if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Limited) + else if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Basic || + Op.FPFeatures.getComplexRange() == LangOptions::CX_Promoted) return EmitAlgebraicDiv(LHSr, LHSi, RHSr, RHSi); - else if (!CGF.getLangOpts().FastMath || - // '-ffast-math' is used in the command line but followed by an - // '-fno-cx-limited-range'. - Op.FPFeatures.getComplexRange() == LangOptions::CX_Full) { + // '-ffast-math' is used in the command line but followed by an + // '-fno-cx-limited-range' or '-fcomplex-arithmetic=full'. + else if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Full) { LHSi = OrigLHSi; // If we have a complex operand on the RHS and FastMath is not allowed, we // delegate to a libcall to handle all of the complexities and minimize diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index bcdf2737bc7a..bc9cc8ce6cf5 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -2687,60 +2687,43 @@ static void CollectArgsForIntegratedAssembler(Compilation &C, } } -static StringRef EnumComplexRangeToStr(LangOptions::ComplexRangeKind Range, - StringRef Option) { +static std::string ComplexRangeKindToStr(LangOptions::ComplexRangeKind Range) { switch (Range) { - case LangOptions::ComplexRangeKind::CX_Limited: - return "-fcx-limited-range"; + case LangOptions::ComplexRangeKind::CX_Full: + return "full"; break; - case LangOptions::ComplexRangeKind::CX_Fortran: - return "-fcx-fortran-rules"; + case LangOptions::ComplexRangeKind::CX_Basic: + return "basic"; break; - default: - return Option; + case LangOptions::ComplexRangeKind::CX_Improved: + return "improved"; + break; + case LangOptions::ComplexRangeKind::CX_Promoted: + return "promoted"; break; + default: + return ""; } } -static void EmitComplexRangeDiag(const Driver &D, - LangOptions::ComplexRangeKind Range1, - LangOptions::ComplexRangeKind Range2, - StringRef Option = StringRef()) { - if (Range1 != Range2 && Range1 != LangOptions::ComplexRangeKind::CX_None) { - bool NegateFortranOption = false; - bool NegateLimitedOption = false; - if (!Option.empty()) { - NegateFortranOption = - Range1 == LangOptions::ComplexRangeKind::CX_Fortran && - Option == "-fno-cx-fortran-rules"; - NegateLimitedOption = - Range1 == LangOptions::ComplexRangeKind::CX_Limited && - Option == "-fno-cx-limited-range"; - } - if (Option.empty() || - (!Option.empty() && !NegateFortranOption && !NegateLimitedOption)) - D.Diag(clang::diag::warn_drv_overriding_option) - << EnumComplexRangeToStr(Range1, Option) - << EnumComplexRangeToStr(Range2, Option); +static std::string ComplexArithmeticStr(LangOptions::ComplexRangeKind Range) { + return (Range == LangOptions::ComplexRangeKind::CX_None) + ? "" + : "-fcomplex-arithmetic=" + ComplexRangeKindToStr(Range); +} + +static void EmitComplexRangeDiag(const Driver &D, std::string str1, + std::string str2) { + if ((str1.compare(str2) != 0) && !str2.empty() && !str1.empty()) { + D.Diag(clang::diag::warn_drv_overriding_option) << str1 << str2; } } static std::string RenderComplexRangeOption(LangOptions::ComplexRangeKind Range) { - std::string ComplexRangeStr = "-complex-range="; - switch (Range) { - case LangOptions::ComplexRangeKind::CX_Full: - ComplexRangeStr += "full"; - break; - case LangOptions::ComplexRangeKind::CX_Limited: - ComplexRangeStr += "limited"; - break; - case LangOptions::ComplexRangeKind::CX_Fortran: - ComplexRangeStr += "fortran"; - break; - default: - assert(0 && "Unexpected range option"); - } + std::string ComplexRangeStr = ComplexRangeKindToStr(Range); + if (!ComplexRangeStr.empty()) + return "-complex-range=" + ComplexRangeStr; return ComplexRangeStr; } @@ -2792,6 +2775,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, StringRef BFloat16ExcessPrecision = ""; LangOptions::ComplexRangeKind Range = LangOptions::ComplexRangeKind::CX_None; std::string ComplexRangeStr = ""; + std::string GccRangeComplexOption = ""; // Lambda to set fast-math options. This is also used by -ffp-model=fast auto applyFastMath = [&]() { @@ -2807,9 +2791,19 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, FPExceptionBehavior = ""; // If fast-math is set then set the fp-contract mode to fast. FPContract = "fast"; - // ffast-math enables limited range rules for complex multiplication and + // ffast-math enables basic range rules for complex multiplication and // division. - Range = LangOptions::ComplexRangeKind::CX_Limited; + // Warn if user expects to perform full implementation of complex + // multiplication or division in the presence of nan or ninf flags. + if (Range == LangOptions::ComplexRangeKind::CX_Full || + Range == LangOptions::ComplexRangeKind::CX_Improved || + Range == LangOptions::ComplexRangeKind::CX_Promoted) + EmitComplexRangeDiag( + D, ComplexArithmeticStr(Range), + !GccRangeComplexOption.empty() + ? GccRangeComplexOption + : ComplexArithmeticStr(LangOptions::ComplexRangeKind::CX_Basic)); + Range = LangOptions::ComplexRangeKind::CX_Basic; SeenUnsafeMathModeOption = true; }; @@ -2824,26 +2818,87 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, switch (optID) { default: break; - case options::OPT_fcx_limited_range: { - EmitComplexRangeDiag(D, Range, LangOptions::ComplexRangeKind::CX_Limited); - Range = LangOptions::ComplexRangeKind::CX_Limited; + case options::OPT_fcx_limited_range: + if (GccRangeComplexOption.empty()) { + if (Range != LangOptions::ComplexRangeKind::CX_Basic) + EmitComplexRangeDiag(D, RenderComplexRangeOption(Range), + "-fcx-limited-range"); + } else { + if (GccRangeComplexOption != "-fno-cx-limited-range") + EmitComplexRangeDiag(D, GccRangeComplexOption, "-fcx-limited-range"); + } + GccRangeComplexOption = "-fcx-limited-range"; + Range = LangOptions::ComplexRangeKind::CX_Basic; break; - } case options::OPT_fno_cx_limited_range: - EmitComplexRangeDiag(D, Range, LangOptions::ComplexRangeKind::CX_Full, - "-fno-cx-limited-range"); + if (GccRangeComplexOption.empty()) { + EmitComplexRangeDiag(D, RenderComplexRangeOption(Range), + "-fno-cx-limited-range"); + } else { + if (GccRangeComplexOption.compare("-fcx-limited-range") != 0 && + GccRangeComplexOption.compare("-fno-cx-fortran-rules") != 0) + EmitComplexRangeDiag(D, GccRangeComplexOption, + "-fno-cx-limited-range"); + } + GccRangeComplexOption = "-fno-cx-limited-range"; Range = LangOptions::ComplexRangeKind::CX_Full; break; - case options::OPT_fcx_fortran_rules: { - EmitComplexRangeDiag(D, Range, LangOptions::ComplexRangeKind::CX_Fortran); - Range = LangOptions::ComplexRangeKind::CX_Fortran; + case options::OPT_fcx_fortran_rules: + if (GccRangeComplexOption.empty()) + EmitComplexRangeDiag(D, RenderComplexRangeOption(Range), + "-fcx-fortran-rules"); + else + EmitComplexRangeDiag(D, GccRangeComplexOption, "-fcx-fortran-rules"); + GccRangeComplexOption = "-fcx-fortran-rules"; + Range = LangOptions::ComplexRangeKind::CX_Improved; break; - } case options::OPT_fno_cx_fortran_rules: - EmitComplexRangeDiag(D, Range, LangOptions::ComplexRangeKind::CX_Full, - "-fno-cx-fortran-rules"); + if (GccRangeComplexOption.empty()) { + EmitComplexRangeDiag(D, RenderComplexRangeOption(Range), + "-fno-cx-fortran-rules"); + } else { + if (GccRangeComplexOption != "-fno-cx-limited-range") + EmitComplexRangeDiag(D, GccRangeComplexOption, + "-fno-cx-fortran-rules"); + } + GccRangeComplexOption = "-fno-cx-fortran-rules"; Range = LangOptions::ComplexRangeKind::CX_Full; break; + case options::OPT_fcomplex_arithmetic_EQ: { + LangOptions::ComplexRangeKind RangeVal; + StringRef Val = A->getValue(); + if (Val.equals("full")) + RangeVal = LangOptions::ComplexRangeKind::CX_Full; + else if (Val.equals("improved")) + RangeVal = LangOptions::ComplexRangeKind::CX_Improved; + else if (Val.equals("promoted")) + RangeVal = LangOptions::ComplexRangeKind::CX_Promoted; + else if (Val.equals("basic")) + RangeVal = LangOptions::ComplexRangeKind::CX_Basic; + else { + D.Diag(diag::err_drv_unsupported_option_argument) + << A->getSpelling() << Val; + break; + } + if (!GccRangeComplexOption.empty()) { + if (GccRangeComplexOption.compare("-fcx-limited-range") != 0) { + if (GccRangeComplexOption.compare("-fcx-fortran-rules") != 0) { + if (RangeVal != LangOptions::ComplexRangeKind::CX_Improved) + EmitComplexRangeDiag(D, GccRangeComplexOption, + ComplexArithmeticStr(RangeVal)); + } else { + EmitComplexRangeDiag(D, GccRangeComplexOption, + ComplexArithmeticStr(RangeVal)); + } + } else { + if (RangeVal != LangOptions::ComplexRangeKind::CX_Basic) + EmitComplexRangeDiag(D, GccRangeComplexOption, + ComplexArithmeticStr(RangeVal)); + } + } + Range = RangeVal; + break; + } case options::OPT_ffp_model_EQ: { // If -ffp-model= is seen, reset to fno-fast-math HonorINFs = true; @@ -3256,8 +3311,12 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, if (Range != LangOptions::ComplexRangeKind::CX_None) ComplexRangeStr = RenderComplexRangeOption(Range); - if (!ComplexRangeStr.empty()) + if (!ComplexRangeStr.empty()) { CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr)); + if (Args.hasArg(options::OPT_fcomplex_arithmetic_EQ)) + CmdArgs.push_back(Args.MakeArgString("-fcomplex-arithmetic=" + + ComplexRangeKindToStr(Range))); + } if (Args.hasArg(options::OPT_fcx_limited_range)) CmdArgs.push_back("-fcx-limited-range"); if (Args.hasArg(options::OPT_fcx_fortran_rules)) diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp index 730ac1a0fee5..0f692e2146a4 100644 --- a/clang/lib/Parse/ParsePragma.cpp +++ b/clang/lib/Parse/ParsePragma.cpp @@ -844,6 +844,11 @@ void Parser::HandlePragmaFPContract() { FPC = LangOptions::FPM_Off; break; case tok::OOS_DEFAULT: + // According to ISO C99 standard chapter 7.3.4, the default value + // for the pragma is ``off'. '-fcomplex-arithmetic=basic', + // '-fcx-limited-range', '-fcx-fortran-rules' and + // '-fcomplex-arithmetic=improved' control the default value of these + // pragmas. FPC = getLangOpts().getDefaultFPContractMode(); break; } @@ -909,15 +914,15 @@ void Parser::HandlePragmaCXLimitedRange() { LangOptions::ComplexRangeKind Range; switch (OOS) { case tok::OOS_ON: - Range = LangOptions::CX_Limited; + Range = LangOptions::CX_Basic; break; case tok::OOS_OFF: Range = LangOptions::CX_Full; break; case tok::OOS_DEFAULT: // According to ISO C99 standard chapter 7.3.4, the default value - // for the pragma is ``off'. -fcx-limited-range and -fcx-fortran-rules - // control the default value of these pragmas. + // for the pragma is ``off'. -fcomplex-arithmetic controls the default value + // of these pragmas. Range = getLangOpts().getComplexRange(); break; } diff --git a/clang/test/CodeGen/X86/cx-complex-range.c b/clang/test/CodeGen/X86/cx-complex-range.c new file mode 100644 index 000000000000..fa46576266a2 --- /dev/null +++ b/clang/test/CodeGen/X86/cx-complex-range.c @@ -0,0 +1,1425 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4 +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -o - | FileCheck %s --check-prefix=FULL + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -complex-range=basic -o - | FileCheck %s --check-prefix=BASIC + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -fno-cx-limited-range -o - | FileCheck %s --check-prefix=FULL + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -complex-range=improved -o - | FileCheck %s --check-prefix=IMPRVD + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -complex-range=promoted -o - | FileCheck %s --check-prefix=PRMTD + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -complex-range=full -o - | FileCheck %s --check-prefix=FULL + +// RUN: %clang_cc1 -triple x86_64-windows-pc -complex-range=promoted \ +// RUN: -emit-llvm -o - %s | FileCheck %s --check-prefix=X86WINPRMTD + +// Fast math +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu \ +// RUN: -ffast-math -complex-range=basic -emit-llvm -o - %s \ +// RUN: | FileCheck %s --check-prefix=BASIC_FAST + +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu \ +// RUN: -ffast-math -complex-range=full -emit-llvm -o - %s \ +// RUN: | FileCheck %s --check-prefix=FULL_FAST + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -fno-cx-fortran-rules -o - | FileCheck %s --check-prefix=FULL + +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu \ +// RUN: -ffast-math -complex-range=improved -emit-llvm -o - %s \ +// RUN: | FileCheck %s --check-prefix=IMPRVD_FAST + +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu \ +// RUN: -ffast-math -complex-range=promoted -emit-llvm -o - %s \ +// RUN: | FileCheck %s --check-prefix=PRMTD_FAST + +// FULL-LABEL: define dso_local <2 x half> @divf16( +// FULL-SAME: <2 x half> noundef [[A_COERCE:%.*]], <2 x half> noundef [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// FULL-NEXT: entry: +// FULL-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// FULL-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// FULL-NEXT: [[B:%.*]] = alloca { half, half }, align 2 +// FULL-NEXT: [[COERCE:%.*]] = alloca { float, float }, align 4 +// FULL-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// FULL-NEXT: store <2 x half> [[B_COERCE]], ptr [[B]], align 2 +// FULL-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// FULL-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// FULL-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// FULL-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// FULL-NEXT: [[EXT:%.*]] = fpext half [[A_REAL]] to float +// FULL-NEXT: [[EXT1:%.*]] = fpext half [[A_IMAG]] to float +// FULL-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 0 +// FULL-NEXT: [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2 +// FULL-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 1 +// FULL-NEXT: [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2 +// FULL-NEXT: [[EXT2:%.*]] = fpext half [[B_REAL]] to float +// FULL-NEXT: [[EXT3:%.*]] = fpext half [[B_IMAG]] to float +// FULL-NEXT: [[CALL:%.*]] = call <2 x float> @__divsc3(float noundef [[EXT]], float noundef [[EXT1]], float noundef [[EXT2]], float noundef [[EXT3]]) #[[ATTR1:[0-9]+]] +// FULL-NEXT: store <2 x float> [[CALL]], ptr [[COERCE]], align 4 +// FULL-NEXT: [[COERCE_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 0 +// FULL-NEXT: [[COERCE_REAL:%.*]] = load float, ptr [[COERCE_REALP]], align 4 +// FULL-NEXT: [[COERCE_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 1 +// FULL-NEXT: [[COERCE_IMAG:%.*]] = load float, ptr [[COERCE_IMAGP]], align 4 +// FULL-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[COERCE_REAL]] to half +// FULL-NEXT: [[UNPROMOTION4:%.*]] = fptrunc float [[COERCE_IMAG]] to half +// FULL-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// FULL-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// FULL-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// FULL-NEXT: store half [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 2 +// FULL-NEXT: [[TMP0:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// FULL-NEXT: ret <2 x half> [[TMP0]] +// +// BASIC-LABEL: define dso_local <2 x half> @divf16( +// BASIC-SAME: <2 x half> noundef [[A_COERCE:%.*]], <2 x half> noundef [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// BASIC-NEXT: entry: +// BASIC-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// BASIC-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// BASIC-NEXT: [[B:%.*]] = alloca { half, half }, align 2 +// BASIC-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// BASIC-NEXT: store <2 x half> [[B_COERCE]], ptr [[B]], align 2 +// BASIC-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// BASIC-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// BASIC-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// BASIC-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// BASIC-NEXT: [[EXT:%.*]] = fpext half [[A_REAL]] to float +// BASIC-NEXT: [[EXT1:%.*]] = fpext half [[A_IMAG]] to float +// BASIC-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 0 +// BASIC-NEXT: [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2 +// BASIC-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 1 +// BASIC-NEXT: [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2 +// BASIC-NEXT: [[EXT2:%.*]] = fpext half [[B_REAL]] to float +// BASIC-NEXT: [[EXT3:%.*]] = fpext half [[B_IMAG]] to float +// BASIC-NEXT: [[TMP0:%.*]] = fmul float [[EXT]], [[EXT2]] +// BASIC-NEXT: [[TMP1:%.*]] = fmul float [[EXT1]], [[EXT3]] +// BASIC-NEXT: [[TMP2:%.*]] = fadd float [[TMP0]], [[TMP1]] +// BASIC-NEXT: [[TMP3:%.*]] = fmul float [[EXT2]], [[EXT2]] +// BASIC-NEXT: [[TMP4:%.*]] = fmul float [[EXT3]], [[EXT3]] +// BASIC-NEXT: [[TMP5:%.*]] = fadd float [[TMP3]], [[TMP4]] +// BASIC-NEXT: [[TMP6:%.*]] = fmul float [[EXT1]], [[EXT2]] +// BASIC-NEXT: [[TMP7:%.*]] = fmul float [[EXT]], [[EXT3]] +// BASIC-NEXT: [[TMP8:%.*]] = fsub float [[TMP6]], [[TMP7]] +// BASIC-NEXT: [[TMP9:%.*]] = fdiv float [[TMP2]], [[TMP5]] +// BASIC-NEXT: [[TMP10:%.*]] = fdiv float [[TMP8]], [[TMP5]] +// BASIC-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[TMP9]] to half +// BASIC-NEXT: [[UNPROMOTION4:%.*]] = fptrunc float [[TMP10]] to half +// BASIC-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// BASIC-NEXT: store half [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 2 +// BASIC-NEXT: [[TMP11:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// BASIC-NEXT: ret <2 x half> [[TMP11]] +// +// IMPRVD-LABEL: define dso_local <2 x half> @divf16( +// IMPRVD-SAME: <2 x half> noundef [[A_COERCE:%.*]], <2 x half> noundef [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// IMPRVD-NEXT: entry: +// IMPRVD-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// IMPRVD-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// IMPRVD-NEXT: [[B:%.*]] = alloca { half, half }, align 2 +// IMPRVD-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// IMPRVD-NEXT: store <2 x half> [[B_COERCE]], ptr [[B]], align 2 +// IMPRVD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// IMPRVD-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// IMPRVD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// IMPRVD-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// IMPRVD-NEXT: [[EXT:%.*]] = fpext half [[A_REAL]] to float +// IMPRVD-NEXT: [[EXT1:%.*]] = fpext half [[A_IMAG]] to float +// IMPRVD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 0 +// IMPRVD-NEXT: [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2 +// IMPRVD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 1 +// IMPRVD-NEXT: [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2 +// IMPRVD-NEXT: [[EXT2:%.*]] = fpext half [[B_REAL]] to float +// IMPRVD-NEXT: [[EXT3:%.*]] = fpext half [[B_IMAG]] to float +// IMPRVD-NEXT: [[TMP0:%.*]] = call float @llvm.fabs.f32(float [[EXT2]]) +// IMPRVD-NEXT: [[TMP1:%.*]] = call float @llvm.fabs.f32(float [[EXT3]]) +// IMPRVD-NEXT: [[ABS_CMP:%.*]] = fcmp ugt float [[TMP0]], [[TMP1]] +// IMPRVD-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// IMPRVD: abs_rhsr_greater_or_equal_abs_rhsi: +// IMPRVD-NEXT: [[TMP2:%.*]] = fdiv float [[EXT3]], [[EXT2]] +// IMPRVD-NEXT: [[TMP3:%.*]] = fmul float [[TMP2]], [[EXT3]] +// IMPRVD-NEXT: [[TMP4:%.*]] = fadd float [[EXT2]], [[TMP3]] +// IMPRVD-NEXT: [[TMP5:%.*]] = fmul float [[EXT1]], [[TMP2]] +// IMPRVD-NEXT: [[TMP6:%.*]] = fadd float [[EXT]], [[TMP5]] +// IMPRVD-NEXT: [[TMP7:%.*]] = fdiv float [[TMP6]], [[TMP4]] +// IMPRVD-NEXT: [[TMP8:%.*]] = fmul float [[EXT]], [[TMP2]] +// IMPRVD-NEXT: [[TMP9:%.*]] = fsub float [[EXT1]], [[TMP8]] +// IMPRVD-NEXT: [[TMP10:%.*]] = fdiv float [[TMP9]], [[TMP4]] +// IMPRVD-NEXT: br label [[COMPLEX_DIV:%.*]] +// IMPRVD: abs_rhsr_less_than_abs_rhsi: +// IMPRVD-NEXT: [[TMP11:%.*]] = fdiv float [[EXT2]], [[EXT3]] +// IMPRVD-NEXT: [[TMP12:%.*]] = fmul float [[TMP11]], [[EXT2]] +// IMPRVD-NEXT: [[TMP13:%.*]] = fadd float [[EXT3]], [[TMP12]] +// IMPRVD-NEXT: [[TMP14:%.*]] = fmul float [[EXT]], [[TMP11]] +// IMPRVD-NEXT: [[TMP15:%.*]] = fadd float [[TMP14]], [[EXT1]] +// IMPRVD-NEXT: [[TMP16:%.*]] = fdiv float [[TMP15]], [[TMP13]] +// IMPRVD-NEXT: [[TMP17:%.*]] = fmul float [[EXT1]], [[TMP11]] +// IMPRVD-NEXT: [[TMP18:%.*]] = fsub float [[TMP17]], [[EXT]] +// IMPRVD-NEXT: [[TMP19:%.*]] = fdiv float [[TMP18]], [[TMP13]] +// IMPRVD-NEXT: br label [[COMPLEX_DIV]] +// IMPRVD: complex_div: +// IMPRVD-NEXT: [[TMP20:%.*]] = phi float [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD-NEXT: [[TMP21:%.*]] = phi float [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[TMP20]] to half +// IMPRVD-NEXT: [[UNPROMOTION4:%.*]] = fptrunc float [[TMP21]] to half +// IMPRVD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// IMPRVD-NEXT: store half [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 2 +// IMPRVD-NEXT: [[TMP22:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// IMPRVD-NEXT: ret <2 x half> [[TMP22]] +// +// PRMTD-LABEL: define dso_local <2 x half> @divf16( +// PRMTD-SAME: <2 x half> noundef [[A_COERCE:%.*]], <2 x half> noundef [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// PRMTD-NEXT: entry: +// PRMTD-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// PRMTD-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// PRMTD-NEXT: [[B:%.*]] = alloca { half, half }, align 2 +// PRMTD-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// PRMTD-NEXT: store <2 x half> [[B_COERCE]], ptr [[B]], align 2 +// PRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// PRMTD-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// PRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// PRMTD-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// PRMTD-NEXT: [[EXT:%.*]] = fpext half [[A_REAL]] to float +// PRMTD-NEXT: [[EXT1:%.*]] = fpext half [[A_IMAG]] to float +// PRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 0 +// PRMTD-NEXT: [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2 +// PRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 1 +// PRMTD-NEXT: [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2 +// PRMTD-NEXT: [[EXT2:%.*]] = fpext half [[B_REAL]] to float +// PRMTD-NEXT: [[EXT3:%.*]] = fpext half [[B_IMAG]] to float +// PRMTD-NEXT: [[TMP0:%.*]] = fmul float [[EXT]], [[EXT2]] +// PRMTD-NEXT: [[TMP1:%.*]] = fmul float [[EXT1]], [[EXT3]] +// PRMTD-NEXT: [[TMP2:%.*]] = fadd float [[TMP0]], [[TMP1]] +// PRMTD-NEXT: [[TMP3:%.*]] = fmul float [[EXT2]], [[EXT2]] +// PRMTD-NEXT: [[TMP4:%.*]] = fmul float [[EXT3]], [[EXT3]] +// PRMTD-NEXT: [[TMP5:%.*]] = fadd float [[TMP3]], [[TMP4]] +// PRMTD-NEXT: [[TMP6:%.*]] = fmul float [[EXT1]], [[EXT2]] +// PRMTD-NEXT: [[TMP7:%.*]] = fmul float [[EXT]], [[EXT3]] +// PRMTD-NEXT: [[TMP8:%.*]] = fsub float [[TMP6]], [[TMP7]] +// PRMTD-NEXT: [[TMP9:%.*]] = fdiv float [[TMP2]], [[TMP5]] +// PRMTD-NEXT: [[TMP10:%.*]] = fdiv float [[TMP8]], [[TMP5]] +// PRMTD-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[TMP9]] to half +// PRMTD-NEXT: [[UNPROMOTION4:%.*]] = fptrunc float [[TMP10]] to half +// PRMTD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// PRMTD-NEXT: store half [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 2 +// PRMTD-NEXT: [[TMP11:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// PRMTD-NEXT: ret <2 x half> [[TMP11]] +// +// X86WINPRMTD-LABEL: define dso_local i32 @divf16( +// X86WINPRMTD-SAME: i32 noundef [[A_COERCE:%.*]], i32 noundef [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// X86WINPRMTD-NEXT: entry: +// X86WINPRMTD-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// X86WINPRMTD-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// X86WINPRMTD-NEXT: [[B:%.*]] = alloca { half, half }, align 2 +// X86WINPRMTD-NEXT: store i32 [[A_COERCE]], ptr [[A]], align 2 +// X86WINPRMTD-NEXT: store i32 [[B_COERCE]], ptr [[B]], align 2 +// X86WINPRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// X86WINPRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// X86WINPRMTD-NEXT: [[EXT:%.*]] = fpext half [[A_REAL]] to float +// X86WINPRMTD-NEXT: [[EXT1:%.*]] = fpext half [[A_IMAG]] to float +// X86WINPRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2 +// X86WINPRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2 +// X86WINPRMTD-NEXT: [[EXT2:%.*]] = fpext half [[B_REAL]] to float +// X86WINPRMTD-NEXT: [[EXT3:%.*]] = fpext half [[B_IMAG]] to float +// X86WINPRMTD-NEXT: [[TMP0:%.*]] = fmul float [[EXT]], [[EXT2]] +// X86WINPRMTD-NEXT: [[TMP1:%.*]] = fmul float [[EXT1]], [[EXT3]] +// X86WINPRMTD-NEXT: [[TMP2:%.*]] = fadd float [[TMP0]], [[TMP1]] +// X86WINPRMTD-NEXT: [[TMP3:%.*]] = fmul float [[EXT2]], [[EXT2]] +// X86WINPRMTD-NEXT: [[TMP4:%.*]] = fmul float [[EXT3]], [[EXT3]] +// X86WINPRMTD-NEXT: [[TMP5:%.*]] = fadd float [[TMP3]], [[TMP4]] +// X86WINPRMTD-NEXT: [[TMP6:%.*]] = fmul float [[EXT1]], [[EXT2]] +// X86WINPRMTD-NEXT: [[TMP7:%.*]] = fmul float [[EXT]], [[EXT3]] +// X86WINPRMTD-NEXT: [[TMP8:%.*]] = fsub float [[TMP6]], [[TMP7]] +// X86WINPRMTD-NEXT: [[TMP9:%.*]] = fdiv float [[TMP2]], [[TMP5]] +// X86WINPRMTD-NEXT: [[TMP10:%.*]] = fdiv float [[TMP8]], [[TMP5]] +// X86WINPRMTD-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[TMP9]] to half +// X86WINPRMTD-NEXT: [[UNPROMOTION4:%.*]] = fptrunc float [[TMP10]] to half +// X86WINPRMTD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// X86WINPRMTD-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// X86WINPRMTD-NEXT: store half [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 2 +// X86WINPRMTD-NEXT: [[TMP11:%.*]] = load i32, ptr [[RETVAL]], align 2 +// X86WINPRMTD-NEXT: ret i32 [[TMP11]] +// +// BASIC_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x half> @divf16( +// BASIC_FAST-SAME: <2 x half> noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x half> noundef nofpclass(nan inf) [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// BASIC_FAST-NEXT: entry: +// BASIC_FAST-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// BASIC_FAST-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// BASIC_FAST-NEXT: [[B:%.*]] = alloca { half, half }, align 2 +// BASIC_FAST-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// BASIC_FAST-NEXT: store <2 x half> [[B_COERCE]], ptr [[B]], align 2 +// BASIC_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// BASIC_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// BASIC_FAST-NEXT: [[EXT:%.*]] = fpext half [[A_REAL]] to float +// BASIC_FAST-NEXT: [[EXT1:%.*]] = fpext half [[A_IMAG]] to float +// BASIC_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2 +// BASIC_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2 +// BASIC_FAST-NEXT: [[EXT2:%.*]] = fpext half [[B_REAL]] to float +// BASIC_FAST-NEXT: [[EXT3:%.*]] = fpext half [[B_IMAG]] to float +// BASIC_FAST-NEXT: [[TMP0:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT2]] +// BASIC_FAST-NEXT: [[TMP1:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT3]] +// BASIC_FAST-NEXT: [[TMP2:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP0]], [[TMP1]] +// BASIC_FAST-NEXT: [[TMP3:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT2]], [[EXT2]] +// BASIC_FAST-NEXT: [[TMP4:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT3]], [[EXT3]] +// BASIC_FAST-NEXT: [[TMP5:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP3]], [[TMP4]] +// BASIC_FAST-NEXT: [[TMP6:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT2]] +// BASIC_FAST-NEXT: [[TMP7:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT3]] +// BASIC_FAST-NEXT: [[TMP8:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[TMP6]], [[TMP7]] +// BASIC_FAST-NEXT: [[TMP9:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP2]], [[TMP5]] +// BASIC_FAST-NEXT: [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP8]], [[TMP5]] +// BASIC_FAST-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[TMP9]] to half +// BASIC_FAST-NEXT: [[UNPROMOTION4:%.*]] = fptrunc float [[TMP10]] to half +// BASIC_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC_FAST-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// BASIC_FAST-NEXT: store half [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 2 +// BASIC_FAST-NEXT: [[TMP11:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// BASIC_FAST-NEXT: ret <2 x half> [[TMP11]] +// +// FULL_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x half> @divf16( +// FULL_FAST-SAME: <2 x half> noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x half> noundef nofpclass(nan inf) [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// FULL_FAST-NEXT: entry: +// FULL_FAST-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// FULL_FAST-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// FULL_FAST-NEXT: [[B:%.*]] = alloca { half, half }, align 2 +// FULL_FAST-NEXT: [[COERCE:%.*]] = alloca { float, float }, align 4 +// FULL_FAST-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// FULL_FAST-NEXT: store <2 x half> [[B_COERCE]], ptr [[B]], align 2 +// FULL_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// FULL_FAST-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// FULL_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// FULL_FAST-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// FULL_FAST-NEXT: [[EXT:%.*]] = fpext half [[A_REAL]] to float +// FULL_FAST-NEXT: [[EXT1:%.*]] = fpext half [[A_IMAG]] to float +// FULL_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 0 +// FULL_FAST-NEXT: [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2 +// FULL_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 1 +// FULL_FAST-NEXT: [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2 +// FULL_FAST-NEXT: [[EXT2:%.*]] = fpext half [[B_REAL]] to float +// FULL_FAST-NEXT: [[EXT3:%.*]] = fpext half [[B_IMAG]] to float +// FULL_FAST-NEXT: [[CALL:%.*]] = call reassoc nnan ninf nsz arcp afn nofpclass(nan inf) <2 x float> @__divsc3(float noundef nofpclass(nan inf) [[EXT]], float noundef nofpclass(nan inf) [[EXT1]], float noundef nofpclass(nan inf) [[EXT2]], float noundef nofpclass(nan inf) [[EXT3]]) #[[ATTR1:[0-9]+]] +// FULL_FAST-NEXT: store <2 x float> [[CALL]], ptr [[COERCE]], align 4 +// FULL_FAST-NEXT: [[COERCE_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 0 +// FULL_FAST-NEXT: [[COERCE_REAL:%.*]] = load float, ptr [[COERCE_REALP]], align 4 +// FULL_FAST-NEXT: [[COERCE_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 1 +// FULL_FAST-NEXT: [[COERCE_IMAG:%.*]] = load float, ptr [[COERCE_IMAGP]], align 4 +// FULL_FAST-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[COERCE_REAL]] to half +// FULL_FAST-NEXT: [[UNPROMOTION4:%.*]] = fptrunc float [[COERCE_IMAG]] to half +// FULL_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// FULL_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// FULL_FAST-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// FULL_FAST-NEXT: store half [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 2 +// FULL_FAST-NEXT: [[TMP0:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// FULL_FAST-NEXT: ret <2 x half> [[TMP0]] +// +// IMPRVD_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x half> @divf16( +// IMPRVD_FAST-SAME: <2 x half> noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x half> noundef nofpclass(nan inf) [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// IMPRVD_FAST-NEXT: entry: +// IMPRVD_FAST-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// IMPRVD_FAST-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// IMPRVD_FAST-NEXT: [[B:%.*]] = alloca { half, half }, align 2 +// IMPRVD_FAST-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// IMPRVD_FAST-NEXT: store <2 x half> [[B_COERCE]], ptr [[B]], align 2 +// IMPRVD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// IMPRVD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// IMPRVD_FAST-NEXT: [[EXT:%.*]] = fpext half [[A_REAL]] to float +// IMPRVD_FAST-NEXT: [[EXT1:%.*]] = fpext half [[A_IMAG]] to float +// IMPRVD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2 +// IMPRVD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2 +// IMPRVD_FAST-NEXT: [[EXT2:%.*]] = fpext half [[B_REAL]] to float +// IMPRVD_FAST-NEXT: [[EXT3:%.*]] = fpext half [[B_IMAG]] to float +// IMPRVD_FAST-NEXT: [[TMP0:%.*]] = call reassoc nnan ninf nsz arcp afn float @llvm.fabs.f32(float [[EXT2]]) +// IMPRVD_FAST-NEXT: [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn float @llvm.fabs.f32(float [[EXT3]]) +// IMPRVD_FAST-NEXT: [[ABS_CMP:%.*]] = fcmp reassoc nnan ninf nsz arcp afn ugt float [[TMP0]], [[TMP1]] +// IMPRVD_FAST-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// IMPRVD_FAST: abs_rhsr_greater_or_equal_abs_rhsi: +// IMPRVD_FAST-NEXT: [[TMP2:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[EXT3]], [[EXT2]] +// IMPRVD_FAST-NEXT: [[TMP3:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[TMP2]], [[EXT3]] +// IMPRVD_FAST-NEXT: [[TMP4:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[EXT2]], [[TMP3]] +// IMPRVD_FAST-NEXT: [[TMP5:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[TMP2]] +// IMPRVD_FAST-NEXT: [[TMP6:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[EXT]], [[TMP5]] +// IMPRVD_FAST-NEXT: [[TMP7:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP6]], [[TMP4]] +// IMPRVD_FAST-NEXT: [[TMP8:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[TMP2]] +// IMPRVD_FAST-NEXT: [[TMP9:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[EXT1]], [[TMP8]] +// IMPRVD_FAST-NEXT: [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP9]], [[TMP4]] +// IMPRVD_FAST-NEXT: br label [[COMPLEX_DIV:%.*]] +// IMPRVD_FAST: abs_rhsr_less_than_abs_rhsi: +// IMPRVD_FAST-NEXT: [[TMP11:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[EXT2]], [[EXT3]] +// IMPRVD_FAST-NEXT: [[TMP12:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[TMP11]], [[EXT2]] +// IMPRVD_FAST-NEXT: [[TMP13:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[EXT3]], [[TMP12]] +// IMPRVD_FAST-NEXT: [[TMP14:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[TMP11]] +// IMPRVD_FAST-NEXT: [[TMP15:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP14]], [[EXT1]] +// IMPRVD_FAST-NEXT: [[TMP16:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP15]], [[TMP13]] +// IMPRVD_FAST-NEXT: [[TMP17:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[TMP11]] +// IMPRVD_FAST-NEXT: [[TMP18:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[TMP17]], [[EXT]] +// IMPRVD_FAST-NEXT: [[TMP19:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP18]], [[TMP13]] +// IMPRVD_FAST-NEXT: br label [[COMPLEX_DIV]] +// IMPRVD_FAST: complex_div: +// IMPRVD_FAST-NEXT: [[TMP20:%.*]] = phi reassoc nnan ninf nsz arcp afn float [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD_FAST-NEXT: [[TMP21:%.*]] = phi reassoc nnan ninf nsz arcp afn float [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD_FAST-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[TMP20]] to half +// IMPRVD_FAST-NEXT: [[UNPROMOTION4:%.*]] = fptrunc float [[TMP21]] to half +// IMPRVD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// IMPRVD_FAST-NEXT: store half [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 2 +// IMPRVD_FAST-NEXT: [[TMP22:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// IMPRVD_FAST-NEXT: ret <2 x half> [[TMP22]] +// +// PRMTD_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x half> @divf16( +// PRMTD_FAST-SAME: <2 x half> noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x half> noundef nofpclass(nan inf) [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// PRMTD_FAST-NEXT: entry: +// PRMTD_FAST-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// PRMTD_FAST-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// PRMTD_FAST-NEXT: [[B:%.*]] = alloca { half, half }, align 2 +// PRMTD_FAST-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// PRMTD_FAST-NEXT: store <2 x half> [[B_COERCE]], ptr [[B]], align 2 +// PRMTD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// PRMTD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// PRMTD_FAST-NEXT: [[EXT:%.*]] = fpext half [[A_REAL]] to float +// PRMTD_FAST-NEXT: [[EXT1:%.*]] = fpext half [[A_IMAG]] to float +// PRMTD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2 +// PRMTD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2 +// PRMTD_FAST-NEXT: [[EXT2:%.*]] = fpext half [[B_REAL]] to float +// PRMTD_FAST-NEXT: [[EXT3:%.*]] = fpext half [[B_IMAG]] to float +// PRMTD_FAST-NEXT: [[TMP0:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT2]] +// PRMTD_FAST-NEXT: [[TMP1:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT3]] +// PRMTD_FAST-NEXT: [[TMP2:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP0]], [[TMP1]] +// PRMTD_FAST-NEXT: [[TMP3:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT2]], [[EXT2]] +// PRMTD_FAST-NEXT: [[TMP4:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT3]], [[EXT3]] +// PRMTD_FAST-NEXT: [[TMP5:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP3]], [[TMP4]] +// PRMTD_FAST-NEXT: [[TMP6:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT2]] +// PRMTD_FAST-NEXT: [[TMP7:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT3]] +// PRMTD_FAST-NEXT: [[TMP8:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[TMP6]], [[TMP7]] +// PRMTD_FAST-NEXT: [[TMP9:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP2]], [[TMP5]] +// PRMTD_FAST-NEXT: [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP8]], [[TMP5]] +// PRMTD_FAST-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[TMP9]] to half +// PRMTD_FAST-NEXT: [[UNPROMOTION4:%.*]] = fptrunc float [[TMP10]] to half +// PRMTD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD_FAST-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// PRMTD_FAST-NEXT: store half [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 2 +// PRMTD_FAST-NEXT: [[TMP11:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// PRMTD_FAST-NEXT: ret <2 x half> [[TMP11]] +// +_Complex _Float16 divf16(_Complex _Float16 a, _Complex _Float16 b) { + return a / b; +} + +// FULL-LABEL: define dso_local <2 x half> @mulf16( +// FULL-SAME: <2 x half> noundef [[A_COERCE:%.*]], <2 x half> noundef [[B_COERCE:%.*]]) #[[ATTR0]] { +// FULL-NEXT: entry: +// FULL-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// FULL-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// FULL-NEXT: [[B:%.*]] = alloca { half, half }, align 2 +// FULL-NEXT: [[COERCE:%.*]] = alloca { float, float }, align 4 +// FULL-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// FULL-NEXT: store <2 x half> [[B_COERCE]], ptr [[B]], align 2 +// FULL-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// FULL-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// FULL-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// FULL-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// FULL-NEXT: [[EXT:%.*]] = fpext half [[A_REAL]] to float +// FULL-NEXT: [[EXT1:%.*]] = fpext half [[A_IMAG]] to float +// FULL-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 0 +// FULL-NEXT: [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2 +// FULL-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 1 +// FULL-NEXT: [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2 +// FULL-NEXT: [[EXT2:%.*]] = fpext half [[B_REAL]] to float +// FULL-NEXT: [[EXT3:%.*]] = fpext half [[B_IMAG]] to float +// FULL-NEXT: [[MUL_AC:%.*]] = fmul float [[EXT]], [[EXT2]] +// FULL-NEXT: [[MUL_BD:%.*]] = fmul float [[EXT1]], [[EXT3]] +// FULL-NEXT: [[MUL_AD:%.*]] = fmul float [[EXT]], [[EXT3]] +// FULL-NEXT: [[MUL_BC:%.*]] = fmul float [[EXT1]], [[EXT2]] +// FULL-NEXT: [[MUL_R:%.*]] = fsub float [[MUL_AC]], [[MUL_BD]] +// FULL-NEXT: [[MUL_I:%.*]] = fadd float [[MUL_AD]], [[MUL_BC]] +// FULL-NEXT: [[ISNAN_CMP:%.*]] = fcmp uno float [[MUL_R]], [[MUL_R]] +// FULL-NEXT: br i1 [[ISNAN_CMP]], label [[COMPLEX_MUL_IMAG_NAN:%.*]], label [[COMPLEX_MUL_CONT:%.*]], !prof [[PROF2:![0-9]+]] +// FULL: complex_mul_imag_nan: +// FULL-NEXT: [[ISNAN_CMP4:%.*]] = fcmp uno float [[MUL_I]], [[MUL_I]] +// FULL-NEXT: br i1 [[ISNAN_CMP4]], label [[COMPLEX_MUL_LIBCALL:%.*]], label [[COMPLEX_MUL_CONT]], !prof [[PROF2]] +// FULL: complex_mul_libcall: +// FULL-NEXT: [[CALL:%.*]] = call <2 x float> @__mulsc3(float noundef [[EXT]], float noundef [[EXT1]], float noundef [[EXT2]], float noundef [[EXT3]]) #[[ATTR1]] +// FULL-NEXT: store <2 x float> [[CALL]], ptr [[COERCE]], align 4 +// FULL-NEXT: [[COERCE_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 0 +// FULL-NEXT: [[COERCE_REAL:%.*]] = load float, ptr [[COERCE_REALP]], align 4 +// FULL-NEXT: [[COERCE_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 1 +// FULL-NEXT: [[COERCE_IMAG:%.*]] = load float, ptr [[COERCE_IMAGP]], align 4 +// FULL-NEXT: br label [[COMPLEX_MUL_CONT]] +// FULL: complex_mul_cont: +// FULL-NEXT: [[REAL_MUL_PHI:%.*]] = phi float [ [[MUL_R]], [[ENTRY:%.*]] ], [ [[MUL_R]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[COERCE_REAL]], [[COMPLEX_MUL_LIBCALL]] ] +// FULL-NEXT: [[IMAG_MUL_PHI:%.*]] = phi float [ [[MUL_I]], [[ENTRY]] ], [ [[MUL_I]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[COERCE_IMAG]], [[COMPLEX_MUL_LIBCALL]] ] +// FULL-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[REAL_MUL_PHI]] to half +// FULL-NEXT: [[UNPROMOTION5:%.*]] = fptrunc float [[IMAG_MUL_PHI]] to half +// FULL-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// FULL-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// FULL-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// FULL-NEXT: store half [[UNPROMOTION5]], ptr [[RETVAL_IMAGP]], align 2 +// FULL-NEXT: [[TMP0:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// FULL-NEXT: ret <2 x half> [[TMP0]] +// +// BASIC-LABEL: define dso_local <2 x half> @mulf16( +// BASIC-SAME: <2 x half> noundef [[A_COERCE:%.*]], <2 x half> noundef [[B_COERCE:%.*]]) #[[ATTR0]] { +// BASIC-NEXT: entry: +// BASIC-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// BASIC-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// BASIC-NEXT: [[B:%.*]] = alloca { half, half }, align 2 +// BASIC-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// BASIC-NEXT: store <2 x half> [[B_COERCE]], ptr [[B]], align 2 +// BASIC-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// BASIC-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// BASIC-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// BASIC-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// BASIC-NEXT: [[EXT:%.*]] = fpext half [[A_REAL]] to float +// BASIC-NEXT: [[EXT1:%.*]] = fpext half [[A_IMAG]] to float +// BASIC-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 0 +// BASIC-NEXT: [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2 +// BASIC-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 1 +// BASIC-NEXT: [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2 +// BASIC-NEXT: [[EXT2:%.*]] = fpext half [[B_REAL]] to float +// BASIC-NEXT: [[EXT3:%.*]] = fpext half [[B_IMAG]] to float +// BASIC-NEXT: [[MUL_AC:%.*]] = fmul float [[EXT]], [[EXT2]] +// BASIC-NEXT: [[MUL_BD:%.*]] = fmul float [[EXT1]], [[EXT3]] +// BASIC-NEXT: [[MUL_AD:%.*]] = fmul float [[EXT]], [[EXT3]] +// BASIC-NEXT: [[MUL_BC:%.*]] = fmul float [[EXT1]], [[EXT2]] +// BASIC-NEXT: [[MUL_R:%.*]] = fsub float [[MUL_AC]], [[MUL_BD]] +// BASIC-NEXT: [[MUL_I:%.*]] = fadd float [[MUL_AD]], [[MUL_BC]] +// BASIC-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[MUL_R]] to half +// BASIC-NEXT: [[UNPROMOTION4:%.*]] = fptrunc float [[MUL_I]] to half +// BASIC-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// BASIC-NEXT: store half [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 2 +// BASIC-NEXT: [[TMP0:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// BASIC-NEXT: ret <2 x half> [[TMP0]] +// +// IMPRVD-LABEL: define dso_local <2 x half> @mulf16( +// IMPRVD-SAME: <2 x half> noundef [[A_COERCE:%.*]], <2 x half> noundef [[B_COERCE:%.*]]) #[[ATTR0]] { +// IMPRVD-NEXT: entry: +// IMPRVD-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// IMPRVD-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// IMPRVD-NEXT: [[B:%.*]] = alloca { half, half }, align 2 +// IMPRVD-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// IMPRVD-NEXT: store <2 x half> [[B_COERCE]], ptr [[B]], align 2 +// IMPRVD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// IMPRVD-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// IMPRVD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// IMPRVD-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// IMPRVD-NEXT: [[EXT:%.*]] = fpext half [[A_REAL]] to float +// IMPRVD-NEXT: [[EXT1:%.*]] = fpext half [[A_IMAG]] to float +// IMPRVD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 0 +// IMPRVD-NEXT: [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2 +// IMPRVD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 1 +// IMPRVD-NEXT: [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2 +// IMPRVD-NEXT: [[EXT2:%.*]] = fpext half [[B_REAL]] to float +// IMPRVD-NEXT: [[EXT3:%.*]] = fpext half [[B_IMAG]] to float +// IMPRVD-NEXT: [[MUL_AC:%.*]] = fmul float [[EXT]], [[EXT2]] +// IMPRVD-NEXT: [[MUL_BD:%.*]] = fmul float [[EXT1]], [[EXT3]] +// IMPRVD-NEXT: [[MUL_AD:%.*]] = fmul float [[EXT]], [[EXT3]] +// IMPRVD-NEXT: [[MUL_BC:%.*]] = fmul float [[EXT1]], [[EXT2]] +// IMPRVD-NEXT: [[MUL_R:%.*]] = fsub float [[MUL_AC]], [[MUL_BD]] +// IMPRVD-NEXT: [[MUL_I:%.*]] = fadd float [[MUL_AD]], [[MUL_BC]] +// IMPRVD-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[MUL_R]] to half +// IMPRVD-NEXT: [[UNPROMOTION4:%.*]] = fptrunc float [[MUL_I]] to half +// IMPRVD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// IMPRVD-NEXT: store half [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 2 +// IMPRVD-NEXT: [[TMP0:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// IMPRVD-NEXT: ret <2 x half> [[TMP0]] +// +// PRMTD-LABEL: define dso_local <2 x half> @mulf16( +// PRMTD-SAME: <2 x half> noundef [[A_COERCE:%.*]], <2 x half> noundef [[B_COERCE:%.*]]) #[[ATTR0]] { +// PRMTD-NEXT: entry: +// PRMTD-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// PRMTD-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// PRMTD-NEXT: [[B:%.*]] = alloca { half, half }, align 2 +// PRMTD-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// PRMTD-NEXT: store <2 x half> [[B_COERCE]], ptr [[B]], align 2 +// PRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// PRMTD-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// PRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// PRMTD-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// PRMTD-NEXT: [[EXT:%.*]] = fpext half [[A_REAL]] to float +// PRMTD-NEXT: [[EXT1:%.*]] = fpext half [[A_IMAG]] to float +// PRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 0 +// PRMTD-NEXT: [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2 +// PRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 1 +// PRMTD-NEXT: [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2 +// PRMTD-NEXT: [[EXT2:%.*]] = fpext half [[B_REAL]] to float +// PRMTD-NEXT: [[EXT3:%.*]] = fpext half [[B_IMAG]] to float +// PRMTD-NEXT: [[MUL_AC:%.*]] = fmul float [[EXT]], [[EXT2]] +// PRMTD-NEXT: [[MUL_BD:%.*]] = fmul float [[EXT1]], [[EXT3]] +// PRMTD-NEXT: [[MUL_AD:%.*]] = fmul float [[EXT]], [[EXT3]] +// PRMTD-NEXT: [[MUL_BC:%.*]] = fmul float [[EXT1]], [[EXT2]] +// PRMTD-NEXT: [[MUL_R:%.*]] = fsub float [[MUL_AC]], [[MUL_BD]] +// PRMTD-NEXT: [[MUL_I:%.*]] = fadd float [[MUL_AD]], [[MUL_BC]] +// PRMTD-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[MUL_R]] to half +// PRMTD-NEXT: [[UNPROMOTION4:%.*]] = fptrunc float [[MUL_I]] to half +// PRMTD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// PRMTD-NEXT: store half [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 2 +// PRMTD-NEXT: [[TMP0:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// PRMTD-NEXT: ret <2 x half> [[TMP0]] +// +// X86WINPRMTD-LABEL: define dso_local i32 @mulf16( +// X86WINPRMTD-SAME: i32 noundef [[A_COERCE:%.*]], i32 noundef [[B_COERCE:%.*]]) #[[ATTR0]] { +// X86WINPRMTD-NEXT: entry: +// X86WINPRMTD-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// X86WINPRMTD-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// X86WINPRMTD-NEXT: [[B:%.*]] = alloca { half, half }, align 2 +// X86WINPRMTD-NEXT: store i32 [[A_COERCE]], ptr [[A]], align 2 +// X86WINPRMTD-NEXT: store i32 [[B_COERCE]], ptr [[B]], align 2 +// X86WINPRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// X86WINPRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// X86WINPRMTD-NEXT: [[EXT:%.*]] = fpext half [[A_REAL]] to float +// X86WINPRMTD-NEXT: [[EXT1:%.*]] = fpext half [[A_IMAG]] to float +// X86WINPRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2 +// X86WINPRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2 +// X86WINPRMTD-NEXT: [[EXT2:%.*]] = fpext half [[B_REAL]] to float +// X86WINPRMTD-NEXT: [[EXT3:%.*]] = fpext half [[B_IMAG]] to float +// X86WINPRMTD-NEXT: [[MUL_AC:%.*]] = fmul float [[EXT]], [[EXT2]] +// X86WINPRMTD-NEXT: [[MUL_BD:%.*]] = fmul float [[EXT1]], [[EXT3]] +// X86WINPRMTD-NEXT: [[MUL_AD:%.*]] = fmul float [[EXT]], [[EXT3]] +// X86WINPRMTD-NEXT: [[MUL_BC:%.*]] = fmul float [[EXT1]], [[EXT2]] +// X86WINPRMTD-NEXT: [[MUL_R:%.*]] = fsub float [[MUL_AC]], [[MUL_BD]] +// X86WINPRMTD-NEXT: [[MUL_I:%.*]] = fadd float [[MUL_AD]], [[MUL_BC]] +// X86WINPRMTD-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[MUL_R]] to half +// X86WINPRMTD-NEXT: [[UNPROMOTION4:%.*]] = fptrunc float [[MUL_I]] to half +// X86WINPRMTD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// X86WINPRMTD-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// X86WINPRMTD-NEXT: store half [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 2 +// X86WINPRMTD-NEXT: [[TMP0:%.*]] = load i32, ptr [[RETVAL]], align 2 +// X86WINPRMTD-NEXT: ret i32 [[TMP0]] +// +// BASIC_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x half> @mulf16( +// BASIC_FAST-SAME: <2 x half> noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x half> noundef nofpclass(nan inf) [[B_COERCE:%.*]]) #[[ATTR0]] { +// BASIC_FAST-NEXT: entry: +// BASIC_FAST-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// BASIC_FAST-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// BASIC_FAST-NEXT: [[B:%.*]] = alloca { half, half }, align 2 +// BASIC_FAST-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// BASIC_FAST-NEXT: store <2 x half> [[B_COERCE]], ptr [[B]], align 2 +// BASIC_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// BASIC_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// BASIC_FAST-NEXT: [[EXT:%.*]] = fpext half [[A_REAL]] to float +// BASIC_FAST-NEXT: [[EXT1:%.*]] = fpext half [[A_IMAG]] to float +// BASIC_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2 +// BASIC_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2 +// BASIC_FAST-NEXT: [[EXT2:%.*]] = fpext half [[B_REAL]] to float +// BASIC_FAST-NEXT: [[EXT3:%.*]] = fpext half [[B_IMAG]] to float +// BASIC_FAST-NEXT: [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT2]] +// BASIC_FAST-NEXT: [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT3]] +// BASIC_FAST-NEXT: [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT3]] +// BASIC_FAST-NEXT: [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT2]] +// BASIC_FAST-NEXT: [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[MUL_AC]], [[MUL_BD]] +// BASIC_FAST-NEXT: [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[MUL_AD]], [[MUL_BC]] +// BASIC_FAST-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[MUL_R]] to half +// BASIC_FAST-NEXT: [[UNPROMOTION4:%.*]] = fptrunc float [[MUL_I]] to half +// BASIC_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC_FAST-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// BASIC_FAST-NEXT: store half [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 2 +// BASIC_FAST-NEXT: [[TMP0:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// BASIC_FAST-NEXT: ret <2 x half> [[TMP0]] +// +// FULL_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x half> @mulf16( +// FULL_FAST-SAME: <2 x half> noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x half> noundef nofpclass(nan inf) [[B_COERCE:%.*]]) #[[ATTR0]] { +// FULL_FAST-NEXT: entry: +// FULL_FAST-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// FULL_FAST-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// FULL_FAST-NEXT: [[B:%.*]] = alloca { half, half }, align 2 +// FULL_FAST-NEXT: [[COERCE:%.*]] = alloca { float, float }, align 4 +// FULL_FAST-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// FULL_FAST-NEXT: store <2 x half> [[B_COERCE]], ptr [[B]], align 2 +// FULL_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// FULL_FAST-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// FULL_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// FULL_FAST-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// FULL_FAST-NEXT: [[EXT:%.*]] = fpext half [[A_REAL]] to float +// FULL_FAST-NEXT: [[EXT1:%.*]] = fpext half [[A_IMAG]] to float +// FULL_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 0 +// FULL_FAST-NEXT: [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2 +// FULL_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 1 +// FULL_FAST-NEXT: [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2 +// FULL_FAST-NEXT: [[EXT2:%.*]] = fpext half [[B_REAL]] to float +// FULL_FAST-NEXT: [[EXT3:%.*]] = fpext half [[B_IMAG]] to float +// FULL_FAST-NEXT: [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT2]] +// FULL_FAST-NEXT: [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT3]] +// FULL_FAST-NEXT: [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT3]] +// FULL_FAST-NEXT: [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT2]] +// FULL_FAST-NEXT: [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[MUL_AC]], [[MUL_BD]] +// FULL_FAST-NEXT: [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[MUL_AD]], [[MUL_BC]] +// FULL_FAST-NEXT: [[ISNAN_CMP:%.*]] = fcmp reassoc nnan ninf nsz arcp afn uno float [[MUL_R]], [[MUL_R]] +// FULL_FAST-NEXT: br i1 [[ISNAN_CMP]], label [[COMPLEX_MUL_IMAG_NAN:%.*]], label [[COMPLEX_MUL_CONT:%.*]], !prof [[PROF2:![0-9]+]] +// FULL_FAST: complex_mul_imag_nan: +// FULL_FAST-NEXT: [[ISNAN_CMP4:%.*]] = fcmp reassoc nnan ninf nsz arcp afn uno float [[MUL_I]], [[MUL_I]] +// FULL_FAST-NEXT: br i1 [[ISNAN_CMP4]], label [[COMPLEX_MUL_LIBCALL:%.*]], label [[COMPLEX_MUL_CONT]], !prof [[PROF2]] +// FULL_FAST: complex_mul_libcall: +// FULL_FAST-NEXT: [[CALL:%.*]] = call reassoc nnan ninf nsz arcp afn nofpclass(nan inf) <2 x float> @__mulsc3(float noundef nofpclass(nan inf) [[EXT]], float noundef nofpclass(nan inf) [[EXT1]], float noundef nofpclass(nan inf) [[EXT2]], float noundef nofpclass(nan inf) [[EXT3]]) #[[ATTR1]] +// FULL_FAST-NEXT: store <2 x float> [[CALL]], ptr [[COERCE]], align 4 +// FULL_FAST-NEXT: [[COERCE_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 0 +// FULL_FAST-NEXT: [[COERCE_REAL:%.*]] = load float, ptr [[COERCE_REALP]], align 4 +// FULL_FAST-NEXT: [[COERCE_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 1 +// FULL_FAST-NEXT: [[COERCE_IMAG:%.*]] = load float, ptr [[COERCE_IMAGP]], align 4 +// FULL_FAST-NEXT: br label [[COMPLEX_MUL_CONT]] +// FULL_FAST: complex_mul_cont: +// FULL_FAST-NEXT: [[REAL_MUL_PHI:%.*]] = phi reassoc nnan ninf nsz arcp afn float [ [[MUL_R]], [[ENTRY:%.*]] ], [ [[MUL_R]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[COERCE_REAL]], [[COMPLEX_MUL_LIBCALL]] ] +// FULL_FAST-NEXT: [[IMAG_MUL_PHI:%.*]] = phi reassoc nnan ninf nsz arcp afn float [ [[MUL_I]], [[ENTRY]] ], [ [[MUL_I]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[COERCE_IMAG]], [[COMPLEX_MUL_LIBCALL]] ] +// FULL_FAST-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[REAL_MUL_PHI]] to half +// FULL_FAST-NEXT: [[UNPROMOTION5:%.*]] = fptrunc float [[IMAG_MUL_PHI]] to half +// FULL_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// FULL_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// FULL_FAST-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// FULL_FAST-NEXT: store half [[UNPROMOTION5]], ptr [[RETVAL_IMAGP]], align 2 +// FULL_FAST-NEXT: [[TMP0:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// FULL_FAST-NEXT: ret <2 x half> [[TMP0]] +// +// IMPRVD_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x half> @mulf16( +// IMPRVD_FAST-SAME: <2 x half> noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x half> noundef nofpclass(nan inf) [[B_COERCE:%.*]]) #[[ATTR0]] { +// IMPRVD_FAST-NEXT: entry: +// IMPRVD_FAST-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// IMPRVD_FAST-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// IMPRVD_FAST-NEXT: [[B:%.*]] = alloca { half, half }, align 2 +// IMPRVD_FAST-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// IMPRVD_FAST-NEXT: store <2 x half> [[B_COERCE]], ptr [[B]], align 2 +// IMPRVD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// IMPRVD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// IMPRVD_FAST-NEXT: [[EXT:%.*]] = fpext half [[A_REAL]] to float +// IMPRVD_FAST-NEXT: [[EXT1:%.*]] = fpext half [[A_IMAG]] to float +// IMPRVD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2 +// IMPRVD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2 +// IMPRVD_FAST-NEXT: [[EXT2:%.*]] = fpext half [[B_REAL]] to float +// IMPRVD_FAST-NEXT: [[EXT3:%.*]] = fpext half [[B_IMAG]] to float +// IMPRVD_FAST-NEXT: [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT2]] +// IMPRVD_FAST-NEXT: [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT3]] +// IMPRVD_FAST-NEXT: [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT3]] +// IMPRVD_FAST-NEXT: [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT2]] +// IMPRVD_FAST-NEXT: [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[MUL_AC]], [[MUL_BD]] +// IMPRVD_FAST-NEXT: [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[MUL_AD]], [[MUL_BC]] +// IMPRVD_FAST-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[MUL_R]] to half +// IMPRVD_FAST-NEXT: [[UNPROMOTION4:%.*]] = fptrunc float [[MUL_I]] to half +// IMPRVD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// IMPRVD_FAST-NEXT: store half [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 2 +// IMPRVD_FAST-NEXT: [[TMP0:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// IMPRVD_FAST-NEXT: ret <2 x half> [[TMP0]] +// +// PRMTD_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x half> @mulf16( +// PRMTD_FAST-SAME: <2 x half> noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x half> noundef nofpclass(nan inf) [[B_COERCE:%.*]]) #[[ATTR0]] { +// PRMTD_FAST-NEXT: entry: +// PRMTD_FAST-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// PRMTD_FAST-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// PRMTD_FAST-NEXT: [[B:%.*]] = alloca { half, half }, align 2 +// PRMTD_FAST-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// PRMTD_FAST-NEXT: store <2 x half> [[B_COERCE]], ptr [[B]], align 2 +// PRMTD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// PRMTD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// PRMTD_FAST-NEXT: [[EXT:%.*]] = fpext half [[A_REAL]] to float +// PRMTD_FAST-NEXT: [[EXT1:%.*]] = fpext half [[A_IMAG]] to float +// PRMTD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[B_REAL:%.*]] = load half, ptr [[B_REALP]], align 2 +// PRMTD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[B]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[B_IMAG:%.*]] = load half, ptr [[B_IMAGP]], align 2 +// PRMTD_FAST-NEXT: [[EXT2:%.*]] = fpext half [[B_REAL]] to float +// PRMTD_FAST-NEXT: [[EXT3:%.*]] = fpext half [[B_IMAG]] to float +// PRMTD_FAST-NEXT: [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT2]] +// PRMTD_FAST-NEXT: [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT3]] +// PRMTD_FAST-NEXT: [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT3]] +// PRMTD_FAST-NEXT: [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT1]], [[EXT2]] +// PRMTD_FAST-NEXT: [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[MUL_AC]], [[MUL_BD]] +// PRMTD_FAST-NEXT: [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[MUL_AD]], [[MUL_BC]] +// PRMTD_FAST-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[MUL_R]] to half +// PRMTD_FAST-NEXT: [[UNPROMOTION4:%.*]] = fptrunc float [[MUL_I]] to half +// PRMTD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD_FAST-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// PRMTD_FAST-NEXT: store half [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 2 +// PRMTD_FAST-NEXT: [[TMP0:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// PRMTD_FAST-NEXT: ret <2 x half> [[TMP0]] +// +_Complex _Float16 mulf16(_Complex _Float16 a, _Complex _Float16 b) { + return a * b; +} + +// FULL-LABEL: define dso_local <2 x half> @f1( +// FULL-SAME: <2 x half> noundef [[A_COERCE:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]], <2 x half> noundef [[C_COERCE:%.*]]) #[[ATTR0]] { +// FULL-NEXT: entry: +// FULL-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// FULL-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// FULL-NEXT: [[C:%.*]] = alloca { half, half }, align 2 +// FULL-NEXT: [[COERCE:%.*]] = alloca { float, float }, align 4 +// FULL-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// FULL-NEXT: store <2 x half> [[C_COERCE]], ptr [[C]], align 2 +// FULL-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// FULL-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// FULL-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// FULL-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// FULL-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[C]], i32 0, i32 0 +// FULL-NEXT: [[C_REAL:%.*]] = load half, ptr [[C_REALP]], align 2 +// FULL-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[C]], i32 0, i32 1 +// FULL-NEXT: [[C_IMAG:%.*]] = load half, ptr [[C_IMAGP]], align 2 +// FULL-NEXT: [[CONV:%.*]] = fpext half [[C_REAL]] to x86_fp80 +// FULL-NEXT: [[CONV1:%.*]] = fpext half [[C_IMAG]] to x86_fp80 +// FULL-NEXT: [[CALL:%.*]] = call { x86_fp80, x86_fp80 } @__divxc3(x86_fp80 noundef [[B_REAL]], x86_fp80 noundef [[B_IMAG]], x86_fp80 noundef [[CONV]], x86_fp80 noundef [[CONV1]]) #[[ATTR1]] +// FULL-NEXT: [[TMP0:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 0 +// FULL-NEXT: [[TMP1:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 1 +// FULL-NEXT: [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP0]] to half +// FULL-NEXT: [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP1]] to half +// FULL-NEXT: [[EXT:%.*]] = fpext half [[CONV2]] to float +// FULL-NEXT: [[EXT4:%.*]] = fpext half [[CONV3]] to float +// FULL-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// FULL-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// FULL-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// FULL-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// FULL-NEXT: [[EXT5:%.*]] = fpext half [[A_REAL]] to float +// FULL-NEXT: [[EXT6:%.*]] = fpext half [[A_IMAG]] to float +// FULL-NEXT: [[CALL7:%.*]] = call <2 x float> @__divsc3(float noundef [[EXT]], float noundef [[EXT4]], float noundef [[EXT5]], float noundef [[EXT6]]) #[[ATTR1]] +// FULL-NEXT: store <2 x float> [[CALL7]], ptr [[COERCE]], align 4 +// FULL-NEXT: [[COERCE_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 0 +// FULL-NEXT: [[COERCE_REAL:%.*]] = load float, ptr [[COERCE_REALP]], align 4 +// FULL-NEXT: [[COERCE_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 1 +// FULL-NEXT: [[COERCE_IMAG:%.*]] = load float, ptr [[COERCE_IMAGP]], align 4 +// FULL-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[COERCE_REAL]] to half +// FULL-NEXT: [[UNPROMOTION8:%.*]] = fptrunc float [[COERCE_IMAG]] to half +// FULL-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// FULL-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// FULL-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// FULL-NEXT: store half [[UNPROMOTION8]], ptr [[RETVAL_IMAGP]], align 2 +// FULL-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// FULL-NEXT: ret <2 x half> [[TMP2]] +// +// BASIC-LABEL: define dso_local <2 x half> @f1( +// BASIC-SAME: <2 x half> noundef [[A_COERCE:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]], <2 x half> noundef [[C_COERCE:%.*]]) #[[ATTR0]] { +// BASIC-NEXT: entry: +// BASIC-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// BASIC-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// BASIC-NEXT: [[C:%.*]] = alloca { half, half }, align 2 +// BASIC-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// BASIC-NEXT: store <2 x half> [[C_COERCE]], ptr [[C]], align 2 +// BASIC-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// BASIC-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// BASIC-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// BASIC-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// BASIC-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[C]], i32 0, i32 0 +// BASIC-NEXT: [[C_REAL:%.*]] = load half, ptr [[C_REALP]], align 2 +// BASIC-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[C]], i32 0, i32 1 +// BASIC-NEXT: [[C_IMAG:%.*]] = load half, ptr [[C_IMAGP]], align 2 +// BASIC-NEXT: [[CONV:%.*]] = fpext half [[C_REAL]] to x86_fp80 +// BASIC-NEXT: [[CONV1:%.*]] = fpext half [[C_IMAG]] to x86_fp80 +// BASIC-NEXT: [[TMP0:%.*]] = fmul x86_fp80 [[B_REAL]], [[CONV]] +// BASIC-NEXT: [[TMP1:%.*]] = fmul x86_fp80 [[B_IMAG]], [[CONV1]] +// BASIC-NEXT: [[TMP2:%.*]] = fadd x86_fp80 [[TMP0]], [[TMP1]] +// BASIC-NEXT: [[TMP3:%.*]] = fmul x86_fp80 [[CONV]], [[CONV]] +// BASIC-NEXT: [[TMP4:%.*]] = fmul x86_fp80 [[CONV1]], [[CONV1]] +// BASIC-NEXT: [[TMP5:%.*]] = fadd x86_fp80 [[TMP3]], [[TMP4]] +// BASIC-NEXT: [[TMP6:%.*]] = fmul x86_fp80 [[B_IMAG]], [[CONV]] +// BASIC-NEXT: [[TMP7:%.*]] = fmul x86_fp80 [[B_REAL]], [[CONV1]] +// BASIC-NEXT: [[TMP8:%.*]] = fsub x86_fp80 [[TMP6]], [[TMP7]] +// BASIC-NEXT: [[TMP9:%.*]] = fdiv x86_fp80 [[TMP2]], [[TMP5]] +// BASIC-NEXT: [[TMP10:%.*]] = fdiv x86_fp80 [[TMP8]], [[TMP5]] +// BASIC-NEXT: [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP9]] to half +// BASIC-NEXT: [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP10]] to half +// BASIC-NEXT: [[EXT:%.*]] = fpext half [[CONV2]] to float +// BASIC-NEXT: [[EXT4:%.*]] = fpext half [[CONV3]] to float +// BASIC-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// BASIC-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// BASIC-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// BASIC-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// BASIC-NEXT: [[EXT5:%.*]] = fpext half [[A_REAL]] to float +// BASIC-NEXT: [[EXT6:%.*]] = fpext half [[A_IMAG]] to float +// BASIC-NEXT: [[TMP11:%.*]] = fmul float [[EXT]], [[EXT5]] +// BASIC-NEXT: [[TMP12:%.*]] = fmul float [[EXT4]], [[EXT6]] +// BASIC-NEXT: [[TMP13:%.*]] = fadd float [[TMP11]], [[TMP12]] +// BASIC-NEXT: [[TMP14:%.*]] = fmul float [[EXT5]], [[EXT5]] +// BASIC-NEXT: [[TMP15:%.*]] = fmul float [[EXT6]], [[EXT6]] +// BASIC-NEXT: [[TMP16:%.*]] = fadd float [[TMP14]], [[TMP15]] +// BASIC-NEXT: [[TMP17:%.*]] = fmul float [[EXT4]], [[EXT5]] +// BASIC-NEXT: [[TMP18:%.*]] = fmul float [[EXT]], [[EXT6]] +// BASIC-NEXT: [[TMP19:%.*]] = fsub float [[TMP17]], [[TMP18]] +// BASIC-NEXT: [[TMP20:%.*]] = fdiv float [[TMP13]], [[TMP16]] +// BASIC-NEXT: [[TMP21:%.*]] = fdiv float [[TMP19]], [[TMP16]] +// BASIC-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[TMP20]] to half +// BASIC-NEXT: [[UNPROMOTION7:%.*]] = fptrunc float [[TMP21]] to half +// BASIC-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// BASIC-NEXT: store half [[UNPROMOTION7]], ptr [[RETVAL_IMAGP]], align 2 +// BASIC-NEXT: [[TMP22:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// BASIC-NEXT: ret <2 x half> [[TMP22]] +// +// IMPRVD-LABEL: define dso_local <2 x half> @f1( +// IMPRVD-SAME: <2 x half> noundef [[A_COERCE:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]], <2 x half> noundef [[C_COERCE:%.*]]) #[[ATTR0]] { +// IMPRVD-NEXT: entry: +// IMPRVD-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// IMPRVD-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// IMPRVD-NEXT: [[C:%.*]] = alloca { half, half }, align 2 +// IMPRVD-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// IMPRVD-NEXT: store <2 x half> [[C_COERCE]], ptr [[C]], align 2 +// IMPRVD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// IMPRVD-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// IMPRVD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// IMPRVD-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// IMPRVD-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[C]], i32 0, i32 0 +// IMPRVD-NEXT: [[C_REAL:%.*]] = load half, ptr [[C_REALP]], align 2 +// IMPRVD-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[C]], i32 0, i32 1 +// IMPRVD-NEXT: [[C_IMAG:%.*]] = load half, ptr [[C_IMAGP]], align 2 +// IMPRVD-NEXT: [[CONV:%.*]] = fpext half [[C_REAL]] to x86_fp80 +// IMPRVD-NEXT: [[CONV1:%.*]] = fpext half [[C_IMAG]] to x86_fp80 +// IMPRVD-NEXT: [[TMP0:%.*]] = call x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV]]) +// IMPRVD-NEXT: [[TMP1:%.*]] = call x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV1]]) +// IMPRVD-NEXT: [[ABS_CMP:%.*]] = fcmp ugt x86_fp80 [[TMP0]], [[TMP1]] +// IMPRVD-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// IMPRVD: abs_rhsr_greater_or_equal_abs_rhsi: +// IMPRVD-NEXT: [[TMP2:%.*]] = fdiv x86_fp80 [[CONV1]], [[CONV]] +// IMPRVD-NEXT: [[TMP3:%.*]] = fmul x86_fp80 [[TMP2]], [[CONV1]] +// IMPRVD-NEXT: [[TMP4:%.*]] = fadd x86_fp80 [[CONV]], [[TMP3]] +// IMPRVD-NEXT: [[TMP5:%.*]] = fmul x86_fp80 [[B_IMAG]], [[TMP2]] +// IMPRVD-NEXT: [[TMP6:%.*]] = fadd x86_fp80 [[B_REAL]], [[TMP5]] +// IMPRVD-NEXT: [[TMP7:%.*]] = fdiv x86_fp80 [[TMP6]], [[TMP4]] +// IMPRVD-NEXT: [[TMP8:%.*]] = fmul x86_fp80 [[B_REAL]], [[TMP2]] +// IMPRVD-NEXT: [[TMP9:%.*]] = fsub x86_fp80 [[B_IMAG]], [[TMP8]] +// IMPRVD-NEXT: [[TMP10:%.*]] = fdiv x86_fp80 [[TMP9]], [[TMP4]] +// IMPRVD-NEXT: br label [[COMPLEX_DIV:%.*]] +// IMPRVD: abs_rhsr_less_than_abs_rhsi: +// IMPRVD-NEXT: [[TMP11:%.*]] = fdiv x86_fp80 [[CONV]], [[CONV1]] +// IMPRVD-NEXT: [[TMP12:%.*]] = fmul x86_fp80 [[TMP11]], [[CONV]] +// IMPRVD-NEXT: [[TMP13:%.*]] = fadd x86_fp80 [[CONV1]], [[TMP12]] +// IMPRVD-NEXT: [[TMP14:%.*]] = fmul x86_fp80 [[B_REAL]], [[TMP11]] +// IMPRVD-NEXT: [[TMP15:%.*]] = fadd x86_fp80 [[TMP14]], [[B_IMAG]] +// IMPRVD-NEXT: [[TMP16:%.*]] = fdiv x86_fp80 [[TMP15]], [[TMP13]] +// IMPRVD-NEXT: [[TMP17:%.*]] = fmul x86_fp80 [[B_IMAG]], [[TMP11]] +// IMPRVD-NEXT: [[TMP18:%.*]] = fsub x86_fp80 [[TMP17]], [[B_REAL]] +// IMPRVD-NEXT: [[TMP19:%.*]] = fdiv x86_fp80 [[TMP18]], [[TMP13]] +// IMPRVD-NEXT: br label [[COMPLEX_DIV]] +// IMPRVD: complex_div: +// IMPRVD-NEXT: [[TMP20:%.*]] = phi x86_fp80 [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD-NEXT: [[TMP21:%.*]] = phi x86_fp80 [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD-NEXT: [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP20]] to half +// IMPRVD-NEXT: [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP21]] to half +// IMPRVD-NEXT: [[EXT:%.*]] = fpext half [[CONV2]] to float +// IMPRVD-NEXT: [[EXT4:%.*]] = fpext half [[CONV3]] to float +// IMPRVD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// IMPRVD-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// IMPRVD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// IMPRVD-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// IMPRVD-NEXT: [[EXT5:%.*]] = fpext half [[A_REAL]] to float +// IMPRVD-NEXT: [[EXT6:%.*]] = fpext half [[A_IMAG]] to float +// IMPRVD-NEXT: [[TMP22:%.*]] = call float @llvm.fabs.f32(float [[EXT5]]) +// IMPRVD-NEXT: [[TMP23:%.*]] = call float @llvm.fabs.f32(float [[EXT6]]) +// IMPRVD-NEXT: [[ABS_CMP7:%.*]] = fcmp ugt float [[TMP22]], [[TMP23]] +// IMPRVD-NEXT: br i1 [[ABS_CMP7]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI8:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI9:%.*]] +// IMPRVD: abs_rhsr_greater_or_equal_abs_rhsi8: +// IMPRVD-NEXT: [[TMP24:%.*]] = fdiv float [[EXT6]], [[EXT5]] +// IMPRVD-NEXT: [[TMP25:%.*]] = fmul float [[TMP24]], [[EXT6]] +// IMPRVD-NEXT: [[TMP26:%.*]] = fadd float [[EXT5]], [[TMP25]] +// IMPRVD-NEXT: [[TMP27:%.*]] = fmul float [[EXT4]], [[TMP24]] +// IMPRVD-NEXT: [[TMP28:%.*]] = fadd float [[EXT]], [[TMP27]] +// IMPRVD-NEXT: [[TMP29:%.*]] = fdiv float [[TMP28]], [[TMP26]] +// IMPRVD-NEXT: [[TMP30:%.*]] = fmul float [[EXT]], [[TMP24]] +// IMPRVD-NEXT: [[TMP31:%.*]] = fsub float [[EXT4]], [[TMP30]] +// IMPRVD-NEXT: [[TMP32:%.*]] = fdiv float [[TMP31]], [[TMP26]] +// IMPRVD-NEXT: br label [[COMPLEX_DIV10:%.*]] +// IMPRVD: abs_rhsr_less_than_abs_rhsi9: +// IMPRVD-NEXT: [[TMP33:%.*]] = fdiv float [[EXT5]], [[EXT6]] +// IMPRVD-NEXT: [[TMP34:%.*]] = fmul float [[TMP33]], [[EXT5]] +// IMPRVD-NEXT: [[TMP35:%.*]] = fadd float [[EXT6]], [[TMP34]] +// IMPRVD-NEXT: [[TMP36:%.*]] = fmul float [[EXT]], [[TMP33]] +// IMPRVD-NEXT: [[TMP37:%.*]] = fadd float [[TMP36]], [[EXT4]] +// IMPRVD-NEXT: [[TMP38:%.*]] = fdiv float [[TMP37]], [[TMP35]] +// IMPRVD-NEXT: [[TMP39:%.*]] = fmul float [[EXT4]], [[TMP33]] +// IMPRVD-NEXT: [[TMP40:%.*]] = fsub float [[TMP39]], [[EXT]] +// IMPRVD-NEXT: [[TMP41:%.*]] = fdiv float [[TMP40]], [[TMP35]] +// IMPRVD-NEXT: br label [[COMPLEX_DIV10]] +// IMPRVD: complex_div10: +// IMPRVD-NEXT: [[TMP42:%.*]] = phi float [ [[TMP29]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI8]] ], [ [[TMP38]], [[ABS_RHSR_LESS_THAN_ABS_RHSI9]] ] +// IMPRVD-NEXT: [[TMP43:%.*]] = phi float [ [[TMP32]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI8]] ], [ [[TMP41]], [[ABS_RHSR_LESS_THAN_ABS_RHSI9]] ] +// IMPRVD-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[TMP42]] to half +// IMPRVD-NEXT: [[UNPROMOTION11:%.*]] = fptrunc float [[TMP43]] to half +// IMPRVD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// IMPRVD-NEXT: store half [[UNPROMOTION11]], ptr [[RETVAL_IMAGP]], align 2 +// IMPRVD-NEXT: [[TMP44:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// IMPRVD-NEXT: ret <2 x half> [[TMP44]] +// +// PRMTD-LABEL: define dso_local <2 x half> @f1( +// PRMTD-SAME: <2 x half> noundef [[A_COERCE:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]], <2 x half> noundef [[C_COERCE:%.*]]) #[[ATTR0]] { +// PRMTD-NEXT: entry: +// PRMTD-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// PRMTD-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// PRMTD-NEXT: [[C:%.*]] = alloca { half, half }, align 2 +// PRMTD-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// PRMTD-NEXT: store <2 x half> [[C_COERCE]], ptr [[C]], align 2 +// PRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// PRMTD-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// PRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// PRMTD-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// PRMTD-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[C]], i32 0, i32 0 +// PRMTD-NEXT: [[C_REAL:%.*]] = load half, ptr [[C_REALP]], align 2 +// PRMTD-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[C]], i32 0, i32 1 +// PRMTD-NEXT: [[C_IMAG:%.*]] = load half, ptr [[C_IMAGP]], align 2 +// PRMTD-NEXT: [[CONV:%.*]] = fpext half [[C_REAL]] to x86_fp80 +// PRMTD-NEXT: [[CONV1:%.*]] = fpext half [[C_IMAG]] to x86_fp80 +// PRMTD-NEXT: [[TMP0:%.*]] = call x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV]]) +// PRMTD-NEXT: [[TMP1:%.*]] = call x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV1]]) +// PRMTD-NEXT: [[ABS_CMP:%.*]] = fcmp ugt x86_fp80 [[TMP0]], [[TMP1]] +// PRMTD-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// PRMTD: abs_rhsr_greater_or_equal_abs_rhsi: +// PRMTD-NEXT: [[TMP2:%.*]] = fdiv x86_fp80 [[CONV1]], [[CONV]] +// PRMTD-NEXT: [[TMP3:%.*]] = fmul x86_fp80 [[TMP2]], [[CONV1]] +// PRMTD-NEXT: [[TMP4:%.*]] = fadd x86_fp80 [[CONV]], [[TMP3]] +// PRMTD-NEXT: [[TMP5:%.*]] = fmul x86_fp80 [[B_IMAG]], [[TMP2]] +// PRMTD-NEXT: [[TMP6:%.*]] = fadd x86_fp80 [[B_REAL]], [[TMP5]] +// PRMTD-NEXT: [[TMP7:%.*]] = fdiv x86_fp80 [[TMP6]], [[TMP4]] +// PRMTD-NEXT: [[TMP8:%.*]] = fmul x86_fp80 [[B_REAL]], [[TMP2]] +// PRMTD-NEXT: [[TMP9:%.*]] = fsub x86_fp80 [[B_IMAG]], [[TMP8]] +// PRMTD-NEXT: [[TMP10:%.*]] = fdiv x86_fp80 [[TMP9]], [[TMP4]] +// PRMTD-NEXT: br label [[COMPLEX_DIV:%.*]] +// PRMTD: abs_rhsr_less_than_abs_rhsi: +// PRMTD-NEXT: [[TMP11:%.*]] = fdiv x86_fp80 [[CONV]], [[CONV1]] +// PRMTD-NEXT: [[TMP12:%.*]] = fmul x86_fp80 [[TMP11]], [[CONV]] +// PRMTD-NEXT: [[TMP13:%.*]] = fadd x86_fp80 [[CONV1]], [[TMP12]] +// PRMTD-NEXT: [[TMP14:%.*]] = fmul x86_fp80 [[B_REAL]], [[TMP11]] +// PRMTD-NEXT: [[TMP15:%.*]] = fadd x86_fp80 [[TMP14]], [[B_IMAG]] +// PRMTD-NEXT: [[TMP16:%.*]] = fdiv x86_fp80 [[TMP15]], [[TMP13]] +// PRMTD-NEXT: [[TMP17:%.*]] = fmul x86_fp80 [[B_IMAG]], [[TMP11]] +// PRMTD-NEXT: [[TMP18:%.*]] = fsub x86_fp80 [[TMP17]], [[B_REAL]] +// PRMTD-NEXT: [[TMP19:%.*]] = fdiv x86_fp80 [[TMP18]], [[TMP13]] +// PRMTD-NEXT: br label [[COMPLEX_DIV]] +// PRMTD: complex_div: +// PRMTD-NEXT: [[TMP20:%.*]] = phi x86_fp80 [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// PRMTD-NEXT: [[TMP21:%.*]] = phi x86_fp80 [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// PRMTD-NEXT: [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP20]] to half +// PRMTD-NEXT: [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP21]] to half +// PRMTD-NEXT: [[EXT:%.*]] = fpext half [[CONV2]] to float +// PRMTD-NEXT: [[EXT4:%.*]] = fpext half [[CONV3]] to float +// PRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// PRMTD-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// PRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// PRMTD-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// PRMTD-NEXT: [[EXT5:%.*]] = fpext half [[A_REAL]] to float +// PRMTD-NEXT: [[EXT6:%.*]] = fpext half [[A_IMAG]] to float +// PRMTD-NEXT: [[TMP22:%.*]] = fmul float [[EXT]], [[EXT5]] +// PRMTD-NEXT: [[TMP23:%.*]] = fmul float [[EXT4]], [[EXT6]] +// PRMTD-NEXT: [[TMP24:%.*]] = fadd float [[TMP22]], [[TMP23]] +// PRMTD-NEXT: [[TMP25:%.*]] = fmul float [[EXT5]], [[EXT5]] +// PRMTD-NEXT: [[TMP26:%.*]] = fmul float [[EXT6]], [[EXT6]] +// PRMTD-NEXT: [[TMP27:%.*]] = fadd float [[TMP25]], [[TMP26]] +// PRMTD-NEXT: [[TMP28:%.*]] = fmul float [[EXT4]], [[EXT5]] +// PRMTD-NEXT: [[TMP29:%.*]] = fmul float [[EXT]], [[EXT6]] +// PRMTD-NEXT: [[TMP30:%.*]] = fsub float [[TMP28]], [[TMP29]] +// PRMTD-NEXT: [[TMP31:%.*]] = fdiv float [[TMP24]], [[TMP27]] +// PRMTD-NEXT: [[TMP32:%.*]] = fdiv float [[TMP30]], [[TMP27]] +// PRMTD-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[TMP31]] to half +// PRMTD-NEXT: [[UNPROMOTION7:%.*]] = fptrunc float [[TMP32]] to half +// PRMTD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// PRMTD-NEXT: store half [[UNPROMOTION7]], ptr [[RETVAL_IMAGP]], align 2 +// PRMTD-NEXT: [[TMP33:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// PRMTD-NEXT: ret <2 x half> [[TMP33]] +// +// X86WINPRMTD-LABEL: define dso_local i32 @f1( +// X86WINPRMTD-SAME: i32 noundef [[A_COERCE:%.*]], ptr noundef [[B:%.*]], i32 noundef [[C_COERCE:%.*]]) #[[ATTR0]] { +// X86WINPRMTD-NEXT: entry: +// X86WINPRMTD-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// X86WINPRMTD-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// X86WINPRMTD-NEXT: [[C:%.*]] = alloca { half, half }, align 2 +// X86WINPRMTD-NEXT: [[B_INDIRECT_ADDR:%.*]] = alloca ptr, align 8 +// X86WINPRMTD-NEXT: store i32 [[A_COERCE]], ptr [[A]], align 2 +// X86WINPRMTD-NEXT: store i32 [[C_COERCE]], ptr [[C]], align 2 +// X86WINPRMTD-NEXT: store ptr [[B]], ptr [[B_INDIRECT_ADDR]], align 8 +// X86WINPRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// X86WINPRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// X86WINPRMTD-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[C]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[C_REAL:%.*]] = load half, ptr [[C_REALP]], align 2 +// X86WINPRMTD-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[C]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[C_IMAG:%.*]] = load half, ptr [[C_IMAGP]], align 2 +// X86WINPRMTD-NEXT: [[CONV:%.*]] = fpext half [[C_REAL]] to double +// X86WINPRMTD-NEXT: [[CONV1:%.*]] = fpext half [[C_IMAG]] to double +// X86WINPRMTD-NEXT: [[TMP0:%.*]] = call double @llvm.fabs.f64(double [[CONV]]) +// X86WINPRMTD-NEXT: [[TMP1:%.*]] = call double @llvm.fabs.f64(double [[CONV1]]) +// X86WINPRMTD-NEXT: [[ABS_CMP:%.*]] = fcmp ugt double [[TMP0]], [[TMP1]] +// X86WINPRMTD-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// X86WINPRMTD: abs_rhsr_greater_or_equal_abs_rhsi: +// X86WINPRMTD-NEXT: [[TMP2:%.*]] = fdiv double [[CONV1]], [[CONV]] +// X86WINPRMTD-NEXT: [[TMP3:%.*]] = fmul double [[TMP2]], [[CONV1]] +// X86WINPRMTD-NEXT: [[TMP4:%.*]] = fadd double [[CONV]], [[TMP3]] +// X86WINPRMTD-NEXT: [[TMP5:%.*]] = fmul double [[B_IMAG]], [[TMP2]] +// X86WINPRMTD-NEXT: [[TMP6:%.*]] = fadd double [[B_REAL]], [[TMP5]] +// X86WINPRMTD-NEXT: [[TMP7:%.*]] = fdiv double [[TMP6]], [[TMP4]] +// X86WINPRMTD-NEXT: [[TMP8:%.*]] = fmul double [[B_REAL]], [[TMP2]] +// X86WINPRMTD-NEXT: [[TMP9:%.*]] = fsub double [[B_IMAG]], [[TMP8]] +// X86WINPRMTD-NEXT: [[TMP10:%.*]] = fdiv double [[TMP9]], [[TMP4]] +// X86WINPRMTD-NEXT: br label [[COMPLEX_DIV:%.*]] +// X86WINPRMTD: abs_rhsr_less_than_abs_rhsi: +// X86WINPRMTD-NEXT: [[TMP11:%.*]] = fdiv double [[CONV]], [[CONV1]] +// X86WINPRMTD-NEXT: [[TMP12:%.*]] = fmul double [[TMP11]], [[CONV]] +// X86WINPRMTD-NEXT: [[TMP13:%.*]] = fadd double [[CONV1]], [[TMP12]] +// X86WINPRMTD-NEXT: [[TMP14:%.*]] = fmul double [[B_REAL]], [[TMP11]] +// X86WINPRMTD-NEXT: [[TMP15:%.*]] = fadd double [[TMP14]], [[B_IMAG]] +// X86WINPRMTD-NEXT: [[TMP16:%.*]] = fdiv double [[TMP15]], [[TMP13]] +// X86WINPRMTD-NEXT: [[TMP17:%.*]] = fmul double [[B_IMAG]], [[TMP11]] +// X86WINPRMTD-NEXT: [[TMP18:%.*]] = fsub double [[TMP17]], [[B_REAL]] +// X86WINPRMTD-NEXT: [[TMP19:%.*]] = fdiv double [[TMP18]], [[TMP13]] +// X86WINPRMTD-NEXT: br label [[COMPLEX_DIV]] +// X86WINPRMTD: complex_div: +// X86WINPRMTD-NEXT: [[TMP20:%.*]] = phi double [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// X86WINPRMTD-NEXT: [[TMP21:%.*]] = phi double [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// X86WINPRMTD-NEXT: [[CONV2:%.*]] = fptrunc double [[TMP20]] to half +// X86WINPRMTD-NEXT: [[CONV3:%.*]] = fptrunc double [[TMP21]] to half +// X86WINPRMTD-NEXT: [[EXT:%.*]] = fpext half [[CONV2]] to float +// X86WINPRMTD-NEXT: [[EXT4:%.*]] = fpext half [[CONV3]] to float +// X86WINPRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// X86WINPRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// X86WINPRMTD-NEXT: [[EXT5:%.*]] = fpext half [[A_REAL]] to float +// X86WINPRMTD-NEXT: [[EXT6:%.*]] = fpext half [[A_IMAG]] to float +// X86WINPRMTD-NEXT: [[TMP22:%.*]] = fmul float [[EXT]], [[EXT5]] +// X86WINPRMTD-NEXT: [[TMP23:%.*]] = fmul float [[EXT4]], [[EXT6]] +// X86WINPRMTD-NEXT: [[TMP24:%.*]] = fadd float [[TMP22]], [[TMP23]] +// X86WINPRMTD-NEXT: [[TMP25:%.*]] = fmul float [[EXT5]], [[EXT5]] +// X86WINPRMTD-NEXT: [[TMP26:%.*]] = fmul float [[EXT6]], [[EXT6]] +// X86WINPRMTD-NEXT: [[TMP27:%.*]] = fadd float [[TMP25]], [[TMP26]] +// X86WINPRMTD-NEXT: [[TMP28:%.*]] = fmul float [[EXT4]], [[EXT5]] +// X86WINPRMTD-NEXT: [[TMP29:%.*]] = fmul float [[EXT]], [[EXT6]] +// X86WINPRMTD-NEXT: [[TMP30:%.*]] = fsub float [[TMP28]], [[TMP29]] +// X86WINPRMTD-NEXT: [[TMP31:%.*]] = fdiv float [[TMP24]], [[TMP27]] +// X86WINPRMTD-NEXT: [[TMP32:%.*]] = fdiv float [[TMP30]], [[TMP27]] +// X86WINPRMTD-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[TMP31]] to half +// X86WINPRMTD-NEXT: [[UNPROMOTION7:%.*]] = fptrunc float [[TMP32]] to half +// X86WINPRMTD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// X86WINPRMTD-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// X86WINPRMTD-NEXT: store half [[UNPROMOTION7]], ptr [[RETVAL_IMAGP]], align 2 +// X86WINPRMTD-NEXT: [[TMP33:%.*]] = load i32, ptr [[RETVAL]], align 2 +// X86WINPRMTD-NEXT: ret i32 [[TMP33]] +// +// BASIC_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x half> @f1( +// BASIC_FAST-SAME: <2 x half> noundef nofpclass(nan inf) [[A_COERCE:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]], <2 x half> noundef nofpclass(nan inf) [[C_COERCE:%.*]]) #[[ATTR0]] { +// BASIC_FAST-NEXT: entry: +// BASIC_FAST-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// BASIC_FAST-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// BASIC_FAST-NEXT: [[C:%.*]] = alloca { half, half }, align 2 +// BASIC_FAST-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// BASIC_FAST-NEXT: store <2 x half> [[C_COERCE]], ptr [[C]], align 2 +// BASIC_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// BASIC_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// BASIC_FAST-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[C]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[C_REAL:%.*]] = load half, ptr [[C_REALP]], align 2 +// BASIC_FAST-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[C]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[C_IMAG:%.*]] = load half, ptr [[C_IMAGP]], align 2 +// BASIC_FAST-NEXT: [[CONV:%.*]] = fpext half [[C_REAL]] to x86_fp80 +// BASIC_FAST-NEXT: [[CONV1:%.*]] = fpext half [[C_IMAG]] to x86_fp80 +// BASIC_FAST-NEXT: [[TMP0:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[CONV]] +// BASIC_FAST-NEXT: [[TMP1:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[CONV1]] +// BASIC_FAST-NEXT: [[TMP2:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP0]], [[TMP1]] +// BASIC_FAST-NEXT: [[TMP3:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV]], [[CONV]] +// BASIC_FAST-NEXT: [[TMP4:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV1]], [[CONV1]] +// BASIC_FAST-NEXT: [[TMP5:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP3]], [[TMP4]] +// BASIC_FAST-NEXT: [[TMP6:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[CONV]] +// BASIC_FAST-NEXT: [[TMP7:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[CONV1]] +// BASIC_FAST-NEXT: [[TMP8:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP6]], [[TMP7]] +// BASIC_FAST-NEXT: [[TMP9:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP2]], [[TMP5]] +// BASIC_FAST-NEXT: [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP8]], [[TMP5]] +// BASIC_FAST-NEXT: [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP9]] to half +// BASIC_FAST-NEXT: [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP10]] to half +// BASIC_FAST-NEXT: [[EXT:%.*]] = fpext half [[CONV2]] to float +// BASIC_FAST-NEXT: [[EXT4:%.*]] = fpext half [[CONV3]] to float +// BASIC_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// BASIC_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// BASIC_FAST-NEXT: [[EXT5:%.*]] = fpext half [[A_REAL]] to float +// BASIC_FAST-NEXT: [[EXT6:%.*]] = fpext half [[A_IMAG]] to float +// BASIC_FAST-NEXT: [[TMP11:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT5]] +// BASIC_FAST-NEXT: [[TMP12:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT4]], [[EXT6]] +// BASIC_FAST-NEXT: [[TMP13:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP11]], [[TMP12]] +// BASIC_FAST-NEXT: [[TMP14:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT5]], [[EXT5]] +// BASIC_FAST-NEXT: [[TMP15:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT6]], [[EXT6]] +// BASIC_FAST-NEXT: [[TMP16:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP14]], [[TMP15]] +// BASIC_FAST-NEXT: [[TMP17:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT4]], [[EXT5]] +// BASIC_FAST-NEXT: [[TMP18:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT6]] +// BASIC_FAST-NEXT: [[TMP19:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[TMP17]], [[TMP18]] +// BASIC_FAST-NEXT: [[TMP20:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP13]], [[TMP16]] +// BASIC_FAST-NEXT: [[TMP21:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP19]], [[TMP16]] +// BASIC_FAST-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[TMP20]] to half +// BASIC_FAST-NEXT: [[UNPROMOTION7:%.*]] = fptrunc float [[TMP21]] to half +// BASIC_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC_FAST-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// BASIC_FAST-NEXT: store half [[UNPROMOTION7]], ptr [[RETVAL_IMAGP]], align 2 +// BASIC_FAST-NEXT: [[TMP22:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// BASIC_FAST-NEXT: ret <2 x half> [[TMP22]] +// +// FULL_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x half> @f1( +// FULL_FAST-SAME: <2 x half> noundef nofpclass(nan inf) [[A_COERCE:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]], <2 x half> noundef nofpclass(nan inf) [[C_COERCE:%.*]]) #[[ATTR0]] { +// FULL_FAST-NEXT: entry: +// FULL_FAST-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// FULL_FAST-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// FULL_FAST-NEXT: [[C:%.*]] = alloca { half, half }, align 2 +// FULL_FAST-NEXT: [[COERCE:%.*]] = alloca { float, float }, align 4 +// FULL_FAST-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// FULL_FAST-NEXT: store <2 x half> [[C_COERCE]], ptr [[C]], align 2 +// FULL_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// FULL_FAST-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// FULL_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// FULL_FAST-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// FULL_FAST-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[C]], i32 0, i32 0 +// FULL_FAST-NEXT: [[C_REAL:%.*]] = load half, ptr [[C_REALP]], align 2 +// FULL_FAST-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[C]], i32 0, i32 1 +// FULL_FAST-NEXT: [[C_IMAG:%.*]] = load half, ptr [[C_IMAGP]], align 2 +// FULL_FAST-NEXT: [[CONV:%.*]] = fpext half [[C_REAL]] to x86_fp80 +// FULL_FAST-NEXT: [[CONV1:%.*]] = fpext half [[C_IMAG]] to x86_fp80 +// FULL_FAST-NEXT: [[CALL:%.*]] = call { x86_fp80, x86_fp80 } @__divxc3(x86_fp80 noundef nofpclass(nan inf) [[B_REAL]], x86_fp80 noundef nofpclass(nan inf) [[B_IMAG]], x86_fp80 noundef nofpclass(nan inf) [[CONV]], x86_fp80 noundef nofpclass(nan inf) [[CONV1]]) #[[ATTR1]] +// FULL_FAST-NEXT: [[TMP0:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 0 +// FULL_FAST-NEXT: [[TMP1:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 1 +// FULL_FAST-NEXT: [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP0]] to half +// FULL_FAST-NEXT: [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP1]] to half +// FULL_FAST-NEXT: [[EXT:%.*]] = fpext half [[CONV2]] to float +// FULL_FAST-NEXT: [[EXT4:%.*]] = fpext half [[CONV3]] to float +// FULL_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// FULL_FAST-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// FULL_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// FULL_FAST-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// FULL_FAST-NEXT: [[EXT5:%.*]] = fpext half [[A_REAL]] to float +// FULL_FAST-NEXT: [[EXT6:%.*]] = fpext half [[A_IMAG]] to float +// FULL_FAST-NEXT: [[CALL7:%.*]] = call reassoc nnan ninf nsz arcp afn nofpclass(nan inf) <2 x float> @__divsc3(float noundef nofpclass(nan inf) [[EXT]], float noundef nofpclass(nan inf) [[EXT4]], float noundef nofpclass(nan inf) [[EXT5]], float noundef nofpclass(nan inf) [[EXT6]]) #[[ATTR1]] +// FULL_FAST-NEXT: store <2 x float> [[CALL7]], ptr [[COERCE]], align 4 +// FULL_FAST-NEXT: [[COERCE_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 0 +// FULL_FAST-NEXT: [[COERCE_REAL:%.*]] = load float, ptr [[COERCE_REALP]], align 4 +// FULL_FAST-NEXT: [[COERCE_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 1 +// FULL_FAST-NEXT: [[COERCE_IMAG:%.*]] = load float, ptr [[COERCE_IMAGP]], align 4 +// FULL_FAST-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[COERCE_REAL]] to half +// FULL_FAST-NEXT: [[UNPROMOTION8:%.*]] = fptrunc float [[COERCE_IMAG]] to half +// FULL_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// FULL_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// FULL_FAST-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// FULL_FAST-NEXT: store half [[UNPROMOTION8]], ptr [[RETVAL_IMAGP]], align 2 +// FULL_FAST-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// FULL_FAST-NEXT: ret <2 x half> [[TMP2]] +// +// IMPRVD_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x half> @f1( +// IMPRVD_FAST-SAME: <2 x half> noundef nofpclass(nan inf) [[A_COERCE:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]], <2 x half> noundef nofpclass(nan inf) [[C_COERCE:%.*]]) #[[ATTR0]] { +// IMPRVD_FAST-NEXT: entry: +// IMPRVD_FAST-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// IMPRVD_FAST-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// IMPRVD_FAST-NEXT: [[C:%.*]] = alloca { half, half }, align 2 +// IMPRVD_FAST-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// IMPRVD_FAST-NEXT: store <2 x half> [[C_COERCE]], ptr [[C]], align 2 +// IMPRVD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// IMPRVD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// IMPRVD_FAST-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[C]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[C_REAL:%.*]] = load half, ptr [[C_REALP]], align 2 +// IMPRVD_FAST-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[C]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[C_IMAG:%.*]] = load half, ptr [[C_IMAGP]], align 2 +// IMPRVD_FAST-NEXT: [[CONV:%.*]] = fpext half [[C_REAL]] to x86_fp80 +// IMPRVD_FAST-NEXT: [[CONV1:%.*]] = fpext half [[C_IMAG]] to x86_fp80 +// IMPRVD_FAST-NEXT: [[TMP0:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV]]) +// IMPRVD_FAST-NEXT: [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV1]]) +// IMPRVD_FAST-NEXT: [[ABS_CMP:%.*]] = fcmp reassoc nnan ninf nsz arcp afn ugt x86_fp80 [[TMP0]], [[TMP1]] +// IMPRVD_FAST-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// IMPRVD_FAST: abs_rhsr_greater_or_equal_abs_rhsi: +// IMPRVD_FAST-NEXT: [[TMP2:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV1]], [[CONV]] +// IMPRVD_FAST-NEXT: [[TMP3:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP2]], [[CONV1]] +// IMPRVD_FAST-NEXT: [[TMP4:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV]], [[TMP3]] +// IMPRVD_FAST-NEXT: [[TMP5:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[TMP2]] +// IMPRVD_FAST-NEXT: [[TMP6:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[TMP5]] +// IMPRVD_FAST-NEXT: [[TMP7:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP6]], [[TMP4]] +// IMPRVD_FAST-NEXT: [[TMP8:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[TMP2]] +// IMPRVD_FAST-NEXT: [[TMP9:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[TMP8]] +// IMPRVD_FAST-NEXT: [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP9]], [[TMP4]] +// IMPRVD_FAST-NEXT: br label [[COMPLEX_DIV:%.*]] +// IMPRVD_FAST: abs_rhsr_less_than_abs_rhsi: +// IMPRVD_FAST-NEXT: [[TMP11:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV]], [[CONV1]] +// IMPRVD_FAST-NEXT: [[TMP12:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP11]], [[CONV]] +// IMPRVD_FAST-NEXT: [[TMP13:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV1]], [[TMP12]] +// IMPRVD_FAST-NEXT: [[TMP14:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[TMP11]] +// IMPRVD_FAST-NEXT: [[TMP15:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP14]], [[B_IMAG]] +// IMPRVD_FAST-NEXT: [[TMP16:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP15]], [[TMP13]] +// IMPRVD_FAST-NEXT: [[TMP17:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[TMP11]] +// IMPRVD_FAST-NEXT: [[TMP18:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP17]], [[B_REAL]] +// IMPRVD_FAST-NEXT: [[TMP19:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP18]], [[TMP13]] +// IMPRVD_FAST-NEXT: br label [[COMPLEX_DIV]] +// IMPRVD_FAST: complex_div: +// IMPRVD_FAST-NEXT: [[TMP20:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD_FAST-NEXT: [[TMP21:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD_FAST-NEXT: [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP20]] to half +// IMPRVD_FAST-NEXT: [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP21]] to half +// IMPRVD_FAST-NEXT: [[EXT:%.*]] = fpext half [[CONV2]] to float +// IMPRVD_FAST-NEXT: [[EXT4:%.*]] = fpext half [[CONV3]] to float +// IMPRVD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// IMPRVD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// IMPRVD_FAST-NEXT: [[EXT5:%.*]] = fpext half [[A_REAL]] to float +// IMPRVD_FAST-NEXT: [[EXT6:%.*]] = fpext half [[A_IMAG]] to float +// IMPRVD_FAST-NEXT: [[TMP22:%.*]] = call reassoc nnan ninf nsz arcp afn float @llvm.fabs.f32(float [[EXT5]]) +// IMPRVD_FAST-NEXT: [[TMP23:%.*]] = call reassoc nnan ninf nsz arcp afn float @llvm.fabs.f32(float [[EXT6]]) +// IMPRVD_FAST-NEXT: [[ABS_CMP7:%.*]] = fcmp reassoc nnan ninf nsz arcp afn ugt float [[TMP22]], [[TMP23]] +// IMPRVD_FAST-NEXT: br i1 [[ABS_CMP7]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI8:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI9:%.*]] +// IMPRVD_FAST: abs_rhsr_greater_or_equal_abs_rhsi8: +// IMPRVD_FAST-NEXT: [[TMP24:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[EXT6]], [[EXT5]] +// IMPRVD_FAST-NEXT: [[TMP25:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[TMP24]], [[EXT6]] +// IMPRVD_FAST-NEXT: [[TMP26:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[EXT5]], [[TMP25]] +// IMPRVD_FAST-NEXT: [[TMP27:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT4]], [[TMP24]] +// IMPRVD_FAST-NEXT: [[TMP28:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[EXT]], [[TMP27]] +// IMPRVD_FAST-NEXT: [[TMP29:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP28]], [[TMP26]] +// IMPRVD_FAST-NEXT: [[TMP30:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[TMP24]] +// IMPRVD_FAST-NEXT: [[TMP31:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[EXT4]], [[TMP30]] +// IMPRVD_FAST-NEXT: [[TMP32:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP31]], [[TMP26]] +// IMPRVD_FAST-NEXT: br label [[COMPLEX_DIV10:%.*]] +// IMPRVD_FAST: abs_rhsr_less_than_abs_rhsi9: +// IMPRVD_FAST-NEXT: [[TMP33:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[EXT5]], [[EXT6]] +// IMPRVD_FAST-NEXT: [[TMP34:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[TMP33]], [[EXT5]] +// IMPRVD_FAST-NEXT: [[TMP35:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[EXT6]], [[TMP34]] +// IMPRVD_FAST-NEXT: [[TMP36:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[TMP33]] +// IMPRVD_FAST-NEXT: [[TMP37:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP36]], [[EXT4]] +// IMPRVD_FAST-NEXT: [[TMP38:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP37]], [[TMP35]] +// IMPRVD_FAST-NEXT: [[TMP39:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT4]], [[TMP33]] +// IMPRVD_FAST-NEXT: [[TMP40:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[TMP39]], [[EXT]] +// IMPRVD_FAST-NEXT: [[TMP41:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP40]], [[TMP35]] +// IMPRVD_FAST-NEXT: br label [[COMPLEX_DIV10]] +// IMPRVD_FAST: complex_div10: +// IMPRVD_FAST-NEXT: [[TMP42:%.*]] = phi reassoc nnan ninf nsz arcp afn float [ [[TMP29]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI8]] ], [ [[TMP38]], [[ABS_RHSR_LESS_THAN_ABS_RHSI9]] ] +// IMPRVD_FAST-NEXT: [[TMP43:%.*]] = phi reassoc nnan ninf nsz arcp afn float [ [[TMP32]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI8]] ], [ [[TMP41]], [[ABS_RHSR_LESS_THAN_ABS_RHSI9]] ] +// IMPRVD_FAST-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[TMP42]] to half +// IMPRVD_FAST-NEXT: [[UNPROMOTION11:%.*]] = fptrunc float [[TMP43]] to half +// IMPRVD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// IMPRVD_FAST-NEXT: store half [[UNPROMOTION11]], ptr [[RETVAL_IMAGP]], align 2 +// IMPRVD_FAST-NEXT: [[TMP44:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// IMPRVD_FAST-NEXT: ret <2 x half> [[TMP44]] +// +// PRMTD_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x half> @f1( +// PRMTD_FAST-SAME: <2 x half> noundef nofpclass(nan inf) [[A_COERCE:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]], <2 x half> noundef nofpclass(nan inf) [[C_COERCE:%.*]]) #[[ATTR0]] { +// PRMTD_FAST-NEXT: entry: +// PRMTD_FAST-NEXT: [[RETVAL:%.*]] = alloca { half, half }, align 2 +// PRMTD_FAST-NEXT: [[A:%.*]] = alloca { half, half }, align 2 +// PRMTD_FAST-NEXT: [[C:%.*]] = alloca { half, half }, align 2 +// PRMTD_FAST-NEXT: store <2 x half> [[A_COERCE]], ptr [[A]], align 2 +// PRMTD_FAST-NEXT: store <2 x half> [[C_COERCE]], ptr [[C]], align 2 +// PRMTD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// PRMTD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// PRMTD_FAST-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[C]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[C_REAL:%.*]] = load half, ptr [[C_REALP]], align 2 +// PRMTD_FAST-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[C]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[C_IMAG:%.*]] = load half, ptr [[C_IMAGP]], align 2 +// PRMTD_FAST-NEXT: [[CONV:%.*]] = fpext half [[C_REAL]] to x86_fp80 +// PRMTD_FAST-NEXT: [[CONV1:%.*]] = fpext half [[C_IMAG]] to x86_fp80 +// PRMTD_FAST-NEXT: [[TMP0:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV]]) +// PRMTD_FAST-NEXT: [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV1]]) +// PRMTD_FAST-NEXT: [[ABS_CMP:%.*]] = fcmp reassoc nnan ninf nsz arcp afn ugt x86_fp80 [[TMP0]], [[TMP1]] +// PRMTD_FAST-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// PRMTD_FAST: abs_rhsr_greater_or_equal_abs_rhsi: +// PRMTD_FAST-NEXT: [[TMP2:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV1]], [[CONV]] +// PRMTD_FAST-NEXT: [[TMP3:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP2]], [[CONV1]] +// PRMTD_FAST-NEXT: [[TMP4:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV]], [[TMP3]] +// PRMTD_FAST-NEXT: [[TMP5:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[TMP2]] +// PRMTD_FAST-NEXT: [[TMP6:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[TMP5]] +// PRMTD_FAST-NEXT: [[TMP7:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP6]], [[TMP4]] +// PRMTD_FAST-NEXT: [[TMP8:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[TMP2]] +// PRMTD_FAST-NEXT: [[TMP9:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[TMP8]] +// PRMTD_FAST-NEXT: [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP9]], [[TMP4]] +// PRMTD_FAST-NEXT: br label [[COMPLEX_DIV:%.*]] +// PRMTD_FAST: abs_rhsr_less_than_abs_rhsi: +// PRMTD_FAST-NEXT: [[TMP11:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV]], [[CONV1]] +// PRMTD_FAST-NEXT: [[TMP12:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP11]], [[CONV]] +// PRMTD_FAST-NEXT: [[TMP13:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV1]], [[TMP12]] +// PRMTD_FAST-NEXT: [[TMP14:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[TMP11]] +// PRMTD_FAST-NEXT: [[TMP15:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP14]], [[B_IMAG]] +// PRMTD_FAST-NEXT: [[TMP16:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP15]], [[TMP13]] +// PRMTD_FAST-NEXT: [[TMP17:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[TMP11]] +// PRMTD_FAST-NEXT: [[TMP18:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP17]], [[B_REAL]] +// PRMTD_FAST-NEXT: [[TMP19:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP18]], [[TMP13]] +// PRMTD_FAST-NEXT: br label [[COMPLEX_DIV]] +// PRMTD_FAST: complex_div: +// PRMTD_FAST-NEXT: [[TMP20:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// PRMTD_FAST-NEXT: [[TMP21:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// PRMTD_FAST-NEXT: [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP20]] to half +// PRMTD_FAST-NEXT: [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP21]] to half +// PRMTD_FAST-NEXT: [[EXT:%.*]] = fpext half [[CONV2]] to float +// PRMTD_FAST-NEXT: [[EXT4:%.*]] = fpext half [[CONV3]] to float +// PRMTD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[A_REAL:%.*]] = load half, ptr [[A_REALP]], align 2 +// PRMTD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[A]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[A_IMAG:%.*]] = load half, ptr [[A_IMAGP]], align 2 +// PRMTD_FAST-NEXT: [[EXT5:%.*]] = fpext half [[A_REAL]] to float +// PRMTD_FAST-NEXT: [[EXT6:%.*]] = fpext half [[A_IMAG]] to float +// PRMTD_FAST-NEXT: [[TMP22:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT5]] +// PRMTD_FAST-NEXT: [[TMP23:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT4]], [[EXT6]] +// PRMTD_FAST-NEXT: [[TMP24:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP22]], [[TMP23]] +// PRMTD_FAST-NEXT: [[TMP25:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT5]], [[EXT5]] +// PRMTD_FAST-NEXT: [[TMP26:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT6]], [[EXT6]] +// PRMTD_FAST-NEXT: [[TMP27:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP25]], [[TMP26]] +// PRMTD_FAST-NEXT: [[TMP28:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT4]], [[EXT5]] +// PRMTD_FAST-NEXT: [[TMP29:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[EXT]], [[EXT6]] +// PRMTD_FAST-NEXT: [[TMP30:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[TMP28]], [[TMP29]] +// PRMTD_FAST-NEXT: [[TMP31:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP24]], [[TMP27]] +// PRMTD_FAST-NEXT: [[TMP32:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP30]], [[TMP27]] +// PRMTD_FAST-NEXT: [[UNPROMOTION:%.*]] = fptrunc float [[TMP31]] to half +// PRMTD_FAST-NEXT: [[UNPROMOTION7:%.*]] = fptrunc float [[TMP32]] to half +// PRMTD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { half, half }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD_FAST-NEXT: store half [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 2 +// PRMTD_FAST-NEXT: store half [[UNPROMOTION7]], ptr [[RETVAL_IMAGP]], align 2 +// PRMTD_FAST-NEXT: [[TMP33:%.*]] = load <2 x half>, ptr [[RETVAL]], align 2 +// PRMTD_FAST-NEXT: ret <2 x half> [[TMP33]] +// +_Complex _Float16 f1(_Complex _Float16 a, _Complex long double b, _Complex _Float16 c) { + return (_Complex _Float16)(b / c) / a; +} diff --git a/clang/test/CodeGen/complex-math.c b/clang/test/CodeGen/complex-math.c index a44aa0014a65..ba00b9cbecd2 100644 --- a/clang/test/CodeGen/complex-math.c +++ b/clang/test/CodeGen/complex-math.c @@ -5,7 +5,7 @@ // RUN: %clang_cc1 %s -O0 -emit-llvm -triple armv7-none-linux-gnueabi -o - | FileCheck %s --check-prefix=ARM // RUN: %clang_cc1 %s -O0 -emit-llvm -triple armv7-none-linux-gnueabihf -o - | FileCheck %s --check-prefix=ARMHF // RUN: %clang_cc1 %s -O0 -emit-llvm -triple thumbv7k-apple-watchos2.0 -o - -target-abi aapcs16 | FileCheck %s --check-prefix=ARM7K -// RUN: %clang_cc1 %s -O0 -emit-llvm -triple aarch64-unknown-unknown -ffast-math -ffp-contract=fast -complex-range=fortran -o - | FileCheck %s --check-prefix=AARCH64-FASTMATH +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple aarch64-unknown-unknown -ffast-math -ffp-contract=fast -complex-range=improved -o - | FileCheck %s --check-prefix=AARCH64-FASTMATH // RUN: %clang_cc1 %s -O0 -emit-llvm -triple spir -o - | FileCheck %s --check-prefix=SPIR float _Complex add_float_rr(float a, float b) { diff --git a/clang/test/CodeGen/cx-complex-range.c b/clang/test/CodeGen/cx-complex-range.c index 2d8507c710f2..9ec80252085b 100644 --- a/clang/test/CodeGen/cx-complex-range.c +++ b/clang/test/CodeGen/cx-complex-range.c @@ -1,124 +1,3451 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4 // RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ // RUN: -o - | FileCheck %s --check-prefix=FULL // RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ -// RUN: -complex-range=limited -o - | FileCheck %s --check-prefix=LMTD +// RUN: -complex-range=basic -o - | FileCheck %s --check-prefix=BASIC // RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ // RUN: -fno-cx-limited-range -o - | FileCheck %s --check-prefix=FULL // RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ -// RUN: -complex-range=fortran -o - | FileCheck %s --check-prefix=FRTRN +// RUN: -complex-range=improved -o - | FileCheck %s --check-prefix=IMPRVD + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -complex-range=promoted -o - | FileCheck %s --check-prefix=PRMTD + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -complex-range=full -o - | FileCheck %s --check-prefix=FULL + +// RUN: %clang_cc1 -triple x86_64-windows-pc -complex-range=promoted \ +// RUN: -emit-llvm -o - %s | FileCheck %s --check-prefix=X86WINPRMTD + +// RUN: %clang_cc1 -triple=avr-unknown-unknown -mdouble=32 \ +// RUN: -complex-range=promoted -emit-llvm -o - %s \ +// RUN: | FileCheck %s --check-prefix=AVRFP32 + +// RUN: %clang_cc1 -triple=avr-unknown-unknown -mdouble=64 \ +// RUN: -complex-range=promoted -emit-llvm -o - %s \ +// RUN: | FileCheck %s --check-prefix=AVRFP64 // Fast math // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu \ -// RUN: -ffast-math -complex-range=limited -emit-llvm -o - %s \ -// RUN: | FileCheck %s --check-prefix=LMTD-FAST +// RUN: -ffast-math -complex-range=basic -emit-llvm -o - %s \ +// RUN: | FileCheck %s --check-prefix=BASIC_FAST // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu \ // RUN: -ffast-math -complex-range=full -emit-llvm -o - %s \ -// RUN: | FileCheck %s --check-prefix=FULL +// RUN: | FileCheck %s --check-prefix=FULL_FAST // RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ // RUN: -fno-cx-fortran-rules -o - | FileCheck %s --check-prefix=FULL -// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ -// RUN: -fcx-limited-range -fno-cx-limited-range -o - \ -// RUN: | FileCheck %s --check-prefix=FULL +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu \ +// RUN: -ffast-math -complex-range=improved -emit-llvm -o - %s \ +// RUN: | FileCheck %s --check-prefix=IMPRVD_FAST -// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ -// RUN: -fno-cx-limited-range -fcx-limited-range -o - \ -// RUN: | FileCheck %s --check-prefix=FULL +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu \ +// RUN: -ffast-math -complex-range=promoted -emit-llvm -o - %s \ +// RUN: | FileCheck %s --check-prefix=PRMTD_FAST -// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ -// RUN: -fno-cx-fortran-rules -fcx-fortran-rules -o - \ -// RUN: | FileCheck %s --check-prefix=FULL - -_Complex float div(_Complex float a, _Complex float b) { - // LABEL: define {{.*}} @div( - // FULL: call {{.*}} @__divsc3 - - // LMTD: fmul float - // LMTD-NEXT: fmul float - // LMTD-NEXT: fadd float - // LMTD-NEXT: fmul float - // LMTD-NEXT: fmul float - // LMTD-NEXT: fadd float - // LMTD-NEXT: fmul float - // LMTD-NEXT: fmul float - // LMTD-NEXT: fsub float - // LMTD-NEXT: fdiv float - // LMTD-NEXT: fdiv float - - // FRTRN: call {{.*}}float @llvm.fabs.f32(float {{.*}}) - // FRTRN-NEXT: call {{.*}}float @llvm.fabs.f32(float {{.*}}) - // FRTRN-NEXT: fcmp {{.*}}ugt float - // FRTRN-NEXT: br i1 {{.*}}, label - // FRTRN: abs_rhsr_greater_or_equal_abs_rhsi: - // FRTRN-NEXT: fdiv {{.*}}float - // FRTRN-NEXT: fmul {{.*}}float - // FRTRN-NEXT: fadd {{.*}}float - // FRTRN-NEXT: fmul {{.*}}float - // FRTRN-NEXT: fadd {{.*}}float - // FRTRN-NEXT: fdiv {{.*}}float - // FRTRN-NEXT: fmul {{.*}}float - // FRTRN-NEXT: fsub {{.*}}float - // FRTRN-NEXT: fdiv {{.*}}float - // FRTRN-NEXT: br label - // FRTRN: abs_rhsr_less_than_abs_rhsi: - // FRTRN-NEXT: fdiv {{.*}}float - // FRTRN-NEXT: fmul {{.*}}float - // FRTRN-NEXT: fadd {{.*}}float - // FRTRN-NEXT: fmul {{.*}}float - // FRTRN-NEXT: fadd {{.*}}float - // FRTRN-NEXT: fdiv {{.*}}float - // FRTRN-NEXT: fmul {{.*}}float - // FRTRN-NEXT: fsub {{.*}}float - // FRTRN-NEXT: fdiv {{.*}}float - // FRTRN-NEXT: br label - // FRTRN: complex_div: - // FRTRN-NEXT: phi {{.*}}float - // FRTRN-NEXT: phi {{.*}}float - - // LMTD-FAST: fmul {{.*}} float - // LMTD-FAST-NEXT: fmul {{.*}} float - // LMTD-FAST-NEXT: fadd {{.*}} float - // LMTD-FAST-NEXT: fmul {{.*}} float - // LMTD-FAST-NEXT: fmul {{.*}} float - // LMTD-FAST-NEXT: fadd {{.*}} float - // LMTD-FAST-NEXT: fmul {{.*}} float - // LMTD-FAST-NEXT: fmul {{.*}} float - // LMTD-FAST-NEXT: fsub {{.*}} float - // LMTD-FAST-NEXT: fdiv {{.*}} float - // LMTD-FAST-NEXT: fdiv {{.*}} float +// FULL-LABEL: define dso_local <2 x float> @divf( +// FULL-SAME: <2 x float> noundef [[A_COERCE:%.*]], <2 x float> noundef [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// FULL-NEXT: entry: +// FULL-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// FULL-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// FULL-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// FULL-NEXT: [[COERCE:%.*]] = alloca { float, float }, align 4 +// FULL-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// FULL-NEXT: store <2 x float> [[B_COERCE]], ptr [[B]], align 4 +// FULL-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// FULL-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// FULL-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// FULL-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// FULL-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// FULL-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// FULL-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// FULL-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// FULL-NEXT: [[CALL:%.*]] = call <2 x float> @__divsc3(float noundef [[A_REAL]], float noundef [[A_IMAG]], float noundef [[B_REAL]], float noundef [[B_IMAG]]) #[[ATTR2:[0-9]+]] +// FULL-NEXT: store <2 x float> [[CALL]], ptr [[COERCE]], align 4 +// FULL-NEXT: [[COERCE_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 0 +// FULL-NEXT: [[COERCE_REAL:%.*]] = load float, ptr [[COERCE_REALP]], align 4 +// FULL-NEXT: [[COERCE_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 1 +// FULL-NEXT: [[COERCE_IMAG:%.*]] = load float, ptr [[COERCE_IMAGP]], align 4 +// FULL-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// FULL-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// FULL-NEXT: store float [[COERCE_REAL]], ptr [[RETVAL_REALP]], align 4 +// FULL-NEXT: store float [[COERCE_IMAG]], ptr [[RETVAL_IMAGP]], align 4 +// FULL-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// FULL-NEXT: ret <2 x float> [[TMP0]] +// +// BASIC-LABEL: define dso_local <2 x float> @divf( +// BASIC-SAME: <2 x float> noundef [[A_COERCE:%.*]], <2 x float> noundef [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// BASIC-NEXT: entry: +// BASIC-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// BASIC-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// BASIC-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// BASIC-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// BASIC-NEXT: store <2 x float> [[B_COERCE]], ptr [[B]], align 4 +// BASIC-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// BASIC-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// BASIC-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// BASIC-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// BASIC-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// BASIC-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// BASIC-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// BASIC-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// BASIC-NEXT: [[TMP0:%.*]] = fmul float [[A_REAL]], [[B_REAL]] +// BASIC-NEXT: [[TMP1:%.*]] = fmul float [[A_IMAG]], [[B_IMAG]] +// BASIC-NEXT: [[TMP2:%.*]] = fadd float [[TMP0]], [[TMP1]] +// BASIC-NEXT: [[TMP3:%.*]] = fmul float [[B_REAL]], [[B_REAL]] +// BASIC-NEXT: [[TMP4:%.*]] = fmul float [[B_IMAG]], [[B_IMAG]] +// BASIC-NEXT: [[TMP5:%.*]] = fadd float [[TMP3]], [[TMP4]] +// BASIC-NEXT: [[TMP6:%.*]] = fmul float [[A_IMAG]], [[B_REAL]] +// BASIC-NEXT: [[TMP7:%.*]] = fmul float [[A_REAL]], [[B_IMAG]] +// BASIC-NEXT: [[TMP8:%.*]] = fsub float [[TMP6]], [[TMP7]] +// BASIC-NEXT: [[TMP9:%.*]] = fdiv float [[TMP2]], [[TMP5]] +// BASIC-NEXT: [[TMP10:%.*]] = fdiv float [[TMP8]], [[TMP5]] +// BASIC-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC-NEXT: store float [[TMP9]], ptr [[RETVAL_REALP]], align 4 +// BASIC-NEXT: store float [[TMP10]], ptr [[RETVAL_IMAGP]], align 4 +// BASIC-NEXT: [[TMP11:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// BASIC-NEXT: ret <2 x float> [[TMP11]] +// +// IMPRVD-LABEL: define dso_local <2 x float> @divf( +// IMPRVD-SAME: <2 x float> noundef [[A_COERCE:%.*]], <2 x float> noundef [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// IMPRVD-NEXT: entry: +// IMPRVD-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// IMPRVD-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// IMPRVD-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// IMPRVD-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// IMPRVD-NEXT: store <2 x float> [[B_COERCE]], ptr [[B]], align 4 +// IMPRVD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// IMPRVD-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// IMPRVD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// IMPRVD-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// IMPRVD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// IMPRVD-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// IMPRVD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// IMPRVD-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// IMPRVD-NEXT: [[TMP0:%.*]] = call float @llvm.fabs.f32(float [[B_REAL]]) +// IMPRVD-NEXT: [[TMP1:%.*]] = call float @llvm.fabs.f32(float [[B_IMAG]]) +// IMPRVD-NEXT: [[ABS_CMP:%.*]] = fcmp ugt float [[TMP0]], [[TMP1]] +// IMPRVD-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// IMPRVD: abs_rhsr_greater_or_equal_abs_rhsi: +// IMPRVD-NEXT: [[TMP2:%.*]] = fdiv float [[B_IMAG]], [[B_REAL]] +// IMPRVD-NEXT: [[TMP3:%.*]] = fmul float [[TMP2]], [[B_IMAG]] +// IMPRVD-NEXT: [[TMP4:%.*]] = fadd float [[B_REAL]], [[TMP3]] +// IMPRVD-NEXT: [[TMP5:%.*]] = fmul float [[A_IMAG]], [[TMP2]] +// IMPRVD-NEXT: [[TMP6:%.*]] = fadd float [[A_REAL]], [[TMP5]] +// IMPRVD-NEXT: [[TMP7:%.*]] = fdiv float [[TMP6]], [[TMP4]] +// IMPRVD-NEXT: [[TMP8:%.*]] = fmul float [[A_REAL]], [[TMP2]] +// IMPRVD-NEXT: [[TMP9:%.*]] = fsub float [[A_IMAG]], [[TMP8]] +// IMPRVD-NEXT: [[TMP10:%.*]] = fdiv float [[TMP9]], [[TMP4]] +// IMPRVD-NEXT: br label [[COMPLEX_DIV:%.*]] +// IMPRVD: abs_rhsr_less_than_abs_rhsi: +// IMPRVD-NEXT: [[TMP11:%.*]] = fdiv float [[B_REAL]], [[B_IMAG]] +// IMPRVD-NEXT: [[TMP12:%.*]] = fmul float [[TMP11]], [[B_REAL]] +// IMPRVD-NEXT: [[TMP13:%.*]] = fadd float [[B_IMAG]], [[TMP12]] +// IMPRVD-NEXT: [[TMP14:%.*]] = fmul float [[A_REAL]], [[TMP11]] +// IMPRVD-NEXT: [[TMP15:%.*]] = fadd float [[TMP14]], [[A_IMAG]] +// IMPRVD-NEXT: [[TMP16:%.*]] = fdiv float [[TMP15]], [[TMP13]] +// IMPRVD-NEXT: [[TMP17:%.*]] = fmul float [[A_IMAG]], [[TMP11]] +// IMPRVD-NEXT: [[TMP18:%.*]] = fsub float [[TMP17]], [[A_REAL]] +// IMPRVD-NEXT: [[TMP19:%.*]] = fdiv float [[TMP18]], [[TMP13]] +// IMPRVD-NEXT: br label [[COMPLEX_DIV]] +// IMPRVD: complex_div: +// IMPRVD-NEXT: [[TMP20:%.*]] = phi float [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD-NEXT: [[TMP21:%.*]] = phi float [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD-NEXT: store float [[TMP20]], ptr [[RETVAL_REALP]], align 4 +// IMPRVD-NEXT: store float [[TMP21]], ptr [[RETVAL_IMAGP]], align 4 +// IMPRVD-NEXT: [[TMP22:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// IMPRVD-NEXT: ret <2 x float> [[TMP22]] +// +// PRMTD-LABEL: define dso_local <2 x float> @divf( +// PRMTD-SAME: <2 x float> noundef [[A_COERCE:%.*]], <2 x float> noundef [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// PRMTD-NEXT: entry: +// PRMTD-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// PRMTD-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// PRMTD-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// PRMTD-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// PRMTD-NEXT: store <2 x float> [[B_COERCE]], ptr [[B]], align 4 +// PRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// PRMTD-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// PRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// PRMTD-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// PRMTD-NEXT: [[EXT:%.*]] = fpext float [[A_REAL]] to double +// PRMTD-NEXT: [[EXT1:%.*]] = fpext float [[A_IMAG]] to double +// PRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// PRMTD-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// PRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// PRMTD-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// PRMTD-NEXT: [[EXT2:%.*]] = fpext float [[B_REAL]] to double +// PRMTD-NEXT: [[EXT3:%.*]] = fpext float [[B_IMAG]] to double +// PRMTD-NEXT: [[TMP0:%.*]] = fmul double [[EXT]], [[EXT2]] +// PRMTD-NEXT: [[TMP1:%.*]] = fmul double [[EXT1]], [[EXT3]] +// PRMTD-NEXT: [[TMP2:%.*]] = fadd double [[TMP0]], [[TMP1]] +// PRMTD-NEXT: [[TMP3:%.*]] = fmul double [[EXT2]], [[EXT2]] +// PRMTD-NEXT: [[TMP4:%.*]] = fmul double [[EXT3]], [[EXT3]] +// PRMTD-NEXT: [[TMP5:%.*]] = fadd double [[TMP3]], [[TMP4]] +// PRMTD-NEXT: [[TMP6:%.*]] = fmul double [[EXT1]], [[EXT2]] +// PRMTD-NEXT: [[TMP7:%.*]] = fmul double [[EXT]], [[EXT3]] +// PRMTD-NEXT: [[TMP8:%.*]] = fsub double [[TMP6]], [[TMP7]] +// PRMTD-NEXT: [[TMP9:%.*]] = fdiv double [[TMP2]], [[TMP5]] +// PRMTD-NEXT: [[TMP10:%.*]] = fdiv double [[TMP8]], [[TMP5]] +// PRMTD-NEXT: [[UNPROMOTION:%.*]] = fptrunc double [[TMP9]] to float +// PRMTD-NEXT: [[UNPROMOTION4:%.*]] = fptrunc double [[TMP10]] to float +// PRMTD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD-NEXT: store float [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 4 +// PRMTD-NEXT: store float [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 4 +// PRMTD-NEXT: [[TMP11:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// PRMTD-NEXT: ret <2 x float> [[TMP11]] +// +// X86WINPRMTD-LABEL: define dso_local i64 @divf( +// X86WINPRMTD-SAME: i64 noundef [[A_COERCE:%.*]], i64 noundef [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// X86WINPRMTD-NEXT: entry: +// X86WINPRMTD-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// X86WINPRMTD-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// X86WINPRMTD-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// X86WINPRMTD-NEXT: store i64 [[A_COERCE]], ptr [[A]], align 4 +// X86WINPRMTD-NEXT: store i64 [[B_COERCE]], ptr [[B]], align 4 +// X86WINPRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// X86WINPRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// X86WINPRMTD-NEXT: [[EXT:%.*]] = fpext float [[A_REAL]] to double +// X86WINPRMTD-NEXT: [[EXT1:%.*]] = fpext float [[A_IMAG]] to double +// X86WINPRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// X86WINPRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// X86WINPRMTD-NEXT: [[EXT2:%.*]] = fpext float [[B_REAL]] to double +// X86WINPRMTD-NEXT: [[EXT3:%.*]] = fpext float [[B_IMAG]] to double +// X86WINPRMTD-NEXT: [[TMP0:%.*]] = fmul double [[EXT]], [[EXT2]] +// X86WINPRMTD-NEXT: [[TMP1:%.*]] = fmul double [[EXT1]], [[EXT3]] +// X86WINPRMTD-NEXT: [[TMP2:%.*]] = fadd double [[TMP0]], [[TMP1]] +// X86WINPRMTD-NEXT: [[TMP3:%.*]] = fmul double [[EXT2]], [[EXT2]] +// X86WINPRMTD-NEXT: [[TMP4:%.*]] = fmul double [[EXT3]], [[EXT3]] +// X86WINPRMTD-NEXT: [[TMP5:%.*]] = fadd double [[TMP3]], [[TMP4]] +// X86WINPRMTD-NEXT: [[TMP6:%.*]] = fmul double [[EXT1]], [[EXT2]] +// X86WINPRMTD-NEXT: [[TMP7:%.*]] = fmul double [[EXT]], [[EXT3]] +// X86WINPRMTD-NEXT: [[TMP8:%.*]] = fsub double [[TMP6]], [[TMP7]] +// X86WINPRMTD-NEXT: [[TMP9:%.*]] = fdiv double [[TMP2]], [[TMP5]] +// X86WINPRMTD-NEXT: [[TMP10:%.*]] = fdiv double [[TMP8]], [[TMP5]] +// X86WINPRMTD-NEXT: [[UNPROMOTION:%.*]] = fptrunc double [[TMP9]] to float +// X86WINPRMTD-NEXT: [[UNPROMOTION4:%.*]] = fptrunc double [[TMP10]] to float +// X86WINPRMTD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// X86WINPRMTD-NEXT: store float [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 4 +// X86WINPRMTD-NEXT: store float [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 4 +// X86WINPRMTD-NEXT: [[TMP11:%.*]] = load i64, ptr [[RETVAL]], align 4 +// X86WINPRMTD-NEXT: ret i64 [[TMP11]] +// +// AVRFP32-LABEL: define dso_local { float, float } @divf( +// AVRFP32-SAME: float noundef [[A_COERCE0:%.*]], float noundef [[A_COERCE1:%.*]], float noundef [[B_COERCE0:%.*]], float noundef [[B_COERCE1:%.*]]) addrspace(1) #[[ATTR0:[0-9]+]] { +// AVRFP32-NEXT: entry: +// AVRFP32-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 1 +// AVRFP32-NEXT: [[A:%.*]] = alloca { float, float }, align 1 +// AVRFP32-NEXT: [[B:%.*]] = alloca { float, float }, align 1 +// AVRFP32-NEXT: [[TMP0:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP32-NEXT: store float [[A_COERCE0]], ptr [[TMP0]], align 1 +// AVRFP32-NEXT: [[TMP1:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[A_COERCE1]], ptr [[TMP1]], align 1 +// AVRFP32-NEXT: [[TMP2:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// AVRFP32-NEXT: store float [[B_COERCE0]], ptr [[TMP2]], align 1 +// AVRFP32-NEXT: [[TMP3:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[B_COERCE1]], ptr [[TMP3]], align 1 +// AVRFP32-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP32-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 1 +// AVRFP32-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP32-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 1 +// AVRFP32-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// AVRFP32-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 1 +// AVRFP32-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// AVRFP32-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 1 +// AVRFP32-NEXT: [[TMP4:%.*]] = call addrspace(1) float @llvm.fabs.f32(float [[B_REAL]]) +// AVRFP32-NEXT: [[TMP5:%.*]] = call addrspace(1) float @llvm.fabs.f32(float [[B_IMAG]]) +// AVRFP32-NEXT: [[ABS_CMP:%.*]] = fcmp ugt float [[TMP4]], [[TMP5]] +// AVRFP32-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// AVRFP32: abs_rhsr_greater_or_equal_abs_rhsi: +// AVRFP32-NEXT: [[TMP6:%.*]] = fdiv float [[B_IMAG]], [[B_REAL]] +// AVRFP32-NEXT: [[TMP7:%.*]] = fmul float [[TMP6]], [[B_IMAG]] +// AVRFP32-NEXT: [[TMP8:%.*]] = fadd float [[B_REAL]], [[TMP7]] +// AVRFP32-NEXT: [[TMP9:%.*]] = fmul float [[A_IMAG]], [[TMP6]] +// AVRFP32-NEXT: [[TMP10:%.*]] = fadd float [[A_REAL]], [[TMP9]] +// AVRFP32-NEXT: [[TMP11:%.*]] = fdiv float [[TMP10]], [[TMP8]] +// AVRFP32-NEXT: [[TMP12:%.*]] = fmul float [[A_REAL]], [[TMP6]] +// AVRFP32-NEXT: [[TMP13:%.*]] = fsub float [[A_IMAG]], [[TMP12]] +// AVRFP32-NEXT: [[TMP14:%.*]] = fdiv float [[TMP13]], [[TMP8]] +// AVRFP32-NEXT: br label [[COMPLEX_DIV:%.*]] +// AVRFP32: abs_rhsr_less_than_abs_rhsi: +// AVRFP32-NEXT: [[TMP15:%.*]] = fdiv float [[B_REAL]], [[B_IMAG]] +// AVRFP32-NEXT: [[TMP16:%.*]] = fmul float [[TMP15]], [[B_REAL]] +// AVRFP32-NEXT: [[TMP17:%.*]] = fadd float [[B_IMAG]], [[TMP16]] +// AVRFP32-NEXT: [[TMP18:%.*]] = fmul float [[A_REAL]], [[TMP15]] +// AVRFP32-NEXT: [[TMP19:%.*]] = fadd float [[TMP18]], [[A_IMAG]] +// AVRFP32-NEXT: [[TMP20:%.*]] = fdiv float [[TMP19]], [[TMP17]] +// AVRFP32-NEXT: [[TMP21:%.*]] = fmul float [[A_IMAG]], [[TMP15]] +// AVRFP32-NEXT: [[TMP22:%.*]] = fsub float [[TMP21]], [[A_REAL]] +// AVRFP32-NEXT: [[TMP23:%.*]] = fdiv float [[TMP22]], [[TMP17]] +// AVRFP32-NEXT: br label [[COMPLEX_DIV]] +// AVRFP32: complex_div: +// AVRFP32-NEXT: [[TMP24:%.*]] = phi float [ [[TMP11]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP20]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// AVRFP32-NEXT: [[TMP25:%.*]] = phi float [ [[TMP14]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP23]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// AVRFP32-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// AVRFP32-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[TMP24]], ptr [[RETVAL_REALP]], align 1 +// AVRFP32-NEXT: store float [[TMP25]], ptr [[RETVAL_IMAGP]], align 1 +// AVRFP32-NEXT: [[TMP26:%.*]] = load { float, float }, ptr [[RETVAL]], align 1 +// AVRFP32-NEXT: ret { float, float } [[TMP26]] +// +// AVRFP64-LABEL: define dso_local { float, float } @divf( +// AVRFP64-SAME: float noundef [[A_COERCE0:%.*]], float noundef [[A_COERCE1:%.*]], float noundef [[B_COERCE0:%.*]], float noundef [[B_COERCE1:%.*]]) addrspace(1) #[[ATTR0:[0-9]+]] { +// AVRFP64-NEXT: entry: +// AVRFP64-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 1 +// AVRFP64-NEXT: [[A:%.*]] = alloca { float, float }, align 1 +// AVRFP64-NEXT: [[B:%.*]] = alloca { float, float }, align 1 +// AVRFP64-NEXT: [[TMP0:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP64-NEXT: store float [[A_COERCE0]], ptr [[TMP0]], align 1 +// AVRFP64-NEXT: [[TMP1:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP64-NEXT: store float [[A_COERCE1]], ptr [[TMP1]], align 1 +// AVRFP64-NEXT: [[TMP2:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// AVRFP64-NEXT: store float [[B_COERCE0]], ptr [[TMP2]], align 1 +// AVRFP64-NEXT: [[TMP3:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// AVRFP64-NEXT: store float [[B_COERCE1]], ptr [[TMP3]], align 1 +// AVRFP64-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP64-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 1 +// AVRFP64-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP64-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 1 +// AVRFP64-NEXT: [[EXT:%.*]] = fpext float [[A_REAL]] to double +// AVRFP64-NEXT: [[EXT1:%.*]] = fpext float [[A_IMAG]] to double +// AVRFP64-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// AVRFP64-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 1 +// AVRFP64-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// AVRFP64-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 1 +// AVRFP64-NEXT: [[EXT2:%.*]] = fpext float [[B_REAL]] to double +// AVRFP64-NEXT: [[EXT3:%.*]] = fpext float [[B_IMAG]] to double +// AVRFP64-NEXT: [[TMP4:%.*]] = fmul double [[EXT]], [[EXT2]] +// AVRFP64-NEXT: [[TMP5:%.*]] = fmul double [[EXT1]], [[EXT3]] +// AVRFP64-NEXT: [[TMP6:%.*]] = fadd double [[TMP4]], [[TMP5]] +// AVRFP64-NEXT: [[TMP7:%.*]] = fmul double [[EXT2]], [[EXT2]] +// AVRFP64-NEXT: [[TMP8:%.*]] = fmul double [[EXT3]], [[EXT3]] +// AVRFP64-NEXT: [[TMP9:%.*]] = fadd double [[TMP7]], [[TMP8]] +// AVRFP64-NEXT: [[TMP10:%.*]] = fmul double [[EXT1]], [[EXT2]] +// AVRFP64-NEXT: [[TMP11:%.*]] = fmul double [[EXT]], [[EXT3]] +// AVRFP64-NEXT: [[TMP12:%.*]] = fsub double [[TMP10]], [[TMP11]] +// AVRFP64-NEXT: [[TMP13:%.*]] = fdiv double [[TMP6]], [[TMP9]] +// AVRFP64-NEXT: [[TMP14:%.*]] = fdiv double [[TMP12]], [[TMP9]] +// AVRFP64-NEXT: [[UNPROMOTION:%.*]] = fptrunc double [[TMP13]] to float +// AVRFP64-NEXT: [[UNPROMOTION4:%.*]] = fptrunc double [[TMP14]] to float +// AVRFP64-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// AVRFP64-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// AVRFP64-NEXT: store float [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 1 +// AVRFP64-NEXT: store float [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 1 +// AVRFP64-NEXT: [[TMP15:%.*]] = load { float, float }, ptr [[RETVAL]], align 1 +// AVRFP64-NEXT: ret { float, float } [[TMP15]] +// +// BASIC_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x float> @divf( +// BASIC_FAST-SAME: <2 x float> noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x float> noundef nofpclass(nan inf) [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// BASIC_FAST-NEXT: entry: +// BASIC_FAST-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// BASIC_FAST-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// BASIC_FAST-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// BASIC_FAST-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// BASIC_FAST-NEXT: store <2 x float> [[B_COERCE]], ptr [[B]], align 4 +// BASIC_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// BASIC_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// BASIC_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// BASIC_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// BASIC_FAST-NEXT: [[TMP0:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_REAL]], [[B_REAL]] +// BASIC_FAST-NEXT: [[TMP1:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_IMAG]], [[B_IMAG]] +// BASIC_FAST-NEXT: [[TMP2:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP0]], [[TMP1]] +// BASIC_FAST-NEXT: [[TMP3:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[B_REAL]], [[B_REAL]] +// BASIC_FAST-NEXT: [[TMP4:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[B_IMAG]], [[B_IMAG]] +// BASIC_FAST-NEXT: [[TMP5:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP3]], [[TMP4]] +// BASIC_FAST-NEXT: [[TMP6:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_IMAG]], [[B_REAL]] +// BASIC_FAST-NEXT: [[TMP7:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_REAL]], [[B_IMAG]] +// BASIC_FAST-NEXT: [[TMP8:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[TMP6]], [[TMP7]] +// BASIC_FAST-NEXT: [[TMP9:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP2]], [[TMP5]] +// BASIC_FAST-NEXT: [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP8]], [[TMP5]] +// BASIC_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC_FAST-NEXT: store float [[TMP9]], ptr [[RETVAL_REALP]], align 4 +// BASIC_FAST-NEXT: store float [[TMP10]], ptr [[RETVAL_IMAGP]], align 4 +// BASIC_FAST-NEXT: [[TMP11:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// BASIC_FAST-NEXT: ret <2 x float> [[TMP11]] +// +// FULL_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x float> @divf( +// FULL_FAST-SAME: <2 x float> noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x float> noundef nofpclass(nan inf) [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// FULL_FAST-NEXT: entry: +// FULL_FAST-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// FULL_FAST-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// FULL_FAST-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// FULL_FAST-NEXT: [[COERCE:%.*]] = alloca { float, float }, align 4 +// FULL_FAST-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// FULL_FAST-NEXT: store <2 x float> [[B_COERCE]], ptr [[B]], align 4 +// FULL_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// FULL_FAST-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// FULL_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// FULL_FAST-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// FULL_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// FULL_FAST-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// FULL_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// FULL_FAST-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// FULL_FAST-NEXT: [[CALL:%.*]] = call reassoc nnan ninf nsz arcp afn nofpclass(nan inf) <2 x float> @__divsc3(float noundef nofpclass(nan inf) [[A_REAL]], float noundef nofpclass(nan inf) [[A_IMAG]], float noundef nofpclass(nan inf) [[B_REAL]], float noundef nofpclass(nan inf) [[B_IMAG]]) #[[ATTR2:[0-9]+]] +// FULL_FAST-NEXT: store <2 x float> [[CALL]], ptr [[COERCE]], align 4 +// FULL_FAST-NEXT: [[COERCE_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 0 +// FULL_FAST-NEXT: [[COERCE_REAL:%.*]] = load float, ptr [[COERCE_REALP]], align 4 +// FULL_FAST-NEXT: [[COERCE_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 1 +// FULL_FAST-NEXT: [[COERCE_IMAG:%.*]] = load float, ptr [[COERCE_IMAGP]], align 4 +// FULL_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// FULL_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// FULL_FAST-NEXT: store float [[COERCE_REAL]], ptr [[RETVAL_REALP]], align 4 +// FULL_FAST-NEXT: store float [[COERCE_IMAG]], ptr [[RETVAL_IMAGP]], align 4 +// FULL_FAST-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// FULL_FAST-NEXT: ret <2 x float> [[TMP0]] +// +// IMPRVD_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x float> @divf( +// IMPRVD_FAST-SAME: <2 x float> noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x float> noundef nofpclass(nan inf) [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// IMPRVD_FAST-NEXT: entry: +// IMPRVD_FAST-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// IMPRVD_FAST-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// IMPRVD_FAST-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// IMPRVD_FAST-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// IMPRVD_FAST-NEXT: store <2 x float> [[B_COERCE]], ptr [[B]], align 4 +// IMPRVD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// IMPRVD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// IMPRVD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// IMPRVD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// IMPRVD_FAST-NEXT: [[TMP0:%.*]] = call reassoc nnan ninf nsz arcp afn float @llvm.fabs.f32(float [[B_REAL]]) +// IMPRVD_FAST-NEXT: [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn float @llvm.fabs.f32(float [[B_IMAG]]) +// IMPRVD_FAST-NEXT: [[ABS_CMP:%.*]] = fcmp reassoc nnan ninf nsz arcp afn ugt float [[TMP0]], [[TMP1]] +// IMPRVD_FAST-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// IMPRVD_FAST: abs_rhsr_greater_or_equal_abs_rhsi: +// IMPRVD_FAST-NEXT: [[TMP2:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[B_IMAG]], [[B_REAL]] +// IMPRVD_FAST-NEXT: [[TMP3:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[TMP2]], [[B_IMAG]] +// IMPRVD_FAST-NEXT: [[TMP4:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[B_REAL]], [[TMP3]] +// IMPRVD_FAST-NEXT: [[TMP5:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_IMAG]], [[TMP2]] +// IMPRVD_FAST-NEXT: [[TMP6:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[A_REAL]], [[TMP5]] +// IMPRVD_FAST-NEXT: [[TMP7:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP6]], [[TMP4]] +// IMPRVD_FAST-NEXT: [[TMP8:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_REAL]], [[TMP2]] +// IMPRVD_FAST-NEXT: [[TMP9:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[A_IMAG]], [[TMP8]] +// IMPRVD_FAST-NEXT: [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP9]], [[TMP4]] +// IMPRVD_FAST-NEXT: br label [[COMPLEX_DIV:%.*]] +// IMPRVD_FAST: abs_rhsr_less_than_abs_rhsi: +// IMPRVD_FAST-NEXT: [[TMP11:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[B_REAL]], [[B_IMAG]] +// IMPRVD_FAST-NEXT: [[TMP12:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[TMP11]], [[B_REAL]] +// IMPRVD_FAST-NEXT: [[TMP13:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[B_IMAG]], [[TMP12]] +// IMPRVD_FAST-NEXT: [[TMP14:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_REAL]], [[TMP11]] +// IMPRVD_FAST-NEXT: [[TMP15:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP14]], [[A_IMAG]] +// IMPRVD_FAST-NEXT: [[TMP16:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP15]], [[TMP13]] +// IMPRVD_FAST-NEXT: [[TMP17:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_IMAG]], [[TMP11]] +// IMPRVD_FAST-NEXT: [[TMP18:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[TMP17]], [[A_REAL]] +// IMPRVD_FAST-NEXT: [[TMP19:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP18]], [[TMP13]] +// IMPRVD_FAST-NEXT: br label [[COMPLEX_DIV]] +// IMPRVD_FAST: complex_div: +// IMPRVD_FAST-NEXT: [[TMP20:%.*]] = phi reassoc nnan ninf nsz arcp afn float [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD_FAST-NEXT: [[TMP21:%.*]] = phi reassoc nnan ninf nsz arcp afn float [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: store float [[TMP20]], ptr [[RETVAL_REALP]], align 4 +// IMPRVD_FAST-NEXT: store float [[TMP21]], ptr [[RETVAL_IMAGP]], align 4 +// IMPRVD_FAST-NEXT: [[TMP22:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// IMPRVD_FAST-NEXT: ret <2 x float> [[TMP22]] +// +// PRMTD_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x float> @divf( +// PRMTD_FAST-SAME: <2 x float> noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x float> noundef nofpclass(nan inf) [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// PRMTD_FAST-NEXT: entry: +// PRMTD_FAST-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// PRMTD_FAST-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// PRMTD_FAST-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// PRMTD_FAST-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// PRMTD_FAST-NEXT: store <2 x float> [[B_COERCE]], ptr [[B]], align 4 +// PRMTD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// PRMTD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// PRMTD_FAST-NEXT: [[EXT:%.*]] = fpext float [[A_REAL]] to double +// PRMTD_FAST-NEXT: [[EXT1:%.*]] = fpext float [[A_IMAG]] to double +// PRMTD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// PRMTD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// PRMTD_FAST-NEXT: [[EXT2:%.*]] = fpext float [[B_REAL]] to double +// PRMTD_FAST-NEXT: [[EXT3:%.*]] = fpext float [[B_IMAG]] to double +// PRMTD_FAST-NEXT: [[TMP0:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[EXT]], [[EXT2]] +// PRMTD_FAST-NEXT: [[TMP1:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[EXT1]], [[EXT3]] +// PRMTD_FAST-NEXT: [[TMP2:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[TMP0]], [[TMP1]] +// PRMTD_FAST-NEXT: [[TMP3:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[EXT2]], [[EXT2]] +// PRMTD_FAST-NEXT: [[TMP4:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[EXT3]], [[EXT3]] +// PRMTD_FAST-NEXT: [[TMP5:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[TMP3]], [[TMP4]] +// PRMTD_FAST-NEXT: [[TMP6:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[EXT1]], [[EXT2]] +// PRMTD_FAST-NEXT: [[TMP7:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[EXT]], [[EXT3]] +// PRMTD_FAST-NEXT: [[TMP8:%.*]] = fsub reassoc nnan ninf nsz arcp afn double [[TMP6]], [[TMP7]] +// PRMTD_FAST-NEXT: [[TMP9:%.*]] = fdiv reassoc nnan ninf nsz arcp afn double [[TMP2]], [[TMP5]] +// PRMTD_FAST-NEXT: [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn double [[TMP8]], [[TMP5]] +// PRMTD_FAST-NEXT: [[UNPROMOTION:%.*]] = fptrunc double [[TMP9]] to float +// PRMTD_FAST-NEXT: [[UNPROMOTION4:%.*]] = fptrunc double [[TMP10]] to float +// PRMTD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD_FAST-NEXT: store float [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 4 +// PRMTD_FAST-NEXT: store float [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 4 +// PRMTD_FAST-NEXT: [[TMP11:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// PRMTD_FAST-NEXT: ret <2 x float> [[TMP11]] +// +_Complex float divf(_Complex float a, _Complex float b) { + return a / b; +} + +// FULL-LABEL: define dso_local <2 x float> @mulf( +// FULL-SAME: <2 x float> noundef [[A_COERCE:%.*]], <2 x float> noundef [[B_COERCE:%.*]]) #[[ATTR0]] { +// FULL-NEXT: entry: +// FULL-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// FULL-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// FULL-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// FULL-NEXT: [[COERCE:%.*]] = alloca { float, float }, align 4 +// FULL-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// FULL-NEXT: store <2 x float> [[B_COERCE]], ptr [[B]], align 4 +// FULL-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// FULL-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// FULL-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// FULL-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// FULL-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// FULL-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// FULL-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// FULL-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// FULL-NEXT: [[MUL_AC:%.*]] = fmul float [[A_REAL]], [[B_REAL]] +// FULL-NEXT: [[MUL_BD:%.*]] = fmul float [[A_IMAG]], [[B_IMAG]] +// FULL-NEXT: [[MUL_AD:%.*]] = fmul float [[A_REAL]], [[B_IMAG]] +// FULL-NEXT: [[MUL_BC:%.*]] = fmul float [[A_IMAG]], [[B_REAL]] +// FULL-NEXT: [[MUL_R:%.*]] = fsub float [[MUL_AC]], [[MUL_BD]] +// FULL-NEXT: [[MUL_I:%.*]] = fadd float [[MUL_AD]], [[MUL_BC]] +// FULL-NEXT: [[ISNAN_CMP:%.*]] = fcmp uno float [[MUL_R]], [[MUL_R]] +// FULL-NEXT: br i1 [[ISNAN_CMP]], label [[COMPLEX_MUL_IMAG_NAN:%.*]], label [[COMPLEX_MUL_CONT:%.*]], !prof [[PROF2:![0-9]+]] +// FULL: complex_mul_imag_nan: +// FULL-NEXT: [[ISNAN_CMP1:%.*]] = fcmp uno float [[MUL_I]], [[MUL_I]] +// FULL-NEXT: br i1 [[ISNAN_CMP1]], label [[COMPLEX_MUL_LIBCALL:%.*]], label [[COMPLEX_MUL_CONT]], !prof [[PROF2]] +// FULL: complex_mul_libcall: +// FULL-NEXT: [[CALL:%.*]] = call <2 x float> @__mulsc3(float noundef [[A_REAL]], float noundef [[A_IMAG]], float noundef [[B_REAL]], float noundef [[B_IMAG]]) #[[ATTR2]] +// FULL-NEXT: store <2 x float> [[CALL]], ptr [[COERCE]], align 4 +// FULL-NEXT: [[COERCE_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 0 +// FULL-NEXT: [[COERCE_REAL:%.*]] = load float, ptr [[COERCE_REALP]], align 4 +// FULL-NEXT: [[COERCE_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 1 +// FULL-NEXT: [[COERCE_IMAG:%.*]] = load float, ptr [[COERCE_IMAGP]], align 4 +// FULL-NEXT: br label [[COMPLEX_MUL_CONT]] +// FULL: complex_mul_cont: +// FULL-NEXT: [[REAL_MUL_PHI:%.*]] = phi float [ [[MUL_R]], [[ENTRY:%.*]] ], [ [[MUL_R]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[COERCE_REAL]], [[COMPLEX_MUL_LIBCALL]] ] +// FULL-NEXT: [[IMAG_MUL_PHI:%.*]] = phi float [ [[MUL_I]], [[ENTRY]] ], [ [[MUL_I]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[COERCE_IMAG]], [[COMPLEX_MUL_LIBCALL]] ] +// FULL-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// FULL-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// FULL-NEXT: store float [[REAL_MUL_PHI]], ptr [[RETVAL_REALP]], align 4 +// FULL-NEXT: store float [[IMAG_MUL_PHI]], ptr [[RETVAL_IMAGP]], align 4 +// FULL-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// FULL-NEXT: ret <2 x float> [[TMP0]] +// +// BASIC-LABEL: define dso_local <2 x float> @mulf( +// BASIC-SAME: <2 x float> noundef [[A_COERCE:%.*]], <2 x float> noundef [[B_COERCE:%.*]]) #[[ATTR0]] { +// BASIC-NEXT: entry: +// BASIC-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// BASIC-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// BASIC-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// BASIC-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// BASIC-NEXT: store <2 x float> [[B_COERCE]], ptr [[B]], align 4 +// BASIC-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// BASIC-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// BASIC-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// BASIC-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// BASIC-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// BASIC-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// BASIC-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// BASIC-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// BASIC-NEXT: [[MUL_AC:%.*]] = fmul float [[A_REAL]], [[B_REAL]] +// BASIC-NEXT: [[MUL_BD:%.*]] = fmul float [[A_IMAG]], [[B_IMAG]] +// BASIC-NEXT: [[MUL_AD:%.*]] = fmul float [[A_REAL]], [[B_IMAG]] +// BASIC-NEXT: [[MUL_BC:%.*]] = fmul float [[A_IMAG]], [[B_REAL]] +// BASIC-NEXT: [[MUL_R:%.*]] = fsub float [[MUL_AC]], [[MUL_BD]] +// BASIC-NEXT: [[MUL_I:%.*]] = fadd float [[MUL_AD]], [[MUL_BC]] +// BASIC-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC-NEXT: store float [[MUL_R]], ptr [[RETVAL_REALP]], align 4 +// BASIC-NEXT: store float [[MUL_I]], ptr [[RETVAL_IMAGP]], align 4 +// BASIC-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// BASIC-NEXT: ret <2 x float> [[TMP0]] +// +// IMPRVD-LABEL: define dso_local <2 x float> @mulf( +// IMPRVD-SAME: <2 x float> noundef [[A_COERCE:%.*]], <2 x float> noundef [[B_COERCE:%.*]]) #[[ATTR0]] { +// IMPRVD-NEXT: entry: +// IMPRVD-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// IMPRVD-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// IMPRVD-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// IMPRVD-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// IMPRVD-NEXT: store <2 x float> [[B_COERCE]], ptr [[B]], align 4 +// IMPRVD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// IMPRVD-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// IMPRVD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// IMPRVD-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// IMPRVD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// IMPRVD-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// IMPRVD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// IMPRVD-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// IMPRVD-NEXT: [[MUL_AC:%.*]] = fmul float [[A_REAL]], [[B_REAL]] +// IMPRVD-NEXT: [[MUL_BD:%.*]] = fmul float [[A_IMAG]], [[B_IMAG]] +// IMPRVD-NEXT: [[MUL_AD:%.*]] = fmul float [[A_REAL]], [[B_IMAG]] +// IMPRVD-NEXT: [[MUL_BC:%.*]] = fmul float [[A_IMAG]], [[B_REAL]] +// IMPRVD-NEXT: [[MUL_R:%.*]] = fsub float [[MUL_AC]], [[MUL_BD]] +// IMPRVD-NEXT: [[MUL_I:%.*]] = fadd float [[MUL_AD]], [[MUL_BC]] +// IMPRVD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD-NEXT: store float [[MUL_R]], ptr [[RETVAL_REALP]], align 4 +// IMPRVD-NEXT: store float [[MUL_I]], ptr [[RETVAL_IMAGP]], align 4 +// IMPRVD-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// IMPRVD-NEXT: ret <2 x float> [[TMP0]] +// +// PRMTD-LABEL: define dso_local <2 x float> @mulf( +// PRMTD-SAME: <2 x float> noundef [[A_COERCE:%.*]], <2 x float> noundef [[B_COERCE:%.*]]) #[[ATTR0]] { +// PRMTD-NEXT: entry: +// PRMTD-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// PRMTD-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// PRMTD-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// PRMTD-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// PRMTD-NEXT: store <2 x float> [[B_COERCE]], ptr [[B]], align 4 +// PRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// PRMTD-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// PRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// PRMTD-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// PRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// PRMTD-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// PRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// PRMTD-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// PRMTD-NEXT: [[MUL_AC:%.*]] = fmul float [[A_REAL]], [[B_REAL]] +// PRMTD-NEXT: [[MUL_BD:%.*]] = fmul float [[A_IMAG]], [[B_IMAG]] +// PRMTD-NEXT: [[MUL_AD:%.*]] = fmul float [[A_REAL]], [[B_IMAG]] +// PRMTD-NEXT: [[MUL_BC:%.*]] = fmul float [[A_IMAG]], [[B_REAL]] +// PRMTD-NEXT: [[MUL_R:%.*]] = fsub float [[MUL_AC]], [[MUL_BD]] +// PRMTD-NEXT: [[MUL_I:%.*]] = fadd float [[MUL_AD]], [[MUL_BC]] +// PRMTD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD-NEXT: store float [[MUL_R]], ptr [[RETVAL_REALP]], align 4 +// PRMTD-NEXT: store float [[MUL_I]], ptr [[RETVAL_IMAGP]], align 4 +// PRMTD-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// PRMTD-NEXT: ret <2 x float> [[TMP0]] +// +// X86WINPRMTD-LABEL: define dso_local i64 @mulf( +// X86WINPRMTD-SAME: i64 noundef [[A_COERCE:%.*]], i64 noundef [[B_COERCE:%.*]]) #[[ATTR0]] { +// X86WINPRMTD-NEXT: entry: +// X86WINPRMTD-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// X86WINPRMTD-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// X86WINPRMTD-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// X86WINPRMTD-NEXT: store i64 [[A_COERCE]], ptr [[A]], align 4 +// X86WINPRMTD-NEXT: store i64 [[B_COERCE]], ptr [[B]], align 4 +// X86WINPRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// X86WINPRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// X86WINPRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// X86WINPRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// X86WINPRMTD-NEXT: [[MUL_AC:%.*]] = fmul float [[A_REAL]], [[B_REAL]] +// X86WINPRMTD-NEXT: [[MUL_BD:%.*]] = fmul float [[A_IMAG]], [[B_IMAG]] +// X86WINPRMTD-NEXT: [[MUL_AD:%.*]] = fmul float [[A_REAL]], [[B_IMAG]] +// X86WINPRMTD-NEXT: [[MUL_BC:%.*]] = fmul float [[A_IMAG]], [[B_REAL]] +// X86WINPRMTD-NEXT: [[MUL_R:%.*]] = fsub float [[MUL_AC]], [[MUL_BD]] +// X86WINPRMTD-NEXT: [[MUL_I:%.*]] = fadd float [[MUL_AD]], [[MUL_BC]] +// X86WINPRMTD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// X86WINPRMTD-NEXT: store float [[MUL_R]], ptr [[RETVAL_REALP]], align 4 +// X86WINPRMTD-NEXT: store float [[MUL_I]], ptr [[RETVAL_IMAGP]], align 4 +// X86WINPRMTD-NEXT: [[TMP0:%.*]] = load i64, ptr [[RETVAL]], align 4 +// X86WINPRMTD-NEXT: ret i64 [[TMP0]] +// +// AVRFP32-LABEL: define dso_local { float, float } @mulf( +// AVRFP32-SAME: float noundef [[A_COERCE0:%.*]], float noundef [[A_COERCE1:%.*]], float noundef [[B_COERCE0:%.*]], float noundef [[B_COERCE1:%.*]]) addrspace(1) #[[ATTR0]] { +// AVRFP32-NEXT: entry: +// AVRFP32-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 1 +// AVRFP32-NEXT: [[A:%.*]] = alloca { float, float }, align 1 +// AVRFP32-NEXT: [[B:%.*]] = alloca { float, float }, align 1 +// AVRFP32-NEXT: [[TMP0:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP32-NEXT: store float [[A_COERCE0]], ptr [[TMP0]], align 1 +// AVRFP32-NEXT: [[TMP1:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[A_COERCE1]], ptr [[TMP1]], align 1 +// AVRFP32-NEXT: [[TMP2:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// AVRFP32-NEXT: store float [[B_COERCE0]], ptr [[TMP2]], align 1 +// AVRFP32-NEXT: [[TMP3:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[B_COERCE1]], ptr [[TMP3]], align 1 +// AVRFP32-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP32-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 1 +// AVRFP32-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP32-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 1 +// AVRFP32-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// AVRFP32-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 1 +// AVRFP32-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// AVRFP32-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 1 +// AVRFP32-NEXT: [[MUL_AC:%.*]] = fmul float [[A_REAL]], [[B_REAL]] +// AVRFP32-NEXT: [[MUL_BD:%.*]] = fmul float [[A_IMAG]], [[B_IMAG]] +// AVRFP32-NEXT: [[MUL_AD:%.*]] = fmul float [[A_REAL]], [[B_IMAG]] +// AVRFP32-NEXT: [[MUL_BC:%.*]] = fmul float [[A_IMAG]], [[B_REAL]] +// AVRFP32-NEXT: [[MUL_R:%.*]] = fsub float [[MUL_AC]], [[MUL_BD]] +// AVRFP32-NEXT: [[MUL_I:%.*]] = fadd float [[MUL_AD]], [[MUL_BC]] +// AVRFP32-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// AVRFP32-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[MUL_R]], ptr [[RETVAL_REALP]], align 1 +// AVRFP32-NEXT: store float [[MUL_I]], ptr [[RETVAL_IMAGP]], align 1 +// AVRFP32-NEXT: [[TMP4:%.*]] = load { float, float }, ptr [[RETVAL]], align 1 +// AVRFP32-NEXT: ret { float, float } [[TMP4]] +// +// AVRFP64-LABEL: define dso_local { float, float } @mulf( +// AVRFP64-SAME: float noundef [[A_COERCE0:%.*]], float noundef [[A_COERCE1:%.*]], float noundef [[B_COERCE0:%.*]], float noundef [[B_COERCE1:%.*]]) addrspace(1) #[[ATTR0]] { +// AVRFP64-NEXT: entry: +// AVRFP64-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 1 +// AVRFP64-NEXT: [[A:%.*]] = alloca { float, float }, align 1 +// AVRFP64-NEXT: [[B:%.*]] = alloca { float, float }, align 1 +// AVRFP64-NEXT: [[TMP0:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP64-NEXT: store float [[A_COERCE0]], ptr [[TMP0]], align 1 +// AVRFP64-NEXT: [[TMP1:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP64-NEXT: store float [[A_COERCE1]], ptr [[TMP1]], align 1 +// AVRFP64-NEXT: [[TMP2:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// AVRFP64-NEXT: store float [[B_COERCE0]], ptr [[TMP2]], align 1 +// AVRFP64-NEXT: [[TMP3:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// AVRFP64-NEXT: store float [[B_COERCE1]], ptr [[TMP3]], align 1 +// AVRFP64-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP64-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 1 +// AVRFP64-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP64-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 1 +// AVRFP64-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// AVRFP64-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 1 +// AVRFP64-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// AVRFP64-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 1 +// AVRFP64-NEXT: [[MUL_AC:%.*]] = fmul float [[A_REAL]], [[B_REAL]] +// AVRFP64-NEXT: [[MUL_BD:%.*]] = fmul float [[A_IMAG]], [[B_IMAG]] +// AVRFP64-NEXT: [[MUL_AD:%.*]] = fmul float [[A_REAL]], [[B_IMAG]] +// AVRFP64-NEXT: [[MUL_BC:%.*]] = fmul float [[A_IMAG]], [[B_REAL]] +// AVRFP64-NEXT: [[MUL_R:%.*]] = fsub float [[MUL_AC]], [[MUL_BD]] +// AVRFP64-NEXT: [[MUL_I:%.*]] = fadd float [[MUL_AD]], [[MUL_BC]] +// AVRFP64-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// AVRFP64-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// AVRFP64-NEXT: store float [[MUL_R]], ptr [[RETVAL_REALP]], align 1 +// AVRFP64-NEXT: store float [[MUL_I]], ptr [[RETVAL_IMAGP]], align 1 +// AVRFP64-NEXT: [[TMP4:%.*]] = load { float, float }, ptr [[RETVAL]], align 1 +// AVRFP64-NEXT: ret { float, float } [[TMP4]] +// +// BASIC_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x float> @mulf( +// BASIC_FAST-SAME: <2 x float> noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x float> noundef nofpclass(nan inf) [[B_COERCE:%.*]]) #[[ATTR0]] { +// BASIC_FAST-NEXT: entry: +// BASIC_FAST-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// BASIC_FAST-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// BASIC_FAST-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// BASIC_FAST-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// BASIC_FAST-NEXT: store <2 x float> [[B_COERCE]], ptr [[B]], align 4 +// BASIC_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// BASIC_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// BASIC_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// BASIC_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// BASIC_FAST-NEXT: [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_REAL]], [[B_REAL]] +// BASIC_FAST-NEXT: [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_IMAG]], [[B_IMAG]] +// BASIC_FAST-NEXT: [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_REAL]], [[B_IMAG]] +// BASIC_FAST-NEXT: [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_IMAG]], [[B_REAL]] +// BASIC_FAST-NEXT: [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[MUL_AC]], [[MUL_BD]] +// BASIC_FAST-NEXT: [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[MUL_AD]], [[MUL_BC]] +// BASIC_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC_FAST-NEXT: store float [[MUL_R]], ptr [[RETVAL_REALP]], align 4 +// BASIC_FAST-NEXT: store float [[MUL_I]], ptr [[RETVAL_IMAGP]], align 4 +// BASIC_FAST-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// BASIC_FAST-NEXT: ret <2 x float> [[TMP0]] +// +// FULL_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x float> @mulf( +// FULL_FAST-SAME: <2 x float> noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x float> noundef nofpclass(nan inf) [[B_COERCE:%.*]]) #[[ATTR0]] { +// FULL_FAST-NEXT: entry: +// FULL_FAST-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// FULL_FAST-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// FULL_FAST-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// FULL_FAST-NEXT: [[COERCE:%.*]] = alloca { float, float }, align 4 +// FULL_FAST-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// FULL_FAST-NEXT: store <2 x float> [[B_COERCE]], ptr [[B]], align 4 +// FULL_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// FULL_FAST-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// FULL_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// FULL_FAST-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// FULL_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// FULL_FAST-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// FULL_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// FULL_FAST-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// FULL_FAST-NEXT: [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_REAL]], [[B_REAL]] +// FULL_FAST-NEXT: [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_IMAG]], [[B_IMAG]] +// FULL_FAST-NEXT: [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_REAL]], [[B_IMAG]] +// FULL_FAST-NEXT: [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_IMAG]], [[B_REAL]] +// FULL_FAST-NEXT: [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[MUL_AC]], [[MUL_BD]] +// FULL_FAST-NEXT: [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[MUL_AD]], [[MUL_BC]] +// FULL_FAST-NEXT: [[ISNAN_CMP:%.*]] = fcmp reassoc nnan ninf nsz arcp afn uno float [[MUL_R]], [[MUL_R]] +// FULL_FAST-NEXT: br i1 [[ISNAN_CMP]], label [[COMPLEX_MUL_IMAG_NAN:%.*]], label [[COMPLEX_MUL_CONT:%.*]], !prof [[PROF2:![0-9]+]] +// FULL_FAST: complex_mul_imag_nan: +// FULL_FAST-NEXT: [[ISNAN_CMP1:%.*]] = fcmp reassoc nnan ninf nsz arcp afn uno float [[MUL_I]], [[MUL_I]] +// FULL_FAST-NEXT: br i1 [[ISNAN_CMP1]], label [[COMPLEX_MUL_LIBCALL:%.*]], label [[COMPLEX_MUL_CONT]], !prof [[PROF2]] +// FULL_FAST: complex_mul_libcall: +// FULL_FAST-NEXT: [[CALL:%.*]] = call reassoc nnan ninf nsz arcp afn nofpclass(nan inf) <2 x float> @__mulsc3(float noundef nofpclass(nan inf) [[A_REAL]], float noundef nofpclass(nan inf) [[A_IMAG]], float noundef nofpclass(nan inf) [[B_REAL]], float noundef nofpclass(nan inf) [[B_IMAG]]) #[[ATTR2]] +// FULL_FAST-NEXT: store <2 x float> [[CALL]], ptr [[COERCE]], align 4 +// FULL_FAST-NEXT: [[COERCE_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 0 +// FULL_FAST-NEXT: [[COERCE_REAL:%.*]] = load float, ptr [[COERCE_REALP]], align 4 +// FULL_FAST-NEXT: [[COERCE_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 1 +// FULL_FAST-NEXT: [[COERCE_IMAG:%.*]] = load float, ptr [[COERCE_IMAGP]], align 4 +// FULL_FAST-NEXT: br label [[COMPLEX_MUL_CONT]] +// FULL_FAST: complex_mul_cont: +// FULL_FAST-NEXT: [[REAL_MUL_PHI:%.*]] = phi reassoc nnan ninf nsz arcp afn float [ [[MUL_R]], [[ENTRY:%.*]] ], [ [[MUL_R]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[COERCE_REAL]], [[COMPLEX_MUL_LIBCALL]] ] +// FULL_FAST-NEXT: [[IMAG_MUL_PHI:%.*]] = phi reassoc nnan ninf nsz arcp afn float [ [[MUL_I]], [[ENTRY]] ], [ [[MUL_I]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[COERCE_IMAG]], [[COMPLEX_MUL_LIBCALL]] ] +// FULL_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// FULL_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// FULL_FAST-NEXT: store float [[REAL_MUL_PHI]], ptr [[RETVAL_REALP]], align 4 +// FULL_FAST-NEXT: store float [[IMAG_MUL_PHI]], ptr [[RETVAL_IMAGP]], align 4 +// FULL_FAST-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// FULL_FAST-NEXT: ret <2 x float> [[TMP0]] +// +// IMPRVD_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x float> @mulf( +// IMPRVD_FAST-SAME: <2 x float> noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x float> noundef nofpclass(nan inf) [[B_COERCE:%.*]]) #[[ATTR0]] { +// IMPRVD_FAST-NEXT: entry: +// IMPRVD_FAST-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// IMPRVD_FAST-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// IMPRVD_FAST-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// IMPRVD_FAST-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// IMPRVD_FAST-NEXT: store <2 x float> [[B_COERCE]], ptr [[B]], align 4 +// IMPRVD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// IMPRVD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// IMPRVD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// IMPRVD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// IMPRVD_FAST-NEXT: [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_REAL]], [[B_REAL]] +// IMPRVD_FAST-NEXT: [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_IMAG]], [[B_IMAG]] +// IMPRVD_FAST-NEXT: [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_REAL]], [[B_IMAG]] +// IMPRVD_FAST-NEXT: [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_IMAG]], [[B_REAL]] +// IMPRVD_FAST-NEXT: [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[MUL_AC]], [[MUL_BD]] +// IMPRVD_FAST-NEXT: [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[MUL_AD]], [[MUL_BC]] +// IMPRVD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: store float [[MUL_R]], ptr [[RETVAL_REALP]], align 4 +// IMPRVD_FAST-NEXT: store float [[MUL_I]], ptr [[RETVAL_IMAGP]], align 4 +// IMPRVD_FAST-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// IMPRVD_FAST-NEXT: ret <2 x float> [[TMP0]] +// +// PRMTD_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x float> @mulf( +// PRMTD_FAST-SAME: <2 x float> noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x float> noundef nofpclass(nan inf) [[B_COERCE:%.*]]) #[[ATTR0]] { +// PRMTD_FAST-NEXT: entry: +// PRMTD_FAST-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// PRMTD_FAST-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// PRMTD_FAST-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// PRMTD_FAST-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// PRMTD_FAST-NEXT: store <2 x float> [[B_COERCE]], ptr [[B]], align 4 +// PRMTD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// PRMTD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// PRMTD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// PRMTD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// PRMTD_FAST-NEXT: [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_REAL]], [[B_REAL]] +// PRMTD_FAST-NEXT: [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_IMAG]], [[B_IMAG]] +// PRMTD_FAST-NEXT: [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_REAL]], [[B_IMAG]] +// PRMTD_FAST-NEXT: [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_IMAG]], [[B_REAL]] +// PRMTD_FAST-NEXT: [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[MUL_AC]], [[MUL_BD]] +// PRMTD_FAST-NEXT: [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[MUL_AD]], [[MUL_BC]] +// PRMTD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD_FAST-NEXT: store float [[MUL_R]], ptr [[RETVAL_REALP]], align 4 +// PRMTD_FAST-NEXT: store float [[MUL_I]], ptr [[RETVAL_IMAGP]], align 4 +// PRMTD_FAST-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// PRMTD_FAST-NEXT: ret <2 x float> [[TMP0]] +// +_Complex float mulf(_Complex float a, _Complex float b) { + return a * b; +} +// FULL-LABEL: define dso_local { double, double } @divd( +// FULL-SAME: double noundef [[A_COERCE0:%.*]], double noundef [[A_COERCE1:%.*]], double noundef [[B_COERCE0:%.*]], double noundef [[B_COERCE1:%.*]]) #[[ATTR1:[0-9]+]] { +// FULL-NEXT: entry: +// FULL-NEXT: [[RETVAL:%.*]] = alloca { double, double }, align 8 +// FULL-NEXT: [[A:%.*]] = alloca { double, double }, align 8 +// FULL-NEXT: [[B:%.*]] = alloca { double, double }, align 8 +// FULL-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// FULL-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 8 +// FULL-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// FULL-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 8 +// FULL-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// FULL-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 8 +// FULL-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// FULL-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 8 +// FULL-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// FULL-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// FULL-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// FULL-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// FULL-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// FULL-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// FULL-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// FULL-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// FULL-NEXT: [[CALL:%.*]] = call { double, double } @__divdc3(double noundef [[A_REAL]], double noundef [[A_IMAG]], double noundef [[B_REAL]], double noundef [[B_IMAG]]) #[[ATTR2]] +// FULL-NEXT: [[TMP4:%.*]] = extractvalue { double, double } [[CALL]], 0 +// FULL-NEXT: [[TMP5:%.*]] = extractvalue { double, double } [[CALL]], 1 +// FULL-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 0 +// FULL-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 1 +// FULL-NEXT: store double [[TMP4]], ptr [[RETVAL_REALP]], align 8 +// FULL-NEXT: store double [[TMP5]], ptr [[RETVAL_IMAGP]], align 8 +// FULL-NEXT: [[TMP6:%.*]] = load { double, double }, ptr [[RETVAL]], align 8 +// FULL-NEXT: ret { double, double } [[TMP6]] +// +// BASIC-LABEL: define dso_local { double, double } @divd( +// BASIC-SAME: double noundef [[A_COERCE0:%.*]], double noundef [[A_COERCE1:%.*]], double noundef [[B_COERCE0:%.*]], double noundef [[B_COERCE1:%.*]]) #[[ATTR1:[0-9]+]] { +// BASIC-NEXT: entry: +// BASIC-NEXT: [[RETVAL:%.*]] = alloca { double, double }, align 8 +// BASIC-NEXT: [[A:%.*]] = alloca { double, double }, align 8 +// BASIC-NEXT: [[B:%.*]] = alloca { double, double }, align 8 +// BASIC-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// BASIC-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 8 +// BASIC-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// BASIC-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 8 +// BASIC-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// BASIC-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 8 +// BASIC-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// BASIC-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 8 +// BASIC-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// BASIC-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// BASIC-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// BASIC-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// BASIC-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// BASIC-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// BASIC-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// BASIC-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// BASIC-NEXT: [[TMP4:%.*]] = fmul double [[A_REAL]], [[B_REAL]] +// BASIC-NEXT: [[TMP5:%.*]] = fmul double [[A_IMAG]], [[B_IMAG]] +// BASIC-NEXT: [[TMP6:%.*]] = fadd double [[TMP4]], [[TMP5]] +// BASIC-NEXT: [[TMP7:%.*]] = fmul double [[B_REAL]], [[B_REAL]] +// BASIC-NEXT: [[TMP8:%.*]] = fmul double [[B_IMAG]], [[B_IMAG]] +// BASIC-NEXT: [[TMP9:%.*]] = fadd double [[TMP7]], [[TMP8]] +// BASIC-NEXT: [[TMP10:%.*]] = fmul double [[A_IMAG]], [[B_REAL]] +// BASIC-NEXT: [[TMP11:%.*]] = fmul double [[A_REAL]], [[B_IMAG]] +// BASIC-NEXT: [[TMP12:%.*]] = fsub double [[TMP10]], [[TMP11]] +// BASIC-NEXT: [[TMP13:%.*]] = fdiv double [[TMP6]], [[TMP9]] +// BASIC-NEXT: [[TMP14:%.*]] = fdiv double [[TMP12]], [[TMP9]] +// BASIC-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC-NEXT: store double [[TMP13]], ptr [[RETVAL_REALP]], align 8 +// BASIC-NEXT: store double [[TMP14]], ptr [[RETVAL_IMAGP]], align 8 +// BASIC-NEXT: [[TMP15:%.*]] = load { double, double }, ptr [[RETVAL]], align 8 +// BASIC-NEXT: ret { double, double } [[TMP15]] +// +// IMPRVD-LABEL: define dso_local { double, double } @divd( +// IMPRVD-SAME: double noundef [[A_COERCE0:%.*]], double noundef [[A_COERCE1:%.*]], double noundef [[B_COERCE0:%.*]], double noundef [[B_COERCE1:%.*]]) #[[ATTR2:[0-9]+]] { +// IMPRVD-NEXT: entry: +// IMPRVD-NEXT: [[RETVAL:%.*]] = alloca { double, double }, align 8 +// IMPRVD-NEXT: [[A:%.*]] = alloca { double, double }, align 8 +// IMPRVD-NEXT: [[B:%.*]] = alloca { double, double }, align 8 +// IMPRVD-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// IMPRVD-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 8 +// IMPRVD-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// IMPRVD-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 8 +// IMPRVD-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// IMPRVD-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 8 +// IMPRVD-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// IMPRVD-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 8 +// IMPRVD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// IMPRVD-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// IMPRVD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// IMPRVD-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// IMPRVD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// IMPRVD-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// IMPRVD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// IMPRVD-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// IMPRVD-NEXT: [[TMP4:%.*]] = call double @llvm.fabs.f64(double [[B_REAL]]) +// IMPRVD-NEXT: [[TMP5:%.*]] = call double @llvm.fabs.f64(double [[B_IMAG]]) +// IMPRVD-NEXT: [[ABS_CMP:%.*]] = fcmp ugt double [[TMP4]], [[TMP5]] +// IMPRVD-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// IMPRVD: abs_rhsr_greater_or_equal_abs_rhsi: +// IMPRVD-NEXT: [[TMP6:%.*]] = fdiv double [[B_IMAG]], [[B_REAL]] +// IMPRVD-NEXT: [[TMP7:%.*]] = fmul double [[TMP6]], [[B_IMAG]] +// IMPRVD-NEXT: [[TMP8:%.*]] = fadd double [[B_REAL]], [[TMP7]] +// IMPRVD-NEXT: [[TMP9:%.*]] = fmul double [[A_IMAG]], [[TMP6]] +// IMPRVD-NEXT: [[TMP10:%.*]] = fadd double [[A_REAL]], [[TMP9]] +// IMPRVD-NEXT: [[TMP11:%.*]] = fdiv double [[TMP10]], [[TMP8]] +// IMPRVD-NEXT: [[TMP12:%.*]] = fmul double [[A_REAL]], [[TMP6]] +// IMPRVD-NEXT: [[TMP13:%.*]] = fsub double [[A_IMAG]], [[TMP12]] +// IMPRVD-NEXT: [[TMP14:%.*]] = fdiv double [[TMP13]], [[TMP8]] +// IMPRVD-NEXT: br label [[COMPLEX_DIV:%.*]] +// IMPRVD: abs_rhsr_less_than_abs_rhsi: +// IMPRVD-NEXT: [[TMP15:%.*]] = fdiv double [[B_REAL]], [[B_IMAG]] +// IMPRVD-NEXT: [[TMP16:%.*]] = fmul double [[TMP15]], [[B_REAL]] +// IMPRVD-NEXT: [[TMP17:%.*]] = fadd double [[B_IMAG]], [[TMP16]] +// IMPRVD-NEXT: [[TMP18:%.*]] = fmul double [[A_REAL]], [[TMP15]] +// IMPRVD-NEXT: [[TMP19:%.*]] = fadd double [[TMP18]], [[A_IMAG]] +// IMPRVD-NEXT: [[TMP20:%.*]] = fdiv double [[TMP19]], [[TMP17]] +// IMPRVD-NEXT: [[TMP21:%.*]] = fmul double [[A_IMAG]], [[TMP15]] +// IMPRVD-NEXT: [[TMP22:%.*]] = fsub double [[TMP21]], [[A_REAL]] +// IMPRVD-NEXT: [[TMP23:%.*]] = fdiv double [[TMP22]], [[TMP17]] +// IMPRVD-NEXT: br label [[COMPLEX_DIV]] +// IMPRVD: complex_div: +// IMPRVD-NEXT: [[TMP24:%.*]] = phi double [ [[TMP11]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP20]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD-NEXT: [[TMP25:%.*]] = phi double [ [[TMP14]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP23]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD-NEXT: store double [[TMP24]], ptr [[RETVAL_REALP]], align 8 +// IMPRVD-NEXT: store double [[TMP25]], ptr [[RETVAL_IMAGP]], align 8 +// IMPRVD-NEXT: [[TMP26:%.*]] = load { double, double }, ptr [[RETVAL]], align 8 +// IMPRVD-NEXT: ret { double, double } [[TMP26]] +// +// PRMTD-LABEL: define dso_local { double, double } @divd( +// PRMTD-SAME: double noundef [[A_COERCE0:%.*]], double noundef [[A_COERCE1:%.*]], double noundef [[B_COERCE0:%.*]], double noundef [[B_COERCE1:%.*]]) #[[ATTR1:[0-9]+]] { +// PRMTD-NEXT: entry: +// PRMTD-NEXT: [[RETVAL:%.*]] = alloca { double, double }, align 8 +// PRMTD-NEXT: [[A:%.*]] = alloca { double, double }, align 8 +// PRMTD-NEXT: [[B:%.*]] = alloca { double, double }, align 8 +// PRMTD-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// PRMTD-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 8 +// PRMTD-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// PRMTD-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 8 +// PRMTD-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// PRMTD-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 8 +// PRMTD-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// PRMTD-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 8 +// PRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// PRMTD-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// PRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// PRMTD-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// PRMTD-NEXT: [[EXT:%.*]] = fpext double [[A_REAL]] to x86_fp80 +// PRMTD-NEXT: [[EXT1:%.*]] = fpext double [[A_IMAG]] to x86_fp80 +// PRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// PRMTD-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// PRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// PRMTD-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// PRMTD-NEXT: [[EXT2:%.*]] = fpext double [[B_REAL]] to x86_fp80 +// PRMTD-NEXT: [[EXT3:%.*]] = fpext double [[B_IMAG]] to x86_fp80 +// PRMTD-NEXT: [[TMP4:%.*]] = fmul x86_fp80 [[EXT]], [[EXT2]] +// PRMTD-NEXT: [[TMP5:%.*]] = fmul x86_fp80 [[EXT1]], [[EXT3]] +// PRMTD-NEXT: [[TMP6:%.*]] = fadd x86_fp80 [[TMP4]], [[TMP5]] +// PRMTD-NEXT: [[TMP7:%.*]] = fmul x86_fp80 [[EXT2]], [[EXT2]] +// PRMTD-NEXT: [[TMP8:%.*]] = fmul x86_fp80 [[EXT3]], [[EXT3]] +// PRMTD-NEXT: [[TMP9:%.*]] = fadd x86_fp80 [[TMP7]], [[TMP8]] +// PRMTD-NEXT: [[TMP10:%.*]] = fmul x86_fp80 [[EXT1]], [[EXT2]] +// PRMTD-NEXT: [[TMP11:%.*]] = fmul x86_fp80 [[EXT]], [[EXT3]] +// PRMTD-NEXT: [[TMP12:%.*]] = fsub x86_fp80 [[TMP10]], [[TMP11]] +// PRMTD-NEXT: [[TMP13:%.*]] = fdiv x86_fp80 [[TMP6]], [[TMP9]] +// PRMTD-NEXT: [[TMP14:%.*]] = fdiv x86_fp80 [[TMP12]], [[TMP9]] +// PRMTD-NEXT: [[UNPROMOTION:%.*]] = fptrunc x86_fp80 [[TMP13]] to double +// PRMTD-NEXT: [[UNPROMOTION4:%.*]] = fptrunc x86_fp80 [[TMP14]] to double +// PRMTD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD-NEXT: store double [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 8 +// PRMTD-NEXT: store double [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 8 +// PRMTD-NEXT: [[TMP15:%.*]] = load { double, double }, ptr [[RETVAL]], align 8 +// PRMTD-NEXT: ret { double, double } [[TMP15]] +// +// X86WINPRMTD-LABEL: define dso_local void @divd( +// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] { +// X86WINPRMTD-NEXT: entry: +// X86WINPRMTD-NEXT: [[RESULT_PTR:%.*]] = alloca ptr, align 8 +// X86WINPRMTD-NEXT: [[B_INDIRECT_ADDR:%.*]] = alloca ptr, align 8 +// X86WINPRMTD-NEXT: [[A_INDIRECT_ADDR:%.*]] = alloca ptr, align 8 +// X86WINPRMTD-NEXT: store ptr [[AGG_RESULT]], ptr [[RESULT_PTR]], align 8 +// X86WINPRMTD-NEXT: store ptr [[B]], ptr [[B_INDIRECT_ADDR]], align 8 +// X86WINPRMTD-NEXT: store ptr [[A]], ptr [[A_INDIRECT_ADDR]], align 8 +// X86WINPRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// X86WINPRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// X86WINPRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// X86WINPRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// X86WINPRMTD-NEXT: [[TMP0:%.*]] = call double @llvm.fabs.f64(double [[B_REAL]]) +// X86WINPRMTD-NEXT: [[TMP1:%.*]] = call double @llvm.fabs.f64(double [[B_IMAG]]) +// X86WINPRMTD-NEXT: [[ABS_CMP:%.*]] = fcmp ugt double [[TMP0]], [[TMP1]] +// X86WINPRMTD-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// X86WINPRMTD: abs_rhsr_greater_or_equal_abs_rhsi: +// X86WINPRMTD-NEXT: [[TMP2:%.*]] = fdiv double [[B_IMAG]], [[B_REAL]] +// X86WINPRMTD-NEXT: [[TMP3:%.*]] = fmul double [[TMP2]], [[B_IMAG]] +// X86WINPRMTD-NEXT: [[TMP4:%.*]] = fadd double [[B_REAL]], [[TMP3]] +// X86WINPRMTD-NEXT: [[TMP5:%.*]] = fmul double [[A_IMAG]], [[TMP2]] +// X86WINPRMTD-NEXT: [[TMP6:%.*]] = fadd double [[A_REAL]], [[TMP5]] +// X86WINPRMTD-NEXT: [[TMP7:%.*]] = fdiv double [[TMP6]], [[TMP4]] +// X86WINPRMTD-NEXT: [[TMP8:%.*]] = fmul double [[A_REAL]], [[TMP2]] +// X86WINPRMTD-NEXT: [[TMP9:%.*]] = fsub double [[A_IMAG]], [[TMP8]] +// X86WINPRMTD-NEXT: [[TMP10:%.*]] = fdiv double [[TMP9]], [[TMP4]] +// X86WINPRMTD-NEXT: br label [[COMPLEX_DIV:%.*]] +// X86WINPRMTD: abs_rhsr_less_than_abs_rhsi: +// X86WINPRMTD-NEXT: [[TMP11:%.*]] = fdiv double [[B_REAL]], [[B_IMAG]] +// X86WINPRMTD-NEXT: [[TMP12:%.*]] = fmul double [[TMP11]], [[B_REAL]] +// X86WINPRMTD-NEXT: [[TMP13:%.*]] = fadd double [[B_IMAG]], [[TMP12]] +// X86WINPRMTD-NEXT: [[TMP14:%.*]] = fmul double [[A_REAL]], [[TMP11]] +// X86WINPRMTD-NEXT: [[TMP15:%.*]] = fadd double [[TMP14]], [[A_IMAG]] +// X86WINPRMTD-NEXT: [[TMP16:%.*]] = fdiv double [[TMP15]], [[TMP13]] +// X86WINPRMTD-NEXT: [[TMP17:%.*]] = fmul double [[A_IMAG]], [[TMP11]] +// X86WINPRMTD-NEXT: [[TMP18:%.*]] = fsub double [[TMP17]], [[A_REAL]] +// X86WINPRMTD-NEXT: [[TMP19:%.*]] = fdiv double [[TMP18]], [[TMP13]] +// X86WINPRMTD-NEXT: br label [[COMPLEX_DIV]] +// X86WINPRMTD: complex_div: +// X86WINPRMTD-NEXT: [[TMP20:%.*]] = phi double [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// X86WINPRMTD-NEXT: [[TMP21:%.*]] = phi double [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// X86WINPRMTD-NEXT: [[AGG_RESULT_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[AGG_RESULT_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// X86WINPRMTD-NEXT: store double [[TMP20]], ptr [[AGG_RESULT_REALP]], align 8 +// X86WINPRMTD-NEXT: store double [[TMP21]], ptr [[AGG_RESULT_IMAGP]], align 8 +// X86WINPRMTD-NEXT: [[AGG_RESULT_REALP1:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[AGG_RESULT_REAL:%.*]] = load double, ptr [[AGG_RESULT_REALP1]], align 8 +// X86WINPRMTD-NEXT: [[AGG_RESULT_IMAGP2:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[AGG_RESULT_IMAG:%.*]] = load double, ptr [[AGG_RESULT_IMAGP2]], align 8 +// X86WINPRMTD-NEXT: [[AGG_RESULT_REALP3:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[AGG_RESULT_IMAGP4:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// X86WINPRMTD-NEXT: store double [[AGG_RESULT_REAL]], ptr [[AGG_RESULT_REALP3]], align 8 +// X86WINPRMTD-NEXT: store double [[AGG_RESULT_IMAG]], ptr [[AGG_RESULT_IMAGP4]], align 8 +// X86WINPRMTD-NEXT: ret void +// +// AVRFP32-LABEL: define dso_local { float, float } @divd( +// AVRFP32-SAME: float noundef [[A_COERCE0:%.*]], float noundef [[A_COERCE1:%.*]], float noundef [[B_COERCE0:%.*]], float noundef [[B_COERCE1:%.*]]) addrspace(1) #[[ATTR0]] { +// AVRFP32-NEXT: entry: +// AVRFP32-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 1 +// AVRFP32-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// AVRFP32-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// AVRFP32-NEXT: [[TMP0:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP32-NEXT: store float [[A_COERCE0]], ptr [[TMP0]], align 4 +// AVRFP32-NEXT: [[TMP1:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[A_COERCE1]], ptr [[TMP1]], align 4 +// AVRFP32-NEXT: [[TMP2:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// AVRFP32-NEXT: store float [[B_COERCE0]], ptr [[TMP2]], align 4 +// AVRFP32-NEXT: [[TMP3:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[B_COERCE1]], ptr [[TMP3]], align 4 +// AVRFP32-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP32-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// AVRFP32-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP32-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// AVRFP32-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// AVRFP32-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// AVRFP32-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// AVRFP32-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// AVRFP32-NEXT: [[TMP4:%.*]] = call addrspace(1) float @llvm.fabs.f32(float [[B_REAL]]) +// AVRFP32-NEXT: [[TMP5:%.*]] = call addrspace(1) float @llvm.fabs.f32(float [[B_IMAG]]) +// AVRFP32-NEXT: [[ABS_CMP:%.*]] = fcmp ugt float [[TMP4]], [[TMP5]] +// AVRFP32-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// AVRFP32: abs_rhsr_greater_or_equal_abs_rhsi: +// AVRFP32-NEXT: [[TMP6:%.*]] = fdiv float [[B_IMAG]], [[B_REAL]] +// AVRFP32-NEXT: [[TMP7:%.*]] = fmul float [[TMP6]], [[B_IMAG]] +// AVRFP32-NEXT: [[TMP8:%.*]] = fadd float [[B_REAL]], [[TMP7]] +// AVRFP32-NEXT: [[TMP9:%.*]] = fmul float [[A_IMAG]], [[TMP6]] +// AVRFP32-NEXT: [[TMP10:%.*]] = fadd float [[A_REAL]], [[TMP9]] +// AVRFP32-NEXT: [[TMP11:%.*]] = fdiv float [[TMP10]], [[TMP8]] +// AVRFP32-NEXT: [[TMP12:%.*]] = fmul float [[A_REAL]], [[TMP6]] +// AVRFP32-NEXT: [[TMP13:%.*]] = fsub float [[A_IMAG]], [[TMP12]] +// AVRFP32-NEXT: [[TMP14:%.*]] = fdiv float [[TMP13]], [[TMP8]] +// AVRFP32-NEXT: br label [[COMPLEX_DIV:%.*]] +// AVRFP32: abs_rhsr_less_than_abs_rhsi: +// AVRFP32-NEXT: [[TMP15:%.*]] = fdiv float [[B_REAL]], [[B_IMAG]] +// AVRFP32-NEXT: [[TMP16:%.*]] = fmul float [[TMP15]], [[B_REAL]] +// AVRFP32-NEXT: [[TMP17:%.*]] = fadd float [[B_IMAG]], [[TMP16]] +// AVRFP32-NEXT: [[TMP18:%.*]] = fmul float [[A_REAL]], [[TMP15]] +// AVRFP32-NEXT: [[TMP19:%.*]] = fadd float [[TMP18]], [[A_IMAG]] +// AVRFP32-NEXT: [[TMP20:%.*]] = fdiv float [[TMP19]], [[TMP17]] +// AVRFP32-NEXT: [[TMP21:%.*]] = fmul float [[A_IMAG]], [[TMP15]] +// AVRFP32-NEXT: [[TMP22:%.*]] = fsub float [[TMP21]], [[A_REAL]] +// AVRFP32-NEXT: [[TMP23:%.*]] = fdiv float [[TMP22]], [[TMP17]] +// AVRFP32-NEXT: br label [[COMPLEX_DIV]] +// AVRFP32: complex_div: +// AVRFP32-NEXT: [[TMP24:%.*]] = phi float [ [[TMP11]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP20]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// AVRFP32-NEXT: [[TMP25:%.*]] = phi float [ [[TMP14]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP23]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// AVRFP32-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// AVRFP32-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[TMP24]], ptr [[RETVAL_REALP]], align 1 +// AVRFP32-NEXT: store float [[TMP25]], ptr [[RETVAL_IMAGP]], align 1 +// AVRFP32-NEXT: [[TMP26:%.*]] = load { float, float }, ptr [[RETVAL]], align 1 +// AVRFP32-NEXT: ret { float, float } [[TMP26]] +// +// AVRFP64-LABEL: define dso_local void @divd( +// AVRFP64-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 1 [[AGG_RESULT:%.*]], double noundef [[A_COERCE0:%.*]], double noundef [[A_COERCE1:%.*]], double noundef [[B_COERCE0:%.*]], double noundef [[B_COERCE1:%.*]]) addrspace(1) #[[ATTR0]] { +// AVRFP64-NEXT: entry: +// AVRFP64-NEXT: [[A:%.*]] = alloca { double, double }, align 8 +// AVRFP64-NEXT: [[B:%.*]] = alloca { double, double }, align 8 +// AVRFP64-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// AVRFP64-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 8 +// AVRFP64-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// AVRFP64-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 8 +// AVRFP64-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// AVRFP64-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 8 +// AVRFP64-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// AVRFP64-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 8 +// AVRFP64-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// AVRFP64-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// AVRFP64-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// AVRFP64-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// AVRFP64-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// AVRFP64-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// AVRFP64-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// AVRFP64-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// AVRFP64-NEXT: [[TMP4:%.*]] = call addrspace(1) double @llvm.fabs.f64(double [[B_REAL]]) +// AVRFP64-NEXT: [[TMP5:%.*]] = call addrspace(1) double @llvm.fabs.f64(double [[B_IMAG]]) +// AVRFP64-NEXT: [[ABS_CMP:%.*]] = fcmp ugt double [[TMP4]], [[TMP5]] +// AVRFP64-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// AVRFP64: abs_rhsr_greater_or_equal_abs_rhsi: +// AVRFP64-NEXT: [[TMP6:%.*]] = fdiv double [[B_IMAG]], [[B_REAL]] +// AVRFP64-NEXT: [[TMP7:%.*]] = fmul double [[TMP6]], [[B_IMAG]] +// AVRFP64-NEXT: [[TMP8:%.*]] = fadd double [[B_REAL]], [[TMP7]] +// AVRFP64-NEXT: [[TMP9:%.*]] = fmul double [[A_IMAG]], [[TMP6]] +// AVRFP64-NEXT: [[TMP10:%.*]] = fadd double [[A_REAL]], [[TMP9]] +// AVRFP64-NEXT: [[TMP11:%.*]] = fdiv double [[TMP10]], [[TMP8]] +// AVRFP64-NEXT: [[TMP12:%.*]] = fmul double [[A_REAL]], [[TMP6]] +// AVRFP64-NEXT: [[TMP13:%.*]] = fsub double [[A_IMAG]], [[TMP12]] +// AVRFP64-NEXT: [[TMP14:%.*]] = fdiv double [[TMP13]], [[TMP8]] +// AVRFP64-NEXT: br label [[COMPLEX_DIV:%.*]] +// AVRFP64: abs_rhsr_less_than_abs_rhsi: +// AVRFP64-NEXT: [[TMP15:%.*]] = fdiv double [[B_REAL]], [[B_IMAG]] +// AVRFP64-NEXT: [[TMP16:%.*]] = fmul double [[TMP15]], [[B_REAL]] +// AVRFP64-NEXT: [[TMP17:%.*]] = fadd double [[B_IMAG]], [[TMP16]] +// AVRFP64-NEXT: [[TMP18:%.*]] = fmul double [[A_REAL]], [[TMP15]] +// AVRFP64-NEXT: [[TMP19:%.*]] = fadd double [[TMP18]], [[A_IMAG]] +// AVRFP64-NEXT: [[TMP20:%.*]] = fdiv double [[TMP19]], [[TMP17]] +// AVRFP64-NEXT: [[TMP21:%.*]] = fmul double [[A_IMAG]], [[TMP15]] +// AVRFP64-NEXT: [[TMP22:%.*]] = fsub double [[TMP21]], [[A_REAL]] +// AVRFP64-NEXT: [[TMP23:%.*]] = fdiv double [[TMP22]], [[TMP17]] +// AVRFP64-NEXT: br label [[COMPLEX_DIV]] +// AVRFP64: complex_div: +// AVRFP64-NEXT: [[TMP24:%.*]] = phi double [ [[TMP11]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP20]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// AVRFP64-NEXT: [[TMP25:%.*]] = phi double [ [[TMP14]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP23]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// AVRFP64-NEXT: [[AGG_RESULT_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// AVRFP64-NEXT: [[AGG_RESULT_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// AVRFP64-NEXT: store double [[TMP24]], ptr [[AGG_RESULT_REALP]], align 1 +// AVRFP64-NEXT: store double [[TMP25]], ptr [[AGG_RESULT_IMAGP]], align 1 +// AVRFP64-NEXT: [[AGG_RESULT_REALP1:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// AVRFP64-NEXT: [[AGG_RESULT_REAL:%.*]] = load double, ptr [[AGG_RESULT_REALP1]], align 1 +// AVRFP64-NEXT: [[AGG_RESULT_IMAGP2:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// AVRFP64-NEXT: [[AGG_RESULT_IMAG:%.*]] = load double, ptr [[AGG_RESULT_IMAGP2]], align 1 +// AVRFP64-NEXT: [[AGG_RESULT_REALP3:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// AVRFP64-NEXT: [[AGG_RESULT_IMAGP4:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// AVRFP64-NEXT: store double [[AGG_RESULT_REAL]], ptr [[AGG_RESULT_REALP3]], align 1 +// AVRFP64-NEXT: store double [[AGG_RESULT_IMAG]], ptr [[AGG_RESULT_IMAGP4]], align 1 +// AVRFP64-NEXT: ret void +// +// BASIC_FAST-LABEL: define dso_local { double, double } @divd( +// BASIC_FAST-SAME: double noundef nofpclass(nan inf) [[A_COERCE0:%.*]], double noundef nofpclass(nan inf) [[A_COERCE1:%.*]], double noundef nofpclass(nan inf) [[B_COERCE0:%.*]], double noundef nofpclass(nan inf) [[B_COERCE1:%.*]]) #[[ATTR1:[0-9]+]] { +// BASIC_FAST-NEXT: entry: +// BASIC_FAST-NEXT: [[RETVAL:%.*]] = alloca { double, double }, align 8 +// BASIC_FAST-NEXT: [[A:%.*]] = alloca { double, double }, align 8 +// BASIC_FAST-NEXT: [[B:%.*]] = alloca { double, double }, align 8 +// BASIC_FAST-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// BASIC_FAST-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 8 +// BASIC_FAST-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// BASIC_FAST-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 8 +// BASIC_FAST-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// BASIC_FAST-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 8 +// BASIC_FAST-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// BASIC_FAST-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 8 +// BASIC_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// BASIC_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// BASIC_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// BASIC_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// BASIC_FAST-NEXT: [[TMP4:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_REAL]], [[B_REAL]] +// BASIC_FAST-NEXT: [[TMP5:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_IMAG]], [[B_IMAG]] +// BASIC_FAST-NEXT: [[TMP6:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[TMP4]], [[TMP5]] +// BASIC_FAST-NEXT: [[TMP7:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[B_REAL]], [[B_REAL]] +// BASIC_FAST-NEXT: [[TMP8:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[B_IMAG]], [[B_IMAG]] +// BASIC_FAST-NEXT: [[TMP9:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[TMP7]], [[TMP8]] +// BASIC_FAST-NEXT: [[TMP10:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_IMAG]], [[B_REAL]] +// BASIC_FAST-NEXT: [[TMP11:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_REAL]], [[B_IMAG]] +// BASIC_FAST-NEXT: [[TMP12:%.*]] = fsub reassoc nnan ninf nsz arcp afn double [[TMP10]], [[TMP11]] +// BASIC_FAST-NEXT: [[TMP13:%.*]] = fdiv reassoc nnan ninf nsz arcp afn double [[TMP6]], [[TMP9]] +// BASIC_FAST-NEXT: [[TMP14:%.*]] = fdiv reassoc nnan ninf nsz arcp afn double [[TMP12]], [[TMP9]] +// BASIC_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC_FAST-NEXT: store double [[TMP13]], ptr [[RETVAL_REALP]], align 8 +// BASIC_FAST-NEXT: store double [[TMP14]], ptr [[RETVAL_IMAGP]], align 8 +// BASIC_FAST-NEXT: [[TMP15:%.*]] = load { double, double }, ptr [[RETVAL]], align 8 +// BASIC_FAST-NEXT: ret { double, double } [[TMP15]] +// +// FULL_FAST-LABEL: define dso_local { double, double } @divd( +// FULL_FAST-SAME: double noundef nofpclass(nan inf) [[A_COERCE0:%.*]], double noundef nofpclass(nan inf) [[A_COERCE1:%.*]], double noundef nofpclass(nan inf) [[B_COERCE0:%.*]], double noundef nofpclass(nan inf) [[B_COERCE1:%.*]]) #[[ATTR1:[0-9]+]] { +// FULL_FAST-NEXT: entry: +// FULL_FAST-NEXT: [[RETVAL:%.*]] = alloca { double, double }, align 8 +// FULL_FAST-NEXT: [[A:%.*]] = alloca { double, double }, align 8 +// FULL_FAST-NEXT: [[B:%.*]] = alloca { double, double }, align 8 +// FULL_FAST-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// FULL_FAST-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 8 +// FULL_FAST-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// FULL_FAST-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 8 +// FULL_FAST-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// FULL_FAST-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 8 +// FULL_FAST-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// FULL_FAST-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 8 +// FULL_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// FULL_FAST-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// FULL_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// FULL_FAST-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// FULL_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// FULL_FAST-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// FULL_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// FULL_FAST-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// FULL_FAST-NEXT: [[CALL:%.*]] = call { double, double } @__divdc3(double noundef nofpclass(nan inf) [[A_REAL]], double noundef nofpclass(nan inf) [[A_IMAG]], double noundef nofpclass(nan inf) [[B_REAL]], double noundef nofpclass(nan inf) [[B_IMAG]]) #[[ATTR2]] +// FULL_FAST-NEXT: [[TMP4:%.*]] = extractvalue { double, double } [[CALL]], 0 +// FULL_FAST-NEXT: [[TMP5:%.*]] = extractvalue { double, double } [[CALL]], 1 +// FULL_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 0 +// FULL_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 1 +// FULL_FAST-NEXT: store double [[TMP4]], ptr [[RETVAL_REALP]], align 8 +// FULL_FAST-NEXT: store double [[TMP5]], ptr [[RETVAL_IMAGP]], align 8 +// FULL_FAST-NEXT: [[TMP6:%.*]] = load { double, double }, ptr [[RETVAL]], align 8 +// FULL_FAST-NEXT: ret { double, double } [[TMP6]] +// +// IMPRVD_FAST-LABEL: define dso_local { double, double } @divd( +// IMPRVD_FAST-SAME: double noundef nofpclass(nan inf) [[A_COERCE0:%.*]], double noundef nofpclass(nan inf) [[A_COERCE1:%.*]], double noundef nofpclass(nan inf) [[B_COERCE0:%.*]], double noundef nofpclass(nan inf) [[B_COERCE1:%.*]]) #[[ATTR2:[0-9]+]] { +// IMPRVD_FAST-NEXT: entry: +// IMPRVD_FAST-NEXT: [[RETVAL:%.*]] = alloca { double, double }, align 8 +// IMPRVD_FAST-NEXT: [[A:%.*]] = alloca { double, double }, align 8 +// IMPRVD_FAST-NEXT: [[B:%.*]] = alloca { double, double }, align 8 +// IMPRVD_FAST-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 8 +// IMPRVD_FAST-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 8 +// IMPRVD_FAST-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 8 +// IMPRVD_FAST-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 8 +// IMPRVD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// IMPRVD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// IMPRVD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// IMPRVD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// IMPRVD_FAST-NEXT: [[TMP4:%.*]] = call reassoc nnan ninf nsz arcp afn double @llvm.fabs.f64(double [[B_REAL]]) +// IMPRVD_FAST-NEXT: [[TMP5:%.*]] = call reassoc nnan ninf nsz arcp afn double @llvm.fabs.f64(double [[B_IMAG]]) +// IMPRVD_FAST-NEXT: [[ABS_CMP:%.*]] = fcmp reassoc nnan ninf nsz arcp afn ugt double [[TMP4]], [[TMP5]] +// IMPRVD_FAST-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// IMPRVD_FAST: abs_rhsr_greater_or_equal_abs_rhsi: +// IMPRVD_FAST-NEXT: [[TMP6:%.*]] = fdiv reassoc nnan ninf nsz arcp afn double [[B_IMAG]], [[B_REAL]] +// IMPRVD_FAST-NEXT: [[TMP7:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[TMP6]], [[B_IMAG]] +// IMPRVD_FAST-NEXT: [[TMP8:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[B_REAL]], [[TMP7]] +// IMPRVD_FAST-NEXT: [[TMP9:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_IMAG]], [[TMP6]] +// IMPRVD_FAST-NEXT: [[TMP10:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[A_REAL]], [[TMP9]] +// IMPRVD_FAST-NEXT: [[TMP11:%.*]] = fdiv reassoc nnan ninf nsz arcp afn double [[TMP10]], [[TMP8]] +// IMPRVD_FAST-NEXT: [[TMP12:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_REAL]], [[TMP6]] +// IMPRVD_FAST-NEXT: [[TMP13:%.*]] = fsub reassoc nnan ninf nsz arcp afn double [[A_IMAG]], [[TMP12]] +// IMPRVD_FAST-NEXT: [[TMP14:%.*]] = fdiv reassoc nnan ninf nsz arcp afn double [[TMP13]], [[TMP8]] +// IMPRVD_FAST-NEXT: br label [[COMPLEX_DIV:%.*]] +// IMPRVD_FAST: abs_rhsr_less_than_abs_rhsi: +// IMPRVD_FAST-NEXT: [[TMP15:%.*]] = fdiv reassoc nnan ninf nsz arcp afn double [[B_REAL]], [[B_IMAG]] +// IMPRVD_FAST-NEXT: [[TMP16:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[TMP15]], [[B_REAL]] +// IMPRVD_FAST-NEXT: [[TMP17:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[B_IMAG]], [[TMP16]] +// IMPRVD_FAST-NEXT: [[TMP18:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_REAL]], [[TMP15]] +// IMPRVD_FAST-NEXT: [[TMP19:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[TMP18]], [[A_IMAG]] +// IMPRVD_FAST-NEXT: [[TMP20:%.*]] = fdiv reassoc nnan ninf nsz arcp afn double [[TMP19]], [[TMP17]] +// IMPRVD_FAST-NEXT: [[TMP21:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_IMAG]], [[TMP15]] +// IMPRVD_FAST-NEXT: [[TMP22:%.*]] = fsub reassoc nnan ninf nsz arcp afn double [[TMP21]], [[A_REAL]] +// IMPRVD_FAST-NEXT: [[TMP23:%.*]] = fdiv reassoc nnan ninf nsz arcp afn double [[TMP22]], [[TMP17]] +// IMPRVD_FAST-NEXT: br label [[COMPLEX_DIV]] +// IMPRVD_FAST: complex_div: +// IMPRVD_FAST-NEXT: [[TMP24:%.*]] = phi reassoc nnan ninf nsz arcp afn double [ [[TMP11]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP20]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD_FAST-NEXT: [[TMP25:%.*]] = phi reassoc nnan ninf nsz arcp afn double [ [[TMP14]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP23]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: store double [[TMP24]], ptr [[RETVAL_REALP]], align 8 +// IMPRVD_FAST-NEXT: store double [[TMP25]], ptr [[RETVAL_IMAGP]], align 8 +// IMPRVD_FAST-NEXT: [[TMP26:%.*]] = load { double, double }, ptr [[RETVAL]], align 8 +// IMPRVD_FAST-NEXT: ret { double, double } [[TMP26]] +// +// PRMTD_FAST-LABEL: define dso_local { double, double } @divd( +// PRMTD_FAST-SAME: double noundef nofpclass(nan inf) [[A_COERCE0:%.*]], double noundef nofpclass(nan inf) [[A_COERCE1:%.*]], double noundef nofpclass(nan inf) [[B_COERCE0:%.*]], double noundef nofpclass(nan inf) [[B_COERCE1:%.*]]) #[[ATTR1:[0-9]+]] { +// PRMTD_FAST-NEXT: entry: +// PRMTD_FAST-NEXT: [[RETVAL:%.*]] = alloca { double, double }, align 8 +// PRMTD_FAST-NEXT: [[A:%.*]] = alloca { double, double }, align 8 +// PRMTD_FAST-NEXT: [[B:%.*]] = alloca { double, double }, align 8 +// PRMTD_FAST-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// PRMTD_FAST-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 8 +// PRMTD_FAST-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// PRMTD_FAST-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 8 +// PRMTD_FAST-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// PRMTD_FAST-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 8 +// PRMTD_FAST-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// PRMTD_FAST-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 8 +// PRMTD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// PRMTD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// PRMTD_FAST-NEXT: [[EXT:%.*]] = fpext double [[A_REAL]] to x86_fp80 +// PRMTD_FAST-NEXT: [[EXT1:%.*]] = fpext double [[A_IMAG]] to x86_fp80 +// PRMTD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// PRMTD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// PRMTD_FAST-NEXT: [[EXT2:%.*]] = fpext double [[B_REAL]] to x86_fp80 +// PRMTD_FAST-NEXT: [[EXT3:%.*]] = fpext double [[B_IMAG]] to x86_fp80 +// PRMTD_FAST-NEXT: [[TMP4:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[EXT]], [[EXT2]] +// PRMTD_FAST-NEXT: [[TMP5:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[EXT1]], [[EXT3]] +// PRMTD_FAST-NEXT: [[TMP6:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP4]], [[TMP5]] +// PRMTD_FAST-NEXT: [[TMP7:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[EXT2]], [[EXT2]] +// PRMTD_FAST-NEXT: [[TMP8:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[EXT3]], [[EXT3]] +// PRMTD_FAST-NEXT: [[TMP9:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP7]], [[TMP8]] +// PRMTD_FAST-NEXT: [[TMP10:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[EXT1]], [[EXT2]] +// PRMTD_FAST-NEXT: [[TMP11:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[EXT]], [[EXT3]] +// PRMTD_FAST-NEXT: [[TMP12:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP10]], [[TMP11]] +// PRMTD_FAST-NEXT: [[TMP13:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP6]], [[TMP9]] +// PRMTD_FAST-NEXT: [[TMP14:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP12]], [[TMP9]] +// PRMTD_FAST-NEXT: [[UNPROMOTION:%.*]] = fptrunc x86_fp80 [[TMP13]] to double +// PRMTD_FAST-NEXT: [[UNPROMOTION4:%.*]] = fptrunc x86_fp80 [[TMP14]] to double +// PRMTD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD_FAST-NEXT: store double [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 8 +// PRMTD_FAST-NEXT: store double [[UNPROMOTION4]], ptr [[RETVAL_IMAGP]], align 8 +// PRMTD_FAST-NEXT: [[TMP15:%.*]] = load { double, double }, ptr [[RETVAL]], align 8 +// PRMTD_FAST-NEXT: ret { double, double } [[TMP15]] +// +_Complex double divd(_Complex double a, _Complex double b) { return a / b; } -_Complex float mul(_Complex float a, _Complex float b) { - // LABEL: define {{.*}} @mul( - // FULL: call {{.*}} @__mulsc3 - - // LMTD: fmul float - // LMTD-NEXT: fmul float - // LMTD-NEXT: fmul float - // LMTD-NEXT: fmul float - // LMTD-NEXT: fsub float - // LMTD-NEXT: fadd float - - // FRTRN: fmul {{.*}}float - // FRTRN-NEXT: fmul {{.*}}float - // FRTRN-NEXT: fmul {{.*}}float - // FRTRN-NEXT: fmul {{.*}}float - // FRTRN-NEXT: fsub {{.*}}float - // FRTRN-NEXT: fadd {{.*}}float - - // LMTD-FAST: fmul {{.*}} float - // LMTD-FAST-NEXT: fmul {{.*}} float - // LMTD-FAST-NEXT: fmul {{.*}} float - // LMTD-FAST-NEXT: fmul {{.*}} float - // LMTD-FAST-NEXT: fsub {{.*}} float - // LMTD-FAST-NEXT: fadd {{.*}} float +// FULL-LABEL: define dso_local { double, double } @muld( +// FULL-SAME: double noundef [[A_COERCE0:%.*]], double noundef [[A_COERCE1:%.*]], double noundef [[B_COERCE0:%.*]], double noundef [[B_COERCE1:%.*]]) #[[ATTR1]] { +// FULL-NEXT: entry: +// FULL-NEXT: [[RETVAL:%.*]] = alloca { double, double }, align 8 +// FULL-NEXT: [[A:%.*]] = alloca { double, double }, align 8 +// FULL-NEXT: [[B:%.*]] = alloca { double, double }, align 8 +// FULL-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// FULL-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 8 +// FULL-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// FULL-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 8 +// FULL-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// FULL-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 8 +// FULL-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// FULL-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 8 +// FULL-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// FULL-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// FULL-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// FULL-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// FULL-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// FULL-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// FULL-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// FULL-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// FULL-NEXT: [[MUL_AC:%.*]] = fmul double [[A_REAL]], [[B_REAL]] +// FULL-NEXT: [[MUL_BD:%.*]] = fmul double [[A_IMAG]], [[B_IMAG]] +// FULL-NEXT: [[MUL_AD:%.*]] = fmul double [[A_REAL]], [[B_IMAG]] +// FULL-NEXT: [[MUL_BC:%.*]] = fmul double [[A_IMAG]], [[B_REAL]] +// FULL-NEXT: [[MUL_R:%.*]] = fsub double [[MUL_AC]], [[MUL_BD]] +// FULL-NEXT: [[MUL_I:%.*]] = fadd double [[MUL_AD]], [[MUL_BC]] +// FULL-NEXT: [[ISNAN_CMP:%.*]] = fcmp uno double [[MUL_R]], [[MUL_R]] +// FULL-NEXT: br i1 [[ISNAN_CMP]], label [[COMPLEX_MUL_IMAG_NAN:%.*]], label [[COMPLEX_MUL_CONT:%.*]], !prof [[PROF2]] +// FULL: complex_mul_imag_nan: +// FULL-NEXT: [[ISNAN_CMP1:%.*]] = fcmp uno double [[MUL_I]], [[MUL_I]] +// FULL-NEXT: br i1 [[ISNAN_CMP1]], label [[COMPLEX_MUL_LIBCALL:%.*]], label [[COMPLEX_MUL_CONT]], !prof [[PROF2]] +// FULL: complex_mul_libcall: +// FULL-NEXT: [[CALL:%.*]] = call { double, double } @__muldc3(double noundef [[A_REAL]], double noundef [[A_IMAG]], double noundef [[B_REAL]], double noundef [[B_IMAG]]) #[[ATTR2]] +// FULL-NEXT: [[TMP4:%.*]] = extractvalue { double, double } [[CALL]], 0 +// FULL-NEXT: [[TMP5:%.*]] = extractvalue { double, double } [[CALL]], 1 +// FULL-NEXT: br label [[COMPLEX_MUL_CONT]] +// FULL: complex_mul_cont: +// FULL-NEXT: [[REAL_MUL_PHI:%.*]] = phi double [ [[MUL_R]], [[ENTRY:%.*]] ], [ [[MUL_R]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[TMP4]], [[COMPLEX_MUL_LIBCALL]] ] +// FULL-NEXT: [[IMAG_MUL_PHI:%.*]] = phi double [ [[MUL_I]], [[ENTRY]] ], [ [[MUL_I]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[TMP5]], [[COMPLEX_MUL_LIBCALL]] ] +// FULL-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 0 +// FULL-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 1 +// FULL-NEXT: store double [[REAL_MUL_PHI]], ptr [[RETVAL_REALP]], align 8 +// FULL-NEXT: store double [[IMAG_MUL_PHI]], ptr [[RETVAL_IMAGP]], align 8 +// FULL-NEXT: [[TMP6:%.*]] = load { double, double }, ptr [[RETVAL]], align 8 +// FULL-NEXT: ret { double, double } [[TMP6]] +// +// BASIC-LABEL: define dso_local { double, double } @muld( +// BASIC-SAME: double noundef [[A_COERCE0:%.*]], double noundef [[A_COERCE1:%.*]], double noundef [[B_COERCE0:%.*]], double noundef [[B_COERCE1:%.*]]) #[[ATTR1]] { +// BASIC-NEXT: entry: +// BASIC-NEXT: [[RETVAL:%.*]] = alloca { double, double }, align 8 +// BASIC-NEXT: [[A:%.*]] = alloca { double, double }, align 8 +// BASIC-NEXT: [[B:%.*]] = alloca { double, double }, align 8 +// BASIC-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// BASIC-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 8 +// BASIC-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// BASIC-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 8 +// BASIC-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// BASIC-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 8 +// BASIC-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// BASIC-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 8 +// BASIC-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// BASIC-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// BASIC-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// BASIC-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// BASIC-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// BASIC-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// BASIC-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// BASIC-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// BASIC-NEXT: [[MUL_AC:%.*]] = fmul double [[A_REAL]], [[B_REAL]] +// BASIC-NEXT: [[MUL_BD:%.*]] = fmul double [[A_IMAG]], [[B_IMAG]] +// BASIC-NEXT: [[MUL_AD:%.*]] = fmul double [[A_REAL]], [[B_IMAG]] +// BASIC-NEXT: [[MUL_BC:%.*]] = fmul double [[A_IMAG]], [[B_REAL]] +// BASIC-NEXT: [[MUL_R:%.*]] = fsub double [[MUL_AC]], [[MUL_BD]] +// BASIC-NEXT: [[MUL_I:%.*]] = fadd double [[MUL_AD]], [[MUL_BC]] +// BASIC-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC-NEXT: store double [[MUL_R]], ptr [[RETVAL_REALP]], align 8 +// BASIC-NEXT: store double [[MUL_I]], ptr [[RETVAL_IMAGP]], align 8 +// BASIC-NEXT: [[TMP4:%.*]] = load { double, double }, ptr [[RETVAL]], align 8 +// BASIC-NEXT: ret { double, double } [[TMP4]] +// +// IMPRVD-LABEL: define dso_local { double, double } @muld( +// IMPRVD-SAME: double noundef [[A_COERCE0:%.*]], double noundef [[A_COERCE1:%.*]], double noundef [[B_COERCE0:%.*]], double noundef [[B_COERCE1:%.*]]) #[[ATTR2]] { +// IMPRVD-NEXT: entry: +// IMPRVD-NEXT: [[RETVAL:%.*]] = alloca { double, double }, align 8 +// IMPRVD-NEXT: [[A:%.*]] = alloca { double, double }, align 8 +// IMPRVD-NEXT: [[B:%.*]] = alloca { double, double }, align 8 +// IMPRVD-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// IMPRVD-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 8 +// IMPRVD-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// IMPRVD-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 8 +// IMPRVD-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// IMPRVD-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 8 +// IMPRVD-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// IMPRVD-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 8 +// IMPRVD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// IMPRVD-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// IMPRVD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// IMPRVD-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// IMPRVD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// IMPRVD-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// IMPRVD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// IMPRVD-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// IMPRVD-NEXT: [[MUL_AC:%.*]] = fmul double [[A_REAL]], [[B_REAL]] +// IMPRVD-NEXT: [[MUL_BD:%.*]] = fmul double [[A_IMAG]], [[B_IMAG]] +// IMPRVD-NEXT: [[MUL_AD:%.*]] = fmul double [[A_REAL]], [[B_IMAG]] +// IMPRVD-NEXT: [[MUL_BC:%.*]] = fmul double [[A_IMAG]], [[B_REAL]] +// IMPRVD-NEXT: [[MUL_R:%.*]] = fsub double [[MUL_AC]], [[MUL_BD]] +// IMPRVD-NEXT: [[MUL_I:%.*]] = fadd double [[MUL_AD]], [[MUL_BC]] +// IMPRVD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD-NEXT: store double [[MUL_R]], ptr [[RETVAL_REALP]], align 8 +// IMPRVD-NEXT: store double [[MUL_I]], ptr [[RETVAL_IMAGP]], align 8 +// IMPRVD-NEXT: [[TMP4:%.*]] = load { double, double }, ptr [[RETVAL]], align 8 +// IMPRVD-NEXT: ret { double, double } [[TMP4]] +// +// PRMTD-LABEL: define dso_local { double, double } @muld( +// PRMTD-SAME: double noundef [[A_COERCE0:%.*]], double noundef [[A_COERCE1:%.*]], double noundef [[B_COERCE0:%.*]], double noundef [[B_COERCE1:%.*]]) #[[ATTR1]] { +// PRMTD-NEXT: entry: +// PRMTD-NEXT: [[RETVAL:%.*]] = alloca { double, double }, align 8 +// PRMTD-NEXT: [[A:%.*]] = alloca { double, double }, align 8 +// PRMTD-NEXT: [[B:%.*]] = alloca { double, double }, align 8 +// PRMTD-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// PRMTD-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 8 +// PRMTD-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// PRMTD-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 8 +// PRMTD-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// PRMTD-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 8 +// PRMTD-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// PRMTD-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 8 +// PRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// PRMTD-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// PRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// PRMTD-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// PRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// PRMTD-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// PRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// PRMTD-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// PRMTD-NEXT: [[MUL_AC:%.*]] = fmul double [[A_REAL]], [[B_REAL]] +// PRMTD-NEXT: [[MUL_BD:%.*]] = fmul double [[A_IMAG]], [[B_IMAG]] +// PRMTD-NEXT: [[MUL_AD:%.*]] = fmul double [[A_REAL]], [[B_IMAG]] +// PRMTD-NEXT: [[MUL_BC:%.*]] = fmul double [[A_IMAG]], [[B_REAL]] +// PRMTD-NEXT: [[MUL_R:%.*]] = fsub double [[MUL_AC]], [[MUL_BD]] +// PRMTD-NEXT: [[MUL_I:%.*]] = fadd double [[MUL_AD]], [[MUL_BC]] +// PRMTD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD-NEXT: store double [[MUL_R]], ptr [[RETVAL_REALP]], align 8 +// PRMTD-NEXT: store double [[MUL_I]], ptr [[RETVAL_IMAGP]], align 8 +// PRMTD-NEXT: [[TMP4:%.*]] = load { double, double }, ptr [[RETVAL]], align 8 +// PRMTD-NEXT: ret { double, double } [[TMP4]] +// +// X86WINPRMTD-LABEL: define dso_local void @muld( +// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] { +// X86WINPRMTD-NEXT: entry: +// X86WINPRMTD-NEXT: [[RESULT_PTR:%.*]] = alloca ptr, align 8 +// X86WINPRMTD-NEXT: [[B_INDIRECT_ADDR:%.*]] = alloca ptr, align 8 +// X86WINPRMTD-NEXT: [[A_INDIRECT_ADDR:%.*]] = alloca ptr, align 8 +// X86WINPRMTD-NEXT: store ptr [[AGG_RESULT]], ptr [[RESULT_PTR]], align 8 +// X86WINPRMTD-NEXT: store ptr [[B]], ptr [[B_INDIRECT_ADDR]], align 8 +// X86WINPRMTD-NEXT: store ptr [[A]], ptr [[A_INDIRECT_ADDR]], align 8 +// X86WINPRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// X86WINPRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// X86WINPRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// X86WINPRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// X86WINPRMTD-NEXT: [[MUL_AC:%.*]] = fmul double [[A_REAL]], [[B_REAL]] +// X86WINPRMTD-NEXT: [[MUL_BD:%.*]] = fmul double [[A_IMAG]], [[B_IMAG]] +// X86WINPRMTD-NEXT: [[MUL_AD:%.*]] = fmul double [[A_REAL]], [[B_IMAG]] +// X86WINPRMTD-NEXT: [[MUL_BC:%.*]] = fmul double [[A_IMAG]], [[B_REAL]] +// X86WINPRMTD-NEXT: [[MUL_R:%.*]] = fsub double [[MUL_AC]], [[MUL_BD]] +// X86WINPRMTD-NEXT: [[MUL_I:%.*]] = fadd double [[MUL_AD]], [[MUL_BC]] +// X86WINPRMTD-NEXT: [[AGG_RESULT_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[AGG_RESULT_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// X86WINPRMTD-NEXT: store double [[MUL_R]], ptr [[AGG_RESULT_REALP]], align 8 +// X86WINPRMTD-NEXT: store double [[MUL_I]], ptr [[AGG_RESULT_IMAGP]], align 8 +// X86WINPRMTD-NEXT: [[AGG_RESULT_REALP1:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[AGG_RESULT_REAL:%.*]] = load double, ptr [[AGG_RESULT_REALP1]], align 8 +// X86WINPRMTD-NEXT: [[AGG_RESULT_IMAGP2:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[AGG_RESULT_IMAG:%.*]] = load double, ptr [[AGG_RESULT_IMAGP2]], align 8 +// X86WINPRMTD-NEXT: [[AGG_RESULT_REALP3:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[AGG_RESULT_IMAGP4:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// X86WINPRMTD-NEXT: store double [[AGG_RESULT_REAL]], ptr [[AGG_RESULT_REALP3]], align 8 +// X86WINPRMTD-NEXT: store double [[AGG_RESULT_IMAG]], ptr [[AGG_RESULT_IMAGP4]], align 8 +// X86WINPRMTD-NEXT: ret void +// +// AVRFP32-LABEL: define dso_local { float, float } @muld( +// AVRFP32-SAME: float noundef [[A_COERCE0:%.*]], float noundef [[A_COERCE1:%.*]], float noundef [[B_COERCE0:%.*]], float noundef [[B_COERCE1:%.*]]) addrspace(1) #[[ATTR0]] { +// AVRFP32-NEXT: entry: +// AVRFP32-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 1 +// AVRFP32-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// AVRFP32-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// AVRFP32-NEXT: [[TMP0:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP32-NEXT: store float [[A_COERCE0]], ptr [[TMP0]], align 4 +// AVRFP32-NEXT: [[TMP1:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[A_COERCE1]], ptr [[TMP1]], align 4 +// AVRFP32-NEXT: [[TMP2:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// AVRFP32-NEXT: store float [[B_COERCE0]], ptr [[TMP2]], align 4 +// AVRFP32-NEXT: [[TMP3:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[B_COERCE1]], ptr [[TMP3]], align 4 +// AVRFP32-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP32-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// AVRFP32-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP32-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// AVRFP32-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// AVRFP32-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// AVRFP32-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// AVRFP32-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// AVRFP32-NEXT: [[MUL_AC:%.*]] = fmul float [[A_REAL]], [[B_REAL]] +// AVRFP32-NEXT: [[MUL_BD:%.*]] = fmul float [[A_IMAG]], [[B_IMAG]] +// AVRFP32-NEXT: [[MUL_AD:%.*]] = fmul float [[A_REAL]], [[B_IMAG]] +// AVRFP32-NEXT: [[MUL_BC:%.*]] = fmul float [[A_IMAG]], [[B_REAL]] +// AVRFP32-NEXT: [[MUL_R:%.*]] = fsub float [[MUL_AC]], [[MUL_BD]] +// AVRFP32-NEXT: [[MUL_I:%.*]] = fadd float [[MUL_AD]], [[MUL_BC]] +// AVRFP32-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// AVRFP32-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[MUL_R]], ptr [[RETVAL_REALP]], align 1 +// AVRFP32-NEXT: store float [[MUL_I]], ptr [[RETVAL_IMAGP]], align 1 +// AVRFP32-NEXT: [[TMP4:%.*]] = load { float, float }, ptr [[RETVAL]], align 1 +// AVRFP32-NEXT: ret { float, float } [[TMP4]] +// +// AVRFP64-LABEL: define dso_local void @muld( +// AVRFP64-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 1 [[AGG_RESULT:%.*]], double noundef [[A_COERCE0:%.*]], double noundef [[A_COERCE1:%.*]], double noundef [[B_COERCE0:%.*]], double noundef [[B_COERCE1:%.*]]) addrspace(1) #[[ATTR0]] { +// AVRFP64-NEXT: entry: +// AVRFP64-NEXT: [[A:%.*]] = alloca { double, double }, align 8 +// AVRFP64-NEXT: [[B:%.*]] = alloca { double, double }, align 8 +// AVRFP64-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// AVRFP64-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 8 +// AVRFP64-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// AVRFP64-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 8 +// AVRFP64-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// AVRFP64-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 8 +// AVRFP64-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// AVRFP64-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 8 +// AVRFP64-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// AVRFP64-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// AVRFP64-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// AVRFP64-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// AVRFP64-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// AVRFP64-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// AVRFP64-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// AVRFP64-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// AVRFP64-NEXT: [[MUL_AC:%.*]] = fmul double [[A_REAL]], [[B_REAL]] +// AVRFP64-NEXT: [[MUL_BD:%.*]] = fmul double [[A_IMAG]], [[B_IMAG]] +// AVRFP64-NEXT: [[MUL_AD:%.*]] = fmul double [[A_REAL]], [[B_IMAG]] +// AVRFP64-NEXT: [[MUL_BC:%.*]] = fmul double [[A_IMAG]], [[B_REAL]] +// AVRFP64-NEXT: [[MUL_R:%.*]] = fsub double [[MUL_AC]], [[MUL_BD]] +// AVRFP64-NEXT: [[MUL_I:%.*]] = fadd double [[MUL_AD]], [[MUL_BC]] +// AVRFP64-NEXT: [[AGG_RESULT_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// AVRFP64-NEXT: [[AGG_RESULT_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// AVRFP64-NEXT: store double [[MUL_R]], ptr [[AGG_RESULT_REALP]], align 1 +// AVRFP64-NEXT: store double [[MUL_I]], ptr [[AGG_RESULT_IMAGP]], align 1 +// AVRFP64-NEXT: [[AGG_RESULT_REALP1:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// AVRFP64-NEXT: [[AGG_RESULT_REAL:%.*]] = load double, ptr [[AGG_RESULT_REALP1]], align 1 +// AVRFP64-NEXT: [[AGG_RESULT_IMAGP2:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// AVRFP64-NEXT: [[AGG_RESULT_IMAG:%.*]] = load double, ptr [[AGG_RESULT_IMAGP2]], align 1 +// AVRFP64-NEXT: [[AGG_RESULT_REALP3:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// AVRFP64-NEXT: [[AGG_RESULT_IMAGP4:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// AVRFP64-NEXT: store double [[AGG_RESULT_REAL]], ptr [[AGG_RESULT_REALP3]], align 1 +// AVRFP64-NEXT: store double [[AGG_RESULT_IMAG]], ptr [[AGG_RESULT_IMAGP4]], align 1 +// AVRFP64-NEXT: ret void +// +// BASIC_FAST-LABEL: define dso_local { double, double } @muld( +// BASIC_FAST-SAME: double noundef nofpclass(nan inf) [[A_COERCE0:%.*]], double noundef nofpclass(nan inf) [[A_COERCE1:%.*]], double noundef nofpclass(nan inf) [[B_COERCE0:%.*]], double noundef nofpclass(nan inf) [[B_COERCE1:%.*]]) #[[ATTR1]] { +// BASIC_FAST-NEXT: entry: +// BASIC_FAST-NEXT: [[RETVAL:%.*]] = alloca { double, double }, align 8 +// BASIC_FAST-NEXT: [[A:%.*]] = alloca { double, double }, align 8 +// BASIC_FAST-NEXT: [[B:%.*]] = alloca { double, double }, align 8 +// BASIC_FAST-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// BASIC_FAST-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 8 +// BASIC_FAST-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// BASIC_FAST-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 8 +// BASIC_FAST-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// BASIC_FAST-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 8 +// BASIC_FAST-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// BASIC_FAST-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 8 +// BASIC_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// BASIC_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// BASIC_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// BASIC_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// BASIC_FAST-NEXT: [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_REAL]], [[B_REAL]] +// BASIC_FAST-NEXT: [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_IMAG]], [[B_IMAG]] +// BASIC_FAST-NEXT: [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_REAL]], [[B_IMAG]] +// BASIC_FAST-NEXT: [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_IMAG]], [[B_REAL]] +// BASIC_FAST-NEXT: [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn double [[MUL_AC]], [[MUL_BD]] +// BASIC_FAST-NEXT: [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[MUL_AD]], [[MUL_BC]] +// BASIC_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC_FAST-NEXT: store double [[MUL_R]], ptr [[RETVAL_REALP]], align 8 +// BASIC_FAST-NEXT: store double [[MUL_I]], ptr [[RETVAL_IMAGP]], align 8 +// BASIC_FAST-NEXT: [[TMP4:%.*]] = load { double, double }, ptr [[RETVAL]], align 8 +// BASIC_FAST-NEXT: ret { double, double } [[TMP4]] +// +// FULL_FAST-LABEL: define dso_local { double, double } @muld( +// FULL_FAST-SAME: double noundef nofpclass(nan inf) [[A_COERCE0:%.*]], double noundef nofpclass(nan inf) [[A_COERCE1:%.*]], double noundef nofpclass(nan inf) [[B_COERCE0:%.*]], double noundef nofpclass(nan inf) [[B_COERCE1:%.*]]) #[[ATTR1]] { +// FULL_FAST-NEXT: entry: +// FULL_FAST-NEXT: [[RETVAL:%.*]] = alloca { double, double }, align 8 +// FULL_FAST-NEXT: [[A:%.*]] = alloca { double, double }, align 8 +// FULL_FAST-NEXT: [[B:%.*]] = alloca { double, double }, align 8 +// FULL_FAST-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// FULL_FAST-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 8 +// FULL_FAST-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// FULL_FAST-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 8 +// FULL_FAST-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// FULL_FAST-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 8 +// FULL_FAST-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// FULL_FAST-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 8 +// FULL_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// FULL_FAST-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// FULL_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// FULL_FAST-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// FULL_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// FULL_FAST-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// FULL_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// FULL_FAST-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// FULL_FAST-NEXT: [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_REAL]], [[B_REAL]] +// FULL_FAST-NEXT: [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_IMAG]], [[B_IMAG]] +// FULL_FAST-NEXT: [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_REAL]], [[B_IMAG]] +// FULL_FAST-NEXT: [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_IMAG]], [[B_REAL]] +// FULL_FAST-NEXT: [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn double [[MUL_AC]], [[MUL_BD]] +// FULL_FAST-NEXT: [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[MUL_AD]], [[MUL_BC]] +// FULL_FAST-NEXT: [[ISNAN_CMP:%.*]] = fcmp reassoc nnan ninf nsz arcp afn uno double [[MUL_R]], [[MUL_R]] +// FULL_FAST-NEXT: br i1 [[ISNAN_CMP]], label [[COMPLEX_MUL_IMAG_NAN:%.*]], label [[COMPLEX_MUL_CONT:%.*]], !prof [[PROF2]] +// FULL_FAST: complex_mul_imag_nan: +// FULL_FAST-NEXT: [[ISNAN_CMP1:%.*]] = fcmp reassoc nnan ninf nsz arcp afn uno double [[MUL_I]], [[MUL_I]] +// FULL_FAST-NEXT: br i1 [[ISNAN_CMP1]], label [[COMPLEX_MUL_LIBCALL:%.*]], label [[COMPLEX_MUL_CONT]], !prof [[PROF2]] +// FULL_FAST: complex_mul_libcall: +// FULL_FAST-NEXT: [[CALL:%.*]] = call { double, double } @__muldc3(double noundef nofpclass(nan inf) [[A_REAL]], double noundef nofpclass(nan inf) [[A_IMAG]], double noundef nofpclass(nan inf) [[B_REAL]], double noundef nofpclass(nan inf) [[B_IMAG]]) #[[ATTR2]] +// FULL_FAST-NEXT: [[TMP4:%.*]] = extractvalue { double, double } [[CALL]], 0 +// FULL_FAST-NEXT: [[TMP5:%.*]] = extractvalue { double, double } [[CALL]], 1 +// FULL_FAST-NEXT: br label [[COMPLEX_MUL_CONT]] +// FULL_FAST: complex_mul_cont: +// FULL_FAST-NEXT: [[REAL_MUL_PHI:%.*]] = phi reassoc nnan ninf nsz arcp afn double [ [[MUL_R]], [[ENTRY:%.*]] ], [ [[MUL_R]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[TMP4]], [[COMPLEX_MUL_LIBCALL]] ] +// FULL_FAST-NEXT: [[IMAG_MUL_PHI:%.*]] = phi reassoc nnan ninf nsz arcp afn double [ [[MUL_I]], [[ENTRY]] ], [ [[MUL_I]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[TMP5]], [[COMPLEX_MUL_LIBCALL]] ] +// FULL_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 0 +// FULL_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 1 +// FULL_FAST-NEXT: store double [[REAL_MUL_PHI]], ptr [[RETVAL_REALP]], align 8 +// FULL_FAST-NEXT: store double [[IMAG_MUL_PHI]], ptr [[RETVAL_IMAGP]], align 8 +// FULL_FAST-NEXT: [[TMP6:%.*]] = load { double, double }, ptr [[RETVAL]], align 8 +// FULL_FAST-NEXT: ret { double, double } [[TMP6]] +// +// IMPRVD_FAST-LABEL: define dso_local { double, double } @muld( +// IMPRVD_FAST-SAME: double noundef nofpclass(nan inf) [[A_COERCE0:%.*]], double noundef nofpclass(nan inf) [[A_COERCE1:%.*]], double noundef nofpclass(nan inf) [[B_COERCE0:%.*]], double noundef nofpclass(nan inf) [[B_COERCE1:%.*]]) #[[ATTR2]] { +// IMPRVD_FAST-NEXT: entry: +// IMPRVD_FAST-NEXT: [[RETVAL:%.*]] = alloca { double, double }, align 8 +// IMPRVD_FAST-NEXT: [[A:%.*]] = alloca { double, double }, align 8 +// IMPRVD_FAST-NEXT: [[B:%.*]] = alloca { double, double }, align 8 +// IMPRVD_FAST-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 8 +// IMPRVD_FAST-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 8 +// IMPRVD_FAST-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 8 +// IMPRVD_FAST-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 8 +// IMPRVD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// IMPRVD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// IMPRVD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// IMPRVD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// IMPRVD_FAST-NEXT: [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_REAL]], [[B_REAL]] +// IMPRVD_FAST-NEXT: [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_IMAG]], [[B_IMAG]] +// IMPRVD_FAST-NEXT: [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_REAL]], [[B_IMAG]] +// IMPRVD_FAST-NEXT: [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_IMAG]], [[B_REAL]] +// IMPRVD_FAST-NEXT: [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn double [[MUL_AC]], [[MUL_BD]] +// IMPRVD_FAST-NEXT: [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[MUL_AD]], [[MUL_BC]] +// IMPRVD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: store double [[MUL_R]], ptr [[RETVAL_REALP]], align 8 +// IMPRVD_FAST-NEXT: store double [[MUL_I]], ptr [[RETVAL_IMAGP]], align 8 +// IMPRVD_FAST-NEXT: [[TMP4:%.*]] = load { double, double }, ptr [[RETVAL]], align 8 +// IMPRVD_FAST-NEXT: ret { double, double } [[TMP4]] +// +// PRMTD_FAST-LABEL: define dso_local { double, double } @muld( +// PRMTD_FAST-SAME: double noundef nofpclass(nan inf) [[A_COERCE0:%.*]], double noundef nofpclass(nan inf) [[A_COERCE1:%.*]], double noundef nofpclass(nan inf) [[B_COERCE0:%.*]], double noundef nofpclass(nan inf) [[B_COERCE1:%.*]]) #[[ATTR1]] { +// PRMTD_FAST-NEXT: entry: +// PRMTD_FAST-NEXT: [[RETVAL:%.*]] = alloca { double, double }, align 8 +// PRMTD_FAST-NEXT: [[A:%.*]] = alloca { double, double }, align 8 +// PRMTD_FAST-NEXT: [[B:%.*]] = alloca { double, double }, align 8 +// PRMTD_FAST-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// PRMTD_FAST-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 8 +// PRMTD_FAST-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// PRMTD_FAST-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 8 +// PRMTD_FAST-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// PRMTD_FAST-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 8 +// PRMTD_FAST-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// PRMTD_FAST-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 8 +// PRMTD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// PRMTD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// PRMTD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// PRMTD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// PRMTD_FAST-NEXT: [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_REAL]], [[B_REAL]] +// PRMTD_FAST-NEXT: [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_IMAG]], [[B_IMAG]] +// PRMTD_FAST-NEXT: [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_REAL]], [[B_IMAG]] +// PRMTD_FAST-NEXT: [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[A_IMAG]], [[B_REAL]] +// PRMTD_FAST-NEXT: [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn double [[MUL_AC]], [[MUL_BD]] +// PRMTD_FAST-NEXT: [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[MUL_AD]], [[MUL_BC]] +// PRMTD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD_FAST-NEXT: store double [[MUL_R]], ptr [[RETVAL_REALP]], align 8 +// PRMTD_FAST-NEXT: store double [[MUL_I]], ptr [[RETVAL_IMAGP]], align 8 +// PRMTD_FAST-NEXT: [[TMP4:%.*]] = load { double, double }, ptr [[RETVAL]], align 8 +// PRMTD_FAST-NEXT: ret { double, double } [[TMP4]] +// +_Complex double muld(_Complex double a, _Complex double b) { + return a * b; +} + +// FULL-LABEL: define dso_local { x86_fp80, x86_fp80 } @divld( +// FULL-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR1]] { +// FULL-NEXT: entry: +// FULL-NEXT: [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16 +// FULL-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 0 +// FULL-NEXT: [[A_REAL:%.*]] = load x86_fp80, ptr [[A_REALP]], align 16 +// FULL-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 1 +// FULL-NEXT: [[A_IMAG:%.*]] = load x86_fp80, ptr [[A_IMAGP]], align 16 +// FULL-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// FULL-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// FULL-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// FULL-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// FULL-NEXT: [[CALL:%.*]] = call { x86_fp80, x86_fp80 } @__divxc3(x86_fp80 noundef [[A_REAL]], x86_fp80 noundef [[A_IMAG]], x86_fp80 noundef [[B_REAL]], x86_fp80 noundef [[B_IMAG]]) #[[ATTR2]] +// FULL-NEXT: [[TMP0:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 0 +// FULL-NEXT: [[TMP1:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 1 +// FULL-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 0 +// FULL-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 1 +// FULL-NEXT: store x86_fp80 [[TMP0]], ptr [[RETVAL_REALP]], align 16 +// FULL-NEXT: store x86_fp80 [[TMP1]], ptr [[RETVAL_IMAGP]], align 16 +// FULL-NEXT: [[TMP2:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16 +// FULL-NEXT: ret { x86_fp80, x86_fp80 } [[TMP2]] +// +// BASIC-LABEL: define dso_local { x86_fp80, x86_fp80 } @divld( +// BASIC-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR1]] { +// BASIC-NEXT: entry: +// BASIC-NEXT: [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16 +// BASIC-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 0 +// BASIC-NEXT: [[A_REAL:%.*]] = load x86_fp80, ptr [[A_REALP]], align 16 +// BASIC-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 1 +// BASIC-NEXT: [[A_IMAG:%.*]] = load x86_fp80, ptr [[A_IMAGP]], align 16 +// BASIC-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// BASIC-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// BASIC-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// BASIC-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// BASIC-NEXT: [[TMP0:%.*]] = fmul x86_fp80 [[A_REAL]], [[B_REAL]] +// BASIC-NEXT: [[TMP1:%.*]] = fmul x86_fp80 [[A_IMAG]], [[B_IMAG]] +// BASIC-NEXT: [[TMP2:%.*]] = fadd x86_fp80 [[TMP0]], [[TMP1]] +// BASIC-NEXT: [[TMP3:%.*]] = fmul x86_fp80 [[B_REAL]], [[B_REAL]] +// BASIC-NEXT: [[TMP4:%.*]] = fmul x86_fp80 [[B_IMAG]], [[B_IMAG]] +// BASIC-NEXT: [[TMP5:%.*]] = fadd x86_fp80 [[TMP3]], [[TMP4]] +// BASIC-NEXT: [[TMP6:%.*]] = fmul x86_fp80 [[A_IMAG]], [[B_REAL]] +// BASIC-NEXT: [[TMP7:%.*]] = fmul x86_fp80 [[A_REAL]], [[B_IMAG]] +// BASIC-NEXT: [[TMP8:%.*]] = fsub x86_fp80 [[TMP6]], [[TMP7]] +// BASIC-NEXT: [[TMP9:%.*]] = fdiv x86_fp80 [[TMP2]], [[TMP5]] +// BASIC-NEXT: [[TMP10:%.*]] = fdiv x86_fp80 [[TMP8]], [[TMP5]] +// BASIC-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC-NEXT: store x86_fp80 [[TMP9]], ptr [[RETVAL_REALP]], align 16 +// BASIC-NEXT: store x86_fp80 [[TMP10]], ptr [[RETVAL_IMAGP]], align 16 +// BASIC-NEXT: [[TMP11:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16 +// BASIC-NEXT: ret { x86_fp80, x86_fp80 } [[TMP11]] +// +// IMPRVD-LABEL: define dso_local { x86_fp80, x86_fp80 } @divld( +// IMPRVD-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR2]] { +// IMPRVD-NEXT: entry: +// IMPRVD-NEXT: [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16 +// IMPRVD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 0 +// IMPRVD-NEXT: [[A_REAL:%.*]] = load x86_fp80, ptr [[A_REALP]], align 16 +// IMPRVD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 1 +// IMPRVD-NEXT: [[A_IMAG:%.*]] = load x86_fp80, ptr [[A_IMAGP]], align 16 +// IMPRVD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// IMPRVD-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// IMPRVD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// IMPRVD-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// IMPRVD-NEXT: [[TMP0:%.*]] = call x86_fp80 @llvm.fabs.f80(x86_fp80 [[B_REAL]]) +// IMPRVD-NEXT: [[TMP1:%.*]] = call x86_fp80 @llvm.fabs.f80(x86_fp80 [[B_IMAG]]) +// IMPRVD-NEXT: [[ABS_CMP:%.*]] = fcmp ugt x86_fp80 [[TMP0]], [[TMP1]] +// IMPRVD-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// IMPRVD: abs_rhsr_greater_or_equal_abs_rhsi: +// IMPRVD-NEXT: [[TMP2:%.*]] = fdiv x86_fp80 [[B_IMAG]], [[B_REAL]] +// IMPRVD-NEXT: [[TMP3:%.*]] = fmul x86_fp80 [[TMP2]], [[B_IMAG]] +// IMPRVD-NEXT: [[TMP4:%.*]] = fadd x86_fp80 [[B_REAL]], [[TMP3]] +// IMPRVD-NEXT: [[TMP5:%.*]] = fmul x86_fp80 [[A_IMAG]], [[TMP2]] +// IMPRVD-NEXT: [[TMP6:%.*]] = fadd x86_fp80 [[A_REAL]], [[TMP5]] +// IMPRVD-NEXT: [[TMP7:%.*]] = fdiv x86_fp80 [[TMP6]], [[TMP4]] +// IMPRVD-NEXT: [[TMP8:%.*]] = fmul x86_fp80 [[A_REAL]], [[TMP2]] +// IMPRVD-NEXT: [[TMP9:%.*]] = fsub x86_fp80 [[A_IMAG]], [[TMP8]] +// IMPRVD-NEXT: [[TMP10:%.*]] = fdiv x86_fp80 [[TMP9]], [[TMP4]] +// IMPRVD-NEXT: br label [[COMPLEX_DIV:%.*]] +// IMPRVD: abs_rhsr_less_than_abs_rhsi: +// IMPRVD-NEXT: [[TMP11:%.*]] = fdiv x86_fp80 [[B_REAL]], [[B_IMAG]] +// IMPRVD-NEXT: [[TMP12:%.*]] = fmul x86_fp80 [[TMP11]], [[B_REAL]] +// IMPRVD-NEXT: [[TMP13:%.*]] = fadd x86_fp80 [[B_IMAG]], [[TMP12]] +// IMPRVD-NEXT: [[TMP14:%.*]] = fmul x86_fp80 [[A_REAL]], [[TMP11]] +// IMPRVD-NEXT: [[TMP15:%.*]] = fadd x86_fp80 [[TMP14]], [[A_IMAG]] +// IMPRVD-NEXT: [[TMP16:%.*]] = fdiv x86_fp80 [[TMP15]], [[TMP13]] +// IMPRVD-NEXT: [[TMP17:%.*]] = fmul x86_fp80 [[A_IMAG]], [[TMP11]] +// IMPRVD-NEXT: [[TMP18:%.*]] = fsub x86_fp80 [[TMP17]], [[A_REAL]] +// IMPRVD-NEXT: [[TMP19:%.*]] = fdiv x86_fp80 [[TMP18]], [[TMP13]] +// IMPRVD-NEXT: br label [[COMPLEX_DIV]] +// IMPRVD: complex_div: +// IMPRVD-NEXT: [[TMP20:%.*]] = phi x86_fp80 [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD-NEXT: [[TMP21:%.*]] = phi x86_fp80 [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD-NEXT: store x86_fp80 [[TMP20]], ptr [[RETVAL_REALP]], align 16 +// IMPRVD-NEXT: store x86_fp80 [[TMP21]], ptr [[RETVAL_IMAGP]], align 16 +// IMPRVD-NEXT: [[TMP22:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16 +// IMPRVD-NEXT: ret { x86_fp80, x86_fp80 } [[TMP22]] +// +// PRMTD-LABEL: define dso_local { x86_fp80, x86_fp80 } @divld( +// PRMTD-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR1]] { +// PRMTD-NEXT: entry: +// PRMTD-NEXT: [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16 +// PRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 0 +// PRMTD-NEXT: [[A_REAL:%.*]] = load x86_fp80, ptr [[A_REALP]], align 16 +// PRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 1 +// PRMTD-NEXT: [[A_IMAG:%.*]] = load x86_fp80, ptr [[A_IMAGP]], align 16 +// PRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// PRMTD-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// PRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// PRMTD-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// PRMTD-NEXT: [[TMP0:%.*]] = call x86_fp80 @llvm.fabs.f80(x86_fp80 [[B_REAL]]) +// PRMTD-NEXT: [[TMP1:%.*]] = call x86_fp80 @llvm.fabs.f80(x86_fp80 [[B_IMAG]]) +// PRMTD-NEXT: [[ABS_CMP:%.*]] = fcmp ugt x86_fp80 [[TMP0]], [[TMP1]] +// PRMTD-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// PRMTD: abs_rhsr_greater_or_equal_abs_rhsi: +// PRMTD-NEXT: [[TMP2:%.*]] = fdiv x86_fp80 [[B_IMAG]], [[B_REAL]] +// PRMTD-NEXT: [[TMP3:%.*]] = fmul x86_fp80 [[TMP2]], [[B_IMAG]] +// PRMTD-NEXT: [[TMP4:%.*]] = fadd x86_fp80 [[B_REAL]], [[TMP3]] +// PRMTD-NEXT: [[TMP5:%.*]] = fmul x86_fp80 [[A_IMAG]], [[TMP2]] +// PRMTD-NEXT: [[TMP6:%.*]] = fadd x86_fp80 [[A_REAL]], [[TMP5]] +// PRMTD-NEXT: [[TMP7:%.*]] = fdiv x86_fp80 [[TMP6]], [[TMP4]] +// PRMTD-NEXT: [[TMP8:%.*]] = fmul x86_fp80 [[A_REAL]], [[TMP2]] +// PRMTD-NEXT: [[TMP9:%.*]] = fsub x86_fp80 [[A_IMAG]], [[TMP8]] +// PRMTD-NEXT: [[TMP10:%.*]] = fdiv x86_fp80 [[TMP9]], [[TMP4]] +// PRMTD-NEXT: br label [[COMPLEX_DIV:%.*]] +// PRMTD: abs_rhsr_less_than_abs_rhsi: +// PRMTD-NEXT: [[TMP11:%.*]] = fdiv x86_fp80 [[B_REAL]], [[B_IMAG]] +// PRMTD-NEXT: [[TMP12:%.*]] = fmul x86_fp80 [[TMP11]], [[B_REAL]] +// PRMTD-NEXT: [[TMP13:%.*]] = fadd x86_fp80 [[B_IMAG]], [[TMP12]] +// PRMTD-NEXT: [[TMP14:%.*]] = fmul x86_fp80 [[A_REAL]], [[TMP11]] +// PRMTD-NEXT: [[TMP15:%.*]] = fadd x86_fp80 [[TMP14]], [[A_IMAG]] +// PRMTD-NEXT: [[TMP16:%.*]] = fdiv x86_fp80 [[TMP15]], [[TMP13]] +// PRMTD-NEXT: [[TMP17:%.*]] = fmul x86_fp80 [[A_IMAG]], [[TMP11]] +// PRMTD-NEXT: [[TMP18:%.*]] = fsub x86_fp80 [[TMP17]], [[A_REAL]] +// PRMTD-NEXT: [[TMP19:%.*]] = fdiv x86_fp80 [[TMP18]], [[TMP13]] +// PRMTD-NEXT: br label [[COMPLEX_DIV]] +// PRMTD: complex_div: +// PRMTD-NEXT: [[TMP20:%.*]] = phi x86_fp80 [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// PRMTD-NEXT: [[TMP21:%.*]] = phi x86_fp80 [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// PRMTD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD-NEXT: store x86_fp80 [[TMP20]], ptr [[RETVAL_REALP]], align 16 +// PRMTD-NEXT: store x86_fp80 [[TMP21]], ptr [[RETVAL_IMAGP]], align 16 +// PRMTD-NEXT: [[TMP22:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16 +// PRMTD-NEXT: ret { x86_fp80, x86_fp80 } [[TMP22]] +// +// X86WINPRMTD-LABEL: define dso_local void @divld( +// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] { +// X86WINPRMTD-NEXT: entry: +// X86WINPRMTD-NEXT: [[RESULT_PTR:%.*]] = alloca ptr, align 8 +// X86WINPRMTD-NEXT: [[B_INDIRECT_ADDR:%.*]] = alloca ptr, align 8 +// X86WINPRMTD-NEXT: [[A_INDIRECT_ADDR:%.*]] = alloca ptr, align 8 +// X86WINPRMTD-NEXT: store ptr [[AGG_RESULT]], ptr [[RESULT_PTR]], align 8 +// X86WINPRMTD-NEXT: store ptr [[B]], ptr [[B_INDIRECT_ADDR]], align 8 +// X86WINPRMTD-NEXT: store ptr [[A]], ptr [[A_INDIRECT_ADDR]], align 8 +// X86WINPRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// X86WINPRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// X86WINPRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// X86WINPRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// X86WINPRMTD-NEXT: [[TMP0:%.*]] = call double @llvm.fabs.f64(double [[B_REAL]]) +// X86WINPRMTD-NEXT: [[TMP1:%.*]] = call double @llvm.fabs.f64(double [[B_IMAG]]) +// X86WINPRMTD-NEXT: [[ABS_CMP:%.*]] = fcmp ugt double [[TMP0]], [[TMP1]] +// X86WINPRMTD-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// X86WINPRMTD: abs_rhsr_greater_or_equal_abs_rhsi: +// X86WINPRMTD-NEXT: [[TMP2:%.*]] = fdiv double [[B_IMAG]], [[B_REAL]] +// X86WINPRMTD-NEXT: [[TMP3:%.*]] = fmul double [[TMP2]], [[B_IMAG]] +// X86WINPRMTD-NEXT: [[TMP4:%.*]] = fadd double [[B_REAL]], [[TMP3]] +// X86WINPRMTD-NEXT: [[TMP5:%.*]] = fmul double [[A_IMAG]], [[TMP2]] +// X86WINPRMTD-NEXT: [[TMP6:%.*]] = fadd double [[A_REAL]], [[TMP5]] +// X86WINPRMTD-NEXT: [[TMP7:%.*]] = fdiv double [[TMP6]], [[TMP4]] +// X86WINPRMTD-NEXT: [[TMP8:%.*]] = fmul double [[A_REAL]], [[TMP2]] +// X86WINPRMTD-NEXT: [[TMP9:%.*]] = fsub double [[A_IMAG]], [[TMP8]] +// X86WINPRMTD-NEXT: [[TMP10:%.*]] = fdiv double [[TMP9]], [[TMP4]] +// X86WINPRMTD-NEXT: br label [[COMPLEX_DIV:%.*]] +// X86WINPRMTD: abs_rhsr_less_than_abs_rhsi: +// X86WINPRMTD-NEXT: [[TMP11:%.*]] = fdiv double [[B_REAL]], [[B_IMAG]] +// X86WINPRMTD-NEXT: [[TMP12:%.*]] = fmul double [[TMP11]], [[B_REAL]] +// X86WINPRMTD-NEXT: [[TMP13:%.*]] = fadd double [[B_IMAG]], [[TMP12]] +// X86WINPRMTD-NEXT: [[TMP14:%.*]] = fmul double [[A_REAL]], [[TMP11]] +// X86WINPRMTD-NEXT: [[TMP15:%.*]] = fadd double [[TMP14]], [[A_IMAG]] +// X86WINPRMTD-NEXT: [[TMP16:%.*]] = fdiv double [[TMP15]], [[TMP13]] +// X86WINPRMTD-NEXT: [[TMP17:%.*]] = fmul double [[A_IMAG]], [[TMP11]] +// X86WINPRMTD-NEXT: [[TMP18:%.*]] = fsub double [[TMP17]], [[A_REAL]] +// X86WINPRMTD-NEXT: [[TMP19:%.*]] = fdiv double [[TMP18]], [[TMP13]] +// X86WINPRMTD-NEXT: br label [[COMPLEX_DIV]] +// X86WINPRMTD: complex_div: +// X86WINPRMTD-NEXT: [[TMP20:%.*]] = phi double [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// X86WINPRMTD-NEXT: [[TMP21:%.*]] = phi double [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// X86WINPRMTD-NEXT: [[AGG_RESULT_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[AGG_RESULT_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// X86WINPRMTD-NEXT: store double [[TMP20]], ptr [[AGG_RESULT_REALP]], align 8 +// X86WINPRMTD-NEXT: store double [[TMP21]], ptr [[AGG_RESULT_IMAGP]], align 8 +// X86WINPRMTD-NEXT: [[AGG_RESULT_REALP1:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[AGG_RESULT_REAL:%.*]] = load double, ptr [[AGG_RESULT_REALP1]], align 8 +// X86WINPRMTD-NEXT: [[AGG_RESULT_IMAGP2:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[AGG_RESULT_IMAG:%.*]] = load double, ptr [[AGG_RESULT_IMAGP2]], align 8 +// X86WINPRMTD-NEXT: [[AGG_RESULT_REALP3:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[AGG_RESULT_IMAGP4:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// X86WINPRMTD-NEXT: store double [[AGG_RESULT_REAL]], ptr [[AGG_RESULT_REALP3]], align 8 +// X86WINPRMTD-NEXT: store double [[AGG_RESULT_IMAG]], ptr [[AGG_RESULT_IMAGP4]], align 8 +// X86WINPRMTD-NEXT: ret void +// +// AVRFP32-LABEL: define dso_local { float, float } @divld( +// AVRFP32-SAME: float noundef [[A_COERCE0:%.*]], float noundef [[A_COERCE1:%.*]], float noundef [[B_COERCE0:%.*]], float noundef [[B_COERCE1:%.*]]) addrspace(1) #[[ATTR0]] { +// AVRFP32-NEXT: entry: +// AVRFP32-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 1 +// AVRFP32-NEXT: [[A:%.*]] = alloca { float, float }, align 1 +// AVRFP32-NEXT: [[B:%.*]] = alloca { float, float }, align 1 +// AVRFP32-NEXT: [[TMP0:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP32-NEXT: store float [[A_COERCE0]], ptr [[TMP0]], align 1 +// AVRFP32-NEXT: [[TMP1:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[A_COERCE1]], ptr [[TMP1]], align 1 +// AVRFP32-NEXT: [[TMP2:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// AVRFP32-NEXT: store float [[B_COERCE0]], ptr [[TMP2]], align 1 +// AVRFP32-NEXT: [[TMP3:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[B_COERCE1]], ptr [[TMP3]], align 1 +// AVRFP32-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP32-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 1 +// AVRFP32-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP32-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 1 +// AVRFP32-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// AVRFP32-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 1 +// AVRFP32-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// AVRFP32-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 1 +// AVRFP32-NEXT: [[TMP4:%.*]] = call addrspace(1) float @llvm.fabs.f32(float [[B_REAL]]) +// AVRFP32-NEXT: [[TMP5:%.*]] = call addrspace(1) float @llvm.fabs.f32(float [[B_IMAG]]) +// AVRFP32-NEXT: [[ABS_CMP:%.*]] = fcmp ugt float [[TMP4]], [[TMP5]] +// AVRFP32-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// AVRFP32: abs_rhsr_greater_or_equal_abs_rhsi: +// AVRFP32-NEXT: [[TMP6:%.*]] = fdiv float [[B_IMAG]], [[B_REAL]] +// AVRFP32-NEXT: [[TMP7:%.*]] = fmul float [[TMP6]], [[B_IMAG]] +// AVRFP32-NEXT: [[TMP8:%.*]] = fadd float [[B_REAL]], [[TMP7]] +// AVRFP32-NEXT: [[TMP9:%.*]] = fmul float [[A_IMAG]], [[TMP6]] +// AVRFP32-NEXT: [[TMP10:%.*]] = fadd float [[A_REAL]], [[TMP9]] +// AVRFP32-NEXT: [[TMP11:%.*]] = fdiv float [[TMP10]], [[TMP8]] +// AVRFP32-NEXT: [[TMP12:%.*]] = fmul float [[A_REAL]], [[TMP6]] +// AVRFP32-NEXT: [[TMP13:%.*]] = fsub float [[A_IMAG]], [[TMP12]] +// AVRFP32-NEXT: [[TMP14:%.*]] = fdiv float [[TMP13]], [[TMP8]] +// AVRFP32-NEXT: br label [[COMPLEX_DIV:%.*]] +// AVRFP32: abs_rhsr_less_than_abs_rhsi: +// AVRFP32-NEXT: [[TMP15:%.*]] = fdiv float [[B_REAL]], [[B_IMAG]] +// AVRFP32-NEXT: [[TMP16:%.*]] = fmul float [[TMP15]], [[B_REAL]] +// AVRFP32-NEXT: [[TMP17:%.*]] = fadd float [[B_IMAG]], [[TMP16]] +// AVRFP32-NEXT: [[TMP18:%.*]] = fmul float [[A_REAL]], [[TMP15]] +// AVRFP32-NEXT: [[TMP19:%.*]] = fadd float [[TMP18]], [[A_IMAG]] +// AVRFP32-NEXT: [[TMP20:%.*]] = fdiv float [[TMP19]], [[TMP17]] +// AVRFP32-NEXT: [[TMP21:%.*]] = fmul float [[A_IMAG]], [[TMP15]] +// AVRFP32-NEXT: [[TMP22:%.*]] = fsub float [[TMP21]], [[A_REAL]] +// AVRFP32-NEXT: [[TMP23:%.*]] = fdiv float [[TMP22]], [[TMP17]] +// AVRFP32-NEXT: br label [[COMPLEX_DIV]] +// AVRFP32: complex_div: +// AVRFP32-NEXT: [[TMP24:%.*]] = phi float [ [[TMP11]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP20]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// AVRFP32-NEXT: [[TMP25:%.*]] = phi float [ [[TMP14]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP23]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// AVRFP32-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// AVRFP32-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[TMP24]], ptr [[RETVAL_REALP]], align 1 +// AVRFP32-NEXT: store float [[TMP25]], ptr [[RETVAL_IMAGP]], align 1 +// AVRFP32-NEXT: [[TMP26:%.*]] = load { float, float }, ptr [[RETVAL]], align 1 +// AVRFP32-NEXT: ret { float, float } [[TMP26]] +// +// AVRFP64-LABEL: define dso_local void @divld( +// AVRFP64-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 1 [[AGG_RESULT:%.*]], double noundef [[A_COERCE0:%.*]], double noundef [[A_COERCE1:%.*]], double noundef [[B_COERCE0:%.*]], double noundef [[B_COERCE1:%.*]]) addrspace(1) #[[ATTR0]] { +// AVRFP64-NEXT: entry: +// AVRFP64-NEXT: [[A:%.*]] = alloca { double, double }, align 1 +// AVRFP64-NEXT: [[B:%.*]] = alloca { double, double }, align 1 +// AVRFP64-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// AVRFP64-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 1 +// AVRFP64-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// AVRFP64-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 1 +// AVRFP64-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// AVRFP64-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 1 +// AVRFP64-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// AVRFP64-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 1 +// AVRFP64-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// AVRFP64-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 1 +// AVRFP64-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// AVRFP64-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 1 +// AVRFP64-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// AVRFP64-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 1 +// AVRFP64-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// AVRFP64-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 1 +// AVRFP64-NEXT: [[TMP4:%.*]] = call addrspace(1) double @llvm.fabs.f64(double [[B_REAL]]) +// AVRFP64-NEXT: [[TMP5:%.*]] = call addrspace(1) double @llvm.fabs.f64(double [[B_IMAG]]) +// AVRFP64-NEXT: [[ABS_CMP:%.*]] = fcmp ugt double [[TMP4]], [[TMP5]] +// AVRFP64-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// AVRFP64: abs_rhsr_greater_or_equal_abs_rhsi: +// AVRFP64-NEXT: [[TMP6:%.*]] = fdiv double [[B_IMAG]], [[B_REAL]] +// AVRFP64-NEXT: [[TMP7:%.*]] = fmul double [[TMP6]], [[B_IMAG]] +// AVRFP64-NEXT: [[TMP8:%.*]] = fadd double [[B_REAL]], [[TMP7]] +// AVRFP64-NEXT: [[TMP9:%.*]] = fmul double [[A_IMAG]], [[TMP6]] +// AVRFP64-NEXT: [[TMP10:%.*]] = fadd double [[A_REAL]], [[TMP9]] +// AVRFP64-NEXT: [[TMP11:%.*]] = fdiv double [[TMP10]], [[TMP8]] +// AVRFP64-NEXT: [[TMP12:%.*]] = fmul double [[A_REAL]], [[TMP6]] +// AVRFP64-NEXT: [[TMP13:%.*]] = fsub double [[A_IMAG]], [[TMP12]] +// AVRFP64-NEXT: [[TMP14:%.*]] = fdiv double [[TMP13]], [[TMP8]] +// AVRFP64-NEXT: br label [[COMPLEX_DIV:%.*]] +// AVRFP64: abs_rhsr_less_than_abs_rhsi: +// AVRFP64-NEXT: [[TMP15:%.*]] = fdiv double [[B_REAL]], [[B_IMAG]] +// AVRFP64-NEXT: [[TMP16:%.*]] = fmul double [[TMP15]], [[B_REAL]] +// AVRFP64-NEXT: [[TMP17:%.*]] = fadd double [[B_IMAG]], [[TMP16]] +// AVRFP64-NEXT: [[TMP18:%.*]] = fmul double [[A_REAL]], [[TMP15]] +// AVRFP64-NEXT: [[TMP19:%.*]] = fadd double [[TMP18]], [[A_IMAG]] +// AVRFP64-NEXT: [[TMP20:%.*]] = fdiv double [[TMP19]], [[TMP17]] +// AVRFP64-NEXT: [[TMP21:%.*]] = fmul double [[A_IMAG]], [[TMP15]] +// AVRFP64-NEXT: [[TMP22:%.*]] = fsub double [[TMP21]], [[A_REAL]] +// AVRFP64-NEXT: [[TMP23:%.*]] = fdiv double [[TMP22]], [[TMP17]] +// AVRFP64-NEXT: br label [[COMPLEX_DIV]] +// AVRFP64: complex_div: +// AVRFP64-NEXT: [[TMP24:%.*]] = phi double [ [[TMP11]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP20]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// AVRFP64-NEXT: [[TMP25:%.*]] = phi double [ [[TMP14]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP23]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// AVRFP64-NEXT: [[AGG_RESULT_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// AVRFP64-NEXT: [[AGG_RESULT_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// AVRFP64-NEXT: store double [[TMP24]], ptr [[AGG_RESULT_REALP]], align 1 +// AVRFP64-NEXT: store double [[TMP25]], ptr [[AGG_RESULT_IMAGP]], align 1 +// AVRFP64-NEXT: [[AGG_RESULT_REALP1:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// AVRFP64-NEXT: [[AGG_RESULT_REAL:%.*]] = load double, ptr [[AGG_RESULT_REALP1]], align 1 +// AVRFP64-NEXT: [[AGG_RESULT_IMAGP2:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// AVRFP64-NEXT: [[AGG_RESULT_IMAG:%.*]] = load double, ptr [[AGG_RESULT_IMAGP2]], align 1 +// AVRFP64-NEXT: [[AGG_RESULT_REALP3:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// AVRFP64-NEXT: [[AGG_RESULT_IMAGP4:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// AVRFP64-NEXT: store double [[AGG_RESULT_REAL]], ptr [[AGG_RESULT_REALP3]], align 1 +// AVRFP64-NEXT: store double [[AGG_RESULT_IMAG]], ptr [[AGG_RESULT_IMAGP4]], align 1 +// AVRFP64-NEXT: ret void +// +// BASIC_FAST-LABEL: define dso_local { x86_fp80, x86_fp80 } @divld( +// BASIC_FAST-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR1]] { +// BASIC_FAST-NEXT: entry: +// BASIC_FAST-NEXT: [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16 +// BASIC_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[A_REAL:%.*]] = load x86_fp80, ptr [[A_REALP]], align 16 +// BASIC_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[A_IMAG:%.*]] = load x86_fp80, ptr [[A_IMAGP]], align 16 +// BASIC_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// BASIC_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// BASIC_FAST-NEXT: [[TMP0:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_REAL]], [[B_REAL]] +// BASIC_FAST-NEXT: [[TMP1:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_IMAG]], [[B_IMAG]] +// BASIC_FAST-NEXT: [[TMP2:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP0]], [[TMP1]] +// BASIC_FAST-NEXT: [[TMP3:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[B_REAL]] +// BASIC_FAST-NEXT: [[TMP4:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[B_IMAG]] +// BASIC_FAST-NEXT: [[TMP5:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP3]], [[TMP4]] +// BASIC_FAST-NEXT: [[TMP6:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_IMAG]], [[B_REAL]] +// BASIC_FAST-NEXT: [[TMP7:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_REAL]], [[B_IMAG]] +// BASIC_FAST-NEXT: [[TMP8:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP6]], [[TMP7]] +// BASIC_FAST-NEXT: [[TMP9:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP2]], [[TMP5]] +// BASIC_FAST-NEXT: [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP8]], [[TMP5]] +// BASIC_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC_FAST-NEXT: store x86_fp80 [[TMP9]], ptr [[RETVAL_REALP]], align 16 +// BASIC_FAST-NEXT: store x86_fp80 [[TMP10]], ptr [[RETVAL_IMAGP]], align 16 +// BASIC_FAST-NEXT: [[TMP11:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16 +// BASIC_FAST-NEXT: ret { x86_fp80, x86_fp80 } [[TMP11]] +// +// FULL_FAST-LABEL: define dso_local { x86_fp80, x86_fp80 } @divld( +// FULL_FAST-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR1]] { +// FULL_FAST-NEXT: entry: +// FULL_FAST-NEXT: [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16 +// FULL_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 0 +// FULL_FAST-NEXT: [[A_REAL:%.*]] = load x86_fp80, ptr [[A_REALP]], align 16 +// FULL_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 1 +// FULL_FAST-NEXT: [[A_IMAG:%.*]] = load x86_fp80, ptr [[A_IMAGP]], align 16 +// FULL_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// FULL_FAST-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// FULL_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// FULL_FAST-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// FULL_FAST-NEXT: [[CALL:%.*]] = call { x86_fp80, x86_fp80 } @__divxc3(x86_fp80 noundef nofpclass(nan inf) [[A_REAL]], x86_fp80 noundef nofpclass(nan inf) [[A_IMAG]], x86_fp80 noundef nofpclass(nan inf) [[B_REAL]], x86_fp80 noundef nofpclass(nan inf) [[B_IMAG]]) #[[ATTR2]] +// FULL_FAST-NEXT: [[TMP0:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 0 +// FULL_FAST-NEXT: [[TMP1:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 1 +// FULL_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 0 +// FULL_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 1 +// FULL_FAST-NEXT: store x86_fp80 [[TMP0]], ptr [[RETVAL_REALP]], align 16 +// FULL_FAST-NEXT: store x86_fp80 [[TMP1]], ptr [[RETVAL_IMAGP]], align 16 +// FULL_FAST-NEXT: [[TMP2:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16 +// FULL_FAST-NEXT: ret { x86_fp80, x86_fp80 } [[TMP2]] +// +// IMPRVD_FAST-LABEL: define dso_local { x86_fp80, x86_fp80 } @divld( +// IMPRVD_FAST-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR2]] { +// IMPRVD_FAST-NEXT: entry: +// IMPRVD_FAST-NEXT: [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16 +// IMPRVD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[A_REAL:%.*]] = load x86_fp80, ptr [[A_REALP]], align 16 +// IMPRVD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[A_IMAG:%.*]] = load x86_fp80, ptr [[A_IMAGP]], align 16 +// IMPRVD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// IMPRVD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// IMPRVD_FAST-NEXT: [[TMP0:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[B_REAL]]) +// IMPRVD_FAST-NEXT: [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[B_IMAG]]) +// IMPRVD_FAST-NEXT: [[ABS_CMP:%.*]] = fcmp reassoc nnan ninf nsz arcp afn ugt x86_fp80 [[TMP0]], [[TMP1]] +// IMPRVD_FAST-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// IMPRVD_FAST: abs_rhsr_greater_or_equal_abs_rhsi: +// IMPRVD_FAST-NEXT: [[TMP2:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[B_REAL]] +// IMPRVD_FAST-NEXT: [[TMP3:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP2]], [[B_IMAG]] +// IMPRVD_FAST-NEXT: [[TMP4:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[TMP3]] +// IMPRVD_FAST-NEXT: [[TMP5:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_IMAG]], [[TMP2]] +// IMPRVD_FAST-NEXT: [[TMP6:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[A_REAL]], [[TMP5]] +// IMPRVD_FAST-NEXT: [[TMP7:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP6]], [[TMP4]] +// IMPRVD_FAST-NEXT: [[TMP8:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_REAL]], [[TMP2]] +// IMPRVD_FAST-NEXT: [[TMP9:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[A_IMAG]], [[TMP8]] +// IMPRVD_FAST-NEXT: [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP9]], [[TMP4]] +// IMPRVD_FAST-NEXT: br label [[COMPLEX_DIV:%.*]] +// IMPRVD_FAST: abs_rhsr_less_than_abs_rhsi: +// IMPRVD_FAST-NEXT: [[TMP11:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[B_IMAG]] +// IMPRVD_FAST-NEXT: [[TMP12:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP11]], [[B_REAL]] +// IMPRVD_FAST-NEXT: [[TMP13:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[TMP12]] +// IMPRVD_FAST-NEXT: [[TMP14:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_REAL]], [[TMP11]] +// IMPRVD_FAST-NEXT: [[TMP15:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP14]], [[A_IMAG]] +// IMPRVD_FAST-NEXT: [[TMP16:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP15]], [[TMP13]] +// IMPRVD_FAST-NEXT: [[TMP17:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_IMAG]], [[TMP11]] +// IMPRVD_FAST-NEXT: [[TMP18:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP17]], [[A_REAL]] +// IMPRVD_FAST-NEXT: [[TMP19:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP18]], [[TMP13]] +// IMPRVD_FAST-NEXT: br label [[COMPLEX_DIV]] +// IMPRVD_FAST: complex_div: +// IMPRVD_FAST-NEXT: [[TMP20:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD_FAST-NEXT: [[TMP21:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: store x86_fp80 [[TMP20]], ptr [[RETVAL_REALP]], align 16 +// IMPRVD_FAST-NEXT: store x86_fp80 [[TMP21]], ptr [[RETVAL_IMAGP]], align 16 +// IMPRVD_FAST-NEXT: [[TMP22:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16 +// IMPRVD_FAST-NEXT: ret { x86_fp80, x86_fp80 } [[TMP22]] +// +// PRMTD_FAST-LABEL: define dso_local { x86_fp80, x86_fp80 } @divld( +// PRMTD_FAST-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR1]] { +// PRMTD_FAST-NEXT: entry: +// PRMTD_FAST-NEXT: [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16 +// PRMTD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[A_REAL:%.*]] = load x86_fp80, ptr [[A_REALP]], align 16 +// PRMTD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[A_IMAG:%.*]] = load x86_fp80, ptr [[A_IMAGP]], align 16 +// PRMTD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// PRMTD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// PRMTD_FAST-NEXT: [[TMP0:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[B_REAL]]) +// PRMTD_FAST-NEXT: [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[B_IMAG]]) +// PRMTD_FAST-NEXT: [[ABS_CMP:%.*]] = fcmp reassoc nnan ninf nsz arcp afn ugt x86_fp80 [[TMP0]], [[TMP1]] +// PRMTD_FAST-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// PRMTD_FAST: abs_rhsr_greater_or_equal_abs_rhsi: +// PRMTD_FAST-NEXT: [[TMP2:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[B_REAL]] +// PRMTD_FAST-NEXT: [[TMP3:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP2]], [[B_IMAG]] +// PRMTD_FAST-NEXT: [[TMP4:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[TMP3]] +// PRMTD_FAST-NEXT: [[TMP5:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_IMAG]], [[TMP2]] +// PRMTD_FAST-NEXT: [[TMP6:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[A_REAL]], [[TMP5]] +// PRMTD_FAST-NEXT: [[TMP7:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP6]], [[TMP4]] +// PRMTD_FAST-NEXT: [[TMP8:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_REAL]], [[TMP2]] +// PRMTD_FAST-NEXT: [[TMP9:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[A_IMAG]], [[TMP8]] +// PRMTD_FAST-NEXT: [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP9]], [[TMP4]] +// PRMTD_FAST-NEXT: br label [[COMPLEX_DIV:%.*]] +// PRMTD_FAST: abs_rhsr_less_than_abs_rhsi: +// PRMTD_FAST-NEXT: [[TMP11:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[B_IMAG]] +// PRMTD_FAST-NEXT: [[TMP12:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP11]], [[B_REAL]] +// PRMTD_FAST-NEXT: [[TMP13:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[TMP12]] +// PRMTD_FAST-NEXT: [[TMP14:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_REAL]], [[TMP11]] +// PRMTD_FAST-NEXT: [[TMP15:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP14]], [[A_IMAG]] +// PRMTD_FAST-NEXT: [[TMP16:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP15]], [[TMP13]] +// PRMTD_FAST-NEXT: [[TMP17:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_IMAG]], [[TMP11]] +// PRMTD_FAST-NEXT: [[TMP18:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP17]], [[A_REAL]] +// PRMTD_FAST-NEXT: [[TMP19:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP18]], [[TMP13]] +// PRMTD_FAST-NEXT: br label [[COMPLEX_DIV]] +// PRMTD_FAST: complex_div: +// PRMTD_FAST-NEXT: [[TMP20:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// PRMTD_FAST-NEXT: [[TMP21:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// PRMTD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD_FAST-NEXT: store x86_fp80 [[TMP20]], ptr [[RETVAL_REALP]], align 16 +// PRMTD_FAST-NEXT: store x86_fp80 [[TMP21]], ptr [[RETVAL_IMAGP]], align 16 +// PRMTD_FAST-NEXT: [[TMP22:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16 +// PRMTD_FAST-NEXT: ret { x86_fp80, x86_fp80 } [[TMP22]] +// +_Complex long double divld(_Complex long double a, _Complex long double b) { + return a / b; +} +// FULL-LABEL: define dso_local { x86_fp80, x86_fp80 } @mulld( +// FULL-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR1]] { +// FULL-NEXT: entry: +// FULL-NEXT: [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16 +// FULL-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 0 +// FULL-NEXT: [[A_REAL:%.*]] = load x86_fp80, ptr [[A_REALP]], align 16 +// FULL-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 1 +// FULL-NEXT: [[A_IMAG:%.*]] = load x86_fp80, ptr [[A_IMAGP]], align 16 +// FULL-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// FULL-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// FULL-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// FULL-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// FULL-NEXT: [[MUL_AC:%.*]] = fmul x86_fp80 [[A_REAL]], [[B_REAL]] +// FULL-NEXT: [[MUL_BD:%.*]] = fmul x86_fp80 [[A_IMAG]], [[B_IMAG]] +// FULL-NEXT: [[MUL_AD:%.*]] = fmul x86_fp80 [[A_REAL]], [[B_IMAG]] +// FULL-NEXT: [[MUL_BC:%.*]] = fmul x86_fp80 [[A_IMAG]], [[B_REAL]] +// FULL-NEXT: [[MUL_R:%.*]] = fsub x86_fp80 [[MUL_AC]], [[MUL_BD]] +// FULL-NEXT: [[MUL_I:%.*]] = fadd x86_fp80 [[MUL_AD]], [[MUL_BC]] +// FULL-NEXT: [[ISNAN_CMP:%.*]] = fcmp uno x86_fp80 [[MUL_R]], [[MUL_R]] +// FULL-NEXT: br i1 [[ISNAN_CMP]], label [[COMPLEX_MUL_IMAG_NAN:%.*]], label [[COMPLEX_MUL_CONT:%.*]], !prof [[PROF2]] +// FULL: complex_mul_imag_nan: +// FULL-NEXT: [[ISNAN_CMP1:%.*]] = fcmp uno x86_fp80 [[MUL_I]], [[MUL_I]] +// FULL-NEXT: br i1 [[ISNAN_CMP1]], label [[COMPLEX_MUL_LIBCALL:%.*]], label [[COMPLEX_MUL_CONT]], !prof [[PROF2]] +// FULL: complex_mul_libcall: +// FULL-NEXT: [[CALL:%.*]] = call { x86_fp80, x86_fp80 } @__mulxc3(x86_fp80 noundef [[A_REAL]], x86_fp80 noundef [[A_IMAG]], x86_fp80 noundef [[B_REAL]], x86_fp80 noundef [[B_IMAG]]) #[[ATTR2]] +// FULL-NEXT: [[TMP0:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 0 +// FULL-NEXT: [[TMP1:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 1 +// FULL-NEXT: br label [[COMPLEX_MUL_CONT]] +// FULL: complex_mul_cont: +// FULL-NEXT: [[REAL_MUL_PHI:%.*]] = phi x86_fp80 [ [[MUL_R]], [[ENTRY:%.*]] ], [ [[MUL_R]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[TMP0]], [[COMPLEX_MUL_LIBCALL]] ] +// FULL-NEXT: [[IMAG_MUL_PHI:%.*]] = phi x86_fp80 [ [[MUL_I]], [[ENTRY]] ], [ [[MUL_I]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[TMP1]], [[COMPLEX_MUL_LIBCALL]] ] +// FULL-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 0 +// FULL-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 1 +// FULL-NEXT: store x86_fp80 [[REAL_MUL_PHI]], ptr [[RETVAL_REALP]], align 16 +// FULL-NEXT: store x86_fp80 [[IMAG_MUL_PHI]], ptr [[RETVAL_IMAGP]], align 16 +// FULL-NEXT: [[TMP2:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16 +// FULL-NEXT: ret { x86_fp80, x86_fp80 } [[TMP2]] +// +// BASIC-LABEL: define dso_local { x86_fp80, x86_fp80 } @mulld( +// BASIC-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR1]] { +// BASIC-NEXT: entry: +// BASIC-NEXT: [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16 +// BASIC-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 0 +// BASIC-NEXT: [[A_REAL:%.*]] = load x86_fp80, ptr [[A_REALP]], align 16 +// BASIC-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 1 +// BASIC-NEXT: [[A_IMAG:%.*]] = load x86_fp80, ptr [[A_IMAGP]], align 16 +// BASIC-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// BASIC-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// BASIC-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// BASIC-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// BASIC-NEXT: [[MUL_AC:%.*]] = fmul x86_fp80 [[A_REAL]], [[B_REAL]] +// BASIC-NEXT: [[MUL_BD:%.*]] = fmul x86_fp80 [[A_IMAG]], [[B_IMAG]] +// BASIC-NEXT: [[MUL_AD:%.*]] = fmul x86_fp80 [[A_REAL]], [[B_IMAG]] +// BASIC-NEXT: [[MUL_BC:%.*]] = fmul x86_fp80 [[A_IMAG]], [[B_REAL]] +// BASIC-NEXT: [[MUL_R:%.*]] = fsub x86_fp80 [[MUL_AC]], [[MUL_BD]] +// BASIC-NEXT: [[MUL_I:%.*]] = fadd x86_fp80 [[MUL_AD]], [[MUL_BC]] +// BASIC-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC-NEXT: store x86_fp80 [[MUL_R]], ptr [[RETVAL_REALP]], align 16 +// BASIC-NEXT: store x86_fp80 [[MUL_I]], ptr [[RETVAL_IMAGP]], align 16 +// BASIC-NEXT: [[TMP0:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16 +// BASIC-NEXT: ret { x86_fp80, x86_fp80 } [[TMP0]] +// +// IMPRVD-LABEL: define dso_local { x86_fp80, x86_fp80 } @mulld( +// IMPRVD-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR2]] { +// IMPRVD-NEXT: entry: +// IMPRVD-NEXT: [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16 +// IMPRVD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 0 +// IMPRVD-NEXT: [[A_REAL:%.*]] = load x86_fp80, ptr [[A_REALP]], align 16 +// IMPRVD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 1 +// IMPRVD-NEXT: [[A_IMAG:%.*]] = load x86_fp80, ptr [[A_IMAGP]], align 16 +// IMPRVD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// IMPRVD-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// IMPRVD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// IMPRVD-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// IMPRVD-NEXT: [[MUL_AC:%.*]] = fmul x86_fp80 [[A_REAL]], [[B_REAL]] +// IMPRVD-NEXT: [[MUL_BD:%.*]] = fmul x86_fp80 [[A_IMAG]], [[B_IMAG]] +// IMPRVD-NEXT: [[MUL_AD:%.*]] = fmul x86_fp80 [[A_REAL]], [[B_IMAG]] +// IMPRVD-NEXT: [[MUL_BC:%.*]] = fmul x86_fp80 [[A_IMAG]], [[B_REAL]] +// IMPRVD-NEXT: [[MUL_R:%.*]] = fsub x86_fp80 [[MUL_AC]], [[MUL_BD]] +// IMPRVD-NEXT: [[MUL_I:%.*]] = fadd x86_fp80 [[MUL_AD]], [[MUL_BC]] +// IMPRVD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD-NEXT: store x86_fp80 [[MUL_R]], ptr [[RETVAL_REALP]], align 16 +// IMPRVD-NEXT: store x86_fp80 [[MUL_I]], ptr [[RETVAL_IMAGP]], align 16 +// IMPRVD-NEXT: [[TMP0:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16 +// IMPRVD-NEXT: ret { x86_fp80, x86_fp80 } [[TMP0]] +// +// PRMTD-LABEL: define dso_local { x86_fp80, x86_fp80 } @mulld( +// PRMTD-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR1]] { +// PRMTD-NEXT: entry: +// PRMTD-NEXT: [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16 +// PRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 0 +// PRMTD-NEXT: [[A_REAL:%.*]] = load x86_fp80, ptr [[A_REALP]], align 16 +// PRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 1 +// PRMTD-NEXT: [[A_IMAG:%.*]] = load x86_fp80, ptr [[A_IMAGP]], align 16 +// PRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// PRMTD-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// PRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// PRMTD-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// PRMTD-NEXT: [[MUL_AC:%.*]] = fmul x86_fp80 [[A_REAL]], [[B_REAL]] +// PRMTD-NEXT: [[MUL_BD:%.*]] = fmul x86_fp80 [[A_IMAG]], [[B_IMAG]] +// PRMTD-NEXT: [[MUL_AD:%.*]] = fmul x86_fp80 [[A_REAL]], [[B_IMAG]] +// PRMTD-NEXT: [[MUL_BC:%.*]] = fmul x86_fp80 [[A_IMAG]], [[B_REAL]] +// PRMTD-NEXT: [[MUL_R:%.*]] = fsub x86_fp80 [[MUL_AC]], [[MUL_BD]] +// PRMTD-NEXT: [[MUL_I:%.*]] = fadd x86_fp80 [[MUL_AD]], [[MUL_BC]] +// PRMTD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD-NEXT: store x86_fp80 [[MUL_R]], ptr [[RETVAL_REALP]], align 16 +// PRMTD-NEXT: store x86_fp80 [[MUL_I]], ptr [[RETVAL_IMAGP]], align 16 +// PRMTD-NEXT: [[TMP0:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16 +// PRMTD-NEXT: ret { x86_fp80, x86_fp80 } [[TMP0]] +// +// X86WINPRMTD-LABEL: define dso_local void @mulld( +// X86WINPRMTD-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 8 [[AGG_RESULT:%.*]], ptr noundef [[A:%.*]], ptr noundef [[B:%.*]]) #[[ATTR0]] { +// X86WINPRMTD-NEXT: entry: +// X86WINPRMTD-NEXT: [[RESULT_PTR:%.*]] = alloca ptr, align 8 +// X86WINPRMTD-NEXT: [[B_INDIRECT_ADDR:%.*]] = alloca ptr, align 8 +// X86WINPRMTD-NEXT: [[A_INDIRECT_ADDR:%.*]] = alloca ptr, align 8 +// X86WINPRMTD-NEXT: store ptr [[AGG_RESULT]], ptr [[RESULT_PTR]], align 8 +// X86WINPRMTD-NEXT: store ptr [[B]], ptr [[B_INDIRECT_ADDR]], align 8 +// X86WINPRMTD-NEXT: store ptr [[A]], ptr [[A_INDIRECT_ADDR]], align 8 +// X86WINPRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 8 +// X86WINPRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 8 +// X86WINPRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// X86WINPRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// X86WINPRMTD-NEXT: [[MUL_AC:%.*]] = fmul double [[A_REAL]], [[B_REAL]] +// X86WINPRMTD-NEXT: [[MUL_BD:%.*]] = fmul double [[A_IMAG]], [[B_IMAG]] +// X86WINPRMTD-NEXT: [[MUL_AD:%.*]] = fmul double [[A_REAL]], [[B_IMAG]] +// X86WINPRMTD-NEXT: [[MUL_BC:%.*]] = fmul double [[A_IMAG]], [[B_REAL]] +// X86WINPRMTD-NEXT: [[MUL_R:%.*]] = fsub double [[MUL_AC]], [[MUL_BD]] +// X86WINPRMTD-NEXT: [[MUL_I:%.*]] = fadd double [[MUL_AD]], [[MUL_BC]] +// X86WINPRMTD-NEXT: [[AGG_RESULT_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[AGG_RESULT_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// X86WINPRMTD-NEXT: store double [[MUL_R]], ptr [[AGG_RESULT_REALP]], align 8 +// X86WINPRMTD-NEXT: store double [[MUL_I]], ptr [[AGG_RESULT_IMAGP]], align 8 +// X86WINPRMTD-NEXT: [[AGG_RESULT_REALP1:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[AGG_RESULT_REAL:%.*]] = load double, ptr [[AGG_RESULT_REALP1]], align 8 +// X86WINPRMTD-NEXT: [[AGG_RESULT_IMAGP2:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[AGG_RESULT_IMAG:%.*]] = load double, ptr [[AGG_RESULT_IMAGP2]], align 8 +// X86WINPRMTD-NEXT: [[AGG_RESULT_REALP3:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[AGG_RESULT_IMAGP4:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// X86WINPRMTD-NEXT: store double [[AGG_RESULT_REAL]], ptr [[AGG_RESULT_REALP3]], align 8 +// X86WINPRMTD-NEXT: store double [[AGG_RESULT_IMAG]], ptr [[AGG_RESULT_IMAGP4]], align 8 +// X86WINPRMTD-NEXT: ret void +// +// AVRFP32-LABEL: define dso_local { float, float } @mulld( +// AVRFP32-SAME: float noundef [[A_COERCE0:%.*]], float noundef [[A_COERCE1:%.*]], float noundef [[B_COERCE0:%.*]], float noundef [[B_COERCE1:%.*]]) addrspace(1) #[[ATTR0]] { +// AVRFP32-NEXT: entry: +// AVRFP32-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 1 +// AVRFP32-NEXT: [[A:%.*]] = alloca { float, float }, align 1 +// AVRFP32-NEXT: [[B:%.*]] = alloca { float, float }, align 1 +// AVRFP32-NEXT: [[TMP0:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP32-NEXT: store float [[A_COERCE0]], ptr [[TMP0]], align 1 +// AVRFP32-NEXT: [[TMP1:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[A_COERCE1]], ptr [[TMP1]], align 1 +// AVRFP32-NEXT: [[TMP2:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// AVRFP32-NEXT: store float [[B_COERCE0]], ptr [[TMP2]], align 1 +// AVRFP32-NEXT: [[TMP3:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[B_COERCE1]], ptr [[TMP3]], align 1 +// AVRFP32-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP32-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 1 +// AVRFP32-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP32-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 1 +// AVRFP32-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// AVRFP32-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 1 +// AVRFP32-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// AVRFP32-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 1 +// AVRFP32-NEXT: [[MUL_AC:%.*]] = fmul float [[A_REAL]], [[B_REAL]] +// AVRFP32-NEXT: [[MUL_BD:%.*]] = fmul float [[A_IMAG]], [[B_IMAG]] +// AVRFP32-NEXT: [[MUL_AD:%.*]] = fmul float [[A_REAL]], [[B_IMAG]] +// AVRFP32-NEXT: [[MUL_BC:%.*]] = fmul float [[A_IMAG]], [[B_REAL]] +// AVRFP32-NEXT: [[MUL_R:%.*]] = fsub float [[MUL_AC]], [[MUL_BD]] +// AVRFP32-NEXT: [[MUL_I:%.*]] = fadd float [[MUL_AD]], [[MUL_BC]] +// AVRFP32-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// AVRFP32-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[MUL_R]], ptr [[RETVAL_REALP]], align 1 +// AVRFP32-NEXT: store float [[MUL_I]], ptr [[RETVAL_IMAGP]], align 1 +// AVRFP32-NEXT: [[TMP4:%.*]] = load { float, float }, ptr [[RETVAL]], align 1 +// AVRFP32-NEXT: ret { float, float } [[TMP4]] +// +// AVRFP64-LABEL: define dso_local void @mulld( +// AVRFP64-SAME: ptr dead_on_unwind noalias writable sret({ double, double }) align 1 [[AGG_RESULT:%.*]], double noundef [[A_COERCE0:%.*]], double noundef [[A_COERCE1:%.*]], double noundef [[B_COERCE0:%.*]], double noundef [[B_COERCE1:%.*]]) addrspace(1) #[[ATTR0]] { +// AVRFP64-NEXT: entry: +// AVRFP64-NEXT: [[A:%.*]] = alloca { double, double }, align 1 +// AVRFP64-NEXT: [[B:%.*]] = alloca { double, double }, align 1 +// AVRFP64-NEXT: [[TMP0:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// AVRFP64-NEXT: store double [[A_COERCE0]], ptr [[TMP0]], align 1 +// AVRFP64-NEXT: [[TMP1:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// AVRFP64-NEXT: store double [[A_COERCE1]], ptr [[TMP1]], align 1 +// AVRFP64-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// AVRFP64-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 1 +// AVRFP64-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// AVRFP64-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 1 +// AVRFP64-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 0 +// AVRFP64-NEXT: [[A_REAL:%.*]] = load double, ptr [[A_REALP]], align 1 +// AVRFP64-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[A]], i32 0, i32 1 +// AVRFP64-NEXT: [[A_IMAG:%.*]] = load double, ptr [[A_IMAGP]], align 1 +// AVRFP64-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// AVRFP64-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 1 +// AVRFP64-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// AVRFP64-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 1 +// AVRFP64-NEXT: [[MUL_AC:%.*]] = fmul double [[A_REAL]], [[B_REAL]] +// AVRFP64-NEXT: [[MUL_BD:%.*]] = fmul double [[A_IMAG]], [[B_IMAG]] +// AVRFP64-NEXT: [[MUL_AD:%.*]] = fmul double [[A_REAL]], [[B_IMAG]] +// AVRFP64-NEXT: [[MUL_BC:%.*]] = fmul double [[A_IMAG]], [[B_REAL]] +// AVRFP64-NEXT: [[MUL_R:%.*]] = fsub double [[MUL_AC]], [[MUL_BD]] +// AVRFP64-NEXT: [[MUL_I:%.*]] = fadd double [[MUL_AD]], [[MUL_BC]] +// AVRFP64-NEXT: [[AGG_RESULT_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// AVRFP64-NEXT: [[AGG_RESULT_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// AVRFP64-NEXT: store double [[MUL_R]], ptr [[AGG_RESULT_REALP]], align 1 +// AVRFP64-NEXT: store double [[MUL_I]], ptr [[AGG_RESULT_IMAGP]], align 1 +// AVRFP64-NEXT: [[AGG_RESULT_REALP1:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// AVRFP64-NEXT: [[AGG_RESULT_REAL:%.*]] = load double, ptr [[AGG_RESULT_REALP1]], align 1 +// AVRFP64-NEXT: [[AGG_RESULT_IMAGP2:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// AVRFP64-NEXT: [[AGG_RESULT_IMAG:%.*]] = load double, ptr [[AGG_RESULT_IMAGP2]], align 1 +// AVRFP64-NEXT: [[AGG_RESULT_REALP3:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 0 +// AVRFP64-NEXT: [[AGG_RESULT_IMAGP4:%.*]] = getelementptr inbounds { double, double }, ptr [[AGG_RESULT]], i32 0, i32 1 +// AVRFP64-NEXT: store double [[AGG_RESULT_REAL]], ptr [[AGG_RESULT_REALP3]], align 1 +// AVRFP64-NEXT: store double [[AGG_RESULT_IMAG]], ptr [[AGG_RESULT_IMAGP4]], align 1 +// AVRFP64-NEXT: ret void +// +// BASIC_FAST-LABEL: define dso_local { x86_fp80, x86_fp80 } @mulld( +// BASIC_FAST-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR1]] { +// BASIC_FAST-NEXT: entry: +// BASIC_FAST-NEXT: [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16 +// BASIC_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[A_REAL:%.*]] = load x86_fp80, ptr [[A_REALP]], align 16 +// BASIC_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[A_IMAG:%.*]] = load x86_fp80, ptr [[A_IMAGP]], align 16 +// BASIC_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// BASIC_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// BASIC_FAST-NEXT: [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_REAL]], [[B_REAL]] +// BASIC_FAST-NEXT: [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_IMAG]], [[B_IMAG]] +// BASIC_FAST-NEXT: [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_REAL]], [[B_IMAG]] +// BASIC_FAST-NEXT: [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_IMAG]], [[B_REAL]] +// BASIC_FAST-NEXT: [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[MUL_AC]], [[MUL_BD]] +// BASIC_FAST-NEXT: [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[MUL_AD]], [[MUL_BC]] +// BASIC_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC_FAST-NEXT: store x86_fp80 [[MUL_R]], ptr [[RETVAL_REALP]], align 16 +// BASIC_FAST-NEXT: store x86_fp80 [[MUL_I]], ptr [[RETVAL_IMAGP]], align 16 +// BASIC_FAST-NEXT: [[TMP0:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16 +// BASIC_FAST-NEXT: ret { x86_fp80, x86_fp80 } [[TMP0]] +// +// FULL_FAST-LABEL: define dso_local { x86_fp80, x86_fp80 } @mulld( +// FULL_FAST-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR1]] { +// FULL_FAST-NEXT: entry: +// FULL_FAST-NEXT: [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16 +// FULL_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 0 +// FULL_FAST-NEXT: [[A_REAL:%.*]] = load x86_fp80, ptr [[A_REALP]], align 16 +// FULL_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 1 +// FULL_FAST-NEXT: [[A_IMAG:%.*]] = load x86_fp80, ptr [[A_IMAGP]], align 16 +// FULL_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// FULL_FAST-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// FULL_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// FULL_FAST-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// FULL_FAST-NEXT: [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_REAL]], [[B_REAL]] +// FULL_FAST-NEXT: [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_IMAG]], [[B_IMAG]] +// FULL_FAST-NEXT: [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_REAL]], [[B_IMAG]] +// FULL_FAST-NEXT: [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_IMAG]], [[B_REAL]] +// FULL_FAST-NEXT: [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[MUL_AC]], [[MUL_BD]] +// FULL_FAST-NEXT: [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[MUL_AD]], [[MUL_BC]] +// FULL_FAST-NEXT: [[ISNAN_CMP:%.*]] = fcmp reassoc nnan ninf nsz arcp afn uno x86_fp80 [[MUL_R]], [[MUL_R]] +// FULL_FAST-NEXT: br i1 [[ISNAN_CMP]], label [[COMPLEX_MUL_IMAG_NAN:%.*]], label [[COMPLEX_MUL_CONT:%.*]], !prof [[PROF2]] +// FULL_FAST: complex_mul_imag_nan: +// FULL_FAST-NEXT: [[ISNAN_CMP1:%.*]] = fcmp reassoc nnan ninf nsz arcp afn uno x86_fp80 [[MUL_I]], [[MUL_I]] +// FULL_FAST-NEXT: br i1 [[ISNAN_CMP1]], label [[COMPLEX_MUL_LIBCALL:%.*]], label [[COMPLEX_MUL_CONT]], !prof [[PROF2]] +// FULL_FAST: complex_mul_libcall: +// FULL_FAST-NEXT: [[CALL:%.*]] = call { x86_fp80, x86_fp80 } @__mulxc3(x86_fp80 noundef nofpclass(nan inf) [[A_REAL]], x86_fp80 noundef nofpclass(nan inf) [[A_IMAG]], x86_fp80 noundef nofpclass(nan inf) [[B_REAL]], x86_fp80 noundef nofpclass(nan inf) [[B_IMAG]]) #[[ATTR2]] +// FULL_FAST-NEXT: [[TMP0:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 0 +// FULL_FAST-NEXT: [[TMP1:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 1 +// FULL_FAST-NEXT: br label [[COMPLEX_MUL_CONT]] +// FULL_FAST: complex_mul_cont: +// FULL_FAST-NEXT: [[REAL_MUL_PHI:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[MUL_R]], [[ENTRY:%.*]] ], [ [[MUL_R]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[TMP0]], [[COMPLEX_MUL_LIBCALL]] ] +// FULL_FAST-NEXT: [[IMAG_MUL_PHI:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[MUL_I]], [[ENTRY]] ], [ [[MUL_I]], [[COMPLEX_MUL_IMAG_NAN]] ], [ [[TMP1]], [[COMPLEX_MUL_LIBCALL]] ] +// FULL_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 0 +// FULL_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 1 +// FULL_FAST-NEXT: store x86_fp80 [[REAL_MUL_PHI]], ptr [[RETVAL_REALP]], align 16 +// FULL_FAST-NEXT: store x86_fp80 [[IMAG_MUL_PHI]], ptr [[RETVAL_IMAGP]], align 16 +// FULL_FAST-NEXT: [[TMP2:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16 +// FULL_FAST-NEXT: ret { x86_fp80, x86_fp80 } [[TMP2]] +// +// IMPRVD_FAST-LABEL: define dso_local { x86_fp80, x86_fp80 } @mulld( +// IMPRVD_FAST-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR2]] { +// IMPRVD_FAST-NEXT: entry: +// IMPRVD_FAST-NEXT: [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16 +// IMPRVD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[A_REAL:%.*]] = load x86_fp80, ptr [[A_REALP]], align 16 +// IMPRVD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[A_IMAG:%.*]] = load x86_fp80, ptr [[A_IMAGP]], align 16 +// IMPRVD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// IMPRVD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// IMPRVD_FAST-NEXT: [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_REAL]], [[B_REAL]] +// IMPRVD_FAST-NEXT: [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_IMAG]], [[B_IMAG]] +// IMPRVD_FAST-NEXT: [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_REAL]], [[B_IMAG]] +// IMPRVD_FAST-NEXT: [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_IMAG]], [[B_REAL]] +// IMPRVD_FAST-NEXT: [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[MUL_AC]], [[MUL_BD]] +// IMPRVD_FAST-NEXT: [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[MUL_AD]], [[MUL_BC]] +// IMPRVD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: store x86_fp80 [[MUL_R]], ptr [[RETVAL_REALP]], align 16 +// IMPRVD_FAST-NEXT: store x86_fp80 [[MUL_I]], ptr [[RETVAL_IMAGP]], align 16 +// IMPRVD_FAST-NEXT: [[TMP0:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16 +// IMPRVD_FAST-NEXT: ret { x86_fp80, x86_fp80 } [[TMP0]] +// +// PRMTD_FAST-LABEL: define dso_local { x86_fp80, x86_fp80 } @mulld( +// PRMTD_FAST-SAME: ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[A:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]]) #[[ATTR1]] { +// PRMTD_FAST-NEXT: entry: +// PRMTD_FAST-NEXT: [[RETVAL:%.*]] = alloca { x86_fp80, x86_fp80 }, align 16 +// PRMTD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[A_REAL:%.*]] = load x86_fp80, ptr [[A_REALP]], align 16 +// PRMTD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[A]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[A_IMAG:%.*]] = load x86_fp80, ptr [[A_IMAGP]], align 16 +// PRMTD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// PRMTD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// PRMTD_FAST-NEXT: [[MUL_AC:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_REAL]], [[B_REAL]] +// PRMTD_FAST-NEXT: [[MUL_BD:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_IMAG]], [[B_IMAG]] +// PRMTD_FAST-NEXT: [[MUL_AD:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_REAL]], [[B_IMAG]] +// PRMTD_FAST-NEXT: [[MUL_BC:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[A_IMAG]], [[B_REAL]] +// PRMTD_FAST-NEXT: [[MUL_R:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[MUL_AC]], [[MUL_BD]] +// PRMTD_FAST-NEXT: [[MUL_I:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[MUL_AD]], [[MUL_BC]] +// PRMTD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD_FAST-NEXT: store x86_fp80 [[MUL_R]], ptr [[RETVAL_REALP]], align 16 +// PRMTD_FAST-NEXT: store x86_fp80 [[MUL_I]], ptr [[RETVAL_IMAGP]], align 16 +// PRMTD_FAST-NEXT: [[TMP0:%.*]] = load { x86_fp80, x86_fp80 }, ptr [[RETVAL]], align 16 +// PRMTD_FAST-NEXT: ret { x86_fp80, x86_fp80 } [[TMP0]] +// +_Complex long double mulld(_Complex long double a, _Complex long double b) { return a * b; } + +// FULL-LABEL: define dso_local <2 x float> @f1( +// FULL-SAME: <2 x float> noundef [[A_COERCE:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]], <2 x float> noundef [[C_COERCE:%.*]]) #[[ATTR0]] { +// FULL-NEXT: entry: +// FULL-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// FULL-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// FULL-NEXT: [[C:%.*]] = alloca { float, float }, align 4 +// FULL-NEXT: [[COERCE:%.*]] = alloca { float, float }, align 4 +// FULL-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// FULL-NEXT: store <2 x float> [[C_COERCE]], ptr [[C]], align 4 +// FULL-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// FULL-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// FULL-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// FULL-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// FULL-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 0 +// FULL-NEXT: [[C_REAL:%.*]] = load float, ptr [[C_REALP]], align 4 +// FULL-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 1 +// FULL-NEXT: [[C_IMAG:%.*]] = load float, ptr [[C_IMAGP]], align 4 +// FULL-NEXT: [[CONV:%.*]] = fpext float [[C_REAL]] to x86_fp80 +// FULL-NEXT: [[CONV1:%.*]] = fpext float [[C_IMAG]] to x86_fp80 +// FULL-NEXT: [[CALL:%.*]] = call { x86_fp80, x86_fp80 } @__divxc3(x86_fp80 noundef [[B_REAL]], x86_fp80 noundef [[B_IMAG]], x86_fp80 noundef [[CONV]], x86_fp80 noundef [[CONV1]]) #[[ATTR2]] +// FULL-NEXT: [[TMP0:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 0 +// FULL-NEXT: [[TMP1:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 1 +// FULL-NEXT: [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP0]] to float +// FULL-NEXT: [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP1]] to float +// FULL-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// FULL-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// FULL-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// FULL-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// FULL-NEXT: [[CALL4:%.*]] = call <2 x float> @__divsc3(float noundef [[CONV2]], float noundef [[CONV3]], float noundef [[A_REAL]], float noundef [[A_IMAG]]) #[[ATTR2]] +// FULL-NEXT: store <2 x float> [[CALL4]], ptr [[COERCE]], align 4 +// FULL-NEXT: [[COERCE_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 0 +// FULL-NEXT: [[COERCE_REAL:%.*]] = load float, ptr [[COERCE_REALP]], align 4 +// FULL-NEXT: [[COERCE_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 1 +// FULL-NEXT: [[COERCE_IMAG:%.*]] = load float, ptr [[COERCE_IMAGP]], align 4 +// FULL-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// FULL-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// FULL-NEXT: store float [[COERCE_REAL]], ptr [[RETVAL_REALP]], align 4 +// FULL-NEXT: store float [[COERCE_IMAG]], ptr [[RETVAL_IMAGP]], align 4 +// FULL-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// FULL-NEXT: ret <2 x float> [[TMP2]] +// +// BASIC-LABEL: define dso_local <2 x float> @f1( +// BASIC-SAME: <2 x float> noundef [[A_COERCE:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]], <2 x float> noundef [[C_COERCE:%.*]]) #[[ATTR0]] { +// BASIC-NEXT: entry: +// BASIC-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// BASIC-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// BASIC-NEXT: [[C:%.*]] = alloca { float, float }, align 4 +// BASIC-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// BASIC-NEXT: store <2 x float> [[C_COERCE]], ptr [[C]], align 4 +// BASIC-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// BASIC-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// BASIC-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// BASIC-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// BASIC-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 0 +// BASIC-NEXT: [[C_REAL:%.*]] = load float, ptr [[C_REALP]], align 4 +// BASIC-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 1 +// BASIC-NEXT: [[C_IMAG:%.*]] = load float, ptr [[C_IMAGP]], align 4 +// BASIC-NEXT: [[CONV:%.*]] = fpext float [[C_REAL]] to x86_fp80 +// BASIC-NEXT: [[CONV1:%.*]] = fpext float [[C_IMAG]] to x86_fp80 +// BASIC-NEXT: [[TMP0:%.*]] = fmul x86_fp80 [[B_REAL]], [[CONV]] +// BASIC-NEXT: [[TMP1:%.*]] = fmul x86_fp80 [[B_IMAG]], [[CONV1]] +// BASIC-NEXT: [[TMP2:%.*]] = fadd x86_fp80 [[TMP0]], [[TMP1]] +// BASIC-NEXT: [[TMP3:%.*]] = fmul x86_fp80 [[CONV]], [[CONV]] +// BASIC-NEXT: [[TMP4:%.*]] = fmul x86_fp80 [[CONV1]], [[CONV1]] +// BASIC-NEXT: [[TMP5:%.*]] = fadd x86_fp80 [[TMP3]], [[TMP4]] +// BASIC-NEXT: [[TMP6:%.*]] = fmul x86_fp80 [[B_IMAG]], [[CONV]] +// BASIC-NEXT: [[TMP7:%.*]] = fmul x86_fp80 [[B_REAL]], [[CONV1]] +// BASIC-NEXT: [[TMP8:%.*]] = fsub x86_fp80 [[TMP6]], [[TMP7]] +// BASIC-NEXT: [[TMP9:%.*]] = fdiv x86_fp80 [[TMP2]], [[TMP5]] +// BASIC-NEXT: [[TMP10:%.*]] = fdiv x86_fp80 [[TMP8]], [[TMP5]] +// BASIC-NEXT: [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP9]] to float +// BASIC-NEXT: [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP10]] to float +// BASIC-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// BASIC-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// BASIC-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// BASIC-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// BASIC-NEXT: [[TMP11:%.*]] = fmul float [[CONV2]], [[A_REAL]] +// BASIC-NEXT: [[TMP12:%.*]] = fmul float [[CONV3]], [[A_IMAG]] +// BASIC-NEXT: [[TMP13:%.*]] = fadd float [[TMP11]], [[TMP12]] +// BASIC-NEXT: [[TMP14:%.*]] = fmul float [[A_REAL]], [[A_REAL]] +// BASIC-NEXT: [[TMP15:%.*]] = fmul float [[A_IMAG]], [[A_IMAG]] +// BASIC-NEXT: [[TMP16:%.*]] = fadd float [[TMP14]], [[TMP15]] +// BASIC-NEXT: [[TMP17:%.*]] = fmul float [[CONV3]], [[A_REAL]] +// BASIC-NEXT: [[TMP18:%.*]] = fmul float [[CONV2]], [[A_IMAG]] +// BASIC-NEXT: [[TMP19:%.*]] = fsub float [[TMP17]], [[TMP18]] +// BASIC-NEXT: [[TMP20:%.*]] = fdiv float [[TMP13]], [[TMP16]] +// BASIC-NEXT: [[TMP21:%.*]] = fdiv float [[TMP19]], [[TMP16]] +// BASIC-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC-NEXT: store float [[TMP20]], ptr [[RETVAL_REALP]], align 4 +// BASIC-NEXT: store float [[TMP21]], ptr [[RETVAL_IMAGP]], align 4 +// BASIC-NEXT: [[TMP22:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// BASIC-NEXT: ret <2 x float> [[TMP22]] +// +// IMPRVD-LABEL: define dso_local <2 x float> @f1( +// IMPRVD-SAME: <2 x float> noundef [[A_COERCE:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]], <2 x float> noundef [[C_COERCE:%.*]]) #[[ATTR0]] { +// IMPRVD-NEXT: entry: +// IMPRVD-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// IMPRVD-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// IMPRVD-NEXT: [[C:%.*]] = alloca { float, float }, align 4 +// IMPRVD-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// IMPRVD-NEXT: store <2 x float> [[C_COERCE]], ptr [[C]], align 4 +// IMPRVD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// IMPRVD-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// IMPRVD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// IMPRVD-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// IMPRVD-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 0 +// IMPRVD-NEXT: [[C_REAL:%.*]] = load float, ptr [[C_REALP]], align 4 +// IMPRVD-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 1 +// IMPRVD-NEXT: [[C_IMAG:%.*]] = load float, ptr [[C_IMAGP]], align 4 +// IMPRVD-NEXT: [[CONV:%.*]] = fpext float [[C_REAL]] to x86_fp80 +// IMPRVD-NEXT: [[CONV1:%.*]] = fpext float [[C_IMAG]] to x86_fp80 +// IMPRVD-NEXT: [[TMP0:%.*]] = call x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV]]) +// IMPRVD-NEXT: [[TMP1:%.*]] = call x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV1]]) +// IMPRVD-NEXT: [[ABS_CMP:%.*]] = fcmp ugt x86_fp80 [[TMP0]], [[TMP1]] +// IMPRVD-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// IMPRVD: abs_rhsr_greater_or_equal_abs_rhsi: +// IMPRVD-NEXT: [[TMP2:%.*]] = fdiv x86_fp80 [[CONV1]], [[CONV]] +// IMPRVD-NEXT: [[TMP3:%.*]] = fmul x86_fp80 [[TMP2]], [[CONV1]] +// IMPRVD-NEXT: [[TMP4:%.*]] = fadd x86_fp80 [[CONV]], [[TMP3]] +// IMPRVD-NEXT: [[TMP5:%.*]] = fmul x86_fp80 [[B_IMAG]], [[TMP2]] +// IMPRVD-NEXT: [[TMP6:%.*]] = fadd x86_fp80 [[B_REAL]], [[TMP5]] +// IMPRVD-NEXT: [[TMP7:%.*]] = fdiv x86_fp80 [[TMP6]], [[TMP4]] +// IMPRVD-NEXT: [[TMP8:%.*]] = fmul x86_fp80 [[B_REAL]], [[TMP2]] +// IMPRVD-NEXT: [[TMP9:%.*]] = fsub x86_fp80 [[B_IMAG]], [[TMP8]] +// IMPRVD-NEXT: [[TMP10:%.*]] = fdiv x86_fp80 [[TMP9]], [[TMP4]] +// IMPRVD-NEXT: br label [[COMPLEX_DIV:%.*]] +// IMPRVD: abs_rhsr_less_than_abs_rhsi: +// IMPRVD-NEXT: [[TMP11:%.*]] = fdiv x86_fp80 [[CONV]], [[CONV1]] +// IMPRVD-NEXT: [[TMP12:%.*]] = fmul x86_fp80 [[TMP11]], [[CONV]] +// IMPRVD-NEXT: [[TMP13:%.*]] = fadd x86_fp80 [[CONV1]], [[TMP12]] +// IMPRVD-NEXT: [[TMP14:%.*]] = fmul x86_fp80 [[B_REAL]], [[TMP11]] +// IMPRVD-NEXT: [[TMP15:%.*]] = fadd x86_fp80 [[TMP14]], [[B_IMAG]] +// IMPRVD-NEXT: [[TMP16:%.*]] = fdiv x86_fp80 [[TMP15]], [[TMP13]] +// IMPRVD-NEXT: [[TMP17:%.*]] = fmul x86_fp80 [[B_IMAG]], [[TMP11]] +// IMPRVD-NEXT: [[TMP18:%.*]] = fsub x86_fp80 [[TMP17]], [[B_REAL]] +// IMPRVD-NEXT: [[TMP19:%.*]] = fdiv x86_fp80 [[TMP18]], [[TMP13]] +// IMPRVD-NEXT: br label [[COMPLEX_DIV]] +// IMPRVD: complex_div: +// IMPRVD-NEXT: [[TMP20:%.*]] = phi x86_fp80 [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD-NEXT: [[TMP21:%.*]] = phi x86_fp80 [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD-NEXT: [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP20]] to float +// IMPRVD-NEXT: [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP21]] to float +// IMPRVD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// IMPRVD-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// IMPRVD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// IMPRVD-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// IMPRVD-NEXT: [[TMP22:%.*]] = call float @llvm.fabs.f32(float [[A_REAL]]) +// IMPRVD-NEXT: [[TMP23:%.*]] = call float @llvm.fabs.f32(float [[A_IMAG]]) +// IMPRVD-NEXT: [[ABS_CMP4:%.*]] = fcmp ugt float [[TMP22]], [[TMP23]] +// IMPRVD-NEXT: br i1 [[ABS_CMP4]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI5:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI6:%.*]] +// IMPRVD: abs_rhsr_greater_or_equal_abs_rhsi5: +// IMPRVD-NEXT: [[TMP24:%.*]] = fdiv float [[A_IMAG]], [[A_REAL]] +// IMPRVD-NEXT: [[TMP25:%.*]] = fmul float [[TMP24]], [[A_IMAG]] +// IMPRVD-NEXT: [[TMP26:%.*]] = fadd float [[A_REAL]], [[TMP25]] +// IMPRVD-NEXT: [[TMP27:%.*]] = fmul float [[CONV3]], [[TMP24]] +// IMPRVD-NEXT: [[TMP28:%.*]] = fadd float [[CONV2]], [[TMP27]] +// IMPRVD-NEXT: [[TMP29:%.*]] = fdiv float [[TMP28]], [[TMP26]] +// IMPRVD-NEXT: [[TMP30:%.*]] = fmul float [[CONV2]], [[TMP24]] +// IMPRVD-NEXT: [[TMP31:%.*]] = fsub float [[CONV3]], [[TMP30]] +// IMPRVD-NEXT: [[TMP32:%.*]] = fdiv float [[TMP31]], [[TMP26]] +// IMPRVD-NEXT: br label [[COMPLEX_DIV7:%.*]] +// IMPRVD: abs_rhsr_less_than_abs_rhsi6: +// IMPRVD-NEXT: [[TMP33:%.*]] = fdiv float [[A_REAL]], [[A_IMAG]] +// IMPRVD-NEXT: [[TMP34:%.*]] = fmul float [[TMP33]], [[A_REAL]] +// IMPRVD-NEXT: [[TMP35:%.*]] = fadd float [[A_IMAG]], [[TMP34]] +// IMPRVD-NEXT: [[TMP36:%.*]] = fmul float [[CONV2]], [[TMP33]] +// IMPRVD-NEXT: [[TMP37:%.*]] = fadd float [[TMP36]], [[CONV3]] +// IMPRVD-NEXT: [[TMP38:%.*]] = fdiv float [[TMP37]], [[TMP35]] +// IMPRVD-NEXT: [[TMP39:%.*]] = fmul float [[CONV3]], [[TMP33]] +// IMPRVD-NEXT: [[TMP40:%.*]] = fsub float [[TMP39]], [[CONV2]] +// IMPRVD-NEXT: [[TMP41:%.*]] = fdiv float [[TMP40]], [[TMP35]] +// IMPRVD-NEXT: br label [[COMPLEX_DIV7]] +// IMPRVD: complex_div7: +// IMPRVD-NEXT: [[TMP42:%.*]] = phi float [ [[TMP29]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI5]] ], [ [[TMP38]], [[ABS_RHSR_LESS_THAN_ABS_RHSI6]] ] +// IMPRVD-NEXT: [[TMP43:%.*]] = phi float [ [[TMP32]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI5]] ], [ [[TMP41]], [[ABS_RHSR_LESS_THAN_ABS_RHSI6]] ] +// IMPRVD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD-NEXT: store float [[TMP42]], ptr [[RETVAL_REALP]], align 4 +// IMPRVD-NEXT: store float [[TMP43]], ptr [[RETVAL_IMAGP]], align 4 +// IMPRVD-NEXT: [[TMP44:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// IMPRVD-NEXT: ret <2 x float> [[TMP44]] +// +// PRMTD-LABEL: define dso_local <2 x float> @f1( +// PRMTD-SAME: <2 x float> noundef [[A_COERCE:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]], <2 x float> noundef [[C_COERCE:%.*]]) #[[ATTR0]] { +// PRMTD-NEXT: entry: +// PRMTD-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// PRMTD-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// PRMTD-NEXT: [[C:%.*]] = alloca { float, float }, align 4 +// PRMTD-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// PRMTD-NEXT: store <2 x float> [[C_COERCE]], ptr [[C]], align 4 +// PRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// PRMTD-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// PRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// PRMTD-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// PRMTD-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 0 +// PRMTD-NEXT: [[C_REAL:%.*]] = load float, ptr [[C_REALP]], align 4 +// PRMTD-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 1 +// PRMTD-NEXT: [[C_IMAG:%.*]] = load float, ptr [[C_IMAGP]], align 4 +// PRMTD-NEXT: [[CONV:%.*]] = fpext float [[C_REAL]] to x86_fp80 +// PRMTD-NEXT: [[CONV1:%.*]] = fpext float [[C_IMAG]] to x86_fp80 +// PRMTD-NEXT: [[TMP0:%.*]] = call x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV]]) +// PRMTD-NEXT: [[TMP1:%.*]] = call x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV1]]) +// PRMTD-NEXT: [[ABS_CMP:%.*]] = fcmp ugt x86_fp80 [[TMP0]], [[TMP1]] +// PRMTD-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// PRMTD: abs_rhsr_greater_or_equal_abs_rhsi: +// PRMTD-NEXT: [[TMP2:%.*]] = fdiv x86_fp80 [[CONV1]], [[CONV]] +// PRMTD-NEXT: [[TMP3:%.*]] = fmul x86_fp80 [[TMP2]], [[CONV1]] +// PRMTD-NEXT: [[TMP4:%.*]] = fadd x86_fp80 [[CONV]], [[TMP3]] +// PRMTD-NEXT: [[TMP5:%.*]] = fmul x86_fp80 [[B_IMAG]], [[TMP2]] +// PRMTD-NEXT: [[TMP6:%.*]] = fadd x86_fp80 [[B_REAL]], [[TMP5]] +// PRMTD-NEXT: [[TMP7:%.*]] = fdiv x86_fp80 [[TMP6]], [[TMP4]] +// PRMTD-NEXT: [[TMP8:%.*]] = fmul x86_fp80 [[B_REAL]], [[TMP2]] +// PRMTD-NEXT: [[TMP9:%.*]] = fsub x86_fp80 [[B_IMAG]], [[TMP8]] +// PRMTD-NEXT: [[TMP10:%.*]] = fdiv x86_fp80 [[TMP9]], [[TMP4]] +// PRMTD-NEXT: br label [[COMPLEX_DIV:%.*]] +// PRMTD: abs_rhsr_less_than_abs_rhsi: +// PRMTD-NEXT: [[TMP11:%.*]] = fdiv x86_fp80 [[CONV]], [[CONV1]] +// PRMTD-NEXT: [[TMP12:%.*]] = fmul x86_fp80 [[TMP11]], [[CONV]] +// PRMTD-NEXT: [[TMP13:%.*]] = fadd x86_fp80 [[CONV1]], [[TMP12]] +// PRMTD-NEXT: [[TMP14:%.*]] = fmul x86_fp80 [[B_REAL]], [[TMP11]] +// PRMTD-NEXT: [[TMP15:%.*]] = fadd x86_fp80 [[TMP14]], [[B_IMAG]] +// PRMTD-NEXT: [[TMP16:%.*]] = fdiv x86_fp80 [[TMP15]], [[TMP13]] +// PRMTD-NEXT: [[TMP17:%.*]] = fmul x86_fp80 [[B_IMAG]], [[TMP11]] +// PRMTD-NEXT: [[TMP18:%.*]] = fsub x86_fp80 [[TMP17]], [[B_REAL]] +// PRMTD-NEXT: [[TMP19:%.*]] = fdiv x86_fp80 [[TMP18]], [[TMP13]] +// PRMTD-NEXT: br label [[COMPLEX_DIV]] +// PRMTD: complex_div: +// PRMTD-NEXT: [[TMP20:%.*]] = phi x86_fp80 [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// PRMTD-NEXT: [[TMP21:%.*]] = phi x86_fp80 [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// PRMTD-NEXT: [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP20]] to float +// PRMTD-NEXT: [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP21]] to float +// PRMTD-NEXT: [[EXT:%.*]] = fpext float [[CONV2]] to double +// PRMTD-NEXT: [[EXT4:%.*]] = fpext float [[CONV3]] to double +// PRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// PRMTD-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// PRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// PRMTD-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// PRMTD-NEXT: [[EXT5:%.*]] = fpext float [[A_REAL]] to double +// PRMTD-NEXT: [[EXT6:%.*]] = fpext float [[A_IMAG]] to double +// PRMTD-NEXT: [[TMP22:%.*]] = fmul double [[EXT]], [[EXT5]] +// PRMTD-NEXT: [[TMP23:%.*]] = fmul double [[EXT4]], [[EXT6]] +// PRMTD-NEXT: [[TMP24:%.*]] = fadd double [[TMP22]], [[TMP23]] +// PRMTD-NEXT: [[TMP25:%.*]] = fmul double [[EXT5]], [[EXT5]] +// PRMTD-NEXT: [[TMP26:%.*]] = fmul double [[EXT6]], [[EXT6]] +// PRMTD-NEXT: [[TMP27:%.*]] = fadd double [[TMP25]], [[TMP26]] +// PRMTD-NEXT: [[TMP28:%.*]] = fmul double [[EXT4]], [[EXT5]] +// PRMTD-NEXT: [[TMP29:%.*]] = fmul double [[EXT]], [[EXT6]] +// PRMTD-NEXT: [[TMP30:%.*]] = fsub double [[TMP28]], [[TMP29]] +// PRMTD-NEXT: [[TMP31:%.*]] = fdiv double [[TMP24]], [[TMP27]] +// PRMTD-NEXT: [[TMP32:%.*]] = fdiv double [[TMP30]], [[TMP27]] +// PRMTD-NEXT: [[UNPROMOTION:%.*]] = fptrunc double [[TMP31]] to float +// PRMTD-NEXT: [[UNPROMOTION7:%.*]] = fptrunc double [[TMP32]] to float +// PRMTD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD-NEXT: store float [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 4 +// PRMTD-NEXT: store float [[UNPROMOTION7]], ptr [[RETVAL_IMAGP]], align 4 +// PRMTD-NEXT: [[TMP33:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// PRMTD-NEXT: ret <2 x float> [[TMP33]] +// +// X86WINPRMTD-LABEL: define dso_local i64 @f1( +// X86WINPRMTD-SAME: i64 noundef [[A_COERCE:%.*]], ptr noundef [[B:%.*]], i64 noundef [[C_COERCE:%.*]]) #[[ATTR0]] { +// X86WINPRMTD-NEXT: entry: +// X86WINPRMTD-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// X86WINPRMTD-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// X86WINPRMTD-NEXT: [[C:%.*]] = alloca { float, float }, align 4 +// X86WINPRMTD-NEXT: [[B_INDIRECT_ADDR:%.*]] = alloca ptr, align 8 +// X86WINPRMTD-NEXT: store i64 [[A_COERCE]], ptr [[A]], align 4 +// X86WINPRMTD-NEXT: store i64 [[C_COERCE]], ptr [[C]], align 4 +// X86WINPRMTD-NEXT: store ptr [[B]], ptr [[B_INDIRECT_ADDR]], align 8 +// X86WINPRMTD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 8 +// X86WINPRMTD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 8 +// X86WINPRMTD-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[C_REAL:%.*]] = load float, ptr [[C_REALP]], align 4 +// X86WINPRMTD-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[C_IMAG:%.*]] = load float, ptr [[C_IMAGP]], align 4 +// X86WINPRMTD-NEXT: [[CONV:%.*]] = fpext float [[C_REAL]] to double +// X86WINPRMTD-NEXT: [[CONV1:%.*]] = fpext float [[C_IMAG]] to double +// X86WINPRMTD-NEXT: [[TMP0:%.*]] = call double @llvm.fabs.f64(double [[CONV]]) +// X86WINPRMTD-NEXT: [[TMP1:%.*]] = call double @llvm.fabs.f64(double [[CONV1]]) +// X86WINPRMTD-NEXT: [[ABS_CMP:%.*]] = fcmp ugt double [[TMP0]], [[TMP1]] +// X86WINPRMTD-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// X86WINPRMTD: abs_rhsr_greater_or_equal_abs_rhsi: +// X86WINPRMTD-NEXT: [[TMP2:%.*]] = fdiv double [[CONV1]], [[CONV]] +// X86WINPRMTD-NEXT: [[TMP3:%.*]] = fmul double [[TMP2]], [[CONV1]] +// X86WINPRMTD-NEXT: [[TMP4:%.*]] = fadd double [[CONV]], [[TMP3]] +// X86WINPRMTD-NEXT: [[TMP5:%.*]] = fmul double [[B_IMAG]], [[TMP2]] +// X86WINPRMTD-NEXT: [[TMP6:%.*]] = fadd double [[B_REAL]], [[TMP5]] +// X86WINPRMTD-NEXT: [[TMP7:%.*]] = fdiv double [[TMP6]], [[TMP4]] +// X86WINPRMTD-NEXT: [[TMP8:%.*]] = fmul double [[B_REAL]], [[TMP2]] +// X86WINPRMTD-NEXT: [[TMP9:%.*]] = fsub double [[B_IMAG]], [[TMP8]] +// X86WINPRMTD-NEXT: [[TMP10:%.*]] = fdiv double [[TMP9]], [[TMP4]] +// X86WINPRMTD-NEXT: br label [[COMPLEX_DIV:%.*]] +// X86WINPRMTD: abs_rhsr_less_than_abs_rhsi: +// X86WINPRMTD-NEXT: [[TMP11:%.*]] = fdiv double [[CONV]], [[CONV1]] +// X86WINPRMTD-NEXT: [[TMP12:%.*]] = fmul double [[TMP11]], [[CONV]] +// X86WINPRMTD-NEXT: [[TMP13:%.*]] = fadd double [[CONV1]], [[TMP12]] +// X86WINPRMTD-NEXT: [[TMP14:%.*]] = fmul double [[B_REAL]], [[TMP11]] +// X86WINPRMTD-NEXT: [[TMP15:%.*]] = fadd double [[TMP14]], [[B_IMAG]] +// X86WINPRMTD-NEXT: [[TMP16:%.*]] = fdiv double [[TMP15]], [[TMP13]] +// X86WINPRMTD-NEXT: [[TMP17:%.*]] = fmul double [[B_IMAG]], [[TMP11]] +// X86WINPRMTD-NEXT: [[TMP18:%.*]] = fsub double [[TMP17]], [[B_REAL]] +// X86WINPRMTD-NEXT: [[TMP19:%.*]] = fdiv double [[TMP18]], [[TMP13]] +// X86WINPRMTD-NEXT: br label [[COMPLEX_DIV]] +// X86WINPRMTD: complex_div: +// X86WINPRMTD-NEXT: [[TMP20:%.*]] = phi double [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// X86WINPRMTD-NEXT: [[TMP21:%.*]] = phi double [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// X86WINPRMTD-NEXT: [[CONV2:%.*]] = fptrunc double [[TMP20]] to float +// X86WINPRMTD-NEXT: [[CONV3:%.*]] = fptrunc double [[TMP21]] to float +// X86WINPRMTD-NEXT: [[EXT:%.*]] = fpext float [[CONV2]] to double +// X86WINPRMTD-NEXT: [[EXT4:%.*]] = fpext float [[CONV3]] to double +// X86WINPRMTD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// X86WINPRMTD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// X86WINPRMTD-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// X86WINPRMTD-NEXT: [[EXT5:%.*]] = fpext float [[A_REAL]] to double +// X86WINPRMTD-NEXT: [[EXT6:%.*]] = fpext float [[A_IMAG]] to double +// X86WINPRMTD-NEXT: [[TMP22:%.*]] = fmul double [[EXT]], [[EXT5]] +// X86WINPRMTD-NEXT: [[TMP23:%.*]] = fmul double [[EXT4]], [[EXT6]] +// X86WINPRMTD-NEXT: [[TMP24:%.*]] = fadd double [[TMP22]], [[TMP23]] +// X86WINPRMTD-NEXT: [[TMP25:%.*]] = fmul double [[EXT5]], [[EXT5]] +// X86WINPRMTD-NEXT: [[TMP26:%.*]] = fmul double [[EXT6]], [[EXT6]] +// X86WINPRMTD-NEXT: [[TMP27:%.*]] = fadd double [[TMP25]], [[TMP26]] +// X86WINPRMTD-NEXT: [[TMP28:%.*]] = fmul double [[EXT4]], [[EXT5]] +// X86WINPRMTD-NEXT: [[TMP29:%.*]] = fmul double [[EXT]], [[EXT6]] +// X86WINPRMTD-NEXT: [[TMP30:%.*]] = fsub double [[TMP28]], [[TMP29]] +// X86WINPRMTD-NEXT: [[TMP31:%.*]] = fdiv double [[TMP24]], [[TMP27]] +// X86WINPRMTD-NEXT: [[TMP32:%.*]] = fdiv double [[TMP30]], [[TMP27]] +// X86WINPRMTD-NEXT: [[UNPROMOTION:%.*]] = fptrunc double [[TMP31]] to float +// X86WINPRMTD-NEXT: [[UNPROMOTION7:%.*]] = fptrunc double [[TMP32]] to float +// X86WINPRMTD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// X86WINPRMTD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// X86WINPRMTD-NEXT: store float [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 4 +// X86WINPRMTD-NEXT: store float [[UNPROMOTION7]], ptr [[RETVAL_IMAGP]], align 4 +// X86WINPRMTD-NEXT: [[TMP33:%.*]] = load i64, ptr [[RETVAL]], align 4 +// X86WINPRMTD-NEXT: ret i64 [[TMP33]] +// +// AVRFP32-LABEL: define dso_local { float, float } @f1( +// AVRFP32-SAME: float noundef [[A_COERCE0:%.*]], float noundef [[A_COERCE1:%.*]], float noundef [[B_COERCE0:%.*]], float noundef [[B_COERCE1:%.*]], float noundef [[C_COERCE0:%.*]], float noundef [[C_COERCE1:%.*]]) addrspace(1) #[[ATTR0]] { +// AVRFP32-NEXT: entry: +// AVRFP32-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 1 +// AVRFP32-NEXT: [[A:%.*]] = alloca { float, float }, align 1 +// AVRFP32-NEXT: [[B:%.*]] = alloca { float, float }, align 1 +// AVRFP32-NEXT: [[C:%.*]] = alloca { float, float }, align 1 +// AVRFP32-NEXT: [[TMP0:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP32-NEXT: store float [[A_COERCE0]], ptr [[TMP0]], align 1 +// AVRFP32-NEXT: [[TMP1:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[A_COERCE1]], ptr [[TMP1]], align 1 +// AVRFP32-NEXT: [[TMP2:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// AVRFP32-NEXT: store float [[B_COERCE0]], ptr [[TMP2]], align 1 +// AVRFP32-NEXT: [[TMP3:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[B_COERCE1]], ptr [[TMP3]], align 1 +// AVRFP32-NEXT: [[TMP4:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 0 +// AVRFP32-NEXT: store float [[C_COERCE0]], ptr [[TMP4]], align 1 +// AVRFP32-NEXT: [[TMP5:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[C_COERCE1]], ptr [[TMP5]], align 1 +// AVRFP32-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// AVRFP32-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 1 +// AVRFP32-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// AVRFP32-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 1 +// AVRFP32-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 0 +// AVRFP32-NEXT: [[C_REAL:%.*]] = load float, ptr [[C_REALP]], align 1 +// AVRFP32-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 1 +// AVRFP32-NEXT: [[C_IMAG:%.*]] = load float, ptr [[C_IMAGP]], align 1 +// AVRFP32-NEXT: [[TMP6:%.*]] = call addrspace(1) float @llvm.fabs.f32(float [[C_REAL]]) +// AVRFP32-NEXT: [[TMP7:%.*]] = call addrspace(1) float @llvm.fabs.f32(float [[C_IMAG]]) +// AVRFP32-NEXT: [[ABS_CMP:%.*]] = fcmp ugt float [[TMP6]], [[TMP7]] +// AVRFP32-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// AVRFP32: abs_rhsr_greater_or_equal_abs_rhsi: +// AVRFP32-NEXT: [[TMP8:%.*]] = fdiv float [[C_IMAG]], [[C_REAL]] +// AVRFP32-NEXT: [[TMP9:%.*]] = fmul float [[TMP8]], [[C_IMAG]] +// AVRFP32-NEXT: [[TMP10:%.*]] = fadd float [[C_REAL]], [[TMP9]] +// AVRFP32-NEXT: [[TMP11:%.*]] = fmul float [[B_IMAG]], [[TMP8]] +// AVRFP32-NEXT: [[TMP12:%.*]] = fadd float [[B_REAL]], [[TMP11]] +// AVRFP32-NEXT: [[TMP13:%.*]] = fdiv float [[TMP12]], [[TMP10]] +// AVRFP32-NEXT: [[TMP14:%.*]] = fmul float [[B_REAL]], [[TMP8]] +// AVRFP32-NEXT: [[TMP15:%.*]] = fsub float [[B_IMAG]], [[TMP14]] +// AVRFP32-NEXT: [[TMP16:%.*]] = fdiv float [[TMP15]], [[TMP10]] +// AVRFP32-NEXT: br label [[COMPLEX_DIV:%.*]] +// AVRFP32: abs_rhsr_less_than_abs_rhsi: +// AVRFP32-NEXT: [[TMP17:%.*]] = fdiv float [[C_REAL]], [[C_IMAG]] +// AVRFP32-NEXT: [[TMP18:%.*]] = fmul float [[TMP17]], [[C_REAL]] +// AVRFP32-NEXT: [[TMP19:%.*]] = fadd float [[C_IMAG]], [[TMP18]] +// AVRFP32-NEXT: [[TMP20:%.*]] = fmul float [[B_REAL]], [[TMP17]] +// AVRFP32-NEXT: [[TMP21:%.*]] = fadd float [[TMP20]], [[B_IMAG]] +// AVRFP32-NEXT: [[TMP22:%.*]] = fdiv float [[TMP21]], [[TMP19]] +// AVRFP32-NEXT: [[TMP23:%.*]] = fmul float [[B_IMAG]], [[TMP17]] +// AVRFP32-NEXT: [[TMP24:%.*]] = fsub float [[TMP23]], [[B_REAL]] +// AVRFP32-NEXT: [[TMP25:%.*]] = fdiv float [[TMP24]], [[TMP19]] +// AVRFP32-NEXT: br label [[COMPLEX_DIV]] +// AVRFP32: complex_div: +// AVRFP32-NEXT: [[TMP26:%.*]] = phi float [ [[TMP13]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP22]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// AVRFP32-NEXT: [[TMP27:%.*]] = phi float [ [[TMP16]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP25]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// AVRFP32-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP32-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 1 +// AVRFP32-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP32-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 1 +// AVRFP32-NEXT: [[TMP28:%.*]] = call addrspace(1) float @llvm.fabs.f32(float [[A_REAL]]) +// AVRFP32-NEXT: [[TMP29:%.*]] = call addrspace(1) float @llvm.fabs.f32(float [[A_IMAG]]) +// AVRFP32-NEXT: [[ABS_CMP1:%.*]] = fcmp ugt float [[TMP28]], [[TMP29]] +// AVRFP32-NEXT: br i1 [[ABS_CMP1]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI2:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI3:%.*]] +// AVRFP32: abs_rhsr_greater_or_equal_abs_rhsi2: +// AVRFP32-NEXT: [[TMP30:%.*]] = fdiv float [[A_IMAG]], [[A_REAL]] +// AVRFP32-NEXT: [[TMP31:%.*]] = fmul float [[TMP30]], [[A_IMAG]] +// AVRFP32-NEXT: [[TMP32:%.*]] = fadd float [[A_REAL]], [[TMP31]] +// AVRFP32-NEXT: [[TMP33:%.*]] = fmul float [[TMP27]], [[TMP30]] +// AVRFP32-NEXT: [[TMP34:%.*]] = fadd float [[TMP26]], [[TMP33]] +// AVRFP32-NEXT: [[TMP35:%.*]] = fdiv float [[TMP34]], [[TMP32]] +// AVRFP32-NEXT: [[TMP36:%.*]] = fmul float [[TMP26]], [[TMP30]] +// AVRFP32-NEXT: [[TMP37:%.*]] = fsub float [[TMP27]], [[TMP36]] +// AVRFP32-NEXT: [[TMP38:%.*]] = fdiv float [[TMP37]], [[TMP32]] +// AVRFP32-NEXT: br label [[COMPLEX_DIV4:%.*]] +// AVRFP32: abs_rhsr_less_than_abs_rhsi3: +// AVRFP32-NEXT: [[TMP39:%.*]] = fdiv float [[A_REAL]], [[A_IMAG]] +// AVRFP32-NEXT: [[TMP40:%.*]] = fmul float [[TMP39]], [[A_REAL]] +// AVRFP32-NEXT: [[TMP41:%.*]] = fadd float [[A_IMAG]], [[TMP40]] +// AVRFP32-NEXT: [[TMP42:%.*]] = fmul float [[TMP26]], [[TMP39]] +// AVRFP32-NEXT: [[TMP43:%.*]] = fadd float [[TMP42]], [[TMP27]] +// AVRFP32-NEXT: [[TMP44:%.*]] = fdiv float [[TMP43]], [[TMP41]] +// AVRFP32-NEXT: [[TMP45:%.*]] = fmul float [[TMP27]], [[TMP39]] +// AVRFP32-NEXT: [[TMP46:%.*]] = fsub float [[TMP45]], [[TMP26]] +// AVRFP32-NEXT: [[TMP47:%.*]] = fdiv float [[TMP46]], [[TMP41]] +// AVRFP32-NEXT: br label [[COMPLEX_DIV4]] +// AVRFP32: complex_div4: +// AVRFP32-NEXT: [[TMP48:%.*]] = phi float [ [[TMP35]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI2]] ], [ [[TMP44]], [[ABS_RHSR_LESS_THAN_ABS_RHSI3]] ] +// AVRFP32-NEXT: [[TMP49:%.*]] = phi float [ [[TMP38]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI2]] ], [ [[TMP47]], [[ABS_RHSR_LESS_THAN_ABS_RHSI3]] ] +// AVRFP32-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// AVRFP32-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// AVRFP32-NEXT: store float [[TMP48]], ptr [[RETVAL_REALP]], align 1 +// AVRFP32-NEXT: store float [[TMP49]], ptr [[RETVAL_IMAGP]], align 1 +// AVRFP32-NEXT: [[TMP50:%.*]] = load { float, float }, ptr [[RETVAL]], align 1 +// AVRFP32-NEXT: ret { float, float } [[TMP50]] +// +// AVRFP64-LABEL: define dso_local { float, float } @f1( +// AVRFP64-SAME: float noundef [[A_COERCE0:%.*]], float noundef [[A_COERCE1:%.*]], double noundef [[B_COERCE0:%.*]], double noundef [[B_COERCE1:%.*]], float noundef [[C_COERCE0:%.*]], float noundef [[C_COERCE1:%.*]]) addrspace(1) #[[ATTR0]] { +// AVRFP64-NEXT: entry: +// AVRFP64-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 1 +// AVRFP64-NEXT: [[A:%.*]] = alloca { float, float }, align 1 +// AVRFP64-NEXT: [[B:%.*]] = alloca { double, double }, align 1 +// AVRFP64-NEXT: [[C:%.*]] = alloca { float, float }, align 1 +// AVRFP64-NEXT: [[TMP0:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP64-NEXT: store float [[A_COERCE0]], ptr [[TMP0]], align 1 +// AVRFP64-NEXT: [[TMP1:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP64-NEXT: store float [[A_COERCE1]], ptr [[TMP1]], align 1 +// AVRFP64-NEXT: [[TMP2:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// AVRFP64-NEXT: store double [[B_COERCE0]], ptr [[TMP2]], align 1 +// AVRFP64-NEXT: [[TMP3:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// AVRFP64-NEXT: store double [[B_COERCE1]], ptr [[TMP3]], align 1 +// AVRFP64-NEXT: [[TMP4:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 0 +// AVRFP64-NEXT: store float [[C_COERCE0]], ptr [[TMP4]], align 1 +// AVRFP64-NEXT: [[TMP5:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 1 +// AVRFP64-NEXT: store float [[C_COERCE1]], ptr [[TMP5]], align 1 +// AVRFP64-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 0 +// AVRFP64-NEXT: [[B_REAL:%.*]] = load double, ptr [[B_REALP]], align 1 +// AVRFP64-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { double, double }, ptr [[B]], i32 0, i32 1 +// AVRFP64-NEXT: [[B_IMAG:%.*]] = load double, ptr [[B_IMAGP]], align 1 +// AVRFP64-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 0 +// AVRFP64-NEXT: [[C_REAL:%.*]] = load float, ptr [[C_REALP]], align 1 +// AVRFP64-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 1 +// AVRFP64-NEXT: [[C_IMAG:%.*]] = load float, ptr [[C_IMAGP]], align 1 +// AVRFP64-NEXT: [[CONV:%.*]] = fpext float [[C_REAL]] to double +// AVRFP64-NEXT: [[CONV1:%.*]] = fpext float [[C_IMAG]] to double +// AVRFP64-NEXT: [[TMP6:%.*]] = call addrspace(1) double @llvm.fabs.f64(double [[CONV]]) +// AVRFP64-NEXT: [[TMP7:%.*]] = call addrspace(1) double @llvm.fabs.f64(double [[CONV1]]) +// AVRFP64-NEXT: [[ABS_CMP:%.*]] = fcmp ugt double [[TMP6]], [[TMP7]] +// AVRFP64-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// AVRFP64: abs_rhsr_greater_or_equal_abs_rhsi: +// AVRFP64-NEXT: [[TMP8:%.*]] = fdiv double [[CONV1]], [[CONV]] +// AVRFP64-NEXT: [[TMP9:%.*]] = fmul double [[TMP8]], [[CONV1]] +// AVRFP64-NEXT: [[TMP10:%.*]] = fadd double [[CONV]], [[TMP9]] +// AVRFP64-NEXT: [[TMP11:%.*]] = fmul double [[B_IMAG]], [[TMP8]] +// AVRFP64-NEXT: [[TMP12:%.*]] = fadd double [[B_REAL]], [[TMP11]] +// AVRFP64-NEXT: [[TMP13:%.*]] = fdiv double [[TMP12]], [[TMP10]] +// AVRFP64-NEXT: [[TMP14:%.*]] = fmul double [[B_REAL]], [[TMP8]] +// AVRFP64-NEXT: [[TMP15:%.*]] = fsub double [[B_IMAG]], [[TMP14]] +// AVRFP64-NEXT: [[TMP16:%.*]] = fdiv double [[TMP15]], [[TMP10]] +// AVRFP64-NEXT: br label [[COMPLEX_DIV:%.*]] +// AVRFP64: abs_rhsr_less_than_abs_rhsi: +// AVRFP64-NEXT: [[TMP17:%.*]] = fdiv double [[CONV]], [[CONV1]] +// AVRFP64-NEXT: [[TMP18:%.*]] = fmul double [[TMP17]], [[CONV]] +// AVRFP64-NEXT: [[TMP19:%.*]] = fadd double [[CONV1]], [[TMP18]] +// AVRFP64-NEXT: [[TMP20:%.*]] = fmul double [[B_REAL]], [[TMP17]] +// AVRFP64-NEXT: [[TMP21:%.*]] = fadd double [[TMP20]], [[B_IMAG]] +// AVRFP64-NEXT: [[TMP22:%.*]] = fdiv double [[TMP21]], [[TMP19]] +// AVRFP64-NEXT: [[TMP23:%.*]] = fmul double [[B_IMAG]], [[TMP17]] +// AVRFP64-NEXT: [[TMP24:%.*]] = fsub double [[TMP23]], [[B_REAL]] +// AVRFP64-NEXT: [[TMP25:%.*]] = fdiv double [[TMP24]], [[TMP19]] +// AVRFP64-NEXT: br label [[COMPLEX_DIV]] +// AVRFP64: complex_div: +// AVRFP64-NEXT: [[TMP26:%.*]] = phi double [ [[TMP13]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP22]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// AVRFP64-NEXT: [[TMP27:%.*]] = phi double [ [[TMP16]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP25]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// AVRFP64-NEXT: [[CONV2:%.*]] = fptrunc double [[TMP26]] to float +// AVRFP64-NEXT: [[CONV3:%.*]] = fptrunc double [[TMP27]] to float +// AVRFP64-NEXT: [[EXT:%.*]] = fpext float [[CONV2]] to double +// AVRFP64-NEXT: [[EXT4:%.*]] = fpext float [[CONV3]] to double +// AVRFP64-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// AVRFP64-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 1 +// AVRFP64-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// AVRFP64-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 1 +// AVRFP64-NEXT: [[EXT5:%.*]] = fpext float [[A_REAL]] to double +// AVRFP64-NEXT: [[EXT6:%.*]] = fpext float [[A_IMAG]] to double +// AVRFP64-NEXT: [[TMP28:%.*]] = fmul double [[EXT]], [[EXT5]] +// AVRFP64-NEXT: [[TMP29:%.*]] = fmul double [[EXT4]], [[EXT6]] +// AVRFP64-NEXT: [[TMP30:%.*]] = fadd double [[TMP28]], [[TMP29]] +// AVRFP64-NEXT: [[TMP31:%.*]] = fmul double [[EXT5]], [[EXT5]] +// AVRFP64-NEXT: [[TMP32:%.*]] = fmul double [[EXT6]], [[EXT6]] +// AVRFP64-NEXT: [[TMP33:%.*]] = fadd double [[TMP31]], [[TMP32]] +// AVRFP64-NEXT: [[TMP34:%.*]] = fmul double [[EXT4]], [[EXT5]] +// AVRFP64-NEXT: [[TMP35:%.*]] = fmul double [[EXT]], [[EXT6]] +// AVRFP64-NEXT: [[TMP36:%.*]] = fsub double [[TMP34]], [[TMP35]] +// AVRFP64-NEXT: [[TMP37:%.*]] = fdiv double [[TMP30]], [[TMP33]] +// AVRFP64-NEXT: [[TMP38:%.*]] = fdiv double [[TMP36]], [[TMP33]] +// AVRFP64-NEXT: [[UNPROMOTION:%.*]] = fptrunc double [[TMP37]] to float +// AVRFP64-NEXT: [[UNPROMOTION7:%.*]] = fptrunc double [[TMP38]] to float +// AVRFP64-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// AVRFP64-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// AVRFP64-NEXT: store float [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 1 +// AVRFP64-NEXT: store float [[UNPROMOTION7]], ptr [[RETVAL_IMAGP]], align 1 +// AVRFP64-NEXT: [[TMP39:%.*]] = load { float, float }, ptr [[RETVAL]], align 1 +// AVRFP64-NEXT: ret { float, float } [[TMP39]] +// +// BASIC_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x float> @f1( +// BASIC_FAST-SAME: <2 x float> noundef nofpclass(nan inf) [[A_COERCE:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]], <2 x float> noundef nofpclass(nan inf) [[C_COERCE:%.*]]) #[[ATTR0]] { +// BASIC_FAST-NEXT: entry: +// BASIC_FAST-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// BASIC_FAST-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// BASIC_FAST-NEXT: [[C:%.*]] = alloca { float, float }, align 4 +// BASIC_FAST-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// BASIC_FAST-NEXT: store <2 x float> [[C_COERCE]], ptr [[C]], align 4 +// BASIC_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// BASIC_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// BASIC_FAST-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[C_REAL:%.*]] = load float, ptr [[C_REALP]], align 4 +// BASIC_FAST-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[C_IMAG:%.*]] = load float, ptr [[C_IMAGP]], align 4 +// BASIC_FAST-NEXT: [[CONV:%.*]] = fpext float [[C_REAL]] to x86_fp80 +// BASIC_FAST-NEXT: [[CONV1:%.*]] = fpext float [[C_IMAG]] to x86_fp80 +// BASIC_FAST-NEXT: [[TMP0:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[CONV]] +// BASIC_FAST-NEXT: [[TMP1:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[CONV1]] +// BASIC_FAST-NEXT: [[TMP2:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP0]], [[TMP1]] +// BASIC_FAST-NEXT: [[TMP3:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV]], [[CONV]] +// BASIC_FAST-NEXT: [[TMP4:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV1]], [[CONV1]] +// BASIC_FAST-NEXT: [[TMP5:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP3]], [[TMP4]] +// BASIC_FAST-NEXT: [[TMP6:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[CONV]] +// BASIC_FAST-NEXT: [[TMP7:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[CONV1]] +// BASIC_FAST-NEXT: [[TMP8:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP6]], [[TMP7]] +// BASIC_FAST-NEXT: [[TMP9:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP2]], [[TMP5]] +// BASIC_FAST-NEXT: [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP8]], [[TMP5]] +// BASIC_FAST-NEXT: [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP9]] to float +// BASIC_FAST-NEXT: [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP10]] to float +// BASIC_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// BASIC_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// BASIC_FAST-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// BASIC_FAST-NEXT: [[TMP11:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[CONV2]], [[A_REAL]] +// BASIC_FAST-NEXT: [[TMP12:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[CONV3]], [[A_IMAG]] +// BASIC_FAST-NEXT: [[TMP13:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP11]], [[TMP12]] +// BASIC_FAST-NEXT: [[TMP14:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_REAL]], [[A_REAL]] +// BASIC_FAST-NEXT: [[TMP15:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[A_IMAG]], [[A_IMAG]] +// BASIC_FAST-NEXT: [[TMP16:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP14]], [[TMP15]] +// BASIC_FAST-NEXT: [[TMP17:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[CONV3]], [[A_REAL]] +// BASIC_FAST-NEXT: [[TMP18:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[CONV2]], [[A_IMAG]] +// BASIC_FAST-NEXT: [[TMP19:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[TMP17]], [[TMP18]] +// BASIC_FAST-NEXT: [[TMP20:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP13]], [[TMP16]] +// BASIC_FAST-NEXT: [[TMP21:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP19]], [[TMP16]] +// BASIC_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// BASIC_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// BASIC_FAST-NEXT: store float [[TMP20]], ptr [[RETVAL_REALP]], align 4 +// BASIC_FAST-NEXT: store float [[TMP21]], ptr [[RETVAL_IMAGP]], align 4 +// BASIC_FAST-NEXT: [[TMP22:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// BASIC_FAST-NEXT: ret <2 x float> [[TMP22]] +// +// FULL_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x float> @f1( +// FULL_FAST-SAME: <2 x float> noundef nofpclass(nan inf) [[A_COERCE:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]], <2 x float> noundef nofpclass(nan inf) [[C_COERCE:%.*]]) #[[ATTR0]] { +// FULL_FAST-NEXT: entry: +// FULL_FAST-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// FULL_FAST-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// FULL_FAST-NEXT: [[C:%.*]] = alloca { float, float }, align 4 +// FULL_FAST-NEXT: [[COERCE:%.*]] = alloca { float, float }, align 4 +// FULL_FAST-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// FULL_FAST-NEXT: store <2 x float> [[C_COERCE]], ptr [[C]], align 4 +// FULL_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// FULL_FAST-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// FULL_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// FULL_FAST-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// FULL_FAST-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 0 +// FULL_FAST-NEXT: [[C_REAL:%.*]] = load float, ptr [[C_REALP]], align 4 +// FULL_FAST-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 1 +// FULL_FAST-NEXT: [[C_IMAG:%.*]] = load float, ptr [[C_IMAGP]], align 4 +// FULL_FAST-NEXT: [[CONV:%.*]] = fpext float [[C_REAL]] to x86_fp80 +// FULL_FAST-NEXT: [[CONV1:%.*]] = fpext float [[C_IMAG]] to x86_fp80 +// FULL_FAST-NEXT: [[CALL:%.*]] = call { x86_fp80, x86_fp80 } @__divxc3(x86_fp80 noundef nofpclass(nan inf) [[B_REAL]], x86_fp80 noundef nofpclass(nan inf) [[B_IMAG]], x86_fp80 noundef nofpclass(nan inf) [[CONV]], x86_fp80 noundef nofpclass(nan inf) [[CONV1]]) #[[ATTR2]] +// FULL_FAST-NEXT: [[TMP0:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 0 +// FULL_FAST-NEXT: [[TMP1:%.*]] = extractvalue { x86_fp80, x86_fp80 } [[CALL]], 1 +// FULL_FAST-NEXT: [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP0]] to float +// FULL_FAST-NEXT: [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP1]] to float +// FULL_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// FULL_FAST-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// FULL_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// FULL_FAST-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// FULL_FAST-NEXT: [[CALL4:%.*]] = call reassoc nnan ninf nsz arcp afn nofpclass(nan inf) <2 x float> @__divsc3(float noundef nofpclass(nan inf) [[CONV2]], float noundef nofpclass(nan inf) [[CONV3]], float noundef nofpclass(nan inf) [[A_REAL]], float noundef nofpclass(nan inf) [[A_IMAG]]) #[[ATTR2]] +// FULL_FAST-NEXT: store <2 x float> [[CALL4]], ptr [[COERCE]], align 4 +// FULL_FAST-NEXT: [[COERCE_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 0 +// FULL_FAST-NEXT: [[COERCE_REAL:%.*]] = load float, ptr [[COERCE_REALP]], align 4 +// FULL_FAST-NEXT: [[COERCE_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[COERCE]], i32 0, i32 1 +// FULL_FAST-NEXT: [[COERCE_IMAG:%.*]] = load float, ptr [[COERCE_IMAGP]], align 4 +// FULL_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// FULL_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// FULL_FAST-NEXT: store float [[COERCE_REAL]], ptr [[RETVAL_REALP]], align 4 +// FULL_FAST-NEXT: store float [[COERCE_IMAG]], ptr [[RETVAL_IMAGP]], align 4 +// FULL_FAST-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// FULL_FAST-NEXT: ret <2 x float> [[TMP2]] +// +// IMPRVD_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x float> @f1( +// IMPRVD_FAST-SAME: <2 x float> noundef nofpclass(nan inf) [[A_COERCE:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]], <2 x float> noundef nofpclass(nan inf) [[C_COERCE:%.*]]) #[[ATTR0]] { +// IMPRVD_FAST-NEXT: entry: +// IMPRVD_FAST-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// IMPRVD_FAST-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// IMPRVD_FAST-NEXT: [[C:%.*]] = alloca { float, float }, align 4 +// IMPRVD_FAST-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// IMPRVD_FAST-NEXT: store <2 x float> [[C_COERCE]], ptr [[C]], align 4 +// IMPRVD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// IMPRVD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// IMPRVD_FAST-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[C_REAL:%.*]] = load float, ptr [[C_REALP]], align 4 +// IMPRVD_FAST-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[C_IMAG:%.*]] = load float, ptr [[C_IMAGP]], align 4 +// IMPRVD_FAST-NEXT: [[CONV:%.*]] = fpext float [[C_REAL]] to x86_fp80 +// IMPRVD_FAST-NEXT: [[CONV1:%.*]] = fpext float [[C_IMAG]] to x86_fp80 +// IMPRVD_FAST-NEXT: [[TMP0:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV]]) +// IMPRVD_FAST-NEXT: [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV1]]) +// IMPRVD_FAST-NEXT: [[ABS_CMP:%.*]] = fcmp reassoc nnan ninf nsz arcp afn ugt x86_fp80 [[TMP0]], [[TMP1]] +// IMPRVD_FAST-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// IMPRVD_FAST: abs_rhsr_greater_or_equal_abs_rhsi: +// IMPRVD_FAST-NEXT: [[TMP2:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV1]], [[CONV]] +// IMPRVD_FAST-NEXT: [[TMP3:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP2]], [[CONV1]] +// IMPRVD_FAST-NEXT: [[TMP4:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV]], [[TMP3]] +// IMPRVD_FAST-NEXT: [[TMP5:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[TMP2]] +// IMPRVD_FAST-NEXT: [[TMP6:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[TMP5]] +// IMPRVD_FAST-NEXT: [[TMP7:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP6]], [[TMP4]] +// IMPRVD_FAST-NEXT: [[TMP8:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[TMP2]] +// IMPRVD_FAST-NEXT: [[TMP9:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[TMP8]] +// IMPRVD_FAST-NEXT: [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP9]], [[TMP4]] +// IMPRVD_FAST-NEXT: br label [[COMPLEX_DIV:%.*]] +// IMPRVD_FAST: abs_rhsr_less_than_abs_rhsi: +// IMPRVD_FAST-NEXT: [[TMP11:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV]], [[CONV1]] +// IMPRVD_FAST-NEXT: [[TMP12:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP11]], [[CONV]] +// IMPRVD_FAST-NEXT: [[TMP13:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV1]], [[TMP12]] +// IMPRVD_FAST-NEXT: [[TMP14:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[TMP11]] +// IMPRVD_FAST-NEXT: [[TMP15:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP14]], [[B_IMAG]] +// IMPRVD_FAST-NEXT: [[TMP16:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP15]], [[TMP13]] +// IMPRVD_FAST-NEXT: [[TMP17:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[TMP11]] +// IMPRVD_FAST-NEXT: [[TMP18:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP17]], [[B_REAL]] +// IMPRVD_FAST-NEXT: [[TMP19:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP18]], [[TMP13]] +// IMPRVD_FAST-NEXT: br label [[COMPLEX_DIV]] +// IMPRVD_FAST: complex_div: +// IMPRVD_FAST-NEXT: [[TMP20:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD_FAST-NEXT: [[TMP21:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD_FAST-NEXT: [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP20]] to float +// IMPRVD_FAST-NEXT: [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP21]] to float +// IMPRVD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// IMPRVD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// IMPRVD_FAST-NEXT: [[TMP22:%.*]] = call reassoc nnan ninf nsz arcp afn float @llvm.fabs.f32(float [[A_REAL]]) +// IMPRVD_FAST-NEXT: [[TMP23:%.*]] = call reassoc nnan ninf nsz arcp afn float @llvm.fabs.f32(float [[A_IMAG]]) +// IMPRVD_FAST-NEXT: [[ABS_CMP4:%.*]] = fcmp reassoc nnan ninf nsz arcp afn ugt float [[TMP22]], [[TMP23]] +// IMPRVD_FAST-NEXT: br i1 [[ABS_CMP4]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI5:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI6:%.*]] +// IMPRVD_FAST: abs_rhsr_greater_or_equal_abs_rhsi5: +// IMPRVD_FAST-NEXT: [[TMP24:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[A_IMAG]], [[A_REAL]] +// IMPRVD_FAST-NEXT: [[TMP25:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[TMP24]], [[A_IMAG]] +// IMPRVD_FAST-NEXT: [[TMP26:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[A_REAL]], [[TMP25]] +// IMPRVD_FAST-NEXT: [[TMP27:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[CONV3]], [[TMP24]] +// IMPRVD_FAST-NEXT: [[TMP28:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[CONV2]], [[TMP27]] +// IMPRVD_FAST-NEXT: [[TMP29:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP28]], [[TMP26]] +// IMPRVD_FAST-NEXT: [[TMP30:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[CONV2]], [[TMP24]] +// IMPRVD_FAST-NEXT: [[TMP31:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[CONV3]], [[TMP30]] +// IMPRVD_FAST-NEXT: [[TMP32:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP31]], [[TMP26]] +// IMPRVD_FAST-NEXT: br label [[COMPLEX_DIV7:%.*]] +// IMPRVD_FAST: abs_rhsr_less_than_abs_rhsi6: +// IMPRVD_FAST-NEXT: [[TMP33:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[A_REAL]], [[A_IMAG]] +// IMPRVD_FAST-NEXT: [[TMP34:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[TMP33]], [[A_REAL]] +// IMPRVD_FAST-NEXT: [[TMP35:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[A_IMAG]], [[TMP34]] +// IMPRVD_FAST-NEXT: [[TMP36:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[CONV2]], [[TMP33]] +// IMPRVD_FAST-NEXT: [[TMP37:%.*]] = fadd reassoc nnan ninf nsz arcp afn float [[TMP36]], [[CONV3]] +// IMPRVD_FAST-NEXT: [[TMP38:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP37]], [[TMP35]] +// IMPRVD_FAST-NEXT: [[TMP39:%.*]] = fmul reassoc nnan ninf nsz arcp afn float [[CONV3]], [[TMP33]] +// IMPRVD_FAST-NEXT: [[TMP40:%.*]] = fsub reassoc nnan ninf nsz arcp afn float [[TMP39]], [[CONV2]] +// IMPRVD_FAST-NEXT: [[TMP41:%.*]] = fdiv reassoc nnan ninf nsz arcp afn float [[TMP40]], [[TMP35]] +// IMPRVD_FAST-NEXT: br label [[COMPLEX_DIV7]] +// IMPRVD_FAST: complex_div7: +// IMPRVD_FAST-NEXT: [[TMP42:%.*]] = phi reassoc nnan ninf nsz arcp afn float [ [[TMP29]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI5]] ], [ [[TMP38]], [[ABS_RHSR_LESS_THAN_ABS_RHSI6]] ] +// IMPRVD_FAST-NEXT: [[TMP43:%.*]] = phi reassoc nnan ninf nsz arcp afn float [ [[TMP32]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI5]] ], [ [[TMP41]], [[ABS_RHSR_LESS_THAN_ABS_RHSI6]] ] +// IMPRVD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD_FAST-NEXT: store float [[TMP42]], ptr [[RETVAL_REALP]], align 4 +// IMPRVD_FAST-NEXT: store float [[TMP43]], ptr [[RETVAL_IMAGP]], align 4 +// IMPRVD_FAST-NEXT: [[TMP44:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// IMPRVD_FAST-NEXT: ret <2 x float> [[TMP44]] +// +// PRMTD_FAST-LABEL: define dso_local nofpclass(nan inf) <2 x float> @f1( +// PRMTD_FAST-SAME: <2 x float> noundef nofpclass(nan inf) [[A_COERCE:%.*]], ptr noundef byval({ x86_fp80, x86_fp80 }) align 16 [[B:%.*]], <2 x float> noundef nofpclass(nan inf) [[C_COERCE:%.*]]) #[[ATTR0]] { +// PRMTD_FAST-NEXT: entry: +// PRMTD_FAST-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// PRMTD_FAST-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// PRMTD_FAST-NEXT: [[C:%.*]] = alloca { float, float }, align 4 +// PRMTD_FAST-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// PRMTD_FAST-NEXT: store <2 x float> [[C_COERCE]], ptr [[C]], align 4 +// PRMTD_FAST-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[B_REAL:%.*]] = load x86_fp80, ptr [[B_REALP]], align 16 +// PRMTD_FAST-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { x86_fp80, x86_fp80 }, ptr [[B]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[B_IMAG:%.*]] = load x86_fp80, ptr [[B_IMAGP]], align 16 +// PRMTD_FAST-NEXT: [[C_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[C_REAL:%.*]] = load float, ptr [[C_REALP]], align 4 +// PRMTD_FAST-NEXT: [[C_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[C]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[C_IMAG:%.*]] = load float, ptr [[C_IMAGP]], align 4 +// PRMTD_FAST-NEXT: [[CONV:%.*]] = fpext float [[C_REAL]] to x86_fp80 +// PRMTD_FAST-NEXT: [[CONV1:%.*]] = fpext float [[C_IMAG]] to x86_fp80 +// PRMTD_FAST-NEXT: [[TMP0:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV]]) +// PRMTD_FAST-NEXT: [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn x86_fp80 @llvm.fabs.f80(x86_fp80 [[CONV1]]) +// PRMTD_FAST-NEXT: [[ABS_CMP:%.*]] = fcmp reassoc nnan ninf nsz arcp afn ugt x86_fp80 [[TMP0]], [[TMP1]] +// PRMTD_FAST-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// PRMTD_FAST: abs_rhsr_greater_or_equal_abs_rhsi: +// PRMTD_FAST-NEXT: [[TMP2:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV1]], [[CONV]] +// PRMTD_FAST-NEXT: [[TMP3:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP2]], [[CONV1]] +// PRMTD_FAST-NEXT: [[TMP4:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV]], [[TMP3]] +// PRMTD_FAST-NEXT: [[TMP5:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[TMP2]] +// PRMTD_FAST-NEXT: [[TMP6:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[TMP5]] +// PRMTD_FAST-NEXT: [[TMP7:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP6]], [[TMP4]] +// PRMTD_FAST-NEXT: [[TMP8:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[TMP2]] +// PRMTD_FAST-NEXT: [[TMP9:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[TMP8]] +// PRMTD_FAST-NEXT: [[TMP10:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP9]], [[TMP4]] +// PRMTD_FAST-NEXT: br label [[COMPLEX_DIV:%.*]] +// PRMTD_FAST: abs_rhsr_less_than_abs_rhsi: +// PRMTD_FAST-NEXT: [[TMP11:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV]], [[CONV1]] +// PRMTD_FAST-NEXT: [[TMP12:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP11]], [[CONV]] +// PRMTD_FAST-NEXT: [[TMP13:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[CONV1]], [[TMP12]] +// PRMTD_FAST-NEXT: [[TMP14:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_REAL]], [[TMP11]] +// PRMTD_FAST-NEXT: [[TMP15:%.*]] = fadd reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP14]], [[B_IMAG]] +// PRMTD_FAST-NEXT: [[TMP16:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP15]], [[TMP13]] +// PRMTD_FAST-NEXT: [[TMP17:%.*]] = fmul reassoc nnan ninf nsz arcp afn x86_fp80 [[B_IMAG]], [[TMP11]] +// PRMTD_FAST-NEXT: [[TMP18:%.*]] = fsub reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP17]], [[B_REAL]] +// PRMTD_FAST-NEXT: [[TMP19:%.*]] = fdiv reassoc nnan ninf nsz arcp afn x86_fp80 [[TMP18]], [[TMP13]] +// PRMTD_FAST-NEXT: br label [[COMPLEX_DIV]] +// PRMTD_FAST: complex_div: +// PRMTD_FAST-NEXT: [[TMP20:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// PRMTD_FAST-NEXT: [[TMP21:%.*]] = phi reassoc nnan ninf nsz arcp afn x86_fp80 [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// PRMTD_FAST-NEXT: [[CONV2:%.*]] = fptrunc x86_fp80 [[TMP20]] to float +// PRMTD_FAST-NEXT: [[CONV3:%.*]] = fptrunc x86_fp80 [[TMP21]] to float +// PRMTD_FAST-NEXT: [[EXT:%.*]] = fpext float [[CONV2]] to double +// PRMTD_FAST-NEXT: [[EXT4:%.*]] = fpext float [[CONV3]] to double +// PRMTD_FAST-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// PRMTD_FAST-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// PRMTD_FAST-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// PRMTD_FAST-NEXT: [[EXT5:%.*]] = fpext float [[A_REAL]] to double +// PRMTD_FAST-NEXT: [[EXT6:%.*]] = fpext float [[A_IMAG]] to double +// PRMTD_FAST-NEXT: [[TMP22:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[EXT]], [[EXT5]] +// PRMTD_FAST-NEXT: [[TMP23:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[EXT4]], [[EXT6]] +// PRMTD_FAST-NEXT: [[TMP24:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[TMP22]], [[TMP23]] +// PRMTD_FAST-NEXT: [[TMP25:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[EXT5]], [[EXT5]] +// PRMTD_FAST-NEXT: [[TMP26:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[EXT6]], [[EXT6]] +// PRMTD_FAST-NEXT: [[TMP27:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[TMP25]], [[TMP26]] +// PRMTD_FAST-NEXT: [[TMP28:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[EXT4]], [[EXT5]] +// PRMTD_FAST-NEXT: [[TMP29:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[EXT]], [[EXT6]] +// PRMTD_FAST-NEXT: [[TMP30:%.*]] = fsub reassoc nnan ninf nsz arcp afn double [[TMP28]], [[TMP29]] +// PRMTD_FAST-NEXT: [[TMP31:%.*]] = fdiv reassoc nnan ninf nsz arcp afn double [[TMP24]], [[TMP27]] +// PRMTD_FAST-NEXT: [[TMP32:%.*]] = fdiv reassoc nnan ninf nsz arcp afn double [[TMP30]], [[TMP27]] +// PRMTD_FAST-NEXT: [[UNPROMOTION:%.*]] = fptrunc double [[TMP31]] to float +// PRMTD_FAST-NEXT: [[UNPROMOTION7:%.*]] = fptrunc double [[TMP32]] to float +// PRMTD_FAST-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// PRMTD_FAST-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// PRMTD_FAST-NEXT: store float [[UNPROMOTION]], ptr [[RETVAL_REALP]], align 4 +// PRMTD_FAST-NEXT: store float [[UNPROMOTION7]], ptr [[RETVAL_IMAGP]], align 4 +// PRMTD_FAST-NEXT: [[TMP33:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// PRMTD_FAST-NEXT: ret <2 x float> [[TMP33]] +// +_Complex float f1(_Complex float a, _Complex long double b, _Complex float c) { + return (_Complex float)(b / c) / a; +} diff --git a/clang/test/CodeGen/pragma-cx-limited-range.c b/clang/test/CodeGen/pragma-cx-limited-range.c index 926da8afbee5..68615348c187 100644 --- a/clang/test/CodeGen/pragma-cx-limited-range.c +++ b/clang/test/CodeGen/pragma-cx-limited-range.c @@ -2,20 +2,24 @@ // RUN: -o - | FileCheck %s --check-prefix=FULL // RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ -// RUN: -complex-range=limited -o - | FileCheck --check-prefix=LMTD %s +// RUN: -complex-range=basic -o - | FileCheck --check-prefix=BASIC %s // RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ // RUN: -fno-cx-limited-range -o - | FileCheck %s --check-prefix=FULL // RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ -// RUN: -complex-range=fortran -o - | FileCheck --check-prefix=FRTRN %s +// RUN: -complex-range=improved -o - | FileCheck --check-prefix=IMPRVD %s // RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ -// RUN: -fno-cx-fortran-rules -o - | FileCheck --check-prefix=FULL %s +// RUN: -complex-range=promoted -o - | FileCheck --check-prefix=PRMTD %s + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -complex-range=full -o - | FileCheck --check-prefix=FULL %s _Complex float pragma_on_mul(_Complex float a, _Complex float b) { #pragma STDC CX_LIMITED_RANGE ON // LABEL: define {{.*}} @pragma_on_mul( + // FULL: fmul float // FULL-NEXT: fmul float // FULL-NEXT: fmul float @@ -23,19 +27,26 @@ _Complex float pragma_on_mul(_Complex float a, _Complex float b) { // FULL-NEXT: fsub float // FULL-NEXT: fadd float - // LMTD: fmul float - // LMTD-NEXT: fmul float - // LMTD-NEXT: fmul float - // LMTD-NEXT: fmul float - // LMTD-NEXT: fsub float - // LMTD-NEXT: fadd float - - // FRTRN: fmul float - // FRTRN-NEXT: fmul float - // FRTRN-NEXT: fmul float - // FRTRN-NEXT: fmul float - // FRTRN-NEXT: fsub float - // FRTRN-NEXT: fadd float + // BASIC: fmul float + // BASIC-NEXT: fmul float + // BASIC-NEXT: fmul float + // BASIC-NEXT: fmul float + // BASIC-NEXT: fsub float + // BASIC-NEXT: fadd float + + // IMPRVD: fmul float + // IMPRVD-NEXT: fmul float + // IMPRVD-NEXT: fmul float + // IMPRVD-NEXT: fmul float + // IMPRVD-NEXT: fsub float + // IMPRVD-NEXT: fadd float + + // PRMTD: fmul float + // PRMTD-NEXT: fmul float + // PRMTD-NEXT: fmul float + // PRMTD-NEXT: fmul float + // PRMTD-NEXT: fsub float + // PRMTD-NEXT: fadd float return a * b; } @@ -43,11 +54,14 @@ _Complex float pragma_on_mul(_Complex float a, _Complex float b) { _Complex float pragma_off_mul(_Complex float a, _Complex float b) { #pragma STDC CX_LIMITED_RANGE OFF // LABEL: define {{.*}} @pragma_off_mul( + // FULL: call {{.*}} @__mulsc3 - // LMTD: call {{.*}} @__mulsc3 + // BASIC: call {{.*}} @__mulsc3 + + // IMPRVD: call {{.*}} @__mulsc3 - // FRTRN: call {{.*}} @__mulsc3 + // PRMTD: call {{.*}} @__mulsc3 return a * b; } @@ -55,6 +69,7 @@ _Complex float pragma_off_mul(_Complex float a, _Complex float b) { _Complex float pragma_on_div(_Complex float a, _Complex float b) { #pragma STDC CX_LIMITED_RANGE ON // LABEL: define {{.*}} @pragma_on_div( + // FULL: fmul float // FULL-NEXT: fmul float // FULL-NEXT: fadd float @@ -67,29 +82,45 @@ _Complex float pragma_on_div(_Complex float a, _Complex float b) { // FULL-NEXT: fdiv float // FULL: fdiv float - // LMTD: fmul float - // LMTD-NEXT: fmul float - // LMTD-NEXT: fadd float - // LMTD-NEXT: fmul float - // LMTD-NEXT: fmul float - // LMTD-NEXT: fadd float - // LMTD-NEXT: fmul float - // LMTD-NEXT: fmul float - // LMTD-NEXT: fsub float - // LMTD-NEXT: fdiv float - // LMTD-NEXT: fdiv float - - // FRTRN: fmul float - // FRTRN-NEXT: fmul float - // FRTRN-NEXT: fadd float - // FRTRN-NEXT: fmul float - // FRTRN-NEXT: fmul float - // FRTRN-NEXT: fadd float - // FRTRN-NEXT: fmul float - // FRTRN-NEXT: fmul float - // FRTRN-NEXT: fsub float - // FRTRN-NEXT: fdiv float - // FRTRN-NEXT: fdiv float + // BASIC: fmul float + // BASIC-NEXT: fmul float + // BASIC-NEXT: fadd float + // BASIC-NEXT: fmul float + // BASIC-NEXT: fmul float + // BASIC-NEXT: fadd float + // BASIC-NEXT: fmul float + // BASIC-NEXT: fmul float + // BASIC-NEXT: fsub float + // BASIC-NEXT: fdiv float + // BASIC-NEXT: fdiv float + + // IMPRVD: fmul float + // IMPRVD-NEXT: fmul float + // IMPRVD-NEXT: fadd float + // IMPRVD-NEXT: fmul float + // IMPRVD-NEXT: fmul float + // IMPRVD-NEXT: fadd float + // IMPRVD-NEXT: fmul float + // IMPRVD-NEXT: fmul float + // IMPRVD-NEXT: fsub float + // IMPRVD-NEXT: fdiv float + // IMPRVD-NEXT: fdiv float + + // PRMTD: fpext float {{.*}} to double + // PRMTD: fpext float {{.*}} to double + // PRMTD: fmul double + // PRMTD: fmul double + // PRMTD: fadd double + // PRMTD: fmul double + // PRMTD: fmul double + // PRMTD: fadd double + // PRMTD: fmul double + // PRMTD: fmul double + // PRMTD: fsub double + // PRMTD: fdiv double + // PRMTD: fdiv double + // PRMTD: fptrunc double + // PRMTD: fptrunc double return a / b; } @@ -97,11 +128,118 @@ _Complex float pragma_on_div(_Complex float a, _Complex float b) { _Complex float pragma_off_div(_Complex float a, _Complex float b) { #pragma STDC CX_LIMITED_RANGE OFF // LABEL: define {{.*}} @pragma_off_div( + // FULL: call {{.*}} @__divsc3 - // LMTD: call {{.*}} @__divsc3 + // BASIC: call {{.*}} @__divsc3 + + // IMPRVD: call {{.*}} @__divsc3 + + // PRMTD: call {{.*}} @__divdc3 + + return a / b; +} + +_Complex float pragma_default_mul(_Complex float a, _Complex float b) { +#pragma STDC CX_LIMITED_RANGE DEFAULT + // LABEL: define {{.*}} @pragma_on_mul( + + // FULL: fmul float + // FULL-NEXT: fmul float + // FULL-NEXT: fmul float + // FULL-NEXT: fmul float + // FULL-NEXT: fsub float + // FULL-NEXT: fadd float + + // BASIC: fmul float + // BASIC-NEXT: fmul float + // BASIC-NEXT: fmul float + // BASIC-NEXT: fmul float + // BASIC-NEXT: fsub float + // BASIC-NEXT: fadd float + + // IMPRVD: fmul float + // IMPRVD-NEXT: fmul float + // IMPRVD-NEXT: fmul float + // IMPRVD-NEXT: fmul float + // IMPRVD-NEXT: fsub float + // IMPRVD-NEXT: fadd float + + // PRMTD: fmul float + // PRMTD-NEXT: fmul float + // PRMTD-NEXT: fmul float + // PRMTD-NEXT: fmul float + // PRMTD-NEXT: fsub float + // PRMTD-NEXT: fadd float + + return a * b; +} +_Complex float pragma_default_div(_Complex float a, _Complex float b) { +#pragma STDC CX_LIMITED_RANGE DEFAULT + // LABEL: define {{.*}} @pragma_on_divx( + + // FULL: call {{.*}} @__divsc3 - // FRTRN: call {{.*}} @__divsc3 + // BASIC: fmul float + // BASIC-NEXT: fmul float + // BASIC-NEXT: fadd float + // BASIC-NEXT: fmul float + // BASIC-NEXT: fmul float + // BASIC-NEXT: fadd float + // BASIC-NEXT: fmul float + // BASIC-NEXT: fmul float + // BASIC-NEXT: fsub float + // BASIC-NEXT: fdiv float + // BASIC-NEXT: fdiv float + + // IMPRVD: call{{.*}}float @llvm.fabs.f32(float {{.*}}) + // IMPRVD-NEXT: call{{.*}}float @llvm.fabs.f32(float {{.*}}) + // IMPRVD-NEXT: fcmp{{.*}}ugt float {{.*}}, {{.*}} + // IMPRVD-NEXT: br i1 {{.*}}, label + // IMPRVD: abs_rhsr_greater_or_equal_abs_rhsi: + // IMPRVD-NEXT: fdiv float + // IMPRVD-NEXT: fmul float + // IMPRVD-NEXT: fadd float + // IMPRVD-NEXT: fmul float + // IMPRVD-NEXT: fadd float + // IMPRVD-NEXT: fdiv float + // IMPRVD-NEXT: fmul float + // IMPRVD-NEXT: fsub float + // IMPRVD-NEXT: fdiv float + // IMPRVD-NEXT: br label + // IMPRVD: abs_rhsr_less_than_abs_rhsi: + // IMPRVD-NEXT: fdiv float + // IMPRVD-NEXT: fmul float + // IMPRVD-NEXT: fadd float + // IMPRVD-NEXT: fmul float + // IMPRVD-NEXT: fadd float + // IMPRVD-NEXT: fdiv float + // IMPRVD-NEXT: fmul float + // IMPRVD-NEXT: fsub float + // IMPRVD-NEXT: fdiv float + + // PRMTD: load float, ptr {{.*}} + // PRMTD: fpext float {{.*}} to double + // PRMTD-NEXT: fpext float {{.*}} to double + // PRMTD-NEXT: getelementptr inbounds { float, float }, ptr {{.*}}, i32 0, i32 0 + // PRMTD-NEXT: load float, ptr {{.*}} + // PRMTD-NEXT: getelementptr inbounds { float, float }, ptr {{.*}}, i32 0, i32 1 + // PRMTD-NEXT: load float, ptr {{.*}} + // PRMTD-NEXT: fpext float {{.*}} to double + // PRMTD-NEXT: fpext float {{.*}} to double + // PRMTD-NEXT: fmul double + // PRMTD-NEXT: fmul double + // PRMTD-NEXT: fadd double + // PRMTD-NEXT: fmul double + // PRMTD-NEXT: fmul double + // PRMTD-NEXT: fadd double + // PRMTD-NEXT: fmul double + // PRMTD-NEXT: fmul double + // PRMTD-NEXT: fsub double + // PRMTD-NEXT: fdiv double + // PRMTD-NEXT: fdiv double + // PRMTD-NEXT: fptrunc double {{.*}} to float + // PRMTD-NEXT: fptrunc double {{.*}} to float return a / b; } diff --git a/clang/test/CodeGen/smiths-complex-div.c b/clang/test/CodeGen/smiths-complex-div.c index 75775675c923..5882f8b3545f 100644 --- a/clang/test/CodeGen/smiths-complex-div.c +++ b/clang/test/CodeGen/smiths-complex-div.c @@ -1,58 +1,58 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4 // RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ -// RUN: -complex-range=fortran -o - | FileCheck %s --check-prefix=FRTRN +// RUN: -complex-range=improved -o - | FileCheck %s --check-prefix=IMPRVD -// FRTRN-LABEL: define dso_local <2 x float> @div( -// FRTRN-SAME: <2 x float> noundef [[A_COERCE:%.*]], <2 x float> noundef [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { -// FRTRN-NEXT: entry: -// FRTRN-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 -// FRTRN-NEXT: [[A:%.*]] = alloca { float, float }, align 4 -// FRTRN-NEXT: [[B:%.*]] = alloca { float, float }, align 4 -// FRTRN-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 -// FRTRN-NEXT: store <2 x float> [[B_COERCE]], ptr [[B]], align 4 -// FRTRN-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 -// FRTRN-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 -// FRTRN-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 -// FRTRN-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 -// FRTRN-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 -// FRTRN-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 -// FRTRN-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 -// FRTRN-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 -// FRTRN-NEXT: [[TMP0:%.*]] = call float @llvm.fabs.f32(float [[B_REAL]]) -// FRTRN-NEXT: [[TMP1:%.*]] = call float @llvm.fabs.f32(float [[B_IMAG]]) -// FRTRN-NEXT: [[ABS_CMP:%.*]] = fcmp ugt float [[TMP0]], [[TMP1]] -// FRTRN-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] -// FRTRN: abs_rhsr_greater_or_equal_abs_rhsi: -// FRTRN-NEXT: [[TMP2:%.*]] = fdiv float [[B_IMAG]], [[B_REAL]] -// FRTRN-NEXT: [[TMP3:%.*]] = fmul float [[TMP2]], [[B_IMAG]] -// FRTRN-NEXT: [[TMP4:%.*]] = fadd float [[B_REAL]], [[TMP3]] -// FRTRN-NEXT: [[TMP5:%.*]] = fmul float [[A_IMAG]], [[TMP2]] -// FRTRN-NEXT: [[TMP6:%.*]] = fadd float [[A_REAL]], [[TMP5]] -// FRTRN-NEXT: [[TMP7:%.*]] = fdiv float [[TMP6]], [[TMP4]] -// FRTRN-NEXT: [[TMP8:%.*]] = fmul float [[A_REAL]], [[TMP2]] -// FRTRN-NEXT: [[TMP9:%.*]] = fsub float [[A_IMAG]], [[TMP8]] -// FRTRN-NEXT: [[TMP10:%.*]] = fdiv float [[TMP9]], [[TMP4]] -// FRTRN-NEXT: br label [[COMPLEX_DIV:%.*]] -// FRTRN: abs_rhsr_less_than_abs_rhsi: -// FRTRN-NEXT: [[TMP11:%.*]] = fdiv float [[B_REAL]], [[B_IMAG]] -// FRTRN-NEXT: [[TMP12:%.*]] = fmul float [[TMP11]], [[B_REAL]] -// FRTRN-NEXT: [[TMP13:%.*]] = fadd float [[B_IMAG]], [[TMP12]] -// FRTRN-NEXT: [[TMP14:%.*]] = fmul float [[A_REAL]], [[TMP11]] -// FRTRN-NEXT: [[TMP15:%.*]] = fadd float [[TMP14]], [[A_IMAG]] -// FRTRN-NEXT: [[TMP16:%.*]] = fdiv float [[TMP15]], [[TMP13]] -// FRTRN-NEXT: [[TMP17:%.*]] = fmul float [[A_IMAG]], [[TMP11]] -// FRTRN-NEXT: [[TMP18:%.*]] = fsub float [[TMP17]], [[A_REAL]] -// FRTRN-NEXT: [[TMP19:%.*]] = fdiv float [[TMP18]], [[TMP13]] -// FRTRN-NEXT: br label [[COMPLEX_DIV]] -// FRTRN: complex_div: -// FRTRN-NEXT: [[TMP20:%.*]] = phi float [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] -// FRTRN-NEXT: [[TMP21:%.*]] = phi float [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] -// FRTRN-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 -// FRTRN-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 -// FRTRN-NEXT: store float [[TMP20]], ptr [[RETVAL_REALP]], align 4 -// FRTRN-NEXT: store float [[TMP21]], ptr [[RETVAL_IMAGP]], align 4 -// FRTRN-NEXT: [[TMP22:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 -// FRTRN-NEXT: ret <2 x float> [[TMP22]] +// IMPRVD-LABEL: define dso_local <2 x float> @div( +// IMPRVD-SAME: <2 x float> noundef [[A_COERCE:%.*]], <2 x float> noundef [[B_COERCE:%.*]]) #[[ATTR0:[0-9]+]] { +// IMPRVD-NEXT: entry: +// IMPRVD-NEXT: [[RETVAL:%.*]] = alloca { float, float }, align 4 +// IMPRVD-NEXT: [[A:%.*]] = alloca { float, float }, align 4 +// IMPRVD-NEXT: [[B:%.*]] = alloca { float, float }, align 4 +// IMPRVD-NEXT: store <2 x float> [[A_COERCE]], ptr [[A]], align 4 +// IMPRVD-NEXT: store <2 x float> [[B_COERCE]], ptr [[B]], align 4 +// IMPRVD-NEXT: [[A_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 0 +// IMPRVD-NEXT: [[A_REAL:%.*]] = load float, ptr [[A_REALP]], align 4 +// IMPRVD-NEXT: [[A_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[A]], i32 0, i32 1 +// IMPRVD-NEXT: [[A_IMAG:%.*]] = load float, ptr [[A_IMAGP]], align 4 +// IMPRVD-NEXT: [[B_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 0 +// IMPRVD-NEXT: [[B_REAL:%.*]] = load float, ptr [[B_REALP]], align 4 +// IMPRVD-NEXT: [[B_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[B]], i32 0, i32 1 +// IMPRVD-NEXT: [[B_IMAG:%.*]] = load float, ptr [[B_IMAGP]], align 4 +// IMPRVD-NEXT: [[TMP0:%.*]] = call float @llvm.fabs.f32(float [[B_REAL]]) +// IMPRVD-NEXT: [[TMP1:%.*]] = call float @llvm.fabs.f32(float [[B_IMAG]]) +// IMPRVD-NEXT: [[ABS_CMP:%.*]] = fcmp ugt float [[TMP0]], [[TMP1]] +// IMPRVD-NEXT: br i1 [[ABS_CMP]], label [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI:%.*]], label [[ABS_RHSR_LESS_THAN_ABS_RHSI:%.*]] +// IMPRVD: abs_rhsr_greater_or_equal_abs_rhsi: +// IMPRVD-NEXT: [[TMP2:%.*]] = fdiv float [[B_IMAG]], [[B_REAL]] +// IMPRVD-NEXT: [[TMP3:%.*]] = fmul float [[TMP2]], [[B_IMAG]] +// IMPRVD-NEXT: [[TMP4:%.*]] = fadd float [[B_REAL]], [[TMP3]] +// IMPRVD-NEXT: [[TMP5:%.*]] = fmul float [[A_IMAG]], [[TMP2]] +// IMPRVD-NEXT: [[TMP6:%.*]] = fadd float [[A_REAL]], [[TMP5]] +// IMPRVD-NEXT: [[TMP7:%.*]] = fdiv float [[TMP6]], [[TMP4]] +// IMPRVD-NEXT: [[TMP8:%.*]] = fmul float [[A_REAL]], [[TMP2]] +// IMPRVD-NEXT: [[TMP9:%.*]] = fsub float [[A_IMAG]], [[TMP8]] +// IMPRVD-NEXT: [[TMP10:%.*]] = fdiv float [[TMP9]], [[TMP4]] +// IMPRVD-NEXT: br label [[COMPLEX_DIV:%.*]] +// IMPRVD: abs_rhsr_less_than_abs_rhsi: +// IMPRVD-NEXT: [[TMP11:%.*]] = fdiv float [[B_REAL]], [[B_IMAG]] +// IMPRVD-NEXT: [[TMP12:%.*]] = fmul float [[TMP11]], [[B_REAL]] +// IMPRVD-NEXT: [[TMP13:%.*]] = fadd float [[B_IMAG]], [[TMP12]] +// IMPRVD-NEXT: [[TMP14:%.*]] = fmul float [[A_REAL]], [[TMP11]] +// IMPRVD-NEXT: [[TMP15:%.*]] = fadd float [[TMP14]], [[A_IMAG]] +// IMPRVD-NEXT: [[TMP16:%.*]] = fdiv float [[TMP15]], [[TMP13]] +// IMPRVD-NEXT: [[TMP17:%.*]] = fmul float [[A_IMAG]], [[TMP11]] +// IMPRVD-NEXT: [[TMP18:%.*]] = fsub float [[TMP17]], [[A_REAL]] +// IMPRVD-NEXT: [[TMP19:%.*]] = fdiv float [[TMP18]], [[TMP13]] +// IMPRVD-NEXT: br label [[COMPLEX_DIV]] +// IMPRVD: complex_div: +// IMPRVD-NEXT: [[TMP20:%.*]] = phi float [ [[TMP7]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP16]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD-NEXT: [[TMP21:%.*]] = phi float [ [[TMP10]], [[ABS_RHSR_GREATER_OR_EQUAL_ABS_RHSI]] ], [ [[TMP19]], [[ABS_RHSR_LESS_THAN_ABS_RHSI]] ] +// IMPRVD-NEXT: [[RETVAL_REALP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 0 +// IMPRVD-NEXT: [[RETVAL_IMAGP:%.*]] = getelementptr inbounds { float, float }, ptr [[RETVAL]], i32 0, i32 1 +// IMPRVD-NEXT: store float [[TMP20]], ptr [[RETVAL_REALP]], align 4 +// IMPRVD-NEXT: store float [[TMP21]], ptr [[RETVAL_IMAGP]], align 4 +// IMPRVD-NEXT: [[TMP22:%.*]] = load <2 x float>, ptr [[RETVAL]], align 4 +// IMPRVD-NEXT: ret <2 x float> [[TMP22]] // _Complex float div(_Complex float a, _Complex float b) { return a / b; diff --git a/clang/test/Driver/range.c b/clang/test/Driver/range.c index 2d1fd7f9f1a9..da5748d7c723 100644 --- a/clang/test/Driver/range.c +++ b/clang/test/Driver/range.c @@ -1,16 +1,37 @@ // Test range options for complex multiplication and division. // RUN: %clang -### -target x86_64 -fcx-limited-range -c %s 2>&1 \ -// RUN: | FileCheck --check-prefix=LMTD %s +// RUN: | FileCheck --check-prefix=BASIC %s // RUN: %clang -### -target x86_64 -fno-cx-limited-range -c %s 2>&1 \ -// RUN: | FileCheck %s +// RUN: | FileCheck --check-prefix=FULL %s + +// RUN: %clang -### -target x86_64 -fcx-limited-range -fcx-fortran-rules \ +// RUN: -c %s 2>&1 | FileCheck --check-prefix=WARN1 %s + +// RUN: %clang -### -target x86_64 -fno-cx-limited-range -fcx-fortran-rules \ +// RUN: -c %s 2>&1 | FileCheck --check-prefix=WARN2 %s // RUN: %clang -### -target x86_64 -fcx-limited-range -fno-cx-limited-range \ // RUN: -c %s 2>&1 | FileCheck --check-prefix=FULL %s +// RUN: %clang -### -target x86_64 -fno-cx-limited-range -fcx-limited-range \ +// RUN: -c %s 2>&1 | FileCheck --check-prefix=BASIC %s + +// RUN: %clang -### -target x86_64 -fno-cx-limited-range -fno-cx-fortran-rules \ +// RUN: -c %s 2>&1 | FileCheck --check-prefix=FULL %s + +// RUN: %clang -### -target x86_64 -fno-cx-fortran-rules -fno-cx-limited-range \ +// RUN: -c %s 2>&1 | FileCheck --check-prefix=FULL %s + +// RUN: %clang -### -target x86_64 -fcx-limited-range -fno-cx-fortran-rules \ +// RUN: -c %s 2>&1 | FileCheck --check-prefix=WARN4 %s + // RUN: %clang -### -target x86_64 -fcx-fortran-rules -c %s 2>&1 \ -// RUN: | FileCheck --check-prefix=FRTRN %s +// RUN: | FileCheck --check-prefix=IMPRVD %s + +// RUN: %clang -### -target x86_64 -fno-cx-fortran-rules -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=FULL %s // RUN: %clang -### -target x86_64 -fcx-fortran-rules -c %s 2>&1 \ // RUN: -fno-cx-fortran-rules | FileCheck --check-prefix=FULL %s @@ -32,34 +53,148 @@ // RUN: %clang -### -target x86_64 -fcx-fortran-rules \ // RUN: -fcx-limited-range -c %s 2>&1 \ -// RUN: | FileCheck --check-prefix=WARN2 %s +// RUN: | FileCheck --check-prefix=WARN20 %s // RUN: %clang -### -target x86_64 -ffast-math -c %s 2>&1 \ -// RUN: | FileCheck --check-prefix=LMTD %s +// RUN: | FileCheck --check-prefix=BASIC %s // RUN: %clang -### -target x86_64 -ffast-math -fcx-limited-range -c %s 2>&1 \ -// RUN: | FileCheck --check-prefix=LMTD %s +// RUN: | FileCheck --check-prefix=BASIC %s // RUN: %clang -### -target x86_64 -fcx-limited-range -ffast-math -c %s 2>&1 \ -// RUN: | FileCheck --check-prefix=LMTD %s +// RUN: | FileCheck --check-prefix=BASIC %s // RUN: %clang -### -target x86_64 -ffast-math -fno-cx-limited-range \ // RUN: -c %s 2>&1 | FileCheck --check-prefix=FULL %s +// RUN: not %clang -### -target x86_64 -fcomplex-arithmetic=foo -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=ERR %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=basic -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=BASIC %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=improved -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=IMPRVD %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=promoted -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=PRMTD %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=full -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=FULL %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=basic \ +// RUN: -fcx-limited-range -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=BASIC %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=basic \ +// RUN: -fcomplex-arithmetic=improved -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=IMPRVD %s + +// RUN: %clang -### -target x86_64 -fcx-limited-range \ +// RUN: -fcomplex-arithmetic=improved -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=WARN6 %s + +// RUN: %clang -### -target x86_64 -fcx-fortran-rules \ +// RUN: -fcomplex-arithmetic=basic -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=WARN7 %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=basic \ +// RUN: -fcomplex-arithmetic=full -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=FULL %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=basic \ +// RUN: -fcomplex-arithmetic=promoted -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=PRMTD %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=improved \ +// RUN: -fcomplex-arithmetic=basic -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=BASIC %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=improved \ +// RUN: -fcomplex-arithmetic=full -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=FULL %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=improved \ +// RUN: -fcomplex-arithmetic=promoted -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=PRMTD %s + + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=promoted \ +// RUN: -fcomplex-arithmetic=basic -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=BASIC %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=promoted \ +// RUN: -fcx-limited-range -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=WARN14 %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=promoted \ +// RUN: -fcomplex-arithmetic=improved -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=IMPRVD %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=promoted \ +// RUN: -fcomplex-arithmetic=full -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=FULL %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=full \ +// RUN: -fcomplex-arithmetic=basic -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=BASIC %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=full \ +// RUN: -ffast-math -c %s 2>&1 | FileCheck --check-prefix=WARN17 %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=full \ +// RUN: -fcomplex-arithmetic=improved -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=IMPRVD %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=full \ +// RUN: -fcomplex-arithmetic=promoted -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=PRMTD %s + +// RUN: %clang -### -target x86_64 -ffast-math -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=BASIC %s + +// RUN: %clang -### -target x86_64 -ffast-math -fcx-limited-range -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=BASIC %s + +// RUN: %clang -### -target x86_64 -fcx-limited-range -ffast-math -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=BASIC %s + +// RUN: %clang -### -target x86_64 -ffast-math -fno-cx-limited-range -c %s \ +// RUN: 2>&1 | FileCheck --check-prefix=FULL %s + +// RUN: %clang -### -target x86_64 -ffast-math -fcomplex-arithmetic=basic -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=BASIC %s + +// RUN: %clang -### -target x86_64 -fcomplex-arithmetic=basic -ffast-math -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=BASIC %s + // RUN: %clang -### -Werror -target x86_64 -fcx-limited-range -c %s 2>&1 \ -// RUN: | FileCheck --check-prefix=LMTD %s +// RUN: | FileCheck --check-prefix=BASIC %s + +// RUN: %clang -### -target x86_64 -ffast-math -fcomplex-arithmetic=full -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=FULL %s -// RUN: %clang -### -Werror -target x86_64 -fcx-fortran-rules -c %s 2>&1 \ -// RUN: | FileCheck --check-prefix=FRTRN %s +// RUN: %clang -### -target x86_64 -ffast-math -fcomplex-arithmetic=basic -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=BASIC %s -// LMTD: -complex-range=limited +// BASIC: -complex-range=basic // FULL: -complex-range=full -// LMTD-NOT: -complex-range=fortran -// CHECK-NOT: -complex-range=limited -// FRTRN: -complex-range=fortran -// FRTRN-NOT: -complex-range=limited -// CHECK-NOT: -complex-range=fortran +// PRMTD: -complex-range=promoted +// BASIC-NOT: -complex-range=improved +// CHECK-NOT: -complex-range=basic +// IMPRVD: -complex-range=improved +// IMPRVD-NOT: -complex-range=basic +// CHECK-NOT: -complex-range=improved + // WARN1: warning: overriding '-fcx-limited-range' option with '-fcx-fortran-rules' [-Woverriding-option] -// WARN2: warning: overriding '-fcx-fortran-rules' option with '-fcx-limited-range' [-Woverriding-option] +// WARN2: warning: overriding '-fno-cx-limited-range' option with '-fcx-fortran-rules' [-Woverriding-option] // WARN3: warning: overriding '-fcx-fortran-rules' option with '-fno-cx-limited-range' [-Woverriding-option] // WARN4: warning: overriding '-fcx-limited-range' option with '-fno-cx-fortran-rules' [-Woverriding-option] +// WARN5: warning: overriding '-fcomplex-arithmetic=basic' option with '-fcomplex-arithmetic=improved' [-Woverriding-option] +// WARN6: warning: overriding '-fcx-limited-range' option with '-fcomplex-arithmetic=improved' [-Woverriding-option] +// WARN7: warning: overriding '-fcx-fortran-rules' option with '-fcomplex-arithmetic=basic' [-Woverriding-option] +// WARN14: overriding '-complex-range=promoted' option with '-fcx-limited-range' [-Woverriding-option] +// WARN17: warning: overriding '-fcomplex-arithmetic=full' option with '-fcomplex-arithmetic=basic' [-Woverriding-option] +// WARN20: warning: overriding '-fcx-fortran-rules' option with '-fcx-limited-range' [-Woverriding-option] + +// ERR: error: unsupported argument 'foo' to option '-fcomplex-arithmetic=' -- GitLab From 0177a9547e588222acaf6d006747b5f7014e6fd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?= =?UTF-8?q?=E3=83=B3=29?= Date: Wed, 20 Mar 2024 12:58:11 -0700 Subject: [PATCH 064/296] [flang][cuda] Fix fir.cuda_kernel_launch assembly with no args (#85987) When the kernel launch has no arguments, the generated parser was expecting at least a type to be present. Make the last part of the assemble format optional. Add a run line to round-trip the output through fir-opt so we make sure the IR can be parsed and printed correctly. --- flang/include/flang/Optimizer/Dialect/FIROps.td | 2 +- flang/test/Lower/CUDA/cuda-kernel-calls.cuf | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td index 8a7e36e42457..b991ec76fdd9 100644 --- a/flang/include/flang/Optimizer/Dialect/FIROps.td +++ b/flang/include/flang/Optimizer/Dialect/FIROps.td @@ -2466,7 +2466,7 @@ def fir_CUDAKernelLaunch : fir_Op<"cuda_kernel_launch", [CallOpInterface, let assemblyFormat = [{ $callee `<` `<` `<` $grid_x `,` $grid_y `,` $grid_z `,`$block_x `,` $block_y `,` $block_z ( `,` $bytes^ ( `,` $stream^ )? )? `>` `>` `>` - `` `(` $args `)` `:` `(` type($args) `)` attr-dict + `` `(` $args `)` ( `:` `(` type($args)^ `)` )? attr-dict }]; let extraClassDeclaration = [{ diff --git a/flang/test/Lower/CUDA/cuda-kernel-calls.cuf b/flang/test/Lower/CUDA/cuda-kernel-calls.cuf index f4327b326175..7e28fbb2231a 100644 --- a/flang/test/Lower/CUDA/cuda-kernel-calls.cuf +++ b/flang/test/Lower/CUDA/cuda-kernel-calls.cuf @@ -1,4 +1,5 @@ ! RUN: bbc -emit-hlfir -fcuda %s -o - | FileCheck %s +! RUN: bbc -emit-hlfir -fcuda %s -o - | fir-opt | FileCheck %s ! Test lowering of CUDA procedure calls. -- GitLab From 84115494d6475e1aea3cdd1163d3a88243b75f36 Mon Sep 17 00:00:00 2001 From: Krzysztof Parzyszek Date: Wed, 20 Mar 2024 15:00:29 -0500 Subject: [PATCH 065/296] [flang][Lower] Convert OMP Map and related functions to evaluate::Expr (#81626) The related functions are `gatherDataOperandAddrAndBounds` and `genBoundsOps`. The former is used in OpenACC as well, and it was updated to pass evaluate::Expr instead of parser objects. The difference in the test case comes from unfolded conversions of index expressions, which are explicitly of type integer(kind=8). Delete now unused `findRepeatableClause2` and `findClause2`. Add `AsGenericExpr` that takes std::optional. It already returns optional Expr. Making it accept an optional Expr as input would reduce the number of necessary checks when handling frequent optional values in evaluator. [Clause representation 4/6] --- flang/include/flang/Evaluate/tools.h | 8 + flang/lib/Lower/DirectivesCommon.h | 389 ++++++++++++-------- flang/lib/Lower/OpenACC.cpp | 54 ++- flang/lib/Lower/OpenMP/ClauseProcessor.cpp | 44 +-- flang/lib/Lower/OpenMP/ClauseProcessor.h | 59 +-- flang/lib/Lower/OpenMP/OpenMP.cpp | 7 +- flang/test/Lower/OpenACC/acc-bounds.f90 | 2 +- flang/test/Lower/OpenACC/acc-enter-data.f90 | 40 +- 8 files changed, 335 insertions(+), 268 deletions(-) diff --git a/flang/include/flang/Evaluate/tools.h b/flang/include/flang/Evaluate/tools.h index 9a32062440ab..8c872a0579c8 100644 --- a/flang/include/flang/Evaluate/tools.h +++ b/flang/include/flang/Evaluate/tools.h @@ -148,6 +148,14 @@ inline Expr AsGenericExpr(Expr &&x) { return std::move(x); } std::optional> AsGenericExpr(DataRef &&); std::optional> AsGenericExpr(const Symbol &); +// Propagate std::optional from input to output. +template +std::optional> AsGenericExpr(std::optional &&x) { + if (!x) + return std::nullopt; + return AsGenericExpr(std::move(*x)); +} + template common::IfNoLvalue::category>>, A> AsCategoryExpr( A &&x) { diff --git a/flang/lib/Lower/DirectivesCommon.h b/flang/lib/Lower/DirectivesCommon.h index 8d560db34e05..6daa72b84d90 100644 --- a/flang/lib/Lower/DirectivesCommon.h +++ b/flang/lib/Lower/DirectivesCommon.h @@ -808,6 +808,75 @@ genBaseBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc, return bounds; } +namespace detail { +template // +static T &&AsRvalueRef(T &&t) { + return std::move(t); +} +template // +static T AsRvalueRef(T &t) { + return t; +} +template // +static T AsRvalueRef(const T &t) { + return t; +} + +// Helper class for stripping enclosing parentheses and a conversion that +// preserves type category. This is used for triplet elements, which are +// always of type integer(kind=8). The lower/upper bounds are converted to +// an "index" type, which is 64-bit, so the explicit conversion to kind=8 +// (if present) is not needed. When it's present, though, it causes generated +// names to contain "int(..., kind=8)". +struct PeelConvert { + template + static Fortran::semantics::MaybeExpr visit_with_category( + const Fortran::evaluate::Expr> + &expr) { + return std::visit( + [](auto &&s) { return visit_with_category(s); }, + expr.u); + } + template + static Fortran::semantics::MaybeExpr visit_with_category( + const Fortran::evaluate::Convert, + Category> &expr) { + return AsGenericExpr(AsRvalueRef(expr.left())); + } + template + static Fortran::semantics::MaybeExpr visit_with_category(const T &) { + return std::nullopt; // + } + template + static Fortran::semantics::MaybeExpr visit_with_category(const T &) { + return std::nullopt; // + } + + template + static Fortran::semantics::MaybeExpr + visit(const Fortran::evaluate::Expr> + &expr) { + return std::visit([](auto &&s) { return visit_with_category(s); }, + expr.u); + } + static Fortran::semantics::MaybeExpr + visit(const Fortran::evaluate::Expr &expr) { + return std::visit([](auto &&s) { return visit(s); }, expr.u); + } + template // + static Fortran::semantics::MaybeExpr visit(const T &) { + return std::nullopt; + } +}; + +static Fortran::semantics::SomeExpr +peelOuterConvert(Fortran::semantics::SomeExpr &expr) { + if (auto peeled = PeelConvert::visit(expr)) + return *peeled; + return expr; +} +} // namespace detail + /// Generate bounds operations for an array section when subscripts are /// provided. template @@ -815,7 +884,7 @@ llvm::SmallVector genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc, Fortran::lower::AbstractConverter &converter, Fortran::lower::StatementContext &stmtCtx, - const std::list &subscripts, + const std::vector &subscripts, std::stringstream &asFortran, fir::ExtendedValue &dataExv, bool dataExvIsAssumedSize, AddrAndBoundsInfo &info, bool treatIndexAsSection = false) { @@ -828,8 +897,7 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc, mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1); const int dataExvRank = static_cast(dataExv.rank()); for (const auto &subscript : subscripts) { - const auto *triplet{ - std::get_if(&subscript.u)}; + const auto *triplet{std::get_if(&subscript.u)}; if (triplet || treatIndexAsSection) { if (dimension != 0) asFortran << ','; @@ -868,13 +936,17 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc, strideInBytes = true; } - const Fortran::lower::SomeExpr *lower{nullptr}; + Fortran::semantics::MaybeExpr lower; if (triplet) { - if (const auto &tripletLb{std::get<0>(triplet->t)}) - lower = Fortran::semantics::GetExpr(*tripletLb); + lower = Fortran::evaluate::AsGenericExpr(triplet->lower()); } else { - const auto &index{std::get(subscript.u)}; - lower = Fortran::semantics::GetExpr(index); + // Case of IndirectSubscriptIntegerExpr + using IndirectSubscriptIntegerExpr = + Fortran::evaluate::IndirectSubscriptIntegerExpr; + using SubscriptInteger = Fortran::evaluate::SubscriptInteger; + Fortran::evaluate::Expr oneInt = + std::get(subscript.u).value(); + lower = Fortran::evaluate::AsGenericExpr(std::move(oneInt)); if (lower->Rank() > 0) { mlir::emitError( loc, "vector subscript cannot be used for an array section"); @@ -896,7 +968,7 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc, fir::getBase(converter.genExprValue(loc, *lower, stmtCtx)); lb = builder.createConvert(loc, baseLb.getType(), lb); lbound = builder.create(loc, lb, baseLb); - asFortran << lower->AsFortran(); + asFortran << detail::peelOuterConvert(*lower).AsFortran(); } } else { // If the lower bound is not specified, then the section @@ -912,10 +984,11 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc, extent = one; } else { asFortran << ':'; - const auto &upper{std::get<1>(triplet->t)}; + Fortran::semantics::MaybeExpr upper = + Fortran::evaluate::AsGenericExpr(triplet->upper()); if (upper) { - uval = Fortran::semantics::GetIntValue(upper); + uval = Fortran::evaluate::ToInt64(*upper); if (uval) { if (defaultLb) { ubound = builder.createIntegerConstant(loc, idxTy, *uval - 1); @@ -925,22 +998,21 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc, } asFortran << *uval; } else { - const Fortran::lower::SomeExpr *uexpr = - Fortran::semantics::GetExpr(*upper); mlir::Value ub = - fir::getBase(converter.genExprValue(loc, *uexpr, stmtCtx)); + fir::getBase(converter.genExprValue(loc, *upper, stmtCtx)); ub = builder.createConvert(loc, baseLb.getType(), ub); ubound = builder.create(loc, ub, baseLb); - asFortran << uexpr->AsFortran(); + asFortran << detail::peelOuterConvert(*upper).AsFortran(); } } if (lower && upper) { if (lval && uval && *uval < *lval) { mlir::emitError(loc, "zero sized array section"); break; - } else if (std::get<2>(triplet->t)) { - const auto &strideExpr{std::get<2>(triplet->t)}; - if (strideExpr) { + } else { + // Stride is mandatory in evaluate::Triplet. Make sure it's 1. + auto val = Fortran::evaluate::ToInt64(triplet->GetStride()); + if (!val || *val != 1) { mlir::emitError(loc, "stride cannot be specified on " "an array section"); break; @@ -993,150 +1065,157 @@ genBoundsOps(fir::FirOpBuilder &builder, mlir::Location loc, return bounds; } -template +namespace detail { +template // +std::optional getRef(Expr &&expr) { + if constexpr (std::is_same_v, + Fortran::evaluate::DataRef>) { + if (auto *ref = std::get_if(&expr.u)) + return *ref; + return std::nullopt; + } else { + auto maybeRef = Fortran::evaluate::ExtractDataRef(expr); + if (!maybeRef || !std::holds_alternative(maybeRef->u)) + return std::nullopt; + return std::get(maybeRef->u); + } +} +} // namespace detail + +template AddrAndBoundsInfo gatherDataOperandAddrAndBounds( Fortran::lower::AbstractConverter &converter, fir::FirOpBuilder &builder, - Fortran::semantics::SemanticsContext &semanticsContext, - Fortran::lower::StatementContext &stmtCtx, const ObjectType &object, + semantics::SemanticsContext &semaCtx, + Fortran::lower::StatementContext &stmtCtx, + Fortran::semantics::SymbolRef symbol, + const Fortran::semantics::MaybeExpr &maybeDesignator, mlir::Location operandLocation, std::stringstream &asFortran, llvm::SmallVector &bounds, bool treatIndexAsSection = false) { + using namespace Fortran; + AddrAndBoundsInfo info; - std::visit( - Fortran::common::visitors{ - [&](const Fortran::parser::Designator &designator) { - if (auto expr{Fortran::semantics::AnalyzeExpr(semanticsContext, - designator)}) { - if (((*expr).Rank() > 0 || treatIndexAsSection) && - Fortran::parser::Unwrap( - designator)) { - const auto *arrayElement = - Fortran::parser::Unwrap( - designator); - const auto *dataRef = - std::get_if(&designator.u); - fir::ExtendedValue dataExv; - bool dataExvIsAssumedSize = false; - if (Fortran::parser::Unwrap< - Fortran::parser::StructureComponent>( - arrayElement->base)) { - auto exprBase = Fortran::semantics::AnalyzeExpr( - semanticsContext, arrayElement->base); - dataExv = converter.genExprAddr(operandLocation, *exprBase, - stmtCtx); - info.addr = fir::getBase(dataExv); - info.rawInput = info.addr; - asFortran << (*exprBase).AsFortran(); - } else { - const Fortran::parser::Name &name = - Fortran::parser::GetLastName(*dataRef); - dataExvIsAssumedSize = Fortran::semantics::IsAssumedSizeArray( - name.symbol->GetUltimate()); - info = getDataOperandBaseAddr(converter, builder, - *name.symbol, operandLocation); - dataExv = converter.getSymbolExtendedValue(*name.symbol); - asFortran << name.ToString(); - } - - if (!arrayElement->subscripts.empty()) { - asFortran << '('; - bounds = genBoundsOps( - builder, operandLocation, converter, stmtCtx, - arrayElement->subscripts, asFortran, dataExv, - dataExvIsAssumedSize, info, treatIndexAsSection); - } - asFortran << ')'; - } else if (auto structComp = Fortran::parser::Unwrap< - Fortran::parser::StructureComponent>(designator)) { - fir::ExtendedValue compExv = - converter.genExprAddr(operandLocation, *expr, stmtCtx); - info.addr = fir::getBase(compExv); - info.rawInput = info.addr; - if (fir::unwrapRefType(info.addr.getType()) - .isa()) - bounds = genBaseBoundsOps( - builder, operandLocation, converter, compExv, - /*isAssumedSize=*/false); - asFortran << (*expr).AsFortran(); - - bool isOptional = Fortran::semantics::IsOptional( - *Fortran::parser::GetLastName(*structComp).symbol); - if (isOptional) - info.isPresent = builder.create( - operandLocation, builder.getI1Type(), info.rawInput); - - if (auto loadOp = mlir::dyn_cast_or_null( - info.addr.getDefiningOp())) { - if (fir::isAllocatableType(loadOp.getType()) || - fir::isPointerType(loadOp.getType())) - info.addr = builder.create(operandLocation, - info.addr); - info.rawInput = info.addr; - } - - // If the component is an allocatable or pointer the result of - // genExprAddr will be the result of a fir.box_addr operation or - // a fir.box_addr has been inserted just before. - // Retrieve the box so we handle it like other descriptor. - if (auto boxAddrOp = mlir::dyn_cast_or_null( - info.addr.getDefiningOp())) { - info.addr = boxAddrOp.getVal(); - info.rawInput = info.addr; - bounds = genBoundsOpsFromBox( - builder, operandLocation, converter, compExv, info); - } - } else { - if (Fortran::parser::Unwrap( - designator)) { - // Single array element. - const auto *arrayElement = - Fortran::parser::Unwrap( - designator); - (void)arrayElement; - fir::ExtendedValue compExv = - converter.genExprAddr(operandLocation, *expr, stmtCtx); - info.addr = fir::getBase(compExv); - info.rawInput = info.addr; - asFortran << (*expr).AsFortran(); - } else if (const auto *dataRef{ - std::get_if( - &designator.u)}) { - // Scalar or full array. - const Fortran::parser::Name &name = - Fortran::parser::GetLastName(*dataRef); - fir::ExtendedValue dataExv = - converter.getSymbolExtendedValue(*name.symbol); - info = getDataOperandBaseAddr(converter, builder, - *name.symbol, operandLocation); - if (fir::unwrapRefType(info.addr.getType()) - .isa()) { - bounds = genBoundsOpsFromBox( - builder, operandLocation, converter, dataExv, info); - } - bool dataExvIsAssumedSize = - Fortran::semantics::IsAssumedSizeArray( - name.symbol->GetUltimate()); - if (fir::unwrapRefType(info.addr.getType()) - .isa()) - bounds = genBaseBoundsOps( - builder, operandLocation, converter, dataExv, - dataExvIsAssumedSize); - asFortran << name.ToString(); - } else { // Unsupported - llvm::report_fatal_error( - "Unsupported type of OpenACC operand"); - } - } - } - }, - [&](const Fortran::parser::Name &name) { - info = getDataOperandBaseAddr(converter, builder, *name.symbol, - operandLocation); - asFortran << name.ToString(); - }}, - object.u); + + if (!maybeDesignator) { + info = getDataOperandBaseAddr(converter, builder, symbol, operandLocation); + asFortran << symbol->name().ToString(); + return info; + } + + semantics::SomeExpr designator = *maybeDesignator; + + if ((designator.Rank() > 0 || treatIndexAsSection) && + IsArrayElement(designator)) { + auto arrayRef = detail::getRef(designator); + // This shouldn't fail after IsArrayElement(designator). + assert(arrayRef && "Expecting ArrayRef"); + + fir::ExtendedValue dataExv; + bool dataExvIsAssumedSize = false; + + auto toMaybeExpr = [&](auto &&base) { + using BaseType = llvm::remove_cvref_t; + evaluate::ExpressionAnalyzer ea{semaCtx}; + + if constexpr (std::is_same_v) { + if (auto *ref = base.UnwrapSymbolRef()) + return ea.Designate(evaluate::DataRef{*ref}); + if (auto *ref = base.UnwrapComponent()) + return ea.Designate(evaluate::DataRef{*ref}); + llvm_unreachable("Unexpected NamedEntity"); + } else { + static_assert(std::is_same_v); + return ea.Designate(evaluate::DataRef{base}); + } + }; + + auto arrayBase = toMaybeExpr(arrayRef->base()); + assert(arrayBase); + + if (detail::getRef(*arrayBase)) { + dataExv = converter.genExprAddr(operandLocation, *arrayBase, stmtCtx); + info.addr = fir::getBase(dataExv); + info.rawInput = info.addr; + asFortran << arrayBase->AsFortran(); + } else { + const semantics::Symbol &sym = arrayRef->GetLastSymbol(); + dataExvIsAssumedSize = + Fortran::semantics::IsAssumedSizeArray(sym.GetUltimate()); + info = getDataOperandBaseAddr(converter, builder, sym, operandLocation); + dataExv = converter.getSymbolExtendedValue(sym); + asFortran << sym.name().ToString(); + } + + if (!arrayRef->subscript().empty()) { + asFortran << '('; + bounds = genBoundsOps( + builder, operandLocation, converter, stmtCtx, arrayRef->subscript(), + asFortran, dataExv, dataExvIsAssumedSize, info, treatIndexAsSection); + } + asFortran << ')'; + } else if (auto compRef = detail::getRef(designator)) { + fir::ExtendedValue compExv = + converter.genExprAddr(operandLocation, designator, stmtCtx); + info.addr = fir::getBase(compExv); + info.rawInput = info.addr; + if (fir::unwrapRefType(info.addr.getType()).isa()) + bounds = genBaseBoundsOps(builder, operandLocation, + converter, compExv, + /*isAssumedSize=*/false); + asFortran << designator.AsFortran(); + + if (semantics::IsOptional(compRef->GetLastSymbol())) { + info.isPresent = builder.create( + operandLocation, builder.getI1Type(), info.rawInput); + } + + if (auto loadOp = + mlir::dyn_cast_or_null(info.addr.getDefiningOp())) { + if (fir::isAllocatableType(loadOp.getType()) || + fir::isPointerType(loadOp.getType())) + info.addr = builder.create(operandLocation, info.addr); + info.rawInput = info.addr; + } + + // If the component is an allocatable or pointer the result of + // genExprAddr will be the result of a fir.box_addr operation or + // a fir.box_addr has been inserted just before. + // Retrieve the box so we handle it like other descriptor. + if (auto boxAddrOp = + mlir::dyn_cast_or_null(info.addr.getDefiningOp())) { + info.addr = boxAddrOp.getVal(); + info.rawInput = info.addr; + bounds = genBoundsOpsFromBox( + builder, operandLocation, converter, compExv, info); + } + } else { + if (detail::getRef(designator)) { + fir::ExtendedValue compExv = + converter.genExprAddr(operandLocation, designator, stmtCtx); + info.addr = fir::getBase(compExv); + info.rawInput = info.addr; + asFortran << designator.AsFortran(); + } else if (auto symRef = detail::getRef(designator)) { + // Scalar or full array. + fir::ExtendedValue dataExv = converter.getSymbolExtendedValue(*symRef); + info = + getDataOperandBaseAddr(converter, builder, *symRef, operandLocation); + if (fir::unwrapRefType(info.addr.getType()).isa()) { + bounds = genBoundsOpsFromBox( + builder, operandLocation, converter, dataExv, info); + } + bool dataExvIsAssumedSize = + Fortran::semantics::IsAssumedSizeArray(symRef->get().GetUltimate()); + if (fir::unwrapRefType(info.addr.getType()).isa()) + bounds = genBaseBoundsOps( + builder, operandLocation, converter, dataExv, dataExvIsAssumedSize); + asFortran << symRef->get().name().ToString(); + } else { // Unsupported + llvm::report_fatal_error("Unsupported type of OpenACC operand"); + } + } + return info; } - } // namespace lower } // namespace Fortran diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp index 6539de4d8830..7b7e4a875cd8 100644 --- a/flang/lib/Lower/OpenACC.cpp +++ b/flang/lib/Lower/OpenACC.cpp @@ -269,6 +269,11 @@ getSymbolFromAccObject(const Fortran::parser::AccObject &accObject) { Fortran::parser::GetLastName(arrayElement->base); return *name.symbol; } + if (const auto *component = + Fortran::parser::Unwrap( + *designator)) { + return *component->component.symbol; + } } else if (const auto *name = std::get_if(&accObject.u)) { return *name->symbol; @@ -286,17 +291,20 @@ genDataOperandOperations(const Fortran::parser::AccObjectList &objectList, mlir::acc::DataClause dataClause, bool structured, bool implicit, bool setDeclareAttr = false) { fir::FirOpBuilder &builder = converter.getFirOpBuilder(); + Fortran::evaluate::ExpressionAnalyzer ea{semanticsContext}; for (const auto &accObject : objectList.v) { llvm::SmallVector bounds; std::stringstream asFortran; mlir::Location operandLocation = genOperandLocation(converter, accObject); + Fortran::semantics::Symbol &symbol = getSymbolFromAccObject(accObject); + Fortran::semantics::MaybeExpr designator = + std::visit([&](auto &&s) { return ea.Analyze(s); }, accObject.u); Fortran::lower::AddrAndBoundsInfo info = Fortran::lower::gatherDataOperandAddrAndBounds< - Fortran::parser::AccObject, mlir::acc::DataBoundsOp, - mlir::acc::DataBoundsType>(converter, builder, semanticsContext, - stmtCtx, accObject, operandLocation, - asFortran, bounds, - /*treatIndexAsSection=*/true); + mlir::acc::DataBoundsOp, mlir::acc::DataBoundsType>( + converter, builder, semanticsContext, stmtCtx, symbol, designator, + operandLocation, asFortran, bounds, + /*treatIndexAsSection=*/true); // If the input value is optional and is not a descriptor, we use the // rawInput directly. @@ -321,16 +329,19 @@ static void genDeclareDataOperandOperations( llvm::SmallVectorImpl &dataOperands, mlir::acc::DataClause dataClause, bool structured, bool implicit) { fir::FirOpBuilder &builder = converter.getFirOpBuilder(); + Fortran::evaluate::ExpressionAnalyzer ea{semanticsContext}; for (const auto &accObject : objectList.v) { llvm::SmallVector bounds; std::stringstream asFortran; mlir::Location operandLocation = genOperandLocation(converter, accObject); + Fortran::semantics::Symbol &symbol = getSymbolFromAccObject(accObject); + Fortran::semantics::MaybeExpr designator = + std::visit([&](auto &&s) { return ea.Analyze(s); }, accObject.u); Fortran::lower::AddrAndBoundsInfo info = Fortran::lower::gatherDataOperandAddrAndBounds< - Fortran::parser::AccObject, mlir::acc::DataBoundsOp, - mlir::acc::DataBoundsType>(converter, builder, semanticsContext, - stmtCtx, accObject, operandLocation, - asFortran, bounds); + mlir::acc::DataBoundsOp, mlir::acc::DataBoundsType>( + converter, builder, semanticsContext, stmtCtx, symbol, designator, + operandLocation, asFortran, bounds); EntryOp op = createDataEntryOp( builder, operandLocation, info.addr, asFortran, bounds, structured, implicit, dataClause, info.addr.getType()); @@ -339,8 +350,7 @@ static void genDeclareDataOperandOperations( if (mlir::isa(fir::unwrapRefType(info.addr.getType()))) { mlir::OpBuilder modBuilder(builder.getModule().getBodyRegion()); modBuilder.setInsertionPointAfter(builder.getFunction()); - std::string prefix = - converter.mangleName(getSymbolFromAccObject(accObject)); + std::string prefix = converter.mangleName(symbol); createDeclareAllocFuncWithArg( modBuilder, builder, operandLocation, info.addr.getType(), prefix, asFortran, dataClause); @@ -770,16 +780,19 @@ genPrivatizations(const Fortran::parser::AccObjectList &objectList, llvm::SmallVectorImpl &dataOperands, llvm::SmallVector &privatizations) { fir::FirOpBuilder &builder = converter.getFirOpBuilder(); + Fortran::evaluate::ExpressionAnalyzer ea{semanticsContext}; for (const auto &accObject : objectList.v) { llvm::SmallVector bounds; std::stringstream asFortran; mlir::Location operandLocation = genOperandLocation(converter, accObject); + Fortran::semantics::Symbol &symbol = getSymbolFromAccObject(accObject); + Fortran::semantics::MaybeExpr designator = + std::visit([&](auto &&s) { return ea.Analyze(s); }, accObject.u); Fortran::lower::AddrAndBoundsInfo info = Fortran::lower::gatherDataOperandAddrAndBounds< - Fortran::parser::AccObject, mlir::acc::DataBoundsOp, - mlir::acc::DataBoundsType>(converter, builder, semanticsContext, - stmtCtx, accObject, operandLocation, - asFortran, bounds); + mlir::acc::DataBoundsOp, mlir::acc::DataBoundsType>( + converter, builder, semanticsContext, stmtCtx, symbol, designator, + operandLocation, asFortran, bounds); RecipeOp recipe; mlir::Type retTy = getTypeFromBounds(bounds, info.addr.getType()); if constexpr (std::is_same_v) { @@ -1340,16 +1353,19 @@ genReductions(const Fortran::parser::AccObjectListWithReduction &objectList, const auto &op = std::get(objectList.t); mlir::acc::ReductionOperator mlirOp = getReductionOperator(op); + Fortran::evaluate::ExpressionAnalyzer ea{semanticsContext}; for (const auto &accObject : objects.v) { llvm::SmallVector bounds; std::stringstream asFortran; mlir::Location operandLocation = genOperandLocation(converter, accObject); + Fortran::semantics::Symbol &symbol = getSymbolFromAccObject(accObject); + Fortran::semantics::MaybeExpr designator = + std::visit([&](auto &&s) { return ea.Analyze(s); }, accObject.u); Fortran::lower::AddrAndBoundsInfo info = Fortran::lower::gatherDataOperandAddrAndBounds< - Fortran::parser::AccObject, mlir::acc::DataBoundsOp, - mlir::acc::DataBoundsType>(converter, builder, semanticsContext, - stmtCtx, accObject, operandLocation, - asFortran, bounds); + mlir::acc::DataBoundsOp, mlir::acc::DataBoundsType>( + converter, builder, semanticsContext, stmtCtx, symbol, designator, + operandLocation, asFortran, bounds); mlir::Type reductionTy = fir::unwrapRefType(info.addr.getType()); if (auto seqTy = mlir::dyn_cast(reductionTy)) diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp index ae798b5c0a58..95faa0767e36 100644 --- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp +++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp @@ -818,65 +818,61 @@ bool ClauseProcessor::processMap( llvm::SmallVectorImpl *mapSymbols) const { fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); - return findRepeatableClause2( - [&](const ClauseTy::Map *mapClause, + return findRepeatableClause( + [&](const omp::clause::Map &clause, const Fortran::parser::CharBlock &source) { + using Map = omp::clause::Map; mlir::Location clauseLocation = converter.genLocation(source); - const auto &oMapType = - std::get>( - mapClause->v.t); + const auto &oMapType = std::get>(clause.t); llvm::omp::OpenMPOffloadMappingFlags mapTypeBits = llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE; // If the map type is specified, then process it else Tofrom is the // default. if (oMapType) { - const Fortran::parser::OmpMapType::Type &mapType = - std::get(oMapType->t); + const Map::MapType::Type &mapType = + std::get(oMapType->t); switch (mapType) { - case Fortran::parser::OmpMapType::Type::To: + case Map::MapType::Type::To: mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO; break; - case Fortran::parser::OmpMapType::Type::From: + case Map::MapType::Type::From: mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM; break; - case Fortran::parser::OmpMapType::Type::Tofrom: + case Map::MapType::Type::Tofrom: mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO | llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM; break; - case Fortran::parser::OmpMapType::Type::Alloc: - case Fortran::parser::OmpMapType::Type::Release: + case Map::MapType::Type::Alloc: + case Map::MapType::Type::Release: // alloc and release is the default map_type for the Target Data // Ops, i.e. if no bits for map_type is supplied then alloc/release // is implicitly assumed based on the target directive. Default // value for Target Data and Enter Data is alloc and for Exit Data // it is release. break; - case Fortran::parser::OmpMapType::Type::Delete: + case Map::MapType::Type::Delete: mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_DELETE; } - if (std::get>( - oMapType->t)) + if (std::get>(oMapType->t)) mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS; } else { mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO | llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM; } - for (const Fortran::parser::OmpObject &ompObject : - std::get(mapClause->v.t).v) { + for (const omp::Object &object : std::get(clause.t)) { llvm::SmallVector bounds; std::stringstream asFortran; Fortran::lower::AddrAndBoundsInfo info = Fortran::lower::gatherDataOperandAddrAndBounds< - Fortran::parser::OmpObject, mlir::omp::MapBoundsOp, - mlir::omp::MapBoundsType>( - converter, firOpBuilder, semaCtx, stmtCtx, ompObject, - clauseLocation, asFortran, bounds, treatIndexAsSection); + mlir::omp::MapBoundsOp, mlir::omp::MapBoundsType>( + converter, firOpBuilder, semaCtx, stmtCtx, *object.id(), + object.ref(), clauseLocation, asFortran, bounds, + treatIndexAsSection); - auto origSymbol = - converter.getSymbolAddress(*getOmpObjectSymbol(ompObject)); + auto origSymbol = converter.getSymbolAddress(*object.id()); mlir::Value symAddr = info.addr; if (origSymbol && fir::isTypeWithDescriptor(origSymbol.getType())) symAddr = origSymbol; @@ -899,7 +895,7 @@ bool ClauseProcessor::processMap( mapSymLocs->push_back(symAddr.getLoc()); if (mapSymbols) - mapSymbols->push_back(getOmpObjectSymbol(ompObject)); + mapSymbols->push_back(object.id()); } }); } diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h index c2db0cfc3cb7..ffa8a5e05593 100644 --- a/flang/lib/Lower/OpenMP/ClauseProcessor.h +++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h @@ -162,9 +162,6 @@ private: /// Utility to find a clause within a range in the clause list. template static ClauseIterator findClause(ClauseIterator begin, ClauseIterator end); - template - static ClauseIterator2 findClause2(ClauseIterator2 begin, - ClauseIterator2 end); /// Return the first instance of the given clause found in the clause list or /// `nullptr` if not present. If more than one instance is expected, use @@ -179,10 +176,6 @@ private: bool findRepeatableClause( std::function callbackFn) const; - template - bool findRepeatableClause2( - std::function - callbackFn) const; /// Set the `result` to a new `mlir::UnitAttr` if the clause is present. template @@ -198,32 +191,31 @@ template bool ClauseProcessor::processMotionClauses( Fortran::lower::StatementContext &stmtCtx, llvm::SmallVectorImpl &mapOperands) { - return findRepeatableClause2( - [&](const T *motionClause, const Fortran::parser::CharBlock &source) { + return findRepeatableClause( + [&](const T &clause, const Fortran::parser::CharBlock &source) { mlir::Location clauseLocation = converter.genLocation(source); fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); - static_assert(std::is_same_v || - std::is_same_v); + static_assert(std::is_same_v || + std::is_same_v); // TODO Support motion modifiers: present, mapper, iterator. constexpr llvm::omp::OpenMPOffloadMappingFlags mapTypeBits = - std::is_same_v + std::is_same_v ? llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO : llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM; - for (const Fortran::parser::OmpObject &ompObject : motionClause->v.v) { + for (const omp::Object &object : clause.v) { llvm::SmallVector bounds; std::stringstream asFortran; Fortran::lower::AddrAndBoundsInfo info = Fortran::lower::gatherDataOperandAddrAndBounds< - Fortran::parser::OmpObject, mlir::omp::MapBoundsOp, - mlir::omp::MapBoundsType>( - converter, firOpBuilder, semaCtx, stmtCtx, ompObject, - clauseLocation, asFortran, bounds, treatIndexAsSection); + mlir::omp::MapBoundsOp, mlir::omp::MapBoundsType>( + converter, firOpBuilder, semaCtx, stmtCtx, *object.id(), + object.ref(), clauseLocation, asFortran, bounds, + treatIndexAsSection); - auto origSymbol = - converter.getSymbolAddress(*getOmpObjectSymbol(ompObject)); + auto origSymbol = converter.getSymbolAddress(*object.id()); mlir::Value symAddr = info.addr; if (origSymbol && fir::isTypeWithDescriptor(origSymbol.getType())) symAddr = origSymbol; @@ -273,17 +265,6 @@ ClauseProcessor::findClause(ClauseIterator begin, ClauseIterator end) { return end; } -template -ClauseProcessor::ClauseIterator2 -ClauseProcessor::findClause2(ClauseIterator2 begin, ClauseIterator2 end) { - for (ClauseIterator2 it = begin; it != end; ++it) { - if (std::get_if(&it->u)) - return it; - } - - return end; -} - template const T *ClauseProcessor::findUniqueClause( const Fortran::parser::CharBlock **source) const { @@ -314,24 +295,6 @@ bool ClauseProcessor::findRepeatableClause( return found; } -template -bool ClauseProcessor::findRepeatableClause2( - std::function - callbackFn) const { - bool found = false; - ClauseIterator2 nextIt, endIt = clauses2.v.end(); - for (ClauseIterator2 it = clauses2.v.begin(); it != endIt; it = nextIt) { - nextIt = findClause2(it, endIt); - - if (nextIt != endIt) { - callbackFn(&std::get(nextIt->u), nextIt->source); - found = true; - ++nextIt; - } - } - return found; -} - template bool ClauseProcessor::markClauseOccurrence(mlir::UnitAttr &result) const { if (findUniqueClause()) { diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp index 5c4caa1de573..d335129565b4 100644 --- a/flang/lib/Lower/OpenMP/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP/OpenMP.cpp @@ -930,11 +930,8 @@ static OpTy genTargetEnterExitDataUpdateOp( cp.processNowait(nowaitAttr); if constexpr (std::is_same_v) { - cp.processMotionClauses(stmtCtx, - mapOperands); - cp.processMotionClauses(stmtCtx, - mapOperands); - + cp.processMotionClauses(stmtCtx, mapOperands); + cp.processMotionClauses(stmtCtx, mapOperands); } else { cp.processMap(currentLocation, directive, stmtCtx, mapOperands); } diff --git a/flang/test/Lower/OpenACC/acc-bounds.f90 b/flang/test/Lower/OpenACC/acc-bounds.f90 index df97cbcd187d..c275d4f1b1d5 100644 --- a/flang/test/Lower/OpenACC/acc-bounds.f90 +++ b/flang/test/Lower/OpenACC/acc-bounds.f90 @@ -184,7 +184,7 @@ contains ! CHECK: fir.result %c0{{.*}} : index ! CHECK: } ! CHECK: %[[BOUNDS:.*]] = acc.bounds lowerbound(%c0{{.*}} : index) upperbound(%{{.*}} : index) extent(%{{.*}} : index) stride(%[[STRIDE]] : index) startIdx(%c1 : index) {strideInBytes = true} -! CHECK: %[[NOCREATE:.*]] = acc.nocreate varPtr(%[[DECL_A]]#1 : !fir.ref>) bounds(%14) -> !fir.ref> {name = "a(1:n)"} +! CHECK: %[[NOCREATE:.*]] = acc.nocreate varPtr(%[[DECL_A]]#1 : !fir.ref>) bounds(%[[BOUNDS]]) -> !fir.ref> {name = "a(1:n)"} ! CHECK: acc.data dataOperands(%[[NOCREATE]] : !fir.ref>) { end module diff --git a/flang/test/Lower/OpenACC/acc-enter-data.f90 b/flang/test/Lower/OpenACC/acc-enter-data.f90 index 2cf50c1b62f1..251edbf9c2dd 100644 --- a/flang/test/Lower/OpenACC/acc-enter-data.f90 +++ b/flang/test/Lower/OpenACC/acc-enter-data.f90 @@ -234,11 +234,13 @@ subroutine acc_enter_data_dummy(a, b, n, m) !$acc enter data create(b(n:m)) !CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLB]]#0, %c0{{.*}} : (!fir.box>, index) -> (index, index, index) !CHECK: %[[LOAD_N:.*]] = fir.load %[[DECLN]]#0 : !fir.ref -!CHECK: %[[N_CONV:.*]] = fir.convert %[[LOAD_N]] : (i32) -> index -!CHECK: %[[LB:.*]] = arith.subi %[[N_CONV]], %[[N_IDX]] : index +!CHECK: %[[N_CONV1:.*]] = fir.convert %[[LOAD_N]] : (i32) -> i64 +!CHECK: %[[N_CONV2:.*]] = fir.convert %[[N_CONV1]] : (i64) -> index +!CHECK: %[[LB:.*]] = arith.subi %[[N_CONV2]], %[[N_IDX]] : index !CHECK: %[[LOAD_M:.*]] = fir.load %[[DECLM]]#0 : !fir.ref -!CHECK: %[[M_CONV:.*]] = fir.convert %[[LOAD_M]] : (i32) -> index -!CHECK: %[[UB:.*]] = arith.subi %[[M_CONV]], %[[N_IDX]] : index +!CHECK: %[[M_CONV1:.*]] = fir.convert %[[LOAD_M]] : (i32) -> i64 +!CHECK: %[[M_CONV2:.*]] = fir.convert %[[M_CONV1]] : (i64) -> index +!CHECK: %[[UB:.*]] = arith.subi %[[M_CONV2]], %[[N_IDX]] : index !CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXT_B]] : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[N_IDX]] : index) {strideInBytes = true} !CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLB]]#0 : (!fir.box>) -> !fir.ref> !CHECK: %[[CREATE1:.*]] = acc.create varPtr(%[[ADDR]] : !fir.ref>) bounds(%[[BOUND1]]) -> !fir.ref> {name = "b(n:m)", structured = false} @@ -248,8 +250,9 @@ subroutine acc_enter_data_dummy(a, b, n, m) !CHECK: %[[ONE:.*]] = arith.constant 1 : index !CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLB]]#0, %c0_8 : (!fir.box>, index) -> (index, index, index) !CHECK: %[[LOAD_N:.*]] = fir.load %[[DECLN]]#0 : !fir.ref -!CHECK: %[[CONVERT_N:.*]] = fir.convert %[[LOAD_N]] : (i32) -> index -!CHECK: %[[LB:.*]] = arith.subi %[[CONVERT_N]], %[[N_IDX]] : index +!CHECK: %[[CONVERT1_N:.*]] = fir.convert %[[LOAD_N]] : (i32) -> i64 +!CHECK: %[[CONVERT2_N:.*]] = fir.convert %[[CONVERT1_N]] : (i64) -> index +!CHECK: %[[LB:.*]] = arith.subi %[[CONVERT2_N]], %[[N_IDX]] : index !CHECK: %[[UB:.*]] = arith.subi %[[EXT_B]], %c1{{.*}} : index !CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[EXT_B]] : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[N_IDX]] : index) {strideInBytes = true} !CHECK: %[[ADDR:.*]] = fir.box_addr %[[DECLB]]#0 : (!fir.box>) -> !fir.ref> @@ -424,8 +427,9 @@ subroutine acc_enter_data_assumed(a, b, n, m) !CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLA]]#0, %[[C0]] : (!fir.box>, index) -> (index, index, index) !CHECK: %[[LOAD_N:.*]] = fir.load %[[DECLN]]#0 : !fir.ref -!CHECK: %[[CONVERT_N:.*]] = fir.convert %[[LOAD_N]] : (i32) -> index -!CHECK: %[[LB:.*]] = arith.subi %[[CONVERT_N]], %[[ONE]] : index +!CHECK: %[[CONVERT1_N:.*]] = fir.convert %[[LOAD_N]] : (i32) -> i64 +!CHECK: %[[CONVERT2_N:.*]] = fir.convert %[[CONVERT1_N]] : (i64) -> index +!CHECK: %[[LB:.*]] = arith.subi %[[CONVERT2_N]], %[[ONE]] : index !CHECK: %[[C0:.*]] = arith.constant 0 : index !CHECK: %[[DIMS:.*]]:3 = fir.box_dims %[[DECLA]]#1, %[[C0]] : (!fir.box>, index) -> (index, index, index) @@ -444,8 +448,9 @@ subroutine acc_enter_data_assumed(a, b, n, m) !CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLA]]#0, %[[C0]] : (!fir.box>, index) -> (index, index, index) !CHECK: %[[LOAD_M:.*]] = fir.load %[[DECLM]]#0 : !fir.ref -!CHECK: %[[CONVERT_M:.*]] = fir.convert %[[LOAD_M]] : (i32) -> index -!CHECK: %[[UB:.*]] = arith.subi %[[CONVERT_M]], %[[ONE]] : index +!CHECK: %[[CONVERT1_M:.*]] = fir.convert %[[LOAD_M]] : (i32) -> i64 +!CHECK: %[[CONVERT2_M:.*]] = fir.convert %[[CONVERT1_M]] : (i64) -> index +!CHECK: %[[UB:.*]] = arith.subi %[[CONVERT2_M]], %[[ONE]] : index !CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[DECLA]]#1, %{{.*}} : (!fir.box>, index) -> (index, index, index) !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[BASELB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true} @@ -460,12 +465,14 @@ subroutine acc_enter_data_assumed(a, b, n, m) !CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLA]]#0, %[[C0]] : (!fir.box>, index) -> (index, index, index) !CHECK: %[[LOAD_N:.*]] = fir.load %[[DECLN]]#0 : !fir.ref -!CHECK: %[[CONVERT_N:.*]] = fir.convert %[[LOAD_N]] : (i32) -> index -!CHECK: %[[LB:.*]] = arith.subi %[[CONVERT_N]], %[[ONE]] : index +!CHECK: %[[CONVERT1_N:.*]] = fir.convert %[[LOAD_N]] : (i32) -> i64 +!CHECK: %[[CONVERT2_N:.*]] = fir.convert %[[CONVERT1_N]] : (i64) -> index +!CHECK: %[[LB:.*]] = arith.subi %[[CONVERT2_N]], %[[ONE]] : index !CHECK: %[[LOAD_M:.*]] = fir.load %[[DECLM]]#0 : !fir.ref -!CHECK: %[[CONVERT_M:.*]] = fir.convert %[[LOAD_M]] : (i32) -> index -!CHECK: %[[UB:.*]] = arith.subi %[[CONVERT_M]], %[[ONE]] : index +!CHECK: %[[CONVERT1_M:.*]] = fir.convert %[[LOAD_M]] : (i32) -> i64 +!CHECK: %[[CONVERT2_M:.*]] = fir.convert %[[CONVERT1_M]] : (i64) -> index +!CHECK: %[[UB:.*]] = arith.subi %[[CONVERT2_M]], %[[ONE]] : index !CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[DECLA]]#1, %{{.*}} : (!fir.box>, index) -> (index, index, index) !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[ONE]] : index) {strideInBytes = true} @@ -480,8 +487,9 @@ subroutine acc_enter_data_assumed(a, b, n, m) !CHECK: %[[DIMS0:.*]]:3 = fir.box_dims %[[DECLB]]#0, %[[C0]] : (!fir.box>, index) -> (index, index, index) !CHECK: %[[LOAD_M:.*]] = fir.load %[[DECLM]]#0 : !fir.ref -!CHECK: %[[CONVERT_M:.*]] = fir.convert %[[LOAD_M]] : (i32) -> index -!CHECK: %[[UB:.*]] = arith.subi %[[CONVERT_M]], %[[LB_C10_IDX]] : index +!CHECK: %[[CONVERT1_M:.*]] = fir.convert %[[LOAD_M]] : (i32) -> i64 +!CHECK: %[[CONVERT2_M:.*]] = fir.convert %[[CONVERT1_M]] : (i64) -> index +!CHECK: %[[UB:.*]] = arith.subi %[[CONVERT2_M]], %[[LB_C10_IDX]] : index !CHECK: %[[DIMS1:.*]]:3 = fir.box_dims %[[DECLB]]#1, %{{.*}} : (!fir.box>, index) -> (index, index, index) !CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[ZERO]] : index) upperbound(%[[UB]] : index) extent(%[[DIMS1]]#1 : index) stride(%[[DIMS0]]#2 : index) startIdx(%[[LB_C10_IDX]] : index) {strideInBytes = true} -- GitLab From 17af9addbbd0dc675c962bf034ea083b2b61a01a Mon Sep 17 00:00:00 2001 From: Marc Auberer Date: Wed, 20 Mar 2024 21:18:58 +0100 Subject: [PATCH 066/296] [DAG] Add SDPatternMatch m_ZExtOrSelf/m_SExtOrSelf/m_AExtOrSelf/m_TruncOrSelf matchers (#85480) Fixes #85395 --- llvm/include/llvm/CodeGen/SDPatternMatch.h | 33 +++++++++++++++++++ .../CodeGen/SelectionDAGPatternMatchTest.cpp | 32 ++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/llvm/include/llvm/CodeGen/SDPatternMatch.h b/llvm/include/llvm/CodeGen/SDPatternMatch.h index 541c0ecb5be3..4cc7bb9c3b55 100644 --- a/llvm/include/llvm/CodeGen/SDPatternMatch.h +++ b/llvm/include/llvm/CodeGen/SDPatternMatch.h @@ -632,6 +632,38 @@ template inline UnaryOpc_match m_Trunc(const Opnd &Op) { return UnaryOpc_match(ISD::TRUNCATE, Op); } +/// Match a zext or identity +/// Allows to peek through optional extensions +template +inline Or, Opnd> m_ZExtOrSelf(Opnd &&Op) { + return Or, Opnd>(m_ZExt(std::forward(Op)), + std::forward(Op)); +} + +/// Match a sext or identity +/// Allows to peek through optional extensions +template +inline Or, Opnd> m_SExtOrSelf(Opnd &&Op) { + return Or, Opnd>(m_SExt(std::forward(Op)), + std::forward(Op)); +} + +/// Match a aext or identity +/// Allows to peek through optional extensions +template +inline Or, Opnd> m_AExtOrSelf(Opnd &&Op) { + return Or, Opnd>(m_AnyExt(std::forward(Op)), + std::forward(Op)); +} + +/// Match a trunc or identity +/// Allows to peek through optional truncations +template +inline Or, Opnd> m_TruncOrSelf(Opnd &&Op) { + return Or, Opnd>(m_Trunc(std::forward(Op)), + std::forward(Op)); +} + // === Constants === struct ConstantInt_match { APInt *BindVal; @@ -737,6 +769,7 @@ template inline BinaryOpc_match m_Not(const ValTy &V) { return m_Xor(V, m_AllOnes()); } + } // namespace SDPatternMatch } // namespace llvm #endif diff --git a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp index 1967a62bbf9d..a7112cfac63d 100644 --- a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp +++ b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp @@ -251,6 +251,38 @@ TEST_F(SelectionDAGPatternMatchTest, patternCombinators) { EXPECT_TRUE(sd_match(Add, m_AllOf(m_Opc(ISD::ADD), m_OneUse()))); } +TEST_F(SelectionDAGPatternMatchTest, optionalResizing) { + SDLoc DL; + auto Int32VT = EVT::getIntegerVT(Context, 32); + auto Int64VT = EVT::getIntegerVT(Context, 64); + + SDValue Op32 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 1, Int32VT); + SDValue Op64 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 1, Int64VT); + SDValue ZExt = DAG->getNode(ISD::ZERO_EXTEND, DL, Int64VT, Op32); + SDValue SExt = DAG->getNode(ISD::SIGN_EXTEND, DL, Int64VT, Op32); + SDValue AExt = DAG->getNode(ISD::ANY_EXTEND, DL, Int64VT, Op32); + SDValue Trunc = DAG->getNode(ISD::TRUNCATE, DL, Int32VT, Op64); + + using namespace SDPatternMatch; + SDValue A; + EXPECT_TRUE(sd_match(Op32, m_ZExtOrSelf(m_Value(A)))); + EXPECT_TRUE(A == Op32); + EXPECT_TRUE(sd_match(ZExt, m_ZExtOrSelf(m_Value(A)))); + EXPECT_TRUE(A == Op32); + EXPECT_TRUE(sd_match(Op64, m_SExtOrSelf(m_Value(A)))); + EXPECT_TRUE(A == Op64); + EXPECT_TRUE(sd_match(SExt, m_SExtOrSelf(m_Value(A)))); + EXPECT_TRUE(A == Op32); + EXPECT_TRUE(sd_match(Op32, m_AExtOrSelf(m_Value(A)))); + EXPECT_TRUE(A == Op32); + EXPECT_TRUE(sd_match(AExt, m_AExtOrSelf(m_Value(A)))); + EXPECT_TRUE(A == Op32); + EXPECT_TRUE(sd_match(Op64, m_TruncOrSelf(m_Value(A)))); + EXPECT_TRUE(A == Op64); + EXPECT_TRUE(sd_match(Trunc, m_TruncOrSelf(m_Value(A)))); + EXPECT_TRUE(A == Op64); +} + TEST_F(SelectionDAGPatternMatchTest, matchNode) { SDLoc DL; auto Int32VT = EVT::getIntegerVT(Context, 32); -- GitLab From 1f1f569b29f42161ea978328aea60044f16eee49 Mon Sep 17 00:00:00 2001 From: Christudasan Devadasan Date: Wed, 20 Mar 2024 19:59:40 +0530 Subject: [PATCH 067/296] [PowerPC] Clang format (NFC). --- llvm/lib/Target/PowerPC/PPCFrameLowering.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp index 424501c35c04..6dcb59a3a57f 100644 --- a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp @@ -2676,7 +2676,7 @@ bool PPCFrameLowering::restoreCalleeSavedRegisters( Restored.set(Dst); } else { - // Default behavior for non-CR saves. + // Default behavior for non-CR saves. const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg); // Functions without NoUnwind need to preserve the order of elements in -- GitLab From c2fd0e4398a90be2f4640964f3c1f2a73a3a1487 Mon Sep 17 00:00:00 2001 From: Stanislav Mekhanoshin Date: Wed, 20 Mar 2024 13:29:29 -0700 Subject: [PATCH 068/296] [AMDGPU] Copy SOP properties from pseudo to real. NFCI. (#85997) This is to help llvm-obdump to analyze instructions in a future patch. --- llvm/lib/Target/AMDGPU/SOPInstructions.td | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td index 1159c4e0fc2e..d34ee34e5bbf 100644 --- a/llvm/lib/Target/AMDGPU/SOPInstructions.td +++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td @@ -60,6 +60,11 @@ class SOP1_Real op, SOP1_Pseudo ps, string real_name = ps.Mnemonic> : let SchedRW = ps.SchedRW; let mayLoad = ps.mayLoad; let mayStore = ps.mayStore; + let isTerminator = ps.isTerminator; + let isReturn = ps.isReturn; + let isCall = ps.isCall; + let isBranch = ps.isBranch; + let isBarrier = ps.isBarrier; // encoding bits<7> sdst; @@ -977,6 +982,9 @@ class SOPK_Real : let mayStore = ps.mayStore; let isBranch = ps.isBranch; let isCall = ps.isCall; + let isTerminator = ps.isTerminator; + let isReturn = ps.isReturn; + let isBarrier = ps.isBarrier; // encoding bits<7> sdst; @@ -1426,6 +1434,11 @@ class SOPP_Real : let SchedRW = ps.SchedRW; let mayLoad = ps.mayLoad; let mayStore = ps.mayStore; + let isTerminator = ps.isTerminator; + let isReturn = ps.isReturn; + let isCall = ps.isCall; + let isBranch = ps.isBranch; + let isBarrier = ps.isBarrier; bits <16> simm16; } -- GitLab From 294a6c3b650d2411e50487b287b24b7d85847162 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Wed, 20 Mar 2024 13:38:26 -0700 Subject: [PATCH 069/296] [clang] Fix a warning This patch fixes: clang/lib/CodeGen/CGExprComplex.cpp:1037:14: error: unused variable 'ComplexElementTy' [-Werror,-Wunused-variable] --- clang/lib/CodeGen/CGExprComplex.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/clang/lib/CodeGen/CGExprComplex.cpp b/clang/lib/CodeGen/CGExprComplex.cpp index 27ddaacc28f5..b873bc6737bb 100644 --- a/clang/lib/CodeGen/CGExprComplex.cpp +++ b/clang/lib/CodeGen/CGExprComplex.cpp @@ -1034,7 +1034,6 @@ ComplexPairTy ComplexExprEmitter::EmitBinDiv(const BinOpInfo &Op) { llvm::Value *OrigLHSi = LHSi; if (!LHSi) LHSi = llvm::Constant::getNullValue(RHSi->getType()); - QualType ComplexElementTy = Op.Ty->castAs()->getElementType(); if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Improved || (Op.FPFeatures.getComplexRange() == LangOptions::CX_Promoted && FPHasBeenPromoted)) -- GitLab From f6f474c4ef9694a4ca8f08d59fd112c250fb9c73 Mon Sep 17 00:00:00 2001 From: Paul Kirth Date: Wed, 20 Mar 2024 13:39:39 -0700 Subject: [PATCH 070/296] [llvm][lld] Pre-commit tests for RISCV TLSDESC symbols Currently, we mistakenly mark the local labels used in RISC-V TLSDESC as TLS symbols, when they should not be. This patch adds tests with the current incorrect behavior, and subsequent patches will address the issue. Reviewers: MaskRay, topperc Reviewed By: MaskRay Pull Request: https://github.com/llvm/llvm-project/pull/85816 --- lld/test/ELF/riscv-tlsdesc.s | 25 +++++++++++++++++++++++ llvm/test/CodeGen/RISCV/tlsdesc-symbol.ll | 24 ++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 llvm/test/CodeGen/RISCV/tlsdesc-symbol.ll diff --git a/lld/test/ELF/riscv-tlsdesc.s b/lld/test/ELF/riscv-tlsdesc.s index 1738f86256ca..c583e15cf30c 100644 --- a/lld/test/ELF/riscv-tlsdesc.s +++ b/lld/test/ELF/riscv-tlsdesc.s @@ -29,6 +29,12 @@ # RUN: ld.lld -e 0 -z now a.32.o c.32.so -o a.32.ie # RUN: llvm-objdump --no-show-raw-insn -M no-aliases -h -d a.32.ie | FileCheck %s --check-prefix=IE32 +# RUN: llvm-mc -triple=riscv64 -filetype=obj d.s -o d.64.o +# RUN: not ld.lld -shared -soname=d.64.so -o d.64.so d.64.o 2>&1 | FileCheck %s --check-prefix=BADTLSLABEL + +# RUN: llvm-mc -triple=riscv32 -filetype=obj d.s -o d.32.o --defsym ELF32=1 +# RUN: not ld.lld -shared -soname=d.32.so -o d.32.so d.32.o 2>&1 | FileCheck %s --check-prefix=BADTLSLABEL + # GD64-RELA: .rela.dyn { # GD64-RELA-NEXT: 0x2408 R_RISCV_TLSDESC - 0x7FF # GD64-RELA-NEXT: 0x23E8 R_RISCV_TLSDESC a 0x0 @@ -150,6 +156,9 @@ # IE32-NEXT: lw a0, 0x80(a0) # IE32-NEXT: add a0, a0, tp +## FIXME This should not pass, but the code MC layer needs a fix to prevent this. +# BADTLSLABEL: error: d.{{.*}}.o has an STT_TLS symbol but doesn't have an SHF_TLS section + #--- a.s .macro load dst, src .ifdef ELF32 @@ -192,3 +201,19 @@ b: .tbss .globl c c: .zero 4 + +#--- d.s +.macro load dst, src +.ifdef ELF32 +lw \dst, \src +.else +ld \dst, \src +.endif +.endm + +.Ltlsdesc_hi0: + auipc a0, %tlsdesc_hi(foo) + load a1, %tlsdesc_load_lo(.Ltlsdesc_hi0)(a0) + addi a0, a0, %tlsdesc_add_lo(.Ltlsdesc_hi0) + jalr t0, 0(a1), %tlsdesc_call(.Ltlsdesc_hi0) + add a1, a0, tp diff --git a/llvm/test/CodeGen/RISCV/tlsdesc-symbol.ll b/llvm/test/CodeGen/RISCV/tlsdesc-symbol.ll new file mode 100644 index 000000000000..23ba2ffb1ad7 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/tlsdesc-symbol.ll @@ -0,0 +1,24 @@ +;; The test in this file do not appear in tls-models.ll because +;; they are not auto-generated. +; RUN: llc -mtriple=riscv64 -relocation-model=pic -enable-tlsdesc < %s \ +; RUN: | llvm-mc -triple=riscv64 -filetype=obj -o - \ +; RUN: | llvm-readelf --symbols - \ +; RUN: | FileCheck %s + +; RUN: llc -mtriple=riscv32 -relocation-model=pic -enable-tlsdesc < %s \ +; RUN: | llvm-mc -triple=riscv32 -filetype=obj -o - \ +; RUN: | llvm-readelf --symbols - \ +; RUN: | FileCheck %s + +; Check that TLS symbols are lowered correctly based on the specified +; model. Make sure they're external to avoid them all being optimised to Local +; Exec for the executable. + +@unspecified = external thread_local global i32 + +define ptr @f1() nounwind { +entry: + ret ptr @unspecified + ; CHECK: Symbol table '.symtab' contains 7 entries: + ; CHECK: TLS {{.*}} unspecified +} -- GitLab From b7324b6a9c6bd43786ea853bf1a9730486b4bc88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= Date: Wed, 20 Mar 2024 21:04:06 +0000 Subject: [PATCH 071/296] [mlir][vector] Adds pattern rewrite for maskable Ops (#83827) Adds a generic pattern rewrite for maskable Ops, `MaskableOpRewritePattern`, that will work for both masked and un-masked cases, e.g. for both: * `vector.mask {vector.contract}` (masked), and * `vector.contract` (not masked). This helps to reduce code-duplication and standardise how we implement such patterns. Fixes #78787 --- .../mlir/Dialect/Vector/Utils/VectorUtils.h | 58 ++++++ .../Vector/Transforms/LowerVectorContract.cpp | 179 +++++++++--------- 2 files changed, 143 insertions(+), 94 deletions(-) diff --git a/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h b/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h index 3ce16ef361f3..35e76a8b623a 100644 --- a/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h +++ b/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h @@ -112,6 +112,64 @@ SmallVector getMixedSizesXfer(bool hasTensorSemantics, Operation *xfer, RewriterBase &rewriter); +/// A pattern for ops that implement `MaskableOpInterface` and that _might_ be +/// masked (i.e. inside `vector.mask` Op region). In particular: +/// 1. Matches `SourceOp` operation, Op. +/// 2.1. If Op is masked, retrieves the masking Op, maskOp, and updates the +/// insertion point to avoid inserting new ops into the `vector.mask` Op +/// region (which only allows one Op). +/// 2.2 If Op is not masked, this step is skipped. +/// 3. Invokes `matchAndRewriteMaskableOp` on Op and optionally maskOp if +/// found in step 2.1. +/// +/// This wrapper frees patterns from re-implementing the logic to update the +/// insertion point when a maskable Op is masked. Such patterns are still +/// responsible for providing an updated ("rewritten") version of: +/// a. the source Op when mask _is not_ present, +/// b. the source Op and the masking Op when mask _is_ present. +/// Note that the return value from `matchAndRewriteMaskableOp` depends on the +/// case above. +template +struct MaskableOpRewritePattern : OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + +private: + LogicalResult matchAndRewrite(SourceOp sourceOp, + PatternRewriter &rewriter) const final { + auto maskableOp = dyn_cast(sourceOp.getOperation()); + if (!maskableOp) + return failure(); + + Operation *rootOp = sourceOp; + + // If this Op is masked, update the insertion point to avoid inserting into + // the vector.mask Op region. + OpBuilder::InsertionGuard guard(rewriter); + MaskingOpInterface maskOp; + if (maskableOp.isMasked()) { + maskOp = maskableOp.getMaskingOp(); + rewriter.setInsertionPoint(maskOp); + rootOp = maskOp; + } + + FailureOr newOp = + matchAndRewriteMaskableOp(sourceOp, maskOp, rewriter); + if (failed(newOp)) + return failure(); + + rewriter.replaceOp(rootOp, *newOp); + return success(); + } + +public: + // Matches SourceOp that can potentially be masked with `maskingOp`. If the + // latter is present, returns an updated masking op (with a replacement for + // `sourceOp` nested inside). Otherwise, returns an updated `sourceOp`. + virtual FailureOr + matchAndRewriteMaskableOp(SourceOp sourceOp, MaskingOpInterface maskingOp, + PatternRewriter &rewriter) const = 0; +}; + } // namespace vector /// Constructs a permutation map of invariant memref indices to vector diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp index 0eaf9f71a37d..ba1c96805ff8 100644 --- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp +++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorContract.cpp @@ -41,7 +41,6 @@ using namespace mlir::vector; //===----------------------------------------------------------------------===// // Helper functions //===----------------------------------------------------------------------===// - // Helper to find an index in an affine map. static std::optional getResultIndex(AffineMap map, int64_t index) { for (int64_t i = 0, e = map.getNumResults(); i < e; ++i) { @@ -226,9 +225,9 @@ namespace { /// This only kicks in when VectorTransformsOptions is set to OuterProduct and /// the vector.contract op is a row-major matrix multiply. class ContractionOpToMatmulOpLowering - : public OpRewritePattern { + : public vector::MaskableOpRewritePattern { public: - using OpRewritePattern::OpRewritePattern; + using MaskableOpRewritePattern::MaskableOpRewritePattern; using FilterConstraintType = std::function; @@ -241,12 +240,13 @@ public: vector::VectorTransformsOptions vectorTransformOptions, MLIRContext *context, PatternBenefit benefit = 1, FilterConstraintType constraint = defaultFilter) - : OpRewritePattern(context, benefit), + : MaskableOpRewritePattern(context, benefit), vectorTransformOptions(vectorTransformOptions), filter(std::move(constraint)) {} - LogicalResult matchAndRewrite(vector::ContractionOp op, - PatternRewriter &rewriter) const override; + FailureOr + matchAndRewriteMaskableOp(vector::ContractionOp op, MaskingOpInterface maskOp, + PatternRewriter &rewriter) const override; private: /// Options to control the vector patterns. @@ -270,9 +270,9 @@ private: /// This only kicks in when VectorTransformsOptions is set to OuterProduct and /// the vector.contract op is a row-major matrix multiply. class ContractionOpToOuterProductOpLowering - : public OpRewritePattern { + : public MaskableOpRewritePattern { public: - using OpRewritePattern::OpRewritePattern; + using MaskableOpRewritePattern::MaskableOpRewritePattern; using FilterConstraintType = std::function; @@ -285,12 +285,13 @@ public: vector::VectorTransformsOptions vectorTransformOptions, MLIRContext *context, PatternBenefit benefit = 1, FilterConstraintType constraint = defaultFilter) - : OpRewritePattern(context, benefit), + : MaskableOpRewritePattern(context, benefit), vectorTransformOptions(vectorTransformOptions), filter(std::move(constraint)) {} - LogicalResult matchAndRewrite(vector::ContractionOp op, - PatternRewriter &rewriter) const override; + FailureOr + matchAndRewriteMaskableOp(vector::ContractionOp op, MaskingOpInterface maskOp, + PatternRewriter &rewriter) const override; private: /// Options to control the vector patterns. @@ -317,9 +318,9 @@ private: /// This only kicks in when VectorTransformsOptions is set to Dot and /// the vector.contract op is a row-major matmul or matvec. class ContractionOpToDotLowering - : public OpRewritePattern { + : public MaskableOpRewritePattern { public: - using OpRewritePattern::OpRewritePattern; + using MaskableOpRewritePattern::MaskableOpRewritePattern; using FilterConstraintType = std::function; @@ -332,11 +333,12 @@ public: vector::VectorTransformsOptions vectorTransformOptions, MLIRContext *context, PatternBenefit benefit = 1, const FilterConstraintType &constraint = defaultFilter) - : OpRewritePattern(context, benefit), + : MaskableOpRewritePattern(context, benefit), vectorTransformOptions(vectorTransformOptions), filter(defaultFilter) {} - LogicalResult matchAndRewrite(vector::ContractionOp op, - PatternRewriter &rewriter) const override; + FailureOr + matchAndRewriteMaskableOp(vector::ContractionOp op, MaskingOpInterface maskOp, + PatternRewriter &rewriter) const override; private: /// Options to control the vector patterns. @@ -358,9 +360,10 @@ private: /// /// This only kicks in when either VectorTransformsOptions is set /// to Dot or when other contraction patterns fail. -class ContractionOpLowering : public OpRewritePattern { +class ContractionOpLowering + : public MaskableOpRewritePattern { public: - using OpRewritePattern::OpRewritePattern; + using MaskableOpRewritePattern::MaskableOpRewritePattern; using FilterConstraintType = std::function; @@ -371,12 +374,13 @@ public: ContractionOpLowering(vector::VectorTransformsOptions vectorTransformOptions, MLIRContext *context, PatternBenefit benefit = 1, FilterConstraintType constraint = defaultFilter) - : OpRewritePattern(context, benefit), + : MaskableOpRewritePattern(context, benefit), vectorTransformOptions(vectorTransformOptions), filter(std::move(constraint)) {} - LogicalResult matchAndRewrite(vector::ContractionOp op, - PatternRewriter &rewriter) const override; + FailureOr + matchAndRewriteMaskableOp(vector::ContractionOp op, MaskingOpInterface maskOp, + PatternRewriter &rewriter) const override; private: /// Options to control the vector patterns. @@ -634,8 +638,10 @@ private: /// /// This only kicks in when VectorTransformsOptions is set to OuterProduct but /// otherwise supports any layout permutation of the matrix-multiply. -LogicalResult ContractionOpToOuterProductOpLowering::matchAndRewrite( - vector::ContractionOp op, PatternRewriter &rewriter) const { +FailureOr +ContractionOpToOuterProductOpLowering::matchAndRewriteMaskableOp( + vector::ContractionOp op, MaskingOpInterface maskOp, + PatternRewriter &rewriter) const { if (vectorTransformOptions.vectorContractLowering != vector::VectorContractLowering::OuterProduct) return failure(); @@ -643,43 +649,25 @@ LogicalResult ContractionOpToOuterProductOpLowering::matchAndRewrite( if (failed(filter(op))) return failure(); - // Vector mask setup. - OpBuilder::InsertionGuard guard(rewriter); - auto maskableOp = cast(op.getOperation()); - Operation *rootOp; - if (maskableOp.isMasked()) { - rewriter.setInsertionPoint(maskableOp.getMaskingOp()); - rootOp = maskableOp.getMaskingOp(); - } else { - rootOp = op; - } - UnrolledOuterProductGenerator e(rewriter, op); FailureOr matmatRes = e.matmat(); if (succeeded(matmatRes)) { - rewriter.replaceOp(rootOp, *matmatRes); - return success(); + return matmatRes; } FailureOr matvecRes = e.matvec(); if (succeeded(matvecRes)) { - rewriter.replaceOp(rootOp, *matvecRes); - return success(); - } - FailureOr tmatvecRes = e.tmatvec(); - if (succeeded(tmatvecRes)) { - rewriter.replaceOp(rootOp, *tmatvecRes); - return success(); + return matvecRes; } - return failure(); + FailureOr tmatvecRes = e.tmatvec(); + return tmatvecRes; } -LogicalResult -ContractionOpToDotLowering::matchAndRewrite(vector::ContractionOp op, - PatternRewriter &rewriter) const { +FailureOr ContractionOpToDotLowering::matchAndRewriteMaskableOp( + vector::ContractionOp op, MaskingOpInterface maskOp, + PatternRewriter &rewriter) const { // TODO: Support vector.mask. - auto maskableOp = cast(op.getOperation()); - if (maskableOp.isMasked()) + if (maskOp) return failure(); if (failed(filter(op))) @@ -788,15 +776,14 @@ ContractionOpToDotLowering::matchAndRewrite(vector::ContractionOp op, } if (auto acc = op.getAcc()) res = createAdd(op.getLoc(), res, acc, isInt, rewriter); - rewriter.replaceOp(op, res); - return success(); + return res; } /// Lower vector.contract with all size one reduction dimensions to /// elementwise ops when possible. struct ContractOpToElementwise - : public OpRewritePattern { - using OpRewritePattern::OpRewritePattern; + : public MaskableOpRewritePattern { + using MaskableOpRewritePattern::MaskableOpRewritePattern; using FilterConstraintType = std::function; static LogicalResult defaultFilter(vector::ContractionOp op) { @@ -806,14 +793,15 @@ struct ContractOpToElementwise vector::VectorTransformsOptions vectorTransformOptions, MLIRContext *context, PatternBenefit benefit = 1, const FilterConstraintType &constraint = defaultFilter) - : OpRewritePattern(context, benefit), + : MaskableOpRewritePattern(context, benefit), vectorTransformOptions(vectorTransformOptions), filter(defaultFilter) {} - LogicalResult matchAndRewrite(vector::ContractionOp contractOp, - PatternRewriter &rewriter) const override { + FailureOr + matchAndRewriteMaskableOp(vector::ContractionOp contractOp, + MaskingOpInterface maskOp, + PatternRewriter &rewriter) const override { // TODO: Support vector.mask. - auto maskableOp = cast(contractOp.getOperation()); - if (maskableOp.isMasked()) + if (maskOp) return failure(); if (failed(filter(contractOp))) @@ -903,8 +891,10 @@ struct ContractOpToElementwise std::optional result = createContractArithOp(loc, newLhs, newRhs, contractOp.getAcc(), contractOp.getKind(), rewriter, isInt); - rewriter.replaceOp(contractOp, {*result}); - return success(); + if (result) + return *result; + + return failure(); } private: @@ -930,9 +920,9 @@ private: // TODO: break down into transpose/reshape/cast ops // when they become available to avoid code dup // TODO: investigate lowering order impact on performance -LogicalResult -ContractionOpLowering::matchAndRewrite(vector::ContractionOp op, - PatternRewriter &rewriter) const { +FailureOr ContractionOpLowering::matchAndRewriteMaskableOp( + vector::ContractionOp op, MaskingOpInterface maskOp, + PatternRewriter &rewriter) const { if (failed(filter(op))) return failure(); @@ -951,29 +941,36 @@ ContractionOpLowering::matchAndRewrite(vector::ContractionOp op, // TODO: implement benefits, cost models. MLIRContext *ctx = op.getContext(); + ContractionOpToMatmulOpLowering pat1(vectorTransformOptions, ctx); - if (succeeded(pat1.matchAndRewrite(op, rewriter))) - return success(); + FailureOr newVal1 = + pat1.matchAndRewriteMaskableOp(op, maskOp, rewriter); + if (!failed(newVal1)) + return newVal1; + ContractionOpToOuterProductOpLowering pat2(vectorTransformOptions, ctx); - if (succeeded(pat2.matchAndRewrite(op, rewriter))) - return success(); + FailureOr newVal2 = + pat2.matchAndRewriteMaskableOp(op, maskOp, rewriter); + if (!failed(newVal2)) + return newVal2; + ContractionOpToDotLowering pat3(vectorTransformOptions, ctx); - if (succeeded(pat3.matchAndRewrite(op, rewriter))) - return success(); + FailureOr newVal3 = + pat3.matchAndRewriteMaskableOp(op, maskOp, rewriter); + if (!failed(newVal3)) + return newVal3; + ContractOpToElementwise pat4(vectorTransformOptions, ctx); - if (succeeded(pat4.matchAndRewrite(op, rewriter))) - return success(); + FailureOr newVal4 = + pat4.matchAndRewriteMaskableOp(op, maskOp, rewriter); + if (!failed(newVal4)) + return newVal4; // Vector mask setup. - OpBuilder::InsertionGuard guard(rewriter); - Operation *rootOp = op; - Value mask; - if (op.isMasked()) { - rewriter.setInsertionPoint(op.getMaskingOp()); - rootOp = op.getMaskingOp(); - mask = op.getMaskingOp().getMask(); - } + Value mask; + if (maskOp) + mask = maskOp.getMask(); // Find first batch dimension in LHS/RHS, and lower when found. std::vector> batchDimMap = op.getBatchDimMap(); if (!batchDimMap.empty()) { @@ -982,8 +979,7 @@ ContractionOpLowering::matchAndRewrite(vector::ContractionOp op, auto newOp = lowerParallel(rewriter, op, lhsIndex, rhsIndex, mask); if (failed(newOp)) return failure(); - rewriter.replaceOp(rootOp, *newOp); - return success(); + return newOp; } // Collect contracting dimensions. @@ -1003,8 +999,7 @@ ContractionOpLowering::matchAndRewrite(vector::ContractionOp op, auto newOp = lowerParallel(rewriter, op, lhsIndex, /*rhsIndex=*/-1, mask); if (failed(newOp)) return failure(); - rewriter.replaceOp(rootOp, *newOp); - return success(); + return newOp; } } @@ -1015,8 +1010,7 @@ ContractionOpLowering::matchAndRewrite(vector::ContractionOp op, auto newOp = lowerParallel(rewriter, op, /*lhsIndex=*/-1, rhsIndex, mask); if (failed(newOp)) return failure(); - rewriter.replaceOp(rootOp, *newOp); - return success(); + return newOp; } } @@ -1025,8 +1019,7 @@ ContractionOpLowering::matchAndRewrite(vector::ContractionOp op, auto newOp = lowerReduction(rewriter, op, mask); if (failed(newOp)) return failure(); - rewriter.replaceOp(rootOp, *newOp); - return success(); + return newOp; } return failure(); @@ -1291,12 +1284,11 @@ public: /// This only kicks in when VectorTransformsOptions is set to `Matmul`. /// vector.transpose operations are inserted if the vector.contract op is not a /// row-major matrix multiply. -LogicalResult -ContractionOpToMatmulOpLowering::matchAndRewrite(vector::ContractionOp op, - PatternRewriter &rew) const { +FailureOr ContractionOpToMatmulOpLowering::matchAndRewriteMaskableOp( + vector::ContractionOp op, MaskingOpInterface maskOp, + PatternRewriter &rew) const { // TODO: Support vector.mask. - auto maskableOp = cast(op.getOperation()); - if (maskableOp.isMasked()) + if (maskOp) return failure(); if (vectorTransformOptions.vectorContractLowering != @@ -1379,8 +1371,7 @@ ContractionOpToMatmulOpLowering::matchAndRewrite(vector::ContractionOp op, : static_cast( rew.create(loc, op.getAcc(), mul)); - rew.replaceOp(op, res); - return success(); + return res; } } // namespace -- GitLab From 5a9bdd85ee4d8527e2cedf44f3ce26ff414f9b6a Mon Sep 17 00:00:00 2001 From: "Oleksandr \"Alex\" Zinenko" Date: Wed, 20 Mar 2024 22:15:17 +0100 Subject: [PATCH 072/296] [mlir] split transform interfaces into a separate library (#85221) Transform interfaces are implemented, direction or via extensions, in libraries belonging to multiple other dialects. Those dialects don't need to depend on the non-interface part of the transform dialect, which includes the growing number of ops and transitive dependency footprint. Split out the interfaces into a separate library. This in turn requires flipping the dependency from the interface on the dialect that has crept in because both co-existed in one library. The interface shouldn't depend on the transform dialect either. As a consequence of splitting, the capability of the interpreter to automatically walk the payload IR to identify payload ops of a certain kind based on the type used for the entry point symbol argument is disabled. This is a good move by itself as it simplifies the interpreter logic. This functionality can be trivially replaced by a `transform.structured.match` operation. --- mlir/docs/Tutorials/transform/Ch2.md | 4 +- .../transform/Ch2/include/MyExtension.h | 2 +- .../transform/Ch2/include/MyExtension.td | 2 +- .../transform/Ch2/lib/MyExtension.cpp | 2 +- .../transform/Ch3/include/MyExtension.h | 5 +- .../transform/Ch3/include/MyExtension.td | 2 +- .../transform/Ch3/include/MyExtensionTypes.td | 2 +- .../transform/Ch3/lib/MyExtension.cpp | 1 + .../transform/Ch4/include/MyExtension.h | 2 +- .../transform/Ch4/include/MyExtension.td | 2 +- .../AMDGPU/TransformOps/AMDGPUTransformOps.h | 2 +- .../AMDGPU/TransformOps/AMDGPUTransformOps.td | 2 +- .../Affine/TransformOps/AffineTransformOps.h | 2 +- .../Affine/TransformOps/AffineTransformOps.td | 2 +- .../TransformOps/BufferizationTransformOps.h | 2 +- .../TransformOps/BufferizationTransformOps.td | 2 +- .../Func/TransformOps/FuncTransformOps.h | 2 +- .../Func/TransformOps/FuncTransformOps.td | 2 +- .../GPU/TransformOps/GPUTransformOps.h | 2 +- .../GPU/TransformOps/GPUTransformOps.td | 2 +- .../mlir/Dialect/GPU/TransformOps/Utils.h | 2 +- .../Linalg/TransformOps/LinalgTransformOps.h | 5 +- .../Linalg/TransformOps/LinalgTransformOps.td | 2 +- .../MemRef/TransformOps/MemRefTransformOps.h | 5 +- .../MemRef/TransformOps/MemRefTransformOps.td | 2 +- .../NVGPU/TransformOps/NVGPUTransformOps.h | 2 +- .../NVGPU/TransformOps/NVGPUTransformOps.td | 2 +- .../SCF/TransformOps/SCFTransformOps.h | 2 +- .../SCF/TransformOps/SCFTransformOps.td | 2 +- .../TransformOps/SparseTensorTransformOps.h | 2 +- .../Tensor/TransformOps/TensorTransformOps.h | 2 +- .../Tensor/TransformOps/TensorTransformOps.td | 2 +- .../mlir/Dialect/Transform/CMakeLists.txt | 1 + .../DebugExtension/DebugExtensionOps.h | 2 +- .../DebugExtension/DebugExtensionOps.td | 2 +- .../mlir/Dialect/Transform/IR/CMakeLists.txt | 10 ---- .../Dialect/Transform/IR/MatchInterfaces.h | 2 +- .../Dialect/Transform/IR/MatchInterfaces.td | 2 +- .../Dialect/Transform/IR/TransformDialect.td | 8 +--- .../mlir/Dialect/Transform/IR/TransformOps.h | 2 +- .../mlir/Dialect/Transform/IR/TransformOps.td | 2 +- .../Dialect/Transform/IR/TransformTypes.h | 2 +- .../Dialect/Transform/IR/TransformTypes.td | 2 +- .../Transform/Interfaces/CMakeLists.txt | 11 +++++ .../{IR => Interfaces}/TransformInterfaces.h | 11 +++-- .../{IR => Interfaces}/TransformInterfaces.td | 8 ++++ .../LoopExtension/LoopExtensionOps.h | 2 +- .../LoopExtension/LoopExtensionOps.td | 2 +- .../Transform/PDLExtension/PDLExtensionOps.h | 2 +- .../Transform/PDLExtension/PDLExtensionOps.td | 2 +- .../Transforms/TransformInterpreterPassBase.h | 2 +- .../Transforms/TransformInterpreterUtils.h | 2 +- .../Vector/TransformOps/VectorTransformOps.h | 2 +- .../Vector/TransformOps/VectorTransformOps.td | 2 +- .../lib/CAPI/Dialect/TransformInterpreter.cpp | 2 +- .../TransformOps/AffineTransformOps.cpp | 2 +- .../Func/TransformOps/FuncTransformOps.cpp | 2 +- .../GPU/TransformOps/GPUTransformOps.cpp | 2 +- mlir/lib/Dialect/GPU/TransformOps/Utils.cpp | 2 +- .../Linalg/TransformOps/LinalgMatchOps.cpp | 1 + .../TransformOps/LinalgTransformOps.cpp | 2 +- .../TransformOps/MemRefTransformOps.cpp | 3 +- .../SCF/TransformOps/SCFTransformOps.cpp | 2 +- mlir/lib/Dialect/Tensor/IR/TensorDialect.cpp | 2 +- .../TransformOps/TensorTransformOps.cpp | 2 +- mlir/lib/Dialect/Transform/CMakeLists.txt | 1 + mlir/lib/Dialect/Transform/IR/CMakeLists.txt | 3 +- .../Dialect/Transform/IR/TransformDialect.cpp | 5 +- .../lib/Dialect/Transform/IR/TransformOps.cpp | 2 +- .../Dialect/Transform/IR/TransformTypes.cpp | 4 +- .../Transform/Interfaces/CMakeLists.txt | 15 ++++++ .../TransformInterfaces.cpp | 29 +++-------- .../Transform/Transforms/CheckUses.cpp | 2 +- .../Transform/Transforms/InferEffects.cpp | 2 +- .../Transform/Transforms/InterpreterPass.cpp | 2 +- .../TransformInterpreterPassBase.cpp | 2 +- .../Transforms/TransformInterpreterUtils.cpp | 2 +- .../TransformOps/VectorTransformOps.cpp | 2 +- .../test/Dialect/Tensor/decompose-concat.mlir | 3 +- mlir/test/Dialect/Tensor/fold-empty-op.mlir | 6 ++- ...nsor-subset-ops-into-vector-transfers.mlir | 3 +- .../Dialect/Tensor/rewrite-as-constant.mlir | 3 +- ...act-to-outerproduct-matvec-transforms.mlir | 3 +- .../Vector/vector-materialize-mask.mlir | 3 +- .../vector-multi-reduction-lowering.mlir | 3 +- ...vector-multi-reduction-outer-lowering.mlir | 3 +- ...ctor-transfer-drop-unit-dims-patterns.mlir | 3 +- ...fer-full-partial-split-copy-transform.mlir | 11 +++-- .../vector-transfer-full-partial-split.mlir | 15 ++++-- .../vector-transfer-to-vector-load-store.mlir | 9 ++-- .../Vector/vector-transpose-lowering.mlir | 24 ++++++---- .../Dialect/Vector/CPU/test-shuffle16x16.mlir | 3 +- .../Dialect/Tensor/TestTensorTransforms.cpp | 2 +- .../TestTransformDialectExtension.cpp | 2 +- .../Transform/TestTransformDialectExtension.h | 2 +- .../TestTransformDialectExtension.td | 2 +- .../TestTransformDialectInterpreter.cpp | 2 +- .../Transform/TestTransformStateExtension.h | 2 +- .../TestTilingInterfaceTransformOps.cpp | 2 +- .../TestTilingInterfaceTransformOps.td | 2 +- .../llvm-project-overlay/mlir/BUILD.bazel | 48 ++++++++++++++----- .../mlir/test/BUILD.bazel | 3 ++ 102 files changed, 239 insertions(+), 168 deletions(-) create mode 100644 mlir/include/mlir/Dialect/Transform/Interfaces/CMakeLists.txt rename mlir/include/mlir/Dialect/Transform/{IR => Interfaces}/TransformInterfaces.h (99%) rename mlir/include/mlir/Dialect/Transform/{IR => Interfaces}/TransformInterfaces.td (98%) create mode 100644 mlir/lib/Dialect/Transform/Interfaces/CMakeLists.txt rename mlir/lib/Dialect/Transform/{IR => Interfaces}/TransformInterfaces.cpp (98%) diff --git a/mlir/docs/Tutorials/transform/Ch2.md b/mlir/docs/Tutorials/transform/Ch2.md index 1aaefd2f2c30..6a6cefd8785a 100644 --- a/mlir/docs/Tutorials/transform/Ch2.md +++ b/mlir/docs/Tutorials/transform/Ch2.md @@ -62,7 +62,7 @@ The operations themselves can be defined using ODS, exactly in the same way as r #define MY_EXTENSION include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/IR/OpBase.td" include "mlir/Interfaces/SideEffectInterfaces.td" @@ -124,7 +124,7 @@ This will generate two files, `MyExtension.h.inc` and `MyExtension.cpp.inc`, tha ```c++ // In MyExtension.h. #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #define GET_OP_CLASSES #include "MyExtension.h.inc" diff --git a/mlir/examples/transform/Ch2/include/MyExtension.h b/mlir/examples/transform/Ch2/include/MyExtension.h index 03a24a190e15..5ab70a505aee 100644 --- a/mlir/examples/transform/Ch2/include/MyExtension.h +++ b/mlir/examples/transform/Ch2/include/MyExtension.h @@ -13,7 +13,7 @@ #include "mlir/Bytecode/BytecodeOpInterface.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #define GET_OP_CLASSES #include "MyExtension.h.inc" diff --git a/mlir/examples/transform/Ch2/include/MyExtension.td b/mlir/examples/transform/Ch2/include/MyExtension.td index 4824b83e6c18..1abd95237055 100644 --- a/mlir/examples/transform/Ch2/include/MyExtension.td +++ b/mlir/examples/transform/Ch2/include/MyExtension.td @@ -15,7 +15,7 @@ #define MY_EXTENSION include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/IR/OpBase.td" include "mlir/Interfaces/SideEffectInterfaces.td" diff --git a/mlir/examples/transform/Ch2/lib/MyExtension.cpp b/mlir/examples/transform/Ch2/lib/MyExtension.cpp index 031c52c30738..b2955a905b88 100644 --- a/mlir/examples/transform/Ch2/lib/MyExtension.cpp +++ b/mlir/examples/transform/Ch2/lib/MyExtension.cpp @@ -15,8 +15,8 @@ #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/SCF/IR/SCF.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformTypes.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/DialectRegistry.h" #include "mlir/IR/Operation.h" #include "mlir/Interfaces/SideEffectInterfaces.h" diff --git a/mlir/examples/transform/Ch3/include/MyExtension.h b/mlir/examples/transform/Ch3/include/MyExtension.h index 223638eee1c0..086850403e1c 100644 --- a/mlir/examples/transform/Ch3/include/MyExtension.h +++ b/mlir/examples/transform/Ch3/include/MyExtension.h @@ -13,13 +13,16 @@ #include "mlir/Bytecode/BytecodeOpInterface.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" namespace mlir { class CallOpInterface; namespace func { class CallOp; } // namespace func +namespace transform { +class OperationType; +} // namespace transform } // namespace mlir #define GET_TYPEDEF_CLASSES diff --git a/mlir/examples/transform/Ch3/include/MyExtension.td b/mlir/examples/transform/Ch3/include/MyExtension.td index f444df18d69e..5a78186d75c7 100644 --- a/mlir/examples/transform/Ch3/include/MyExtension.td +++ b/mlir/examples/transform/Ch3/include/MyExtension.td @@ -16,7 +16,7 @@ include "MyExtensionTypes.td" include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/Dialect/Transform/IR/TransformTypes.td" include "mlir/IR/OpBase.td" include "mlir/Interfaces/SideEffectInterfaces.td" diff --git a/mlir/examples/transform/Ch3/include/MyExtensionTypes.td b/mlir/examples/transform/Ch3/include/MyExtensionTypes.td index 7d745935d478..8c4b8a9c782b 100644 --- a/mlir/examples/transform/Ch3/include/MyExtensionTypes.td +++ b/mlir/examples/transform/Ch3/include/MyExtensionTypes.td @@ -16,7 +16,7 @@ include "mlir/IR/AttrTypeBase.td" include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" // Transform dialect allows additional types to be defined and injected. def CallOpInterfaceHandle diff --git a/mlir/examples/transform/Ch3/lib/MyExtension.cpp b/mlir/examples/transform/Ch3/lib/MyExtension.cpp index dc0a8a0ab303..2e4388d4cc22 100644 --- a/mlir/examples/transform/Ch3/lib/MyExtension.cpp +++ b/mlir/examples/transform/Ch3/lib/MyExtension.cpp @@ -15,6 +15,7 @@ #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/SCF/IR/SCF.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" +#include "mlir/Dialect/Transform/IR/TransformTypes.h" #include "mlir/IR/DialectImplementation.h" #include "mlir/Interfaces/CallInterfaces.h" #include "llvm/ADT/TypeSwitch.h" diff --git a/mlir/examples/transform/Ch4/include/MyExtension.h b/mlir/examples/transform/Ch4/include/MyExtension.h index 13e5b3c04b02..620ec8f398a5 100644 --- a/mlir/examples/transform/Ch4/include/MyExtension.h +++ b/mlir/examples/transform/Ch4/include/MyExtension.h @@ -13,8 +13,8 @@ #include "mlir/Bytecode/BytecodeOpInterface.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformOps.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" namespace mlir { class CallOpInterface; diff --git a/mlir/examples/transform/Ch4/include/MyExtension.td b/mlir/examples/transform/Ch4/include/MyExtension.td index ae58dc37db43..6c83ff0f46c8 100644 --- a/mlir/examples/transform/Ch4/include/MyExtension.td +++ b/mlir/examples/transform/Ch4/include/MyExtension.td @@ -16,7 +16,7 @@ include "mlir/Dialect/Transform/IR/MatchInterfaces.td" include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/IR/OpBase.td" include "mlir/Interfaces/SideEffectInterfaces.td" diff --git a/mlir/include/mlir/Dialect/AMDGPU/TransformOps/AMDGPUTransformOps.h b/mlir/include/mlir/Dialect/AMDGPU/TransformOps/AMDGPUTransformOps.h index 4fb4ab08a0da..dcf934c71dd1 100644 --- a/mlir/include/mlir/Dialect/AMDGPU/TransformOps/AMDGPUTransformOps.h +++ b/mlir/include/mlir/Dialect/AMDGPU/TransformOps/AMDGPUTransformOps.h @@ -12,7 +12,7 @@ #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/Transform/IR/TransformAttrs.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/OpImplementation.h" #include "mlir/IR/RegionKindInterface.h" diff --git a/mlir/include/mlir/Dialect/AMDGPU/TransformOps/AMDGPUTransformOps.td b/mlir/include/mlir/Dialect/AMDGPU/TransformOps/AMDGPUTransformOps.td index 0eb670506086..8aaa87511a2b 100644 --- a/mlir/include/mlir/Dialect/AMDGPU/TransformOps/AMDGPUTransformOps.td +++ b/mlir/include/mlir/Dialect/AMDGPU/TransformOps/AMDGPUTransformOps.td @@ -11,7 +11,7 @@ include "mlir/Dialect/Transform/IR/TransformAttrs.td" include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/Dialect/Transform/IR/TransformTypes.td" include "mlir/Interfaces/SideEffectInterfaces.td" diff --git a/mlir/include/mlir/Dialect/Affine/TransformOps/AffineTransformOps.h b/mlir/include/mlir/Dialect/Affine/TransformOps/AffineTransformOps.h index f52f04ada036..1001cb52b518 100644 --- a/mlir/include/mlir/Dialect/Affine/TransformOps/AffineTransformOps.h +++ b/mlir/include/mlir/Dialect/Affine/TransformOps/AffineTransformOps.h @@ -10,8 +10,8 @@ #define MLIR_DIALECT_AFFINE_TRANSFORMOPS_AFFINETRANSFORMOPS_H #include "mlir/Bytecode/BytecodeOpInterface.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformTypes.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/OpImplementation.h" namespace mlir { diff --git a/mlir/include/mlir/Dialect/Affine/TransformOps/AffineTransformOps.td b/mlir/include/mlir/Dialect/Affine/TransformOps/AffineTransformOps.td index b74e4af6eedd..70b127fd063c 100644 --- a/mlir/include/mlir/Dialect/Affine/TransformOps/AffineTransformOps.td +++ b/mlir/include/mlir/Dialect/Affine/TransformOps/AffineTransformOps.td @@ -10,7 +10,7 @@ #define AFFINE_TRANSFORM_OPS include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/Dialect/Transform/IR/TransformTypes.td" include "mlir/Interfaces/SideEffectInterfaces.td" include "mlir/IR/OpBase.td" diff --git a/mlir/include/mlir/Dialect/Bufferization/TransformOps/BufferizationTransformOps.h b/mlir/include/mlir/Dialect/Bufferization/TransformOps/BufferizationTransformOps.h index 75ce4b484165..1dbe29b44413 100644 --- a/mlir/include/mlir/Dialect/Bufferization/TransformOps/BufferizationTransformOps.h +++ b/mlir/include/mlir/Dialect/Bufferization/TransformOps/BufferizationTransformOps.h @@ -11,8 +11,8 @@ #include "mlir/Bytecode/BytecodeOpInterface.h" #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformTypes.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/OpImplementation.h" namespace mlir { diff --git a/mlir/include/mlir/Dialect/Bufferization/TransformOps/BufferizationTransformOps.td b/mlir/include/mlir/Dialect/Bufferization/TransformOps/BufferizationTransformOps.td index 9b588eb610e5..5ace9c390e14 100644 --- a/mlir/include/mlir/Dialect/Bufferization/TransformOps/BufferizationTransformOps.td +++ b/mlir/include/mlir/Dialect/Bufferization/TransformOps/BufferizationTransformOps.td @@ -11,7 +11,7 @@ include "mlir/Dialect/Bufferization/IR/BufferizationEnums.td" include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/Dialect/Transform/IR/TransformTypes.td" include "mlir/Interfaces/SideEffectInterfaces.td" include "mlir/IR/OpBase.td" diff --git a/mlir/include/mlir/Dialect/Func/TransformOps/FuncTransformOps.h b/mlir/include/mlir/Dialect/Func/TransformOps/FuncTransformOps.h index 8d0b97da95d4..37f0ea0f2855 100644 --- a/mlir/include/mlir/Dialect/Func/TransformOps/FuncTransformOps.h +++ b/mlir/include/mlir/Dialect/Func/TransformOps/FuncTransformOps.h @@ -10,7 +10,7 @@ #define MLIR_DIALECT_FUNC_TRANSFORMOPS_FUNCTRANSFORMOPS_H #include "mlir/Bytecode/BytecodeOpInterface.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/OpImplementation.h" #define GET_OP_CLASSES diff --git a/mlir/include/mlir/Dialect/Func/TransformOps/FuncTransformOps.td b/mlir/include/mlir/Dialect/Func/TransformOps/FuncTransformOps.td index c36fdd150556..306fbf881de6 100644 --- a/mlir/include/mlir/Dialect/Func/TransformOps/FuncTransformOps.td +++ b/mlir/include/mlir/Dialect/Func/TransformOps/FuncTransformOps.td @@ -10,7 +10,7 @@ #define FUNC_TRANSFORM_OPS include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/Dialect/Transform/IR/TransformTypes.td" include "mlir/Interfaces/SideEffectInterfaces.td" include "mlir/IR/RegionKindInterface.td" diff --git a/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.h b/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.h index d6612c7c0b7f..4b5f68452504 100644 --- a/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.h +++ b/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.h @@ -10,7 +10,7 @@ #define MLIR_DIALECT_GPU_TRANSFORMOPS_GPUTRANSFORMOPS_H #include "mlir/Dialect/SCF/IR/SCF.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/OpImplementation.h" #include "mlir/IR/PatternMatch.h" diff --git a/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td b/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td index 616a82b08a61..80b4547c32c1 100644 --- a/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td +++ b/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td @@ -10,7 +10,7 @@ #define GPU_TRANSFORM_OPS include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/Interfaces/SideEffectInterfaces.td" include "mlir/IR/OpBase.td" diff --git a/mlir/include/mlir/Dialect/GPU/TransformOps/Utils.h b/mlir/include/mlir/Dialect/GPU/TransformOps/Utils.h index c65f522e0187..52fc6f4d5c71 100644 --- a/mlir/include/mlir/Dialect/GPU/TransformOps/Utils.h +++ b/mlir/include/mlir/Dialect/GPU/TransformOps/Utils.h @@ -11,7 +11,7 @@ #include "mlir/Dialect/GPU/IR/GPUDialect.h" #include "mlir/Dialect/SCF/IR/DeviceMappingInterface.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/OpImplementation.h" #include "mlir/IR/PatternMatch.h" diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h index 12923663b3fb..3af642752724 100644 --- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h +++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h @@ -14,7 +14,7 @@ #include "mlir/Dialect/Linalg/IR/Linalg.h" #include "mlir/Dialect/Transform/IR/TransformAttrs.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Dialect/Utils/StructuredOpsUtils.h" #include "mlir/IR/OpImplementation.h" #include "mlir/IR/RegionKindInterface.h" @@ -38,6 +38,9 @@ class UnPackOp; } // namespace tensor namespace transform { +class AnyOpType; +class AnyValueType; +class OperationType; class TransformHandleTypeInterface; // Types needed for builders. struct TileSizesSpec {}; diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td index bdeab55091b9..4f34016066b4 100644 --- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td +++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td @@ -12,7 +12,7 @@ include "mlir/Dialect/Linalg/TransformOps/LinalgTransformEnums.td" include "mlir/Dialect/Transform/IR/TransformAttrs.td" include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/Dialect/Transform/IR/TransformTypes.td" include "mlir/Dialect/SCF/IR/DeviceMappingInterface.td" include "mlir/Interfaces/SideEffectInterfaces.td" diff --git a/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.h b/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.h index ee7c69683148..a87767acdd3b 100644 --- a/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.h +++ b/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.h @@ -10,13 +10,16 @@ #define MLIR_DIALECT_MEMREF_TRANSFORMOPS_MEMREFTRANSFORMOPS_H #include "mlir/Bytecode/BytecodeOpInterface.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/OpImplementation.h" namespace mlir { namespace memref { class AllocOp; } // namespace memref +namespace transform { +class OperationType; +} // namespace transform } // namespace mlir #define GET_OP_CLASSES diff --git a/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.td b/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.td index 29383a3825be..2d060f3c2da6 100644 --- a/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.td +++ b/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.td @@ -10,7 +10,7 @@ #define MEMREF_TRANSFORM_OPS include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/Dialect/Transform/IR/TransformTypes.td" include "mlir/Interfaces/SideEffectInterfaces.td" include "mlir/IR/OpBase.td" diff --git a/mlir/include/mlir/Dialect/NVGPU/TransformOps/NVGPUTransformOps.h b/mlir/include/mlir/Dialect/NVGPU/TransformOps/NVGPUTransformOps.h index 1c30cc4a57d8..5179adeb09db 100644 --- a/mlir/include/mlir/Dialect/NVGPU/TransformOps/NVGPUTransformOps.h +++ b/mlir/include/mlir/Dialect/NVGPU/TransformOps/NVGPUTransformOps.h @@ -11,7 +11,7 @@ #include "mlir/Dialect/Transform/IR/TransformAttrs.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/OpImplementation.h" #include "mlir/IR/RegionKindInterface.h" diff --git a/mlir/include/mlir/Dialect/NVGPU/TransformOps/NVGPUTransformOps.td b/mlir/include/mlir/Dialect/NVGPU/TransformOps/NVGPUTransformOps.td index bce84cb3fdea..0225562baa58 100644 --- a/mlir/include/mlir/Dialect/NVGPU/TransformOps/NVGPUTransformOps.td +++ b/mlir/include/mlir/Dialect/NVGPU/TransformOps/NVGPUTransformOps.td @@ -11,7 +11,7 @@ include "mlir/Dialect/Transform/IR/TransformAttrs.td" include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/Dialect/Transform/IR/TransformTypes.td" include "mlir/Interfaces/SideEffectInterfaces.td" diff --git a/mlir/include/mlir/Dialect/SCF/TransformOps/SCFTransformOps.h b/mlir/include/mlir/Dialect/SCF/TransformOps/SCFTransformOps.h index d14d63e56dc7..65ccd43b56c8 100644 --- a/mlir/include/mlir/Dialect/SCF/TransformOps/SCFTransformOps.h +++ b/mlir/include/mlir/Dialect/SCF/TransformOps/SCFTransformOps.h @@ -10,8 +10,8 @@ #define MLIR_DIALECT_SCF_TRANSFORMOPS_SCFTRANSFORMOPS_H #include "mlir/Bytecode/BytecodeOpInterface.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformTypes.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/OpImplementation.h" #include "mlir/Interfaces/LoopLikeInterface.h" diff --git a/mlir/include/mlir/Dialect/SCF/TransformOps/SCFTransformOps.td b/mlir/include/mlir/Dialect/SCF/TransformOps/SCFTransformOps.td index cef73689c072..6f94cee5b019 100644 --- a/mlir/include/mlir/Dialect/SCF/TransformOps/SCFTransformOps.td +++ b/mlir/include/mlir/Dialect/SCF/TransformOps/SCFTransformOps.td @@ -10,7 +10,7 @@ #define SCF_TRANSFORM_OPS include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/Dialect/Transform/IR/TransformTypes.td" include "mlir/Interfaces/SideEffectInterfaces.td" include "mlir/IR/OpBase.td" diff --git a/mlir/include/mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.h b/mlir/include/mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.h index 1c52e4881dc8..54a9e2aec805 100644 --- a/mlir/include/mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.h +++ b/mlir/include/mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.h @@ -12,7 +12,7 @@ #include "mlir/Dialect/Transform/IR/MatchInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformAttrs.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/OpImplementation.h" #include "mlir/IR/RegionKindInterface.h" diff --git a/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.h b/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.h index 0e5fda6041fa..4fb777df3409 100644 --- a/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.h +++ b/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.h @@ -9,8 +9,8 @@ #ifndef MLIR_DIALECT_TENSOR_TRANSFORMOPS_TENSORTRANSFORMOPS_H #define MLIR_DIALECT_TENSOR_TRANSFORMOPS_TENSORTRANSFORMOPS_H -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformTypes.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/OpImplementation.h" #include "mlir/IR/PatternMatch.h" diff --git a/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td b/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td index 39e1d7fa3494..fea5afa0b7bb 100644 --- a/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td +++ b/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td @@ -10,7 +10,7 @@ #define TENSOR_TRANSFORM_OPS include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/Dialect/Transform/IR/TransformTypes.td" include "mlir/Interfaces/SideEffectInterfaces.td" include "mlir/IR/OpBase.td" diff --git a/mlir/include/mlir/Dialect/Transform/CMakeLists.txt b/mlir/include/mlir/Dialect/Transform/CMakeLists.txt index ef44ab891cc5..0cd71ec6919d 100644 --- a/mlir/include/mlir/Dialect/Transform/CMakeLists.txt +++ b/mlir/include/mlir/Dialect/Transform/CMakeLists.txt @@ -1,4 +1,5 @@ add_subdirectory(DebugExtension) +add_subdirectory(Interfaces) add_subdirectory(IR) add_subdirectory(LoopExtension) add_subdirectory(PDLExtension) diff --git a/mlir/include/mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.h b/mlir/include/mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.h index d660393a71e0..05abe5adbe80 100644 --- a/mlir/include/mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.h +++ b/mlir/include/mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.h @@ -12,7 +12,7 @@ #include "mlir/Bytecode/BytecodeOpInterface.h" #include "mlir/Dialect/Transform/IR/MatchInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/OpDefinition.h" #include "mlir/IR/OpImplementation.h" #include "mlir/Interfaces/SideEffectInterfaces.h" diff --git a/mlir/include/mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.td b/mlir/include/mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.td index 16e2a39044b4..dc9b7c4229ac 100644 --- a/mlir/include/mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.td +++ b/mlir/include/mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.td @@ -17,7 +17,7 @@ include "mlir/Interfaces/SideEffectInterfaces.td" include "mlir/IR/OpBase.td" include "mlir/Dialect/Transform/IR/MatchInterfaces.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/Dialect/Transform/IR/TransformDialect.td" def DebugEmitRemarkAtOp : TransformDialectOp<"debug.emit_remark_at", diff --git a/mlir/include/mlir/Dialect/Transform/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Transform/IR/CMakeLists.txt index 2b8a887257ca..3ccac2f5c165 100644 --- a/mlir/include/mlir/Dialect/Transform/IR/CMakeLists.txt +++ b/mlir/include/mlir/Dialect/Transform/IR/CMakeLists.txt @@ -24,17 +24,7 @@ add_dependencies(mlir-headers MLIRTransformDialectEnumIncGen) add_mlir_dialect(TransformOps transform) add_mlir_doc(TransformOps TransformOps Dialects/ -gen-op-doc -dialect=transform) -# Contrary to what the name claims, this only produces the _op_ interface. -add_mlir_interface(TransformInterfaces) -add_mlir_doc(TransformInterfaces TransformOpInterfaces Dialects/ -gen-op-interface-docs) - add_mlir_interface(MatchInterfaces) add_dependencies(MLIRMatchInterfacesIncGen MLIRTransformInterfacesIncGen) add_mlir_doc(TransformInterfaces MatchOpInterfaces Dialects/ -gen-op-interface-docs) -set(LLVM_TARGET_DEFINITIONS TransformInterfaces.td) -mlir_tablegen(TransformTypeInterfaces.h.inc -gen-type-interface-decls) -mlir_tablegen(TransformTypeInterfaces.cpp.inc -gen-type-interface-defs) -add_public_tablegen_target(MLIRTransformDialectTypeInterfacesIncGen) -add_dependencies(mlir-headers MLIRTransformDialectTypeInterfacesIncGen) -add_mlir_doc(TransformInterfaces TransformTypeInterfaces Dialects/ -gen-type-interface-docs) diff --git a/mlir/include/mlir/Dialect/Transform/IR/MatchInterfaces.h b/mlir/include/mlir/Dialect/Transform/IR/MatchInterfaces.h index 36aeb4583029..13a52b54201e 100644 --- a/mlir/include/mlir/Dialect/Transform/IR/MatchInterfaces.h +++ b/mlir/include/mlir/Dialect/Transform/IR/MatchInterfaces.h @@ -12,7 +12,7 @@ #include #include -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/OpDefinition.h" #include "llvm/ADT/STLExtras.h" diff --git a/mlir/include/mlir/Dialect/Transform/IR/MatchInterfaces.td b/mlir/include/mlir/Dialect/Transform/IR/MatchInterfaces.td index be92e4d91b42..56d2ac648599 100644 --- a/mlir/include/mlir/Dialect/Transform/IR/MatchInterfaces.td +++ b/mlir/include/mlir/Dialect/Transform/IR/MatchInterfaces.td @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// include "mlir/IR/OpBase.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" def MatchOpInterface : OpInterface<"MatchOpInterface", [TransformOpInterface]> { diff --git a/mlir/include/mlir/Dialect/Transform/IR/TransformDialect.td b/mlir/include/mlir/Dialect/Transform/IR/TransformDialect.td index 33c2e7ce0e6d..d03049e186f9 100644 --- a/mlir/include/mlir/Dialect/Transform/IR/TransformDialect.td +++ b/mlir/include/mlir/Dialect/Transform/IR/TransformDialect.td @@ -23,7 +23,7 @@ def Transform_Dialect : Dialect { /// Symbol name for the default entry point "named sequence". constexpr const static ::llvm::StringLiteral kTransformEntryPointSymbolName = "__transform_main"; - + /// Name of the attribute attachable to the symbol table operation /// containing named sequences. This is used to trigger verification. constexpr const static ::llvm::StringLiteral @@ -34,12 +34,6 @@ def Transform_Dialect : Dialect { constexpr const static ::llvm::StringLiteral kTargetTagAttrName = "transform.target_tag"; - /// Name of the attribute attachable to an operation, indicating that - /// TrackingListener failures should be silenced. - constexpr const static ::llvm::StringLiteral - kSilenceTrackingFailuresAttrName = - "transform.silence_tracking_failures"; - /// Names of the attributes indicating whether an argument of an external /// transform dialect symbol is consumed or only read. constexpr const static ::llvm::StringLiteral kArgConsumedAttrName = diff --git a/mlir/include/mlir/Dialect/Transform/IR/TransformOps.h b/mlir/include/mlir/Dialect/Transform/IR/TransformOps.h index 1f46e34171b8..6c10fcf75804 100644 --- a/mlir/include/mlir/Dialect/Transform/IR/TransformOps.h +++ b/mlir/include/mlir/Dialect/Transform/IR/TransformOps.h @@ -13,8 +13,8 @@ #include "mlir/Dialect/Transform/IR/MatchInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformAttrs.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformTypes.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/OpDefinition.h" #include "mlir/IR/OpImplementation.h" #include "mlir/IR/PatternMatch.h" diff --git a/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td b/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td index 1766e4bb875f..9caa7632c177 100644 --- a/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td +++ b/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td @@ -21,7 +21,7 @@ include "mlir/IR/SymbolInterfaces.td" include "mlir/Dialect/Transform/IR/MatchInterfaces.td" include "mlir/Dialect/Transform/IR/TransformAttrs.td" include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" def AlternativesOp : TransformDialectOp<"alternatives", [DeclareOpInterfaceMethods::verifyTrait(Operation *op) { return success(); } -#endif // DIALECT_TRANSFORM_IR_TRANSFORMINTERFACES_H +#endif // DIALECT_TRANSFORM_INTERFACES_TRANSFORMINTERFACES_H diff --git a/mlir/include/mlir/Dialect/Transform/IR/TransformInterfaces.td b/mlir/include/mlir/Dialect/Transform/Interfaces/TransformInterfaces.td similarity index 98% rename from mlir/include/mlir/Dialect/Transform/IR/TransformInterfaces.td rename to mlir/include/mlir/Dialect/Transform/Interfaces/TransformInterfaces.td index 8f7b8f1999e0..c5c4c61bc2fe 100644 --- a/mlir/include/mlir/Dialect/Transform/IR/TransformInterfaces.td +++ b/mlir/include/mlir/Dialect/Transform/Interfaces/TransformInterfaces.td @@ -241,6 +241,14 @@ def FindPayloadReplacementOpInterface /*arguments=*/(ins) >, ]; + + let extraSharedClassDeclaration = [{ + /// Name of the attribute attachable to an operation, indicating that + /// TrackingListener failures should be silenced. + constexpr const static ::llvm::StringLiteral + kSilenceTrackingFailuresAttrName = + "transform.silence_tracking_failures"; + }]; } def PatternDescriptorOpInterface : OpInterface<"PatternDescriptorOpInterface"> { diff --git a/mlir/include/mlir/Dialect/Transform/LoopExtension/LoopExtensionOps.h b/mlir/include/mlir/Dialect/Transform/LoopExtension/LoopExtensionOps.h index 68cc0699d081..f3f696bb0b9c 100644 --- a/mlir/include/mlir/Dialect/Transform/LoopExtension/LoopExtensionOps.h +++ b/mlir/include/mlir/Dialect/Transform/LoopExtension/LoopExtensionOps.h @@ -11,7 +11,7 @@ #include "mlir/Bytecode/BytecodeOpInterface.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/OpDefinition.h" #include "mlir/IR/OpImplementation.h" #include "mlir/Interfaces/LoopLikeInterface.h" diff --git a/mlir/include/mlir/Dialect/Transform/LoopExtension/LoopExtensionOps.td b/mlir/include/mlir/Dialect/Transform/LoopExtension/LoopExtensionOps.td index 78a8c6ad489a..885b55811e62 100644 --- a/mlir/include/mlir/Dialect/Transform/LoopExtension/LoopExtensionOps.td +++ b/mlir/include/mlir/Dialect/Transform/LoopExtension/LoopExtensionOps.td @@ -10,7 +10,7 @@ #define MLIR_DIALECT_TRANSFORM_LOOPEXTENSION_LOOPEXTENSIONOPS include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/Interfaces/SideEffectInterfaces.td" def HoistLoopInvariantSubsetsOp diff --git a/mlir/include/mlir/Dialect/Transform/PDLExtension/PDLExtensionOps.h b/mlir/include/mlir/Dialect/Transform/PDLExtension/PDLExtensionOps.h index 5172bcf204e5..7f52e00ec30b 100644 --- a/mlir/include/mlir/Dialect/Transform/PDLExtension/PDLExtensionOps.h +++ b/mlir/include/mlir/Dialect/Transform/PDLExtension/PDLExtensionOps.h @@ -11,7 +11,7 @@ #include "mlir/Bytecode/BytecodeOpInterface.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/OpDefinition.h" #include "mlir/IR/OpImplementation.h" #include "mlir/IR/SymbolTable.h" diff --git a/mlir/include/mlir/Dialect/Transform/PDLExtension/PDLExtensionOps.td b/mlir/include/mlir/Dialect/Transform/PDLExtension/PDLExtensionOps.td index 206a799690aa..4c4d230d2ae4 100644 --- a/mlir/include/mlir/Dialect/Transform/PDLExtension/PDLExtensionOps.td +++ b/mlir/include/mlir/Dialect/Transform/PDLExtension/PDLExtensionOps.td @@ -10,7 +10,7 @@ #define MLIR_DIALECT_TRANSFORM_PDLEXTENSION_PDLEXTENSIONOPS include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/Interfaces/SideEffectInterfaces.td" include "mlir/IR/OpAsmInterface.td" include "mlir/IR/SymbolInterfaces.td" diff --git a/mlir/include/mlir/Dialect/Transform/Transforms/TransformInterpreterPassBase.h b/mlir/include/mlir/Dialect/Transform/Transforms/TransformInterpreterPassBase.h index 16ef0bc6a739..3a4b391fd7f4 100644 --- a/mlir/include/mlir/Dialect/Transform/Transforms/TransformInterpreterPassBase.h +++ b/mlir/include/mlir/Dialect/Transform/Transforms/TransformInterpreterPassBase.h @@ -14,7 +14,7 @@ #ifndef MLIR_DIALECT_TRANSFORM_TRANSFORMS_TRANSFORMINTERPRETERPASSBASE_H #define MLIR_DIALECT_TRANSFORM_TRANSFORMS_TRANSFORMINTERPRETERPASSBASE_H -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Pass/Pass.h" #include "mlir/Support/LLVM.h" #include diff --git a/mlir/include/mlir/Dialect/Transform/Transforms/TransformInterpreterUtils.h b/mlir/include/mlir/Dialect/Transform/Transforms/TransformInterpreterUtils.h index 738e0c533c6b..4c16d40d368e 100644 --- a/mlir/include/mlir/Dialect/Transform/Transforms/TransformInterpreterUtils.h +++ b/mlir/include/mlir/Dialect/Transform/Transforms/TransformInterpreterUtils.h @@ -10,7 +10,7 @@ #define MLIR_DIALECT_TRANSFORM_TRANSFORMS_TRANSFORMINTERPRETERUTILS_H #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Pass/Pass.h" #include "mlir/Support/LLVM.h" #include diff --git a/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.h b/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.h index 1fa9057a0682..c43a8a0b02c6 100644 --- a/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.h +++ b/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.h @@ -9,7 +9,7 @@ #ifndef MLIR_DIALECT_VECTOR_TRANSFORMOPS_VECTORTRANSFORMOPS_H #define MLIR_DIALECT_VECTOR_TRANSFORMOPS_VECTORTRANSFORMOPS_H -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h" #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h" #include "mlir/IR/OpImplementation.h" diff --git a/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td b/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td index 83df5fe27d7a..f6371f39c394 100644 --- a/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td +++ b/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td @@ -10,7 +10,7 @@ #define VECTOR_TRANSFORM_OPS include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/Dialect/Vector/Transforms/VectorTransformsBase.td" include "mlir/Interfaces/SideEffectInterfaces.td" include "mlir/IR/OpBase.td" diff --git a/mlir/lib/CAPI/Dialect/TransformInterpreter.cpp b/mlir/lib/CAPI/Dialect/TransformInterpreter.cpp index 6a2cfb235fcf..eb6951dc5584 100644 --- a/mlir/lib/CAPI/Dialect/TransformInterpreter.cpp +++ b/mlir/lib/CAPI/Dialect/TransformInterpreter.cpp @@ -15,7 +15,7 @@ #include "mlir/CAPI/IR.h" #include "mlir/CAPI/Support.h" #include "mlir/CAPI/Wrap.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Dialect/Transform/Transforms/TransformInterpreterUtils.h" using namespace mlir; diff --git a/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp b/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp index 0b4570c65c51..e8bfd4421f5c 100644 --- a/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp +++ b/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp @@ -13,7 +13,7 @@ #include "mlir/Dialect/Affine/IR/AffineValueMap.h" #include "mlir/Dialect/Affine/LoopUtils.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Transforms/GreedyPatternRewriteDriver.h" using namespace mlir; diff --git a/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp b/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp index 9e79b086c0be..1e262736226f 100644 --- a/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp +++ b/mlir/lib/Dialect/Func/TransformOps/FuncTransformOps.cpp @@ -13,8 +13,8 @@ #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformOps.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Transforms/DialectConversion.h" using namespace mlir; diff --git a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp index ada985b5979e..fc3a43756945 100644 --- a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp +++ b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp @@ -22,7 +22,7 @@ #include "mlir/Dialect/SCF/IR/DeviceMappingInterface.h" #include "mlir/Dialect/SCF/IR/SCF.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Dialect/Utils/IndexingUtils.h" #include "mlir/Dialect/Vector/IR/VectorOps.h" #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h" diff --git a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp index cc3be7da11e0..e8ecbe16c3f0 100644 --- a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp +++ b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp @@ -17,7 +17,7 @@ #include "mlir/Dialect/SCF/IR/DeviceMappingInterface.h" #include "mlir/Dialect/SCF/IR/SCF.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Dialect/Utils/IndexingUtils.h" #include "mlir/Dialect/Vector/IR/VectorOps.h" #include "mlir/IR/AffineExpr.h" diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp index fb18886c16b1..ae2a34bcf3e5 100644 --- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp +++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp @@ -13,6 +13,7 @@ #include "mlir/Dialect/Linalg/TransformOps/Syntax.h" #include "mlir/Dialect/Linalg/Utils/Utils.h" #include "mlir/Dialect/Transform/IR/MatchInterfaces.h" +#include "mlir/Dialect/Transform/IR/TransformTypes.h" #include "mlir/IR/BuiltinAttributes.h" #include "mlir/Interfaces/FunctionImplementation.h" #include "llvm/Support/Debug.h" diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp index ae28049f02e3..d82a6beb1086 100644 --- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp +++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp @@ -25,9 +25,9 @@ #include "mlir/Dialect/Tensor/IR/Tensor.h" #include "mlir/Dialect/Tensor/Utils/Utils.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformOps.h" #include "mlir/Dialect/Transform/IR/TransformTypes.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Dialect/Transform/Utils/Utils.h" #include "mlir/Dialect/Utils/IndexingUtils.h" #include "mlir/Dialect/Utils/StaticValueUtils.h" diff --git a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp index 8932d6164182..b3481ce1c56b 100644 --- a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp +++ b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp @@ -19,7 +19,8 @@ #include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h" #include "mlir/Dialect/SCF/IR/SCF.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/IR/TransformTypes.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Dialect/Vector/IR/VectorOps.h" #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h" #include "mlir/Interfaces/LoopLikeInterface.h" diff --git a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp index bc2fe5772af9..4d8d93f7aac7 100644 --- a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp +++ b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp @@ -17,8 +17,8 @@ #include "mlir/Dialect/SCF/Transforms/Transforms.h" #include "mlir/Dialect/SCF/Utils/Utils.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformOps.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Dialect/Utils/StaticValueUtils.h" #include "mlir/Dialect/Vector/IR/VectorOps.h" #include "mlir/IR/BuiltinAttributes.h" diff --git a/mlir/lib/Dialect/Tensor/IR/TensorDialect.cpp b/mlir/lib/Dialect/Tensor/IR/TensorDialect.cpp index 5ca9510408c3..4b3156728cc9 100644 --- a/mlir/lib/Dialect/Tensor/IR/TensorDialect.cpp +++ b/mlir/lib/Dialect/Tensor/IR/TensorDialect.cpp @@ -11,7 +11,7 @@ #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h" #include "mlir/Dialect/Complex/IR/Complex.h" #include "mlir/Dialect/Tensor/IR/Tensor.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Interfaces/SubsetOpInterface.h" #include "mlir/Transforms/InliningUtils.h" diff --git a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp index 38f1824a3634..5c6a32ce9a68 100644 --- a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp +++ b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp @@ -14,7 +14,7 @@ #include "mlir/Dialect/Tensor/Transforms/Transforms.h" #include "mlir/Dialect/Tensor/Utils/Utils.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/Builders.h" #include "mlir/Transforms/DialectConversion.h" diff --git a/mlir/lib/Dialect/Transform/CMakeLists.txt b/mlir/lib/Dialect/Transform/CMakeLists.txt index ed05194b7299..64115dcc29d6 100644 --- a/mlir/lib/Dialect/Transform/CMakeLists.txt +++ b/mlir/lib/Dialect/Transform/CMakeLists.txt @@ -1,4 +1,5 @@ add_subdirectory(DebugExtension) +add_subdirectory(Interfaces) add_subdirectory(IR) add_subdirectory(LoopExtension) add_subdirectory(PDLExtension) diff --git a/mlir/lib/Dialect/Transform/IR/CMakeLists.txt b/mlir/lib/Dialect/Transform/IR/CMakeLists.txt index 34083b2fd7aa..f90ac089adaa 100644 --- a/mlir/lib/Dialect/Transform/IR/CMakeLists.txt +++ b/mlir/lib/Dialect/Transform/IR/CMakeLists.txt @@ -2,7 +2,6 @@ add_mlir_dialect_library(MLIRTransformDialect MatchInterfaces.cpp TransformAttrs.cpp TransformDialect.cpp - TransformInterfaces.cpp TransformOps.cpp TransformTypes.cpp Utils.cpp @@ -10,7 +9,6 @@ add_mlir_dialect_library(MLIRTransformDialect DEPENDS MLIRMatchInterfacesIncGen MLIRTransformDialectIncGen - MLIRTransformInterfacesIncGen LINK_LIBS PUBLIC MLIRCastInterfaces @@ -24,5 +22,6 @@ add_mlir_dialect_library(MLIRTransformDialect MLIRRewrite MLIRSideEffectInterfaces MLIRTransforms + MLIRTransformDialectInterfaces MLIRTransformDialectUtils ) diff --git a/mlir/lib/Dialect/Transform/IR/TransformDialect.cpp b/mlir/lib/Dialect/Transform/IR/TransformDialect.cpp index fb355bc97192..e628430ff861 100644 --- a/mlir/lib/Dialect/Transform/IR/TransformDialect.cpp +++ b/mlir/lib/Dialect/Transform/IR/TransformDialect.cpp @@ -8,10 +8,10 @@ #include "mlir/Dialect/Transform/IR/TransformDialect.h" #include "mlir/Analysis/CallGraph.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformOps.h" #include "mlir/Dialect/Transform/IR/TransformTypes.h" #include "mlir/Dialect/Transform/IR/Utils.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/DialectImplementation.h" #include "llvm/ADT/SCCIterator.h" @@ -178,7 +178,8 @@ LogicalResult transform::TransformDialect::verifyOperationAttribute( } return success(); } - if (attribute.getName().getValue() == kSilenceTrackingFailuresAttrName) { + if (attribute.getName().getValue() == + FindPayloadReplacementOpInterface::kSilenceTrackingFailuresAttrName) { if (!llvm::isa(attribute.getValue())) { return op->emitError() << attribute.getName() << " must be a unit attribute"; diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp index ca80899ab073..8d2ed8f6d737 100644 --- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp +++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp @@ -14,8 +14,8 @@ #include "mlir/Dialect/Transform/IR/MatchInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformAttrs.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformTypes.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/BuiltinAttributes.h" #include "mlir/IR/Diagnostics.h" #include "mlir/IR/Dominance.h" diff --git a/mlir/lib/Dialect/Transform/IR/TransformTypes.cpp b/mlir/lib/Dialect/Transform/IR/TransformTypes.cpp index 5f70235c2352..8d9f105d1c5d 100644 --- a/mlir/lib/Dialect/Transform/IR/TransformTypes.cpp +++ b/mlir/lib/Dialect/Transform/IR/TransformTypes.cpp @@ -8,7 +8,7 @@ #include "mlir/Dialect/Transform/IR/TransformTypes.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/Builders.h" #include "mlir/IR/BuiltinTypes.h" #include "mlir/IR/DialectImplementation.h" @@ -19,8 +19,6 @@ using namespace mlir; -#include "mlir/Dialect/Transform/IR/TransformTypeInterfaces.cpp.inc" - // These are automatically generated by ODS but are not used as the Transform // dialect uses a different dispatch mechanism to support dialect extensions. LLVM_ATTRIBUTE_UNUSED static OptionalParseResult diff --git a/mlir/lib/Dialect/Transform/Interfaces/CMakeLists.txt b/mlir/lib/Dialect/Transform/Interfaces/CMakeLists.txt new file mode 100644 index 000000000000..7b837bde0625 --- /dev/null +++ b/mlir/lib/Dialect/Transform/Interfaces/CMakeLists.txt @@ -0,0 +1,15 @@ +add_mlir_library(MLIRTransformDialectInterfaces + TransformInterfaces.cpp + + DEPENDS + MLIRTransformInterfacesIncGen + + LINK_LIBS PUBLIC + MLIRCastInterfaces + MLIRIR + MLIRRewrite + MLIRSideEffectInterfaces + MLIRTransforms + MLIRTransformDialectUtils +) + diff --git a/mlir/lib/Dialect/Transform/IR/TransformInterfaces.cpp b/mlir/lib/Dialect/Transform/Interfaces/TransformInterfaces.cpp similarity index 98% rename from mlir/lib/Dialect/Transform/IR/TransformInterfaces.cpp rename to mlir/lib/Dialect/Transform/Interfaces/TransformInterfaces.cpp index fe2eea535ffd..48f3954b6cf6 100644 --- a/mlir/lib/Dialect/Transform/IR/TransformInterfaces.cpp +++ b/mlir/lib/Dialect/Transform/Interfaces/TransformInterfaces.cpp @@ -6,10 +6,8 @@ // //===----------------------------------------------------------------------===// -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" -#include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformTypes.h" #include "mlir/IR/Diagnostics.h" #include "mlir/IR/Operation.h" #include "mlir/IR/PatternMatch.h" @@ -951,8 +949,8 @@ transform::TransformState::applyTransform(TransformOpInterface transform) { DiagnosedSilenceableFailure trackingFailure = trackingListener.checkAndResetError(); if (!transform->hasTrait() || - transform->hasAttr( - transform::TransformDialect::kSilenceTrackingFailuresAttrName)) { + transform->hasAttr(FindPayloadReplacementOpInterface:: + kSilenceTrackingFailuresAttrName)) { // Only report failures for ReportTrackingListenerFailuresOpTrait ops. Also // do not report failures if the above mentioned attribute is set. if (trackingFailure.isSilenceableFailure()) @@ -1649,23 +1647,7 @@ LogicalResult transform::detail::mapPossibleTopLevelTransformOpBlockArguments( << " were provided to the interpreter"; } - // Top-level transforms can be used for matching. If no concrete operation - // type is specified, the block argument is mapped to the top-level op. - // Otherwise, it is mapped to all ops of the specified type within the - // top-level op (including the top-level op itself). Once an op is added as - // a target, its descendants are not explored any further. - BlockArgument bbArg = region.front().getArgument(0); - if (auto bbArgType = dyn_cast(bbArg.getType())) { - state.getTopLevel()->walk([&](Operation *op) { - if (op->getName().getStringRef() == bbArgType.getOperationName()) { - targets.push_back(op); - return WalkResult::skip(); - } - return WalkResult::advance(); - }); - } else { - targets.push_back(state.getTopLevel()); - } + targets.push_back(state.getTopLevel()); for (unsigned i = 0, e = state.getNumTopLevelMappings(); i < e; ++i) extraMappings.push_back(llvm::to_vector(state.getTopLevelMapping(i))); @@ -2003,4 +1985,5 @@ LogicalResult transform::applyTransforms( // Generated interface implementation. //===----------------------------------------------------------------------===// -#include "mlir/Dialect/Transform/IR/TransformInterfaces.cpp.inc" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.cpp.inc" +#include "mlir/Dialect/Transform/Interfaces/TransformTypeInterfaces.cpp.inc" diff --git a/mlir/lib/Dialect/Transform/Transforms/CheckUses.cpp b/mlir/lib/Dialect/Transform/Transforms/CheckUses.cpp index 45fa644f42ec..561d3d5b05af 100644 --- a/mlir/lib/Dialect/Transform/Transforms/CheckUses.cpp +++ b/mlir/lib/Dialect/Transform/Transforms/CheckUses.cpp @@ -13,7 +13,7 @@ #include "mlir/Dialect/Transform/Transforms/Passes.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Interfaces/SideEffectInterfaces.h" #include "mlir/Pass/Pass.h" #include "llvm/ADT/SetOperations.h" diff --git a/mlir/lib/Dialect/Transform/Transforms/InferEffects.cpp b/mlir/lib/Dialect/Transform/Transforms/InferEffects.cpp index 281c1b9f8fdb..20db09ca9e8d 100644 --- a/mlir/lib/Dialect/Transform/Transforms/InferEffects.cpp +++ b/mlir/lib/Dialect/Transform/Transforms/InferEffects.cpp @@ -9,7 +9,7 @@ #include "mlir/Dialect/Transform/IR/TransformDialect.h" #include "mlir/Dialect/Transform/Transforms/Passes.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/Visitors.h" #include "mlir/Interfaces/FunctionInterfaces.h" #include "mlir/Interfaces/SideEffectInterfaces.h" diff --git a/mlir/lib/Dialect/Transform/Transforms/InterpreterPass.cpp b/mlir/lib/Dialect/Transform/Transforms/InterpreterPass.cpp index 7adf223f3440..19906f15ae85 100644 --- a/mlir/lib/Dialect/Transform/Transforms/InterpreterPass.cpp +++ b/mlir/lib/Dialect/Transform/Transforms/InterpreterPass.cpp @@ -7,7 +7,7 @@ //===----------------------------------------------------------------------===// #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Dialect/Transform/Transforms/Passes.h" #include "mlir/Dialect/Transform/Transforms/TransformInterpreterUtils.h" diff --git a/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterPassBase.cpp b/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterPassBase.cpp index a2f9e502e723..efb9359e1995 100644 --- a/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterPassBase.cpp +++ b/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterPassBase.cpp @@ -13,9 +13,9 @@ #include "mlir/Dialect/Transform/Transforms/TransformInterpreterPassBase.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformOps.h" #include "mlir/Dialect/Transform/IR/Utils.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Dialect/Transform/Transforms/TransformInterpreterUtils.h" #include "mlir/IR/BuiltinOps.h" #include "mlir/IR/Verifier.h" diff --git a/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterUtils.cpp b/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterUtils.cpp index 8a9cd7c52d82..232c9c96dd09 100644 --- a/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterUtils.cpp +++ b/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterUtils.cpp @@ -12,9 +12,9 @@ #include "mlir/Dialect/Transform/Transforms/TransformInterpreterUtils.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformOps.h" #include "mlir/Dialect/Transform/IR/Utils.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/BuiltinOps.h" #include "mlir/IR/Verifier.h" #include "mlir/IR/Visitors.h" diff --git a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp index 6c2cfd8833dd..885644864c0f 100644 --- a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp +++ b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp @@ -13,8 +13,8 @@ #include "mlir/Conversion/VectorToSCF/VectorToSCF.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformOps.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Dialect/Vector/IR/VectorOps.h" #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h" #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h" diff --git a/mlir/test/Dialect/Tensor/decompose-concat.mlir b/mlir/test/Dialect/Tensor/decompose-concat.mlir index 159347b4f7aa..c0f23b8eddbd 100644 --- a/mlir/test/Dialect/Tensor/decompose-concat.mlir +++ b/mlir/test/Dialect/Tensor/decompose-concat.mlir @@ -73,7 +73,8 @@ func.func @decompose_dynamic_into_static_concat_dim(%arg0 : tensor<1x?x?xf32>, // CHECK: return %[[CONCAT]] : tensor<1x?x128xf32> module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root: !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.tensor.decompose_concat } : !transform.op<"func.func"> diff --git a/mlir/test/Dialect/Tensor/fold-empty-op.mlir b/mlir/test/Dialect/Tensor/fold-empty-op.mlir index 057e105f3b57..15f841f2128e 100644 --- a/mlir/test/Dialect/Tensor/fold-empty-op.mlir +++ b/mlir/test/Dialect/Tensor/fold-empty-op.mlir @@ -1,7 +1,8 @@ // RUN: mlir-opt -split-input-file -transform-interpreter %s | FileCheck %s module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.tensor.fold_tensor_empty } : !transform.op<"func.func"> @@ -67,7 +68,8 @@ func.func @rank_reducing_empty_tensor_extract(%sz : index, %idx : index) -> tens // ----- module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.tensor.fold_tensor_empty {fold_single_use_only = true} diff --git a/mlir/test/Dialect/Tensor/fold-tensor-subset-ops-into-vector-transfers.mlir b/mlir/test/Dialect/Tensor/fold-tensor-subset-ops-into-vector-transfers.mlir index 505abc8f3533..6213db3956f9 100644 --- a/mlir/test/Dialect/Tensor/fold-tensor-subset-ops-into-vector-transfers.mlir +++ b/mlir/test/Dialect/Tensor/fold-tensor-subset-ops-into-vector-transfers.mlir @@ -1,7 +1,8 @@ // RUN: mlir-opt -split-input-file -transform-interpreter %s | FileCheck %s module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.tensor.fold_tensor_subset_ops_into_vector_transfers } : !transform.op<"func.func"> diff --git a/mlir/test/Dialect/Tensor/rewrite-as-constant.mlir b/mlir/test/Dialect/Tensor/rewrite-as-constant.mlir index d68a6bd25286..1a1cf9e407d8 100644 --- a/mlir/test/Dialect/Tensor/rewrite-as-constant.mlir +++ b/mlir/test/Dialect/Tensor/rewrite-as-constant.mlir @@ -1,7 +1,8 @@ // RUN: mlir-opt -split-input-file -transform-interpreter %s | FileCheck %s module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.tensor.rewrite_as_constant } : !transform.op<"func.func"> diff --git a/mlir/test/Dialect/Vector/vector-contract-to-outerproduct-matvec-transforms.mlir b/mlir/test/Dialect/Vector/vector-contract-to-outerproduct-matvec-transforms.mlir index 412e95bede3a..0b3636e3b196 100644 --- a/mlir/test/Dialect/Vector/vector-contract-to-outerproduct-matvec-transforms.mlir +++ b/mlir/test/Dialect/Vector/vector-contract-to-outerproduct-matvec-transforms.mlir @@ -657,7 +657,8 @@ func.func @masked_extract_contract2_scalable_reduction_dim(%arg0: vector<[2]x[3] // TD sequence // ============================================================================ module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.lower_contraction lowering_strategy = "outerproduct" } : !transform.op<"func.func"> diff --git a/mlir/test/Dialect/Vector/vector-materialize-mask.mlir b/mlir/test/Dialect/Vector/vector-materialize-mask.mlir index c47d91bb6ed9..a3fd6339492c 100644 --- a/mlir/test/Dialect/Vector/vector-materialize-mask.mlir +++ b/mlir/test/Dialect/Vector/vector-materialize-mask.mlir @@ -8,7 +8,8 @@ func.func @select_single_i1_vector(%cond : i1) -> vector<1xi1> { } module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.materialize_masks } : !transform.op<"func.func"> diff --git a/mlir/test/Dialect/Vector/vector-multi-reduction-lowering.mlir b/mlir/test/Dialect/Vector/vector-multi-reduction-lowering.mlir index 6e06ba1bb14b..22808aa7d6ac 100644 --- a/mlir/test/Dialect/Vector/vector-multi-reduction-lowering.mlir +++ b/mlir/test/Dialect/Vector/vector-multi-reduction-lowering.mlir @@ -282,7 +282,8 @@ func.func private @scalable_dims(%A : vector<8x[4]x2xf32>, %B: vector<8x[4]xf32> // CHECK: return %[[VAL_163]] : vector<8x[4]xf32> module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.lower_multi_reduction lowering_strategy = "innerreduction" } : !transform.op<"func.func"> diff --git a/mlir/test/Dialect/Vector/vector-multi-reduction-outer-lowering.mlir b/mlir/test/Dialect/Vector/vector-multi-reduction-outer-lowering.mlir index 308baa97af9a..33adb5545647 100644 --- a/mlir/test/Dialect/Vector/vector-multi-reduction-outer-lowering.mlir +++ b/mlir/test/Dialect/Vector/vector-multi-reduction-outer-lowering.mlir @@ -189,7 +189,8 @@ func.func @vector_multi_reduction_to_scalar(%arg0: vector<2x3xf32>, %acc: f32) - // CHECK: return %{{.+}} module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.lower_multi_reduction lowering_strategy = "innerparallel" } : !transform.op<"func.func"> diff --git a/mlir/test/Dialect/Vector/vector-transfer-drop-unit-dims-patterns.mlir b/mlir/test/Dialect/Vector/vector-transfer-drop-unit-dims-patterns.mlir index d65708068862..e9d12b044e2c 100644 --- a/mlir/test/Dialect/Vector/vector-transfer-drop-unit-dims-patterns.mlir +++ b/mlir/test/Dialect/Vector/vector-transfer-drop-unit-dims-patterns.mlir @@ -239,7 +239,8 @@ func.func @masked_transfer_read_dynamic_rank_reducing_scalable_unit_dim( // CHECK: vector.transfer_read {{.*}} vector<[16]x[1]xi8> module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.rank_reducing_subview_patterns } : !transform.op<"func.func"> diff --git a/mlir/test/Dialect/Vector/vector-transfer-full-partial-split-copy-transform.mlir b/mlir/test/Dialect/Vector/vector-transfer-full-partial-split-copy-transform.mlir index bcb8e1a10c84..483147c6f6a4 100644 --- a/mlir/test/Dialect/Vector/vector-transfer-full-partial-split-copy-transform.mlir +++ b/mlir/test/Dialect/Vector/vector-transfer-full-partial-split-copy-transform.mlir @@ -63,7 +63,7 @@ func.func @split_vector_transfer_read_strided_2d( %c0 = arith.constant 0 : index %f0 = arith.constant 0.0 : f32 - + // CHECK-DAG: %[[c0:.*]] = arith.constant 0 : index // CHECK-DAG: %[[c4:.*]] = arith.constant 4 : index // CHECK-DAG: %[[c7:.*]] = arith.constant 7 : index @@ -107,7 +107,8 @@ func.func @split_vector_transfer_read_strided_2d( } module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.split_transfer_full_partial split_transfer_strategy = "linalg-copy" } : !transform.op<"func.func"> @@ -170,7 +171,8 @@ func.func @split_vector_transfer_write_2d(%V: vector<4x8xf32>, %A: memref {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.split_transfer_full_partial split_transfer_strategy = "linalg-copy" } : !transform.op<"func.func"> @@ -240,7 +242,8 @@ func.func @split_vector_transfer_write_strided_2d( // CHECK: } module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.split_transfer_full_partial split_transfer_strategy = "linalg-copy" } : !transform.op<"func.func"> diff --git a/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir b/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir index 644de885bbaa..a9c7bf8e8b32 100644 --- a/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir +++ b/mlir/test/Dialect/Vector/vector-transfer-full-partial-split.mlir @@ -133,7 +133,8 @@ func.func @split_vector_transfer_read_mem_space(%A: memref, %i: inde } module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.split_transfer_full_partial split_transfer_strategy = "vector-transfer" } : !transform.op<"func.func"> @@ -193,7 +194,8 @@ func.func @split_vector_transfer_write_2d(%V: vector<4x8xf32>, %A: memref {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.split_transfer_full_partial split_transfer_strategy = "vector-transfer" } : !transform.op<"func.func"> @@ -257,7 +259,8 @@ func.func @split_vector_transfer_write_strided_2d( // CHECK: } module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.split_transfer_full_partial split_transfer_strategy = "vector-transfer" } : !transform.op<"func.func"> @@ -292,7 +295,8 @@ func.func @split_vector_transfer_write_mem_space(%V: vector<4x8xf32>, %A: memref module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.split_transfer_full_partial split_transfer_strategy = "vector-transfer" } : !transform.op<"func.func"> @@ -337,7 +341,8 @@ func.func @transfer_read_within_scf_for(%A : memref, %lb : index, %ub : } module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.split_transfer_full_partial split_transfer_strategy = "vector-transfer" } : !transform.op<"func.func"> diff --git a/mlir/test/Dialect/Vector/vector-transfer-to-vector-load-store.mlir b/mlir/test/Dialect/Vector/vector-transfer-to-vector-load-store.mlir index 7aaaff70e524..2f2bdcaab5b3 100644 --- a/mlir/test/Dialect/Vector/vector-transfer-to-vector-load-store.mlir +++ b/mlir/test/Dialect/Vector/vector-transfer-to-vector-load-store.mlir @@ -239,7 +239,8 @@ func.func @transfer_broadcasting_complex(%mem : memref<10x20x30x8x8xf32>, %i : i module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.lower_transfer max_transfer_rank = 99 transform.apply_patterns.vector.transfer_permutation_patterns @@ -363,7 +364,8 @@ func.func @transfer_write_broadcast_unit_dim( } module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.lower_transfer max_transfer_rank = 99 transform.apply_patterns.vector.transfer_permutation_patterns @@ -391,7 +393,8 @@ func.func @transfer_2D_masked(%mem : memref, %mask : vector<2x4xi1>) -> } module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.lower_transfer max_transfer_rank = 2 } : !transform.op<"func.func"> diff --git a/mlir/test/Dialect/Vector/vector-transpose-lowering.mlir b/mlir/test/Dialect/Vector/vector-transpose-lowering.mlir index 97b698edeb05..628a8ce50959 100644 --- a/mlir/test/Dialect/Vector/vector-transpose-lowering.mlir +++ b/mlir/test/Dialect/Vector/vector-transpose-lowering.mlir @@ -86,7 +86,8 @@ func.func @transpose23_scalable(%arg0: vector<2x[3]xf32>) -> vector<[3]x2xf32> { } module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.lower_transpose lowering_strategy = "eltwise" } : !transform.op<"func.func"> @@ -111,7 +112,8 @@ func.func @transpose(%arg0: vector<2x4xf32>) -> vector<4x2xf32> { module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.lower_transpose lowering_strategy = "shuffle_1d" } : !transform.op<"func.func"> @@ -132,7 +134,8 @@ func.func @transpose(%arg0: vector<2x4xf32>) -> vector<4x2xf32> { module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.lower_transpose lowering_strategy = "flat_transpose" } : !transform.op<"func.func"> @@ -621,7 +624,8 @@ func.func @transpose210_1x8x8xf32(%arg0: vector<1x8x8xf32>) -> vector<8x8x1xf32> } module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.lower_transpose avx2_lowering_strategy = true } : !transform.op<"func.func"> @@ -701,7 +705,8 @@ func.func @transpose_shuffle16x16xf32(%arg0: vector<16x16xf32>) -> vector<16x16x } module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.lower_transpose lowering_strategy = "shuffle_16x16" } : !transform.op<"func.func"> @@ -782,7 +787,8 @@ func.func @transpose021_shuffle16x16xf32(%arg0: vector<1x16x16xf32>) -> vector<1 } module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.lower_transpose lowering_strategy = "shuffle_16x16" } : !transform.op<"func.func"> @@ -842,7 +848,8 @@ func.func @transpose10_nx4xnx1xf32(%arg0: vector<4x[1]xf32>) -> vector<[1]x4xf32 } module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.lower_transpose } : !transform.op<"func.func"> @@ -863,7 +870,8 @@ func.func @transpose_nx8x2xf32(%arg0: vector<[8]x2xf32>) -> vector<2x[8]xf32> { } module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.lower_transpose lowering_strategy = "shuffle_1d" } : !transform.op<"func.func"> diff --git a/mlir/test/Integration/Dialect/Vector/CPU/test-shuffle16x16.mlir b/mlir/test/Integration/Dialect/Vector/CPU/test-shuffle16x16.mlir index 396417bd9b44..f7f0a7267cd0 100644 --- a/mlir/test/Integration/Dialect/Vector/CPU/test-shuffle16x16.mlir +++ b/mlir/test/Integration/Dialect/Vector/CPU/test-shuffle16x16.mlir @@ -30,7 +30,8 @@ func.func @entry() { } module attributes {transform.with_named_sequence} { - transform.named_sequence @__transform_main(%func_op: !transform.op<"func.func"> {transform.readonly}) { + transform.named_sequence @__transform_main(%root : !transform.any_op {transform.readonly}) { + %func_op = transform.structured.match ops{["func.func"]} in %root : (!transform.any_op) -> !transform.op<"func.func"> transform.apply_patterns to %func_op { transform.apply_patterns.vector.lower_transpose lowering_strategy = "shuffle_16x16" } : !transform.op<"func.func"> diff --git a/mlir/test/lib/Dialect/Tensor/TestTensorTransforms.cpp b/mlir/test/lib/Dialect/Tensor/TestTensorTransforms.cpp index b907f77e9108..ae4f77f5873e 100644 --- a/mlir/test/lib/Dialect/Tensor/TestTensorTransforms.cpp +++ b/mlir/test/lib/Dialect/Tensor/TestTensorTransforms.cpp @@ -16,8 +16,8 @@ #include "mlir/Dialect/Tensor/IR/Tensor.h" #include "mlir/Dialect/Tensor/Transforms/TransformUtils.h" #include "mlir/Dialect/Tensor/Transforms/Transforms.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformOps.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Pass/Pass.h" #include "mlir/Transforms/GreedyPatternRewriteDriver.h" diff --git a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp index b9424e06bf03..2b39668035bc 100644 --- a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp +++ b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp @@ -15,8 +15,8 @@ #include "TestTransformStateExtension.h" #include "mlir/Dialect/PDL/IR/PDL.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformOps.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Dialect/Transform/PDLExtension/PDLExtensionOps.h" #include "mlir/IR/OpImplementation.h" #include "mlir/IR/PatternMatch.h" diff --git a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.h b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.h index 95950e4c5af1..ddc38b993564 100644 --- a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.h +++ b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.h @@ -17,8 +17,8 @@ #include "mlir/Bytecode/BytecodeOpInterface.h" #include "mlir/Dialect/PDL/IR/PDLTypes.h" #include "mlir/Dialect/Transform/IR/MatchInterfaces.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformTypes.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/IR/OpImplementation.h" namespace mlir { diff --git a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.td b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.td index c00cc560e83e..75134b25882f 100644 --- a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.td +++ b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.td @@ -19,7 +19,7 @@ include "mlir/IR/AttrTypeBase.td" include "mlir/IR/OpBase.td" include "mlir/Dialect/Transform/IR/MatchInterfaces.td" include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/Dialect/PDL/IR/PDLTypes.td" def TestTransformTestDialectHandleType diff --git a/mlir/test/lib/Dialect/Transform/TestTransformDialectInterpreter.cpp b/mlir/test/lib/Dialect/Transform/TestTransformDialectInterpreter.cpp index 7d7749958564..e936ac5b852b 100644 --- a/mlir/test/lib/Dialect/Transform/TestTransformDialectInterpreter.cpp +++ b/mlir/test/lib/Dialect/Transform/TestTransformDialectInterpreter.cpp @@ -13,8 +13,8 @@ #include "TestTransformDialectExtension.h" #include "mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" #include "mlir/Dialect/Transform/IR/TransformOps.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Dialect/Transform/Transforms/TransformInterpreterPassBase.h" #include "mlir/IR/Builders.h" #include "mlir/IR/BuiltinOps.h" diff --git a/mlir/test/lib/Dialect/Transform/TestTransformStateExtension.h b/mlir/test/lib/Dialect/Transform/TestTransformStateExtension.h index 752b3a78141e..0bfa6bed015c 100644 --- a/mlir/test/lib/Dialect/Transform/TestTransformStateExtension.h +++ b/mlir/test/lib/Dialect/Transform/TestTransformStateExtension.h @@ -14,7 +14,7 @@ #ifndef MLIR_TEST_LIB_DIALECT_TRANSFORM_TESTTRANSFORMSTATEEXTENSION_H #define MLIR_TEST_LIB_DIALECT_TRANSFORM_TESTTRANSFORMSTATEEXTENSION_H -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" using namespace mlir; diff --git a/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp b/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp index b6a0ad84eee0..335db1a61f47 100644 --- a/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp +++ b/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp @@ -16,7 +16,7 @@ #include "mlir/Dialect/SCF/Transforms/TileUsingInterface.h" #include "mlir/Dialect/Transform/IR/TransformAttrs.h" #include "mlir/Dialect/Transform/IR/TransformDialect.h" -#include "mlir/Dialect/Transform/IR/TransformInterfaces.h" +#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" #include "mlir/Dialect/Utils/StaticValueUtils.h" #include "mlir/IR/Dominance.h" #include "mlir/IR/OpImplementation.h" diff --git a/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.td b/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.td index f6e577a5c17a..ef42375e5286 100644 --- a/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.td +++ b/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.td @@ -11,7 +11,7 @@ include "mlir/Dialect/SCF/IR/DeviceMappingInterface.td" include "mlir/Dialect/Transform/IR/TransformDialect.td" -include "mlir/Dialect/Transform/IR/TransformInterfaces.td" +include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td" include "mlir/Dialect/Transform/IR/TransformTypes.td" include "mlir/Interfaces/SideEffectInterfaces.td" include "mlir/IR/OpBase.td" diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index ba3f60380d34..3951a31bae3e 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -760,6 +760,7 @@ mlir_c_api_cc_library( includes = ["include"], deps = [ ":TransformDialect", + ":TransformDialectInterfaces", ":TransformDialectTransforms", ], ) @@ -1464,6 +1465,7 @@ cc_library( ":FuncDialect", ":IR", ":TransformDialect", + ":TransformDialectInterfaces", ":Transforms", ":VectorDialect", ], @@ -1572,6 +1574,7 @@ cc_library( ":FuncDialect", ":IR", ":TransformDialect", + ":TransformDialectInterfaces", ":VectorDialect", ], ) @@ -2922,6 +2925,7 @@ cc_library( ":SCFUtils", ":SideEffectInterfaces", ":TransformDialect", + ":TransformDialectInterfaces", ":VectorDialect", "//llvm:Support", ], @@ -3153,6 +3157,7 @@ cc_library( ":SparseTensorTransformOpsIncGen", ":Support", ":TransformDialect", + ":TransformDialectInterfaces", "//llvm:Support", ], ) @@ -3580,6 +3585,7 @@ cc_library( ":SCFTransforms", ":Support", ":TransformDialect", + ":TransformDialectInterfaces", ":VectorDialect", "//llvm:Support", ], @@ -4854,6 +4860,7 @@ cc_library( ":LLVMCommonConversion", ":LLVMDialect", ":TransformDialect", + ":TransformDialectInterfaces", ":TransformUtils", ], ) @@ -4995,6 +5002,7 @@ cc_library( ":LLVMDialect", ":SideEffectInterfaces", ":TransformDialect", + ":TransformDialectInterfaces", ":TransformUtils", ":VectorDialect", ":VectorEnumsIncGen", @@ -5785,6 +5793,7 @@ cc_library( ":SideEffectInterfaces", ":Support", ":TransformDialect", + ":TransformDialectInterfaces", ":TransformUtils", ":VectorDialect", ":VectorTransforms", @@ -7538,6 +7547,7 @@ cc_library( ":TensorTransforms", ":TensorUtils", ":TransformDialect", + ":TransformDialectInterfaces", ":TransformUtils", ], ) @@ -11070,6 +11080,7 @@ cc_library( ":TensorUtils", ":TilingInterface", ":TransformDialect", + ":TransformDialectInterfaces", ":TransformDialectUtils", ":TransformUtils", ":VectorDialect", @@ -11912,6 +11923,15 @@ cc_library( ], ) +td_library( + name = "TransformInterfacesTdFiles", + srcs = glob(["include/mlir/Dialect/Transform/Interfaces/*.td"]), + deps = [ + ":OpBaseTdFiles", + ":SideEffectInterfacesTdFiles", + ], +) + td_library( name = "TransformDialectTdFiles", srcs = glob(["include/mlir/Dialect/Transform/IR/*.td"]), @@ -11921,6 +11941,7 @@ td_library( ":InferTypeOpInterfaceTdFiles", ":OpBaseTdFiles", ":SideEffectInterfacesTdFiles", + ":TransformInterfacesTdFiles", ], ) @@ -11976,29 +11997,29 @@ gentbl_cc_library( [ "-gen-op-interface-decls", ], - "include/mlir/Dialect/Transform/IR/TransformInterfaces.h.inc", + "include/mlir/Dialect/Transform/Interfaces/TransformInterfaces.h.inc", ), ( [ "-gen-op-interface-defs", ], - "include/mlir/Dialect/Transform/IR/TransformInterfaces.cpp.inc", + "include/mlir/Dialect/Transform/Interfaces/TransformInterfaces.cpp.inc", ), ( [ "-gen-type-interface-decls", ], - "include/mlir/Dialect/Transform/IR/TransformTypeInterfaces.h.inc", + "include/mlir/Dialect/Transform/Interfaces/TransformTypeInterfaces.h.inc", ), ( [ "-gen-type-interface-defs", ], - "include/mlir/Dialect/Transform/IR/TransformTypeInterfaces.cpp.inc", + "include/mlir/Dialect/Transform/Interfaces/TransformTypeInterfaces.cpp.inc", ), ], tblgen = ":mlir-tblgen", - td_file = "include/mlir/Dialect/Transform/IR/TransformInterfaces.td", + td_file = "include/mlir/Dialect/Transform/Interfaces/TransformInterfaces.td", deps = [":TransformDialectTdFiles"], ) @@ -12063,19 +12084,16 @@ gentbl_cc_library( cc_library( name = "TransformDialectInterfaces", - # FIXME: Change this once https://github.com/llvm/llvm-project/pull/85221 lands - hdrs = [ - "include/mlir/Dialect/Transform/IR/TransformInterfaces.h", - "include/mlir/Dialect/Transform/IR/TransformTypes.h", - ], + srcs = glob(["lib/Dialect/Transform/Interfaces/*.cpp"]), + hdrs = glob(["include/mlir/Dialect/Transform/Interfaces/*.h"]), deps = [ ":CastInterfaces", ":IR", ":Rewrite", + ":SideEffectInterfaces", ":Support", ":TransformDialectInterfacesIncGen", ":TransformDialectUtils", - ":TransformTypesIncGen", ":Transforms", "//llvm:Support", ], @@ -12102,7 +12120,7 @@ cc_library( ":Support", ":TransformDialectEnumsIncGen", ":TransformDialectIncGen", - ":TransformDialectInterfacesIncGen", + ":TransformDialectInterfaces", ":TransformDialectMatchInterfacesIncGen", ":TransformDialectUtils", ":TransformOpsIncGen", @@ -12154,6 +12172,7 @@ cc_library( ":SideEffectInterfaces", ":Support", ":TransformDialect", + ":TransformDialectInterfaces", ":TransformPDLExtensionOpsIncGen", "//llvm:Support", ], @@ -12197,6 +12216,7 @@ cc_library( ":Support", ":TransformDebugExtensionOpsIncGen", ":TransformDialect", + ":TransformDialectInterfaces", "//llvm:Support", ], ) @@ -12241,6 +12261,7 @@ cc_library( ":SideEffectInterfaces", ":Support", ":TransformDialect", + ":TransformDialectInterfaces", ":TransformLoopExtensionOpsIncGen", ":Transforms", "//llvm:Support", @@ -12285,6 +12306,7 @@ cc_library( ":SideEffectInterfaces", ":Support", ":TransformDialect", + ":TransformDialectInterfaces", ":TransformDialectTransformsIncGen", "//llvm:Support", ], @@ -13124,6 +13146,7 @@ cc_library( ":NVGPUDialect", ":SCFDialect", ":TransformDialect", + ":TransformDialectInterfaces", ":TransformUtils", ":VectorDialect", ":VectorTransforms", @@ -13556,6 +13579,7 @@ cc_library( ":SideEffectInterfaces", ":TensorDialect", ":TransformDialect", + ":TransformDialectInterfaces", "//llvm:Support", ], ) diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel index ccfef3f24340..771cbcc4eea0 100644 --- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel @@ -361,6 +361,7 @@ cc_library( "//mlir:Pass", "//mlir:TransformDebugExtension", "//mlir:TransformDialect", + "//mlir:TransformDialectInterfaces", "//mlir:TransformDialectTransforms", "//mlir:TransformPDLExtension", ], @@ -505,6 +506,7 @@ cc_library( "//mlir:TensorTilingInterfaceImpl", "//mlir:TilingInterface", "//mlir:TransformDialect", + "//mlir:TransformDialectInterfaces", "//mlir:Transforms", ], ) @@ -1008,6 +1010,7 @@ cc_library( "//mlir:TensorDialect", "//mlir:TensorTransforms", "//mlir:TransformDialect", + "//mlir:TransformDialectInterfaces", "//mlir:Transforms", ], ) -- GitLab From de4ce5dd2bde7f9d7cbfe47a542a308779c43ce3 Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 20 Mar 2024 17:21:37 -0400 Subject: [PATCH 073/296] Rebase swiftasynccall's musttail support onto the [[clang::musttail]] logic (#86011) The old logic expects the call to be the last thing we emitted, and since it kicks in before we emit cleanups, and since `swiftasynccall` functions always return void, that's likely to be true. "Likely" isn't very reassuring when we're talking about slapping attributes on random calls, though. And indeed, while I can't find any way to break the logic directly in current main, our previous (ongoing?) experiments with shortening argument temporary lifetimes definitely broke it wide open. So while this commit is prophylactic for now, it's clearly the right thing to do, and it can cherry-picked to other branches to fix problems. --- clang/lib/CodeGen/CGStmt.cpp | 33 ++++++++++++---------- clang/test/CodeGen/swift-async-call-conv.c | 16 +++++++++++ 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp index 8898e3f22a7d..cb5a004e4f4a 100644 --- a/clang/lib/CodeGen/CGStmt.cpp +++ b/clang/lib/CodeGen/CGStmt.cpp @@ -1341,10 +1341,8 @@ struct SaveRetExprRAII { }; } // namespace -/// If we have 'return f(...);', where both caller and callee are SwiftAsync, -/// codegen it as 'tail call ...; ret void;'. -static void makeTailCallIfSwiftAsync(const CallExpr *CE, CGBuilderTy &Builder, - const CGFunctionInfo *CurFnInfo) { +/// Determine if the given call uses the swiftasync calling convention. +static bool isSwiftAsyncCallee(const CallExpr *CE) { auto calleeQualType = CE->getCallee()->getType(); const FunctionType *calleeType = nullptr; if (calleeQualType->isFunctionPointerType() || @@ -1359,18 +1357,12 @@ static void makeTailCallIfSwiftAsync(const CallExpr *CE, CGBuilderTy &Builder, // getMethodDecl() doesn't handle member pointers at the moment. calleeType = methodDecl->getType()->castAs(); } else { - return; + return false; } } else { - return; - } - if (calleeType->getCallConv() == CallingConv::CC_SwiftAsync && - (CurFnInfo->getASTCallingConvention() == CallingConv::CC_SwiftAsync)) { - auto CI = cast(&Builder.GetInsertBlock()->back()); - CI->setTailCallKind(llvm::CallInst::TCK_MustTail); - Builder.CreateRetVoid(); - Builder.ClearInsertionPoint(); + return false; } + return calleeType->getCallConv() == CallingConv::CC_SwiftAsync; } /// EmitReturnStmt - Note that due to GCC extensions, this can have an operand @@ -1410,6 +1402,19 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) { RunCleanupsScope cleanupScope(*this); if (const auto *EWC = dyn_cast_or_null(RV)) RV = EWC->getSubExpr(); + + // If we're in a swiftasynccall function, and the return expression is a + // call to a swiftasynccall function, mark the call as the musttail call. + std::optional> SaveMustTail; + if (RV && CurFnInfo && + CurFnInfo->getASTCallingConvention() == CallingConv::CC_SwiftAsync) { + if (auto CE = dyn_cast(RV)) { + if (isSwiftAsyncCallee(CE)) { + SaveMustTail.emplace(MustTailCall, CE); + } + } + } + // FIXME: Clean this up by using an LValue for ReturnTemp, // EmitStoreThroughLValue, and EmitAnyExpr. // Check if the NRVO candidate was not globalized in OpenMP mode. @@ -1432,8 +1437,6 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) { // for side effects. if (RV) { EmitAnyExpr(RV); - if (auto *CE = dyn_cast(RV)) - makeTailCallIfSwiftAsync(CE, Builder, CurFnInfo); } } else if (!RV) { // Do nothing (return value is left uninitialized) diff --git a/clang/test/CodeGen/swift-async-call-conv.c b/clang/test/CodeGen/swift-async-call-conv.c index ce32c22fe809..39511698bbae 100644 --- a/clang/test/CodeGen/swift-async-call-conv.c +++ b/clang/test/CodeGen/swift-async-call-conv.c @@ -182,3 +182,19 @@ SWIFTASYNCCALL void async_struct_field_and_methods(int i, S &sref, S *sptr) { // CPPONLY-LABEL: define{{.*}} swifttailcc void @{{.*}}async_nonleaf_method2 // CPPONLY: musttail call swifttailcc void @{{.*}}async_leaf_method #endif + +// Passing this as an argument requires a coerce-and-expand operation, +// which requires a temporary. Make sure that cleaning up that temporary +// doesn't mess around with the musttail handling. +struct coerce_and_expand { + char a,b,c,d; +}; +struct coerce_and_expand return_coerced(void); +SWIFTASYNCCALL void take_coerced_async(struct coerce_and_expand); + +// CHECK-LABEL: swifttailcc void @{{.*}}test_coerced +SWIFTASYNCCALL void test_coerced() { + // CHECK: musttail call swifttailcc void @{{.*}}take_coerced_async + // CHECK-NEXT: ret void + return take_coerced_async(return_coerced()); +} -- GitLab From 2b7289d48a36563a6b33187f4bda581cb021aba7 Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Wed, 20 Mar 2024 14:22:14 -0700 Subject: [PATCH 074/296] [PGO] Use isScopedEHPersonality for funclet check (#85671) This line should be `isScopedEHPersonality` rather than `isFuncletEHPersonality` because this line is used for checking whether we need to add `funclet` op bundles to newly added calls, and Wasm EH needs that too. The new test case is adapted from https://github.com/llvm/llvm-project/blob/main/llvm/test/Transforms/PGOProfile/memop_profile_funclet.ll. --- .../Instrumentation/PGOInstrumentation.cpp | 2 +- .../PGOProfile/memop_profile_funclet_wasm.ll | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 llvm/test/Transforms/PGOProfile/memop_profile_funclet_wasm.ll diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp index 55728709cde5..50eccc69a38a 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -920,7 +920,7 @@ static void instrumentOneFunc( // on the instrumentation call based on the funclet coloring. DenseMap BlockColors; if (F.hasPersonalityFn() && - isFuncletEHPersonality(classifyEHPersonality(F.getPersonalityFn()))) + isScopedEHPersonality(classifyEHPersonality(F.getPersonalityFn()))) BlockColors = colorEHFunclets(F); // For each VP Kind, walk the VP candidates and instrument each one. diff --git a/llvm/test/Transforms/PGOProfile/memop_profile_funclet_wasm.ll b/llvm/test/Transforms/PGOProfile/memop_profile_funclet_wasm.ll new file mode 100644 index 000000000000..f8dcb768c94c --- /dev/null +++ b/llvm/test/Transforms/PGOProfile/memop_profile_funclet_wasm.ll @@ -0,0 +1,48 @@ +; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefixes=CHECK,GEN +; RUN: opt < %s -passes=pgo-instr-gen,instrprof -S | FileCheck %s --check-prefixes=CHECK,LOWER + +target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128" +target triple = "wasm32-unknown-unknown" + +define void @wasm_funclet_op_bundle(ptr %p, ptr %dst, ptr %src) personality ptr @__gxx_wasm_personality_v0 { +entry: + invoke void @foo() + to label %try.cont unwind label %catch.dispatch + +catch.dispatch: ; preds = %entry + %0 = catchswitch within none [label %catch.start] unwind to caller + +catch.start: ; preds = %catch.dispatch + %1 = catchpad within %0 [ptr null] +; CHECK: %[[CATCHPAD:.*]] = catchpad + %2 = call ptr @llvm.wasm.get.exception(token %1) + %3 = call i32 @llvm.wasm.get.ehselector(token %1) + %4 = call ptr @__cxa_begin_catch(ptr %2) #3 [ "funclet"(token %1) ] + %tmp = load i32, ptr %p, align 4 + call void @llvm.memcpy.p0.p0.i32(ptr %dst, ptr %src, i32 %tmp, i1 false) +; GEN: call void @llvm.instrprof.value.profile({{.*}}) [ "funclet"(token %[[CATCHPAD]]) ] +; LOWER: call void @__llvm_profile_instrument_memop({{.*}}) [ "funclet"(token %[[CATCHPAD]]) ] + call void @__cxa_end_catch() [ "funclet"(token %1) ] + catchret from %1 to label %try.cont + +try.cont: ; preds = %catch.start, %entry + ret void +} + +declare void @foo() +declare i32 @__gxx_wasm_personality_v0(...) +; Function Attrs: nocallback nofree nosync nounwind willreturn +declare ptr @llvm.wasm.get.exception(token) #0 +; Function Attrs: nocallback nofree nosync nounwind willreturn +declare i32 @llvm.wasm.get.ehselector(token) #0 +; Function Attrs: nounwind memory(none) +declare i32 @llvm.eh.typeid.for(ptr) #1 +declare ptr @__cxa_begin_catch(ptr) +declare void @__cxa_end_catch() +; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: readwrite) +declare void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1 immarg) #2 + +attributes #0 = { nocallback nofree nosync nounwind willreturn } +attributes #1 = { nounwind memory(none) } +attributes #2 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) } +attributes #3 = { nounwind } -- GitLab From 061b40896470f6f1840d340fe52deb761026d3ef Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Wed, 20 Mar 2024 14:35:47 -0700 Subject: [PATCH 075/296] [BOLT][NFC] Expose YAMLProfileWriter::convert function The function is to be used by YAML profile emission in BAT mode for BinaryFunctions not covered by BAT tables (same as in original binary). Test Plan: NFC Reviewers: rafaelauler, ayermolo, dcci, maksfb Reviewed By: dcci Pull Request: https://github.com/llvm/llvm-project/pull/76909 --- bolt/include/bolt/Profile/YAMLProfileWriter.h | 4 ++++ bolt/lib/Profile/YAMLProfileWriter.cpp | 17 +++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/bolt/include/bolt/Profile/YAMLProfileWriter.h b/bolt/include/bolt/Profile/YAMLProfileWriter.h index 2d3009ca9175..882748627e7f 100644 --- a/bolt/include/bolt/Profile/YAMLProfileWriter.h +++ b/bolt/include/bolt/Profile/YAMLProfileWriter.h @@ -9,6 +9,7 @@ #ifndef BOLT_PROFILE_YAML_PROFILE_WRITER_H #define BOLT_PROFILE_YAML_PROFILE_WRITER_H +#include "bolt/Profile/ProfileYAMLMapping.h" #include "llvm/Support/raw_ostream.h" #include @@ -29,6 +30,9 @@ public: /// Save execution profile for that instance. std::error_code writeProfile(const RewriteInstance &RI); + + static yaml::bolt::BinaryFunctionProfile convert(const BinaryFunction &BF, + bool UseDFS); }; } // namespace bolt diff --git a/bolt/lib/Profile/YAMLProfileWriter.cpp b/bolt/lib/Profile/YAMLProfileWriter.cpp index dffd851a1d6f..0523b6d2e117 100644 --- a/bolt/lib/Profile/YAMLProfileWriter.cpp +++ b/bolt/lib/Profile/YAMLProfileWriter.cpp @@ -10,7 +10,6 @@ #include "bolt/Core/BinaryBasicBlock.h" #include "bolt/Core/BinaryFunction.h" #include "bolt/Profile/ProfileReaderBase.h" -#include "bolt/Profile/ProfileYAMLMapping.h" #include "bolt/Rewrite/RewriteInstance.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FileSystem.h" @@ -26,15 +25,15 @@ extern llvm::cl::opt ProfileUseDFS; namespace llvm { namespace bolt { -namespace { -void convert(const BinaryFunction &BF, - yaml::bolt::BinaryFunctionProfile &YamlBF) { +yaml::bolt::BinaryFunctionProfile +YAMLProfileWriter::convert(const BinaryFunction &BF, bool UseDFS) { + yaml::bolt::BinaryFunctionProfile YamlBF; const BinaryContext &BC = BF.getBinaryContext(); const uint16_t LBRProfile = BF.getProfileFlags() & BinaryFunction::PF_LBR; // Prepare function and block hashes - BF.computeHash(opts::ProfileUseDFS); + BF.computeHash(UseDFS); BF.computeBlockHashes(); YamlBF.Name = BF.getPrintName(); @@ -44,7 +43,7 @@ void convert(const BinaryFunction &BF, YamlBF.ExecCount = BF.getKnownExecutionCount(); BinaryFunction::BasicBlockOrderType Order; - llvm::copy(opts::ProfileUseDFS ? BF.dfs() : BF.getLayout().blocks(), + llvm::copy(UseDFS ? BF.dfs() : BF.getLayout().blocks(), std::back_inserter(Order)); for (const BinaryBasicBlock *BB : Order) { @@ -165,8 +164,8 @@ void convert(const BinaryFunction &BF, YamlBF.Blocks.emplace_back(YamlBB); } + return YamlBF; } -} // end anonymous namespace std::error_code YAMLProfileWriter::writeProfile(const RewriteInstance &RI) { const BinaryContext &BC = RI.getBinaryContext(); @@ -222,9 +221,7 @@ std::error_code YAMLProfileWriter::writeProfile(const RewriteInstance &RI) { if (!BF.hasValidProfile() && !RI.getProfileReader()->isTrustedSource()) continue; - yaml::bolt::BinaryFunctionProfile YamlBF; - convert(BF, YamlBF); - BP.Functions.emplace_back(YamlBF); + BP.Functions.emplace_back(convert(BF, opts::ProfileUseDFS)); } } -- GitLab From de0abc0983d355bbd971c5c571ba4c209a0c63ea Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Wed, 20 Mar 2024 14:39:28 -0700 Subject: [PATCH 076/296] [BOLT][NFC] Simplify YAMLProfileWriter::convert Use `getAnnotationWithDefault` instead of testing if the annotation is set. If the default value is used, and `CSI.Count` is set to zero, the target is discarded by a check below. Test Plan: NFC Reviewers: maksfb, dcci, rafaelauler, ayermolo Reviewed By: ayermolo Pull Request: https://github.com/llvm/llvm-project/pull/82129 --- bolt/lib/Profile/YAMLProfileWriter.cpp | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/bolt/lib/Profile/YAMLProfileWriter.cpp b/bolt/lib/Profile/YAMLProfileWriter.cpp index 0523b6d2e117..6fcc4a956fa1 100644 --- a/bolt/lib/Profile/YAMLProfileWriter.cpp +++ b/bolt/lib/Profile/YAMLProfileWriter.cpp @@ -105,20 +105,14 @@ YAMLProfileWriter::convert(const BinaryFunction &BF, bool UseDFS) { TargetName = Callee->getOneName(); } + auto getAnnotationWithDefault = [&](const MCInst &Inst, StringRef Ann) { + return BC.MIB->getAnnotationWithDefault(Instr, Ann, 0ull); + }; if (BC.MIB->getConditionalTailCall(Instr)) { - auto CTCCount = - BC.MIB->tryGetAnnotationAs(Instr, "CTCTakenCount"); - if (CTCCount) { - CSI.Count = *CTCCount; - auto CTCMispreds = - BC.MIB->tryGetAnnotationAs(Instr, "CTCMispredCount"); - if (CTCMispreds) - CSI.Mispreds = *CTCMispreds; - } + CSI.Count = getAnnotationWithDefault(Instr, "CTCTakenCount"); + CSI.Mispreds = getAnnotationWithDefault(Instr, "CTCMispredCount"); } else { - auto Count = BC.MIB->tryGetAnnotationAs(Instr, "Count"); - if (Count) - CSI.Count = *Count; + CSI.Count = getAnnotationWithDefault(Instr, "Count"); } if (CSI.Count) -- GitLab From 61b24c61a90802e06e40a7ab0aa5e2138486bd73 Mon Sep 17 00:00:00 2001 From: Chao Chen <116223022+chencha3@users.noreply.github.com> Date: Wed, 20 Mar 2024 17:32:30 -0500 Subject: [PATCH 077/296] [MLIR][XeGPU] Adding XeGPU 2d block operators (#85804) This PR adds XeGPU 2D block operators. It contains: 1. TensorDescType and TensorDescAttr definitions 2. MemoryScopeAttr and CacheHintAttr definitions which are used by TensorDescAttr. 3. CreateNdDescOp, PrefetchNdOp, LoadNdOp, and StoreNdOp definitions, and their corresponding testcases for illustration. It cherry-picks daebe5c4f27ba140ac8d13abf41e3fe4db72b91a with asan fix. --------- Co-authored-by: Mehdi Amini --- mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h | 7 +- .../mlir/Dialect/XeGPU/IR/XeGPUAttrs.td | 61 ++++ .../mlir/Dialect/XeGPU/IR/XeGPUDialect.td | 4 +- .../include/mlir/Dialect/XeGPU/IR/XeGPUOps.td | 305 +++++++++++++++++- .../mlir/Dialect/XeGPU/IR/XeGPUTypes.td | 104 +++++- mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp | 73 ++++- mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp | 187 ++++++++++- mlir/test/Dialect/XeGPU/XeGPUOps.mlir | 62 ++++ 8 files changed, 791 insertions(+), 12 deletions(-) create mode 100644 mlir/test/Dialect/XeGPU/XeGPUOps.mlir diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h index 7aaa4ecc7ee7..87aabdc015fe 100644 --- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h +++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h @@ -9,7 +9,12 @@ #ifndef MLIR_DIALECT_XEGPU_IR_XEGPU_H #define MLIR_DIALECT_XEGPU_IR_XEGPU_H -#include +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/Dialect.h" +#include "mlir/Interfaces/ShapedOpInterfaces.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" +#include "mlir/Interfaces/ViewLikeInterface.h" namespace mlir { namespace xegpu { diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td index bb325c272e33..cd38549f1ccf 100644 --- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td +++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td @@ -10,6 +10,7 @@ #define MLIR_DIALECT_XEGPU_IR_XEGPUATTRS_TD include "mlir/Dialect/XeGPU/IR/XeGPUDialect.td" +include "mlir/IR/EnumAttr.td" class XeGPUAttr traits = [], string baseCppClass = "::mlir::Attribute"> @@ -17,4 +18,64 @@ class XeGPUAttr traits = [], let mnemonic = attrMnemonic; } +def XeGPU_TensorDescAttr: XeGPUAttr<"TensorDesc", "tdesc_attr"> { + let parameters = (ins + OptionalParameter<"MemoryScopeAttr">: $memory_scope, + OptionalParameter<"IntegerAttr", "1">: $array_length, + OptionalParameter<"BoolAttr", "true">: $boundary_check + ); + + let builders = [ + AttrBuilder<(ins + CArg<"xegpu::MemoryScope", "xegpu::MemoryScope::Global">:$memory_scope, + CArg<"int", "1">:$array_length, + CArg<"bool", "true">: $boundary_check + )> + ]; + + let assemblyFormat = "`<` struct(params) `>`"; +} + +//===----------------------------------------------------------------------===// +// XeGPU Memory Scope Enums. +//===----------------------------------------------------------------------===// +def XeGPU_MemoryScopeGlobal: I32EnumAttrCase<"Global", 0, "global">; +def XeGPU_MemoryScopeShared: I32EnumAttrCase<"SLM", 1, "slm">; +def XeGPU_MemoryScope: I32EnumAttr<"MemoryScope", + "The address space of the memory the tensor descritor is created for", + [XeGPU_MemoryScopeGlobal, XeGPU_MemoryScopeShared]> { + let genSpecializedAttr = 0; + let cppNamespace = "::mlir::xegpu"; +} + +def XeGPU_MemoryScopeAttr: + EnumAttr { + let assemblyFormat = "$value"; +} + +//===----------------------------------------------------------------------===// +// XeGPU Cache Enums. +//===----------------------------------------------------------------------===// +def XeGPU_CachePolicyCached: I32EnumAttrCase<"CACHED", 0, "cached">; // valid for read and write +def XeGPU_CachePolicyUncached: I32EnumAttrCase<"UNCACHED", 1, "uncached">; // valid for read and write +def XeGPU_CachePolicyStreaming: I32EnumAttrCase<"STREAMING", 2, "streaming">; // valid for read only +def XeGPU_CachePolicyInvalid: I32EnumAttrCase<"READ_INVALIDATE", 3, "read_invalidate">; // valid for read only +def XeGPU_CachePolicyWriteBack: I32EnumAttrCase<"WRITE_BACK", 4, "write_back">; // valid for write only +def XeGPU_CachePolicyWriteThrough: I32EnumAttrCase<"WRITE_THROUGH", 5, "write_through">; // valid for write only + +def XeGPU_CachePolicyEnums : I32EnumAttr<"CachePolicy", "Cache policy", + [XeGPU_CachePolicyCached, XeGPU_CachePolicyUncached, + XeGPU_CachePolicyStreaming, XeGPU_CachePolicyInvalid, + XeGPU_CachePolicyWriteBack, XeGPU_CachePolicyWriteThrough]> { + let genSpecializedAttr = 0; + let cppNamespace = "::mlir::xegpu"; +} + +def XeGPU_CacheHintAttr + : EnumAttr { + let assemblyFormat = "`<` $value `>`"; +} + + + #endif // MLIR_DIALECT_XEGPU_IR_XEGPUATTRS_TD diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUDialect.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUDialect.td index 3851275ad30a..c2f09319c790 100644 --- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUDialect.td +++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUDialect.td @@ -23,8 +23,8 @@ def XeGPU_Dialect : Dialect { the lower-level GPU compiler. }]; - // let useDefaultTypePrinterParser = true; - // let useDefaultAttributePrinterParser = true; + let useDefaultTypePrinterParser = true; + let useDefaultAttributePrinterParser = true; } #endif // MLIR_DIALECT_XEGPU_IR_XEGPUDIALECT_TD diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td index 5825ef9195b0..93c56ad05b43 100644 --- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td +++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td @@ -9,10 +9,13 @@ #ifndef MLIR_DIALECT_XEGPU_IR_XEGPUOPS_TD #define MLIR_DIALECT_XEGPU_IR_XEGPUOPS_TD +include "mlir/IR/AttrTypeBase.td" include "mlir/Dialect/XeGPU/IR/XeGPUAttrs.td" include "mlir/Dialect/XeGPU/IR/XeGPUDialect.td" include "mlir/Dialect/XeGPU/IR/XeGPUTypes.td" - +include "mlir/Interfaces/ShapedOpInterfaces.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "mlir/Interfaces/ViewLikeInterface.td" // Base class for dialect operations. This operation inherits from the base // `Op` class in OpBase.td, and provides: @@ -20,7 +23,305 @@ include "mlir/Dialect/XeGPU/IR/XeGPUTypes.td" // * The mnemonic for the operation, or the name without the dialect prefix. // * A list of traits for the operation. class XeGPU_Op traits = []>: - Op; + Op { + + code extraBaseClassDeclaration = [{ + void printProperties(::mlir::MLIRContext *ctx, + ::mlir::OpAsmPrinter &p, const Properties &prop) { + Attribute propAttr = getPropertiesAsAttr(ctx, prop); + if (propAttr) + p << "<" << propAttr << ">"; + } + + static ::mlir::ParseResult parseProperties(::mlir::OpAsmParser &parser, + ::mlir::OperationState &result) { + if (mlir::succeeded(parser.parseLess())) { + if (parser.parseAttribute(result.propertiesAttr) || parser.parseGreater()) + return failure(); + } + return success(); + } + + }]; +} + + +def XeGPU_CreateNdDescOp: XeGPU_Op<"create_nd_tdesc", [Pure, ViewLikeOpInterface, + AttrSizedOperandSegments, OffsetSizeAndStrideOpInterface]> { + + let summary = "Create nd-tensor descriptor operation"; + let description = [{ + The "create_nd_tdesc" operation creates a TensorDescType which represents + a sub-view of a 2D memory region (It can be extended to support n-D memory + region if needed in future). Elements in the subview continuous in each + dimention. It encodes the following important information for supporting + Intel hardware features: + + * source: an object representing (starting address/pointer of) a 2D memory region. + It can be either a 2D memref object, or simply a pointer represented by uint64_t type. + for the later case, the shape and layout information of the 2D memory region should + be explicitly passed via `dynamic_shape` and `dynamic_strides` parameters. + * offsets: two index values represents offsets from the "source" at the each dimension + at which the subview of the target memory will be created. It is encoded via two + variables, including "dynamic_offsets" and "static_offsets", such that it can + accept various forms, such as, operands (e.g., [%c0, %c]) and attributes (e.g., [2, 4])). + * shape: the shape information of the memory region pointed by the "source". It is + typically encoded via the MemRefType of the source, e.g., memref<4096x4096xf16>. + But if "source" is simply a pointer represented as uint64_t type, or a memref + type without shape information e.g., memref, the shape information has + to be explicitly passed via the "dynamic_shape" argument. Currently "dynamic_shape" + only accepts operands(e.g., [%c4096, %c4096]), not attributes(e.g., [4096, 4096]). + * strides: the strides of the memory region pointed by the "source". Similar to shape, + it is typically encoded via the MemRefType of the source too. But if "source" is + simply a pointer represented as uint64_t type, or a memref type without shape + information e.g., memref, the strides information has to be explicitly + passed via the "dynamic_strides" argument. And it currently only accepts operands two. + + Example 1 (suppose the tensor shape inferred by the compiler is 8x16): + %0 = memref.alloc() : memref<1024x1024xf32> + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %1 = xegpu.create_nd_tdesc %0[%c0, %c0]: memref<1024x1024xf32> -> TensorDesc<8x16xf32> + + Example 2 (suppose the tensor shape inferred by the compiler is 8x16): + %0 = memref.alloc(%h, %w) : memref + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %1 = xegpu.create_nd_tdesc %0[%c0, %c0], [%h, %w], [%w, %c1]: memref -> TensorDesc<8x16xf32> + + Example 3 (suppose the tensor shape inferred by the compiler is 8x16): + %0 = ... : ui64 + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %1 = xegpu.create_nd_tdesc %0[%c0, %c0], [%h, %w], [%w, %c1]: ui64 -> TensorDesc<8x16xf32> + }]; + + let arguments = (ins + XeGPU_BaseAddrType: $source, + Variadic: $offsets, + Variadic: $shape, + Variadic: $strides, + DenseI64ArrayAttr: $const_offsets, + OptionalAttr: $const_shape, + OptionalAttr: $const_strides + ); + let results = (outs XeGPU_TensorDesc: $TensorDesc); + + let assemblyFormat = [{ + $source `` + custom($offsets, $const_offsets) + (`,` custom($shape, $const_shape)^ + `,` custom($strides, $const_strides))? + attr-dict `:` type($source) `->` qualified(type($TensorDesc)) + }]; + + let hasVerifier = 1; + + let builders = [ + OpBuilder<(ins "Type": $tdesc, "TypedValue": $source, + "llvm::ArrayRef": $offsets)>, + + OpBuilder<(ins "Type": $tdesc, "TypedValue ": $source, + "llvm::ArrayRef": $offsets, + "llvm::ArrayRef": $shape, + "llvm::ArrayRef": $strides)> + ]; + + let extraClassDeclaration = extraBaseClassDeclaration # [{ + /// Returns the type of the source memref operand. + Type getSourceType() { + return getSource().getType(); + } + + /// Returns the type of the result TensorDesc. + xegpu::TensorDescType getType() { + return getTensorDesc().getType(); + } + + /// Return the element type of the TensorDesc + Type getElementType() { + return getType().getElementType(); + } + + /// Return the shape of the TensorDesc + llvm::ArrayRef getTensorDescShape() { + return getType().getShape(); + } + + /// wrapper for matching with OffsetSizeAndStrideOpInterface + OperandRange getSizes() { + return getShape(); + } + + ArrayRef getStaticOffsets(){ + return getConstOffsets(); + } + + /// wrapper for matching with OffsetSizeAndStrideOpInterface + /// If source is IntegerType or `const_shape` is filled, + /// it will return `const_shape`, such that mixes of `shape` + /// and `const_shape` will be used to represent the shape of + /// source operand. They overide static shape from source memref type. + ArrayRef getStaticSizes() { + auto attr = getConstShapeAttr(); + if (getSourceType().isa() || attr) + return attr; + + auto memrefType = getSourceType().dyn_cast(); + assert(memrefType && "Incorrect use of getStaticSizes"); + return memrefType.getShape(); + } + + /// wrapper for matching with OffsetSizeAndStrideOpInterface + /// If source is IntegerType or `const_strides` is filled, it + /// will return `const_strides`, such that mixes of `strides` + /// and `const_strides` will be used to represent the strides of + /// source operand. They overide static strides from source memref type. + ArrayRef getStaticStrides() { + auto attr = getConstStridesAttr(); + if (getSourceType().isa() || attr) + return attr; + + auto memrefType = getSourceType().dyn_cast(); + assert(memrefType && "Incorrect use of getStaticStrides"); + auto [strides, offset] = getStridesAndOffset(memrefType); + // reuse the storage of ConstStridesAttr since strides from + // memref is not persistant + setConstStrides(strides); + attr = getConstStridesAttr(); + return attr; + } + + /// Return the expected rank of each of the`static_offsets`, + /// `static_shape` and `static_strides` attributes. + std::array getArrayAttrMaxRanks() { + unsigned rank; + if (auto ty = getSourceType().dyn_cast()) { + rank = ty.getRank(); + } else { + rank = (unsigned)getMixedOffsets().size(); + } + return {rank, rank, rank}; + } + + /// Return the number of leading operands before the `offsets`, + /// `shape` and `strides` operands. + static unsigned getOffsetSizeAndStrideStartOperandIndex() { return 1; } + + mlir::Value getViewSource() { return getSource(); } + }]; +} + +def XeGPU_PrefetchNdOp : XeGPU_Op<"prefetch_nd", []> { + let summary = "prefetches a nD block to cache"; + let description = [{ + It issues an instruction to prefetch the data from memory to each + level of the cache based on their cache policy. + + Example: + ``` + xegpu.prefetch_nd %tdesc {l1_hint = #xegpu.cache_hint, + l2_hint = #xegpu.cache_hint, + l3_hint = #xegpu.cache_hint} + : !xegpu.tensor_desc<8x16xf16> + ``` + + }]; + + let arguments = (ins XeGPU_TensorDesc: $TensorDesc, + OptionalAttr: $l1_hint, + OptionalAttr: $l2_hint, + OptionalAttr: $l3_hint); + + let extraClassDeclaration = extraBaseClassDeclaration; + + let assemblyFormat = "$TensorDesc prop-dict attr-dict `:` qualified(type($TensorDesc))"; +} + + +def XeGPU_LoadNdOp : XeGPU_Op<"load_nd"> { + let summary = "loads a n-D block from memory (represented by TensorDesc)" + "to registers (represented by vector)"; + let description = [{ + LoadNdOp essentially mimics the hardware block read instruction to read + a block of data from memory to register. It takes a set of optional cache + hints for each level of cache, L1, L2 and L3. If hardware does not have a + correspoding cache, Corresponding cache hint attribute will be masked. + vnni transform is an hardware feature for Intel GPU, which is used to + do data packing during the load for B operand of matrix operation, if + the bit width of the data type is less then 32 bits, e.g., fp16. And + transpose is another Intel hardware feature, which will do transpose + operation when loading the data if the bit width of the data type is + fp32 or fp64. It implies that vnni and transpose cannot exit at the + same time. + + Example: + ``` + xegpu.load_nd %1 {transpose = [1, 0], + l1_hint = #xegpu.cache_hint, + l2_hint = #xegpu.cache_hint, + l3_hint = #xegpu.cache_hint} + : !xegpu.tensor_desc<8x16xf32> -> vector<16x8xf32> + ``` + + + }]; + + let arguments = (ins XeGPU_TensorDesc: $TensorDesc, + OptionalAttr: $vnni_axis, + OptionalAttr: $transpose, + OptionalAttr: $l1_hint, + OptionalAttr: $l2_hint, + OptionalAttr: $l3_hint); + + let results = (outs XeGPU_ValueType: $value); + + let extraClassDeclaration = extraBaseClassDeclaration # [{ + VectorType getType() { + return llvm::dyn_cast(getValue().getType()); + } + + xegpu::TensorDescType getTensorDescType() { + return getTensorDesc().getType(); + } + }]; + + let assemblyFormat = "$TensorDesc prop-dict attr-dict `:` qualified(type($TensorDesc)) `->` type($value)"; + let hasVerifier = 1; +} + +def XeGPU_StoreNdOp : XeGPU_Op<"store_nd", []> { + let summary = "stores a n-D block register region back to memory, currently only supports 2D"; + + let description = [{ + StoreNdOp essentially mimics the hardware block write instruction io + write a block of data from register into the memory region as described + by the TensorDesc. It takes a set of optional cache hints for each level + of cache, L1, L2 and L3. If hardware does not have a correspoding cache, + Corresponding cache hint attribute will be masked. + + Example: + ``` + xegpu.store_nd %3, %2 {l1_hint = #xegpu.cache_hint, + l2_hint = #xegpu.cache_hint, + l3_hint = #xegpu.cache_hint} + : vector<8x16xf16>, !xegpu.tensor_desc<8x16xf16> + ``` + + + }]; + + let arguments = (ins XeGPU_ValueType: $value, + XeGPU_TensorDesc: $TensorDesc, + OptionalAttr: $l1_hint, + OptionalAttr: $l2_hint, + OptionalAttr: $l3_hint); + + let extraClassDeclaration = extraBaseClassDeclaration; + let assemblyFormat = [{$value `,` $TensorDesc prop-dict attr-dict + `:` type($value) `,` qualified(type($TensorDesc))}]; + let hasVerifier = 1; +} #endif // MLIR_DIALECT_XEGPU_IR_XEGPUOPS_TD diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td index 1d75bb4e2906..19ac1693712d 100644 --- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td +++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td @@ -9,9 +9,9 @@ #ifndef MLIR_DIALECT_XEGPU_IR_XEGPUTYPES_TD #define MLIR_DIALECT_XEGPU_IR_XEGPUTYPES_TD -include "mlir/IR/BuiltinTypes.td" include "mlir/Dialect/XeGPU/IR/XeGPUAttrs.td" include "mlir/Dialect/XeGPU/IR/XeGPUDialect.td" +include "mlir/IR/BuiltinTypes.td" def XeGPU_IntType: AnyTypeOf<[I1, I8, I16, I32, I64, SI1, SI8, SI16, SI32, SI64, UI1, UI8, UI16, UI32, UI64]>; def XeGPU_FloatType: AnyTypeOf<[F16, F32, F64, BF16, TF32]>; @@ -30,4 +30,106 @@ class XeGPUTypeDef traits = [], let mnemonic = typeMnemonic; } +def XeGPU_TensorDesc: XeGPUTypeDef<"TensorDesc", "tensor_desc", + [ShapedTypeInterface], "::mlir::TensorType"> { + let summary = "TensorDesc describing regions of interested data."; + let description = [{ + TensorDesc is a type designed to describe regions of the interested data as well as some + features that are unique to Intel hardware. Different with the builtin tensor type in MLIR, + it essentially only contains the meta data, and doesn't hold the data by itself. It is designed + to mainly support 2D block load/store and DPAS (matrix multiplication instruction) on Intel GPU. + It encodes the following information: + + * shape: the sizes/shape of the intereted data block, e.g., 8x16 means 8 rows + and each row contains 16 contiguous data element. The rows could be + either contiguous or not, depends on whether the encoding attribute + is set or not. + * element_type: the data type of the data element, e.g., f16, f32. + + Similar to the builtin tensor, it also provides an optinal attribute to encoding + the following information via the TensorDescAttr object: + * memory_scope (xegpu::MemoryScope): [optional] where the data is located, + global memory or shared memory. It is default to Global. + * array_length (int): [optional] The number of contiguous blocks with size as `shape`, + that will be loaded by block load at a time. It is default to 1. + * boundary_check (bool): [optional] indicates whether the operation detects the boundary + and pads with zero for out-of-boundary access. It is default to do boundary check. + + + Syntax: + + ``` + TensorDesc-type ::= `tensor_desc` `<` dim-list element-type (attr-list)? `>` + element-type ::= float-type | integer-type | index-type + dim-list := (static-dim-list `x`)? + static-dim-list ::= decimal-literal `x` decimal-literal + attr-list = (, memory_scope = value)? (, arr_len = value)? (, boundary_check = value)? + ``` + + Examples: + + ```mlir + // A block TensorDesc with 8x16 i32 elements + xegpu.tensor_desc<8x16xi32> + + // A block TensorDesc with 8x16 f32 elements + xegpu.tensor_desc<8x16xf32> + + // A TensorDesc with 8x16 f32 elements for a memory region in shared memory space. + xegpu.tensor_desc<8x16xf32, #xegpu.tdesc_attr> + ``` + }]; + + let parameters = (ins ArrayRefParameter<"int64_t">: $shape, + "mlir::Type": $elementType, + OptionalParameter<"mlir::Attribute">: $encoding); + + let extraClassDeclaration = [{ + using TensorType::clone; + using mlir::ShapedType::Trait::getElementTypeBitWidth; + using mlir::ShapedType::Trait::getRank; + using mlir::ShapedType::Trait::getNumElements; + using mlir::ShapedType::Trait::isDynamicDim; + using mlir::ShapedType::Trait::hasStaticShape; + using mlir::ShapedType::Trait::getNumDynamicDims; + using mlir::ShapedType::Trait::getDimSize; + using mlir::ShapedType::Trait::getDynamicDimIndex; + + TensorDescType clone(::mlir::Type elementType) { + return llvm::cast(cloneWith(getShape(), elementType)); + } + + TensorDescAttr getEncodingAsTensorDescAttr() const { + return llvm::dyn_cast_if_present(getEncoding()); + } + + xegpu::MemoryScope getMemoryScope() const { + auto attr = getEncodingAsTensorDescAttr(); + if (attr && attr.getMemoryScope()) + return attr.getMemoryScope().getValue(); + // return default value + return MemoryScope::Global; + } + + int getArrayLength() { + auto attr = getEncodingAsTensorDescAttr(); + if (attr && attr.getArrayLength()) + return attr.getArrayLength().getInt(); + // return default value + return 1; + } + + bool getBoundaryCheck() { + auto attr = getEncodingAsTensorDescAttr(); + if (attr && attr.getBoundaryCheck()) + return attr.getBoundaryCheck().getValue(); + // return default value + return true; + } + }]; + + let hasCustomAssemblyFormat = true; + +} + #endif // MLIR_DIALECT_XEGPU_IR_XEGPUTYPES_TD diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp index 4f839ee77347..0b3f4b9c9dbe 100644 --- a/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp +++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp @@ -6,7 +6,10 @@ // //===----------------------------------------------------------------------===// -#include +#include "mlir/Dialect/XeGPU/IR/XeGPU.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/DialectImplementation.h" +#include "llvm/ADT/TypeSwitch.h" namespace mlir { namespace xegpu { @@ -26,8 +29,72 @@ void XeGPUDialect::initialize() { >(); } -// this file is for position occupation, -// we will add functions in following PRs. +//===----------------------------------------------------------------------===// +// XeGPU_TensorDescAttr +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// XeGPU_TensorDescType +//===----------------------------------------------------------------------===// +mlir::Type TensorDescType::parse(::mlir::AsmParser &parser) { + llvm::SmallVector shape; + mlir::Type elementType; + mlir::FailureOr encoding; + + // Parse literal '<' + if (parser.parseLess()) + return {}; + + auto shapeLoc = parser.getCurrentLocation(); + if (mlir::failed(parser.parseDimensionList(shape))) { + parser.emitError(shapeLoc, "failed to parse parameter 'shape'"); + return {}; + } + + auto elemTypeLoc = parser.getCurrentLocation(); + if (mlir::failed(parser.parseType(elementType))) { + parser.emitError(elemTypeLoc, "failed to parse parameter 'elementType'"); + return {}; + } + + // parse optional attributes + if (mlir::succeeded(parser.parseOptionalComma())) { + encoding = mlir::FieldParser::parse(parser); + if (mlir::failed(encoding)) { + parser.emitError( + parser.getCurrentLocation(), + "Failed to parse the attribute field for TensorDescType.\n"); + return {}; + } + } + + // Parse literal '>' + if (parser.parseGreater()) + return {}; + + return TensorDescType::get(parser.getContext(), shape, elementType, + encoding.value_or(mlir::Attribute())); +} + +void TensorDescType::print(::mlir::AsmPrinter &printer) const { + printer << "<"; + + auto shape = getShape(); + for (int64_t dim : shape) { + if (mlir::ShapedType::isDynamic(dim)) + printer << '?'; + else + printer << dim; + printer << 'x'; + } + + printer << getElementType(); + + if (auto encoding = getEncoding()) + printer << ", " << encoding; + + printer << ">"; +} } // namespace xegpu } // namespace mlir diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp index b356c397fb83..a0bed513567d 100644 --- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp +++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp @@ -6,15 +6,196 @@ // //===----------------------------------------------------------------------===// -#include +#include "mlir/Dialect/Utils/StaticValueUtils.h" +#include "mlir/Dialect/XeGPU/IR/XeGPU.h" +#include "mlir/IR/Builders.h" #define DEBUG_TYPE "xegpu" namespace mlir { namespace xegpu { -// this file is for position occupation, -// we will add functions in following PRs. +static void transpose(llvm::ArrayRef trans, + std::vector &shape) { + std::vector old = shape; + for (size_t i = 0; i < trans.size(); i++) + shape[i] = old[trans[i]]; +} + +template +static std::string makeString(T array, bool breakline = false) { + std::string buf; + buf.clear(); + llvm::raw_string_ostream os(buf); + os << "["; + for (size_t i = 1; i < array.size(); i++) { + os << array[i - 1] << ", "; + if (breakline) + os << "\n\t\t"; + } + os << array.back() << "]"; + os.flush(); + return buf; +} + +//===----------------------------------------------------------------------===// +// XeGPU_CreateNdDescOp +//===----------------------------------------------------------------------===// +void CreateNdDescOp::build(OpBuilder &builder, OperationState &state, + Type tdesc, TypedValue source, + llvm::ArrayRef offsets) { + auto ty = source.getType(); + assert(ty.hasStaticShape() && offsets.size() == (size_t)ty.getRank()); + + llvm::SmallVector staticOffsets; + llvm::SmallVector dynamicOffsets; + dispatchIndexOpFoldResults(offsets, dynamicOffsets, staticOffsets); + + build(builder, state, tdesc, source, dynamicOffsets /* dynamic offsets */, + ValueRange({}) /* empty dynamic shape */, + ValueRange({}) /* empty dynamic strides */, + staticOffsets /* const offsets */, {} /* empty const shape*/, + {} /* empty const strides*/); +} + +void CreateNdDescOp::build(OpBuilder &builder, OperationState &state, + Type tdesc, TypedValue source, + llvm::ArrayRef offsets, + llvm::ArrayRef shape, + llvm::ArrayRef strides) { + assert(shape.size() && offsets.size() && strides.size() && + shape.size() == strides.size() && shape.size() == offsets.size()); + + llvm::SmallVector staticOffsets; + llvm::SmallVector staticShape; + llvm::SmallVector staticStrides; + llvm::SmallVector dynamicOffsets; + llvm::SmallVector dynamicShape; + llvm::SmallVector dynamicStrides; + + dispatchIndexOpFoldResults(offsets, dynamicOffsets, staticOffsets); + dispatchIndexOpFoldResults(shape, dynamicShape, staticShape); + dispatchIndexOpFoldResults(strides, dynamicStrides, staticOffsets); + + auto staticOffsetsAttr = builder.getDenseI64ArrayAttr(staticOffsets); + auto staticShapeAttr = builder.getDenseI64ArrayAttr(staticShape); + auto staticStridesAttr = builder.getDenseI64ArrayAttr(staticStrides); + + build(builder, state, tdesc, source, dynamicOffsets, dynamicShape, + dynamicStrides, staticOffsetsAttr, staticShapeAttr, staticStridesAttr); +} + +LogicalResult CreateNdDescOp::verify() { + auto rank = (int64_t)getMixedOffsets().size(); + bool invalidRank = (rank != 2); + bool invalidElemTy = false; + + // check source type matches the rank if it is a memref. + // It also should have the same ElementType as TensorDesc. + auto memrefTy = getSourceType().dyn_cast(); + if (memrefTy) { + invalidRank |= (memrefTy.getRank() != rank); + invalidElemTy |= memrefTy.getElementType() != getElementType(); + } + + // check result type matches the rank + invalidRank = (getType().getRank() != rank); + + // mismatches among shape, strides, and offsets are + // already handeled by OffsetSizeAndStrideOpInterface. + // So they are not check here. + if (invalidRank) + return emitOpError( + "Expecting the rank of shape, strides, offsets, " + "source memref type (if source is a memref) and TensorDesc " + "should match with each other. They currenlty are 2D."); + + if (invalidElemTy) + return emitOpError("TensorDesc should have the same element " + "type with the source if it is a memref.\n"); + + return success(); +} + +//===----------------------------------------------------------------------===// +// XeGPU_LoadNdOp +//===----------------------------------------------------------------------===// +LogicalResult LoadNdOp::verify() { + auto tdescTy = getTensorDescType(); + auto valueTy = getType(); + + if (tdescTy.getRank() != 2) + return emitOpError( + "The TensorDesc for LoadNdOp should be a 2D TensorDesc."); + + if (!valueTy) + return emitOpError("Invalid result, it should be a VectorType.\n"); + + auto tdescElemTy = tdescTy.getElementType(); + auto valueElemTy = valueTy.getElementType(); + + if (tdescElemTy != valueElemTy) + return emitOpError( + "Value should have the same element type as TensorDesc."); + + auto array_len = tdescTy.getArrayLength(); + auto tdescShape = tdescTy.getShape().vec(); + auto valueShape = valueTy.getShape().vec(); + + if (getTranspose()) { + auto trans = getTranspose().value(); + if (tdescShape.size() >= trans.size()) + transpose(trans, tdescShape); + else + emitWarning("Invalid transpose attr. It is ignored."); + } + + if (getVnniAxis()) { + auto axis = getVnniAxis().value(); + auto vnni_factor = valueShape.back(); + tdescShape[axis] /= vnni_factor; + tdescShape.push_back(vnni_factor); + } + + if (array_len > 1) { + auto it = tdescShape.begin(); + tdescShape.insert(it, array_len); + } + + if (tdescShape != valueShape) + return emitOpError() << "Result shape doesn't match TensorDesc shape." + << "The expected shape is " << makeString(tdescShape) + << ". But the given shape is " + << makeString(valueShape) << ".\n"; + return success(); +} + +//===----------------------------------------------------------------------===// +// XeGPU_StoreNdOp +//===----------------------------------------------------------------------===// +LogicalResult StoreNdOp::verify() { + auto dstTy = getTensorDesc().getType(); // Tile + auto valTy = getValue().getType().cast(); // Vector + + if (dstTy.getRank() != 2) + return emitOpError("Expecting a 2D TensorDesc shape.\n"); + + if (!valTy) + return emitOpError("Exepcting a VectorType result.\n"); + + auto dstElemTy = dstTy.getElementType(); + auto valElemTy = valTy.getElementType(); + + if (dstElemTy != valElemTy) { + return emitOpError() << "The element type of the value should " + "match the elementtype of the TensorDesc.\n"; + } + + if (dstTy.getShape() != valTy.getShape()) + return emitOpError() + << "The result shape should match the TensorDesc shape.\n"; + return success(); +} } // namespace xegpu } // namespace mlir diff --git a/mlir/test/Dialect/XeGPU/XeGPUOps.mlir b/mlir/test/Dialect/XeGPU/XeGPUOps.mlir new file mode 100644 index 000000000000..039346adbb85 --- /dev/null +++ b/mlir/test/Dialect/XeGPU/XeGPUOps.mlir @@ -0,0 +1,62 @@ +// RUN: mlir-opt %s | FileCheck %s +// Verify the printed output can be parsed. +// RUN: mlir-opt %s | mlir-opt | FileCheck %s +// Verify the generic form can be parsed. +// RUN: mlir-opt -mlir-print-op-generic %s | mlir-opt | FileCheck %s + +// CHECK-LABEL: gpu.module @test { +gpu.module @test { +// CHECK: gpu.func @test_create_nd_tdesc_vc_1(%[[arg0:.*]]: memref<24x32xf32>) { +gpu.func @test_create_nd_tdesc_vc_1(%src: memref<24x32xf32>) { + // CHECK: %[[REG:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<8x16xf32> + %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<8x16xf32> + gpu.return +} + +// CHECK: gpu.func @test_create_nd_tdesc_vc_2(%[[arg0:.*]]: ui64, %[[arg1:.*]]: index, %[[arg2:.*]]: index, %[[arg3:.*]]: index, %[[arg4:.*]]: index) { +gpu.func @test_create_nd_tdesc_vc_2(%src: ui64, %w : index, %h : index, %x : index, %y : index) { + //CHECK: %[[C:.*]] = arith.constant 1 : index + %c1 = arith.constant 1 : index + // CHECK: %[[REG:.*]] = xegpu.create_nd_tdesc %[[arg0]][%[[arg3]], %[[arg4]]], [%[[arg2]], %[[arg1]]], [%[[arg1]], %[[C]]] : ui64 -> !xegpu.tensor_desc<8x16xf32> + %1 = xegpu.create_nd_tdesc %src[%x, %y], [%h, %w], [%w, %c1] : ui64 -> !xegpu.tensor_desc<8x16xf32> + gpu.return +} + +// CHECK: gpu.func @test_create_nd_tdesc_vc_3(%[[arg0:.*]]: memref<24x32xf32>) { +gpu.func @test_create_nd_tdesc_vc_3(%src: memref<24x32xf32>) { + // CHECK: %[[REG:.*]] = xegpu.create_nd_tdesc %[[arg0]][0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<24x16xf32, #xegpu.tdesc_attr + %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<24x16xf32, #xegpu.tdesc_attr> + gpu.return +} + +// CHECK: gpu.func @test_prefetch_nd_vc(%[[arg0:.*]]: memref<24x32xf16>) { +gpu.func @test_prefetch_nd_vc(%src: memref<24x32xf16>) { + // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %[[arg0]][0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<8x16xf16> + %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<8x16xf16> + // CHECK: xegpu.prefetch_nd %[[R0]] <{l1_hint = #xegpu.cache_hint, l2_hint = #xegpu.cache_hint}> : !xegpu.tensor_desc<8x16xf16> + xegpu.prefetch_nd %1 <{l1_hint = #xegpu.cache_hint, l2_hint = #xegpu.cache_hint}>: !xegpu.tensor_desc<8x16xf16> + gpu.return +} + +// CHECK: func @test_load_nd_vc(%[[arg0:.*]]: memref<8x16xf16>) { +gpu.func @test_load_nd_vc(%src: memref<8x16xf16>) { + // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16> + %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16> + // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]] <{l1_hint = #xegpu.cache_hint, l2_hint = #xegpu.cache_hint, vnni_axis = 0 : i64}> : !xegpu.tensor_desc<8x16xf16> -> vector<4x16x2xf16> + %2 = xegpu.load_nd %1 <{vnni_axis = 0, l1_hint = #xegpu.cache_hint, l2_hint = #xegpu.cache_hint}> + : !xegpu.tensor_desc<8x16xf16> -> vector<4x16x2xf16> + gpu.return +} + +// CHECK: func @test_store_nd_vc(%[[arg0:.*]]: memref<24x32xf16>) { +gpu.func @test_store_nd_vc(%dst: memref<24x32xf16>) { + // CHECK: %[[C:.*]] = arith.constant dense<1.000000e+00> : vector<24x32xf16> + %1 = arith.constant dense<1.0>: vector<24x32xf16> + // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %[[arg0]][0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<24x32xf16> + %2 = xegpu.create_nd_tdesc %dst[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<24x32xf16> + // CHECK: xegpu.store_nd %[[C]], %[[R0]] <{l1_hint = #xegpu.cache_hint, l2_hint = #xegpu.cache_hint}> : vector<24x32xf16>, !xegpu.tensor_desc<24x32xf16> + xegpu.store_nd %1, %2 <{l1_hint = #xegpu.cache_hint, l2_hint = #xegpu.cache_hint}>: vector<24x32xf16>, !xegpu.tensor_desc<24x32xf16> + gpu.return +} + +} \ No newline at end of file -- GitLab From a7d5f73a03c81cab8df64dbd099e8acb40f5dfe1 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Wed, 20 Mar 2024 17:41:53 -0500 Subject: [PATCH 078/296] [Libomptarget] Consolidate CPU offloading into 'host' directory (#86014) Summary: All of these CPU targets use the same underlying implementation. We should consolidate them into a single target to make it easier to update this to a static library based approach. I have decided to call this the 'host' target so it can be given a single name. We still only build these if the system processor matches and we are on Linux. --- .../plugins-nextgen/CMakeLists.txt | 99 +--------------- .../plugins-nextgen/aarch64/CMakeLists.txt | 17 --- .../plugins-nextgen/host/CMakeLists.txt | 109 ++++++++++++++++++ .../dynamic_ffi/ffi.cpp | 0 .../dynamic_ffi/ffi.h | 0 .../{generic-elf-64bit => host}/src/rtl.cpp | 11 +- .../plugins-nextgen/ppc64/CMakeLists.txt | 17 --- .../plugins-nextgen/ppc64le/CMakeLists.txt | 17 --- .../plugins-nextgen/s390x/CMakeLists.txt | 17 --- .../plugins-nextgen/x86_64/CMakeLists.txt | 17 --- 10 files changed, 118 insertions(+), 186 deletions(-) delete mode 100644 openmp/libomptarget/plugins-nextgen/aarch64/CMakeLists.txt create mode 100644 openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt rename openmp/libomptarget/plugins-nextgen/{generic-elf-64bit => host}/dynamic_ffi/ffi.cpp (100%) rename openmp/libomptarget/plugins-nextgen/{generic-elf-64bit => host}/dynamic_ffi/ffi.h (100%) rename openmp/libomptarget/plugins-nextgen/{generic-elf-64bit => host}/src/rtl.cpp (97%) delete mode 100644 openmp/libomptarget/plugins-nextgen/ppc64/CMakeLists.txt delete mode 100644 openmp/libomptarget/plugins-nextgen/ppc64le/CMakeLists.txt delete mode 100644 openmp/libomptarget/plugins-nextgen/s390x/CMakeLists.txt delete mode 100644 openmp/libomptarget/plugins-nextgen/x86_64/CMakeLists.txt diff --git a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt index b6fc136e8a17..75540f055844 100644 --- a/openmp/libomptarget/plugins-nextgen/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/CMakeLists.txt @@ -11,106 +11,9 @@ ##===----------------------------------------------------------------------===## add_subdirectory(common) - -# void build_generic_elf64(string tmachine, string tmachine_name, string tmachine_libname, -# string tmachine_llvm, string tmachine_triple, string elf_machine_id); -# - build a plugin for an ELF based generic 64-bit target based on libffi. -# - tmachine: name of the machine processor as used in the cmake build system. -# - tmachine_name: name of the machine to be printed with the debug messages. -# - tmachine_libname: machine name to be appended to the plugin library name. -# - tmachine_llvm: LLVM triple for the processor -# - tmachine_triple: GNU target triple -macro(build_generic_elf64 tmachine tmachine_name tmachine_libname tmachine_llvm tmachine_triple elf_machine_id) -if(CMAKE_SYSTEM_PROCESSOR MATCHES "${tmachine}$") - # Define macro to be used as prefix of the runtime messages for this target. - add_definitions("-DTARGET_NAME=${tmachine_name}") - - # Define debug prefix. TODO: This should be automatized in the Debug.h but - # it requires changing the original plugins. - add_definitions(-DDEBUG_PREFIX="TARGET ${tmachine_name} RTL") - - # Define the macro with the ELF e_machine for this target. - add_definitions("-DTARGET_ELF_ID=${elf_machine_id}") - - # Define target triple - add_definitions("-DLIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE=${tmachine_llvm}") - - add_llvm_library("omptarget.rtl.${tmachine_libname}" - SHARED - - ${CMAKE_CURRENT_SOURCE_DIR}/../generic-elf-64bit/src/rtl.cpp - - ADDITIONAL_HEADER_DIRS - ${LIBOMPTARGET_INCLUDE_DIR} - - LINK_LIBS - PRIVATE - PluginCommon - ${OPENMP_PTHREAD_LIB} - - NO_INSTALL_RPATH - BUILDTREE_ONLY - ) - - if(LIBOMPTARGET_DEP_LIBFFI_FOUND) - libomptarget_say("Building ${tmachine_libname} plugin linked with libffi") - if(FFI_STATIC_LIBRARIES) - target_link_libraries( - "omptarget.rtl.${tmachine_libname}" PRIVATE FFI::ffi_static) - else() - target_link_libraries( - "omptarget.rtl.${tmachine_libname}" PRIVATE FFI::ffi) - endif() - else() - libomptarget_say("Building ${tmachine_libname} plugin for dlopened libffi") - target_sources("omptarget.rtl.${tmachine_libname}" PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/../generic-elf-64bit/dynamic_ffi/ffi.cpp) - target_include_directories("omptarget.rtl.${tmachine_libname}" PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/../generic-elf-64bit/dynamic_ffi) - endif() - - if(OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT) - target_link_libraries("omptarget.rtl.${tmachine_libname}" PRIVATE OMPT) - endif() - - if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG) - target_link_libraries("omptarget.rtl.${tmachine_libname}" PRIVATE - "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports") - endif() - - # Install plugin under the lib destination folder. - install(TARGETS "omptarget.rtl.${tmachine_libname}" - LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}") - set_target_properties("omptarget.rtl.${tmachine_libname}" PROPERTIES - INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/.." - POSITION_INDEPENDENT_CODE ON - CXX_VISIBILITY_PRESET protected) - - target_include_directories("omptarget.rtl.${tmachine_libname}" PRIVATE - ${LIBOMPTARGET_INCLUDE_DIR}) - - if(LIBOMPTARGET_DEP_LIBFFI_FOUND) - list(APPEND LIBOMPTARGET_TESTED_PLUGINS "omptarget.rtl.${tmachine_libname}") - set(LIBOMPTARGET_TESTED_PLUGINS - "${LIBOMPTARGET_TESTED_PLUGINS}" PARENT_SCOPE) - set(LIBOMPTARGET_SYSTEM_TARGETS - "${LIBOMPTARGET_SYSTEM_TARGETS} ${tmachine_triple} - ${tmachine_triple}-LTO" PARENT_SCOPE) - else() - libomptarget_say("Not generating ${tmachine_name} tests. LibFFI not found.") - endif() -else() - libomptarget_say("Not building ${tmachine_name} NextGen offloading plugin: machine not found in the system.") -endif() -endmacro() - -add_subdirectory(aarch64) add_subdirectory(amdgpu) add_subdirectory(cuda) -add_subdirectory(ppc64) -add_subdirectory(ppc64le) -add_subdirectory(x86_64) -add_subdirectory(s390x) +add_subdirectory(host) # Make sure the parent scope can see the plugins that will be created. set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE) diff --git a/openmp/libomptarget/plugins-nextgen/aarch64/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/aarch64/CMakeLists.txt deleted file mode 100644 index 663ab4d60ff9..000000000000 --- a/openmp/libomptarget/plugins-nextgen/aarch64/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -##===----------------------------------------------------------------------===## -# -# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# -##===----------------------------------------------------------------------===## -# -# Build a plugin for an aarch64 machine if available. -# -##===----------------------------------------------------------------------===## - -if(CMAKE_SYSTEM_NAME MATCHES "Linux") - build_generic_elf64("aarch64" "aarch64" "aarch64" "aarch64" "aarch64-unknown-linux-gnu" "183") -else() - libomptarget_say("Not building aarch64 NextGen offloading plugin: machine not found in the system.") -endif() diff --git a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt new file mode 100644 index 000000000000..5ccb20e305e8 --- /dev/null +++ b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt @@ -0,0 +1,109 @@ +if(NOT CMAKE_SYSTEM_NAME MATCHES "Linux") + return() +endif() + + # build_generic_elf64("s390x" "S390X" "s390x" "systemz" "s390x-ibm-linux-gnu" "22") + # build_generic_elf64("aarch64" "aarch64" "aarch64" "aarch64" "aarch64-unknown-linux-gnu" "183") + # build_generic_elf64("ppc64" "PPC64" "ppc64" "ppc64" "powerpc64-ibm-linux-gnu" "21") + # build_generic_elf64("x86_64" "x86_64" "x86_64" "x86_64" "x86_64-pc-linux-gnu" "62") + # build_generic_elf64("ppc64le" "PPC64le" "ppc64" "ppc64le" "powerpc64le-ibm-linux-gnu" "21") +set(supported_targets x86_64 aarch64 ppc64 ppc64le s390x) +if(NOT ${CMAKE_SYSTEM_PROCESSOR} IN_LIST supported_targets) + libomptarget_say("Not building ${machine} NextGen offloading plugin") + return() +endif() + +set(machine ${CMAKE_SYSTEM_PROCESSOR}) +if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le$") + set(machine ppc64) +endif() + +add_llvm_library(omptarget.rtl.${machine} SHARED + src/rtl.cpp + ADDITIONAL_HEADER_DIRS + ${LIBOMPTARGET_INCLUDE_DIR} + LINK_LIBS PRIVATE + PluginCommon + ${OPENMP_PTHREAD_LIB} + NO_INSTALL_RPATH + BUILDTREE_ONLY +) + +if(LIBOMPTARGET_DEP_LIBFFI_FOUND) + libomptarget_say("Building ${machine} plugin linked with libffi") + if(FFI_STATIC_LIBRARIES) + target_link_libraries(omptarget.rtl.${machine} PRIVATE FFI::ffi_static) + else() + target_link_libraries(omptarget.rtl.${machine} PRIVATE FFI::ffi) + endif() +else() + libomptarget_say("Building ${machine} plugin for dlopened libffi") + target_sources(omptarget.rtl.${machine} PRIVATE dynamic_ffi/ffi.cpp) + target_include_directories(omptarget.rtl.${machine} PRIVATE dynamic_ffi) +endif() + +if(OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT) + target_link_libraries(omptarget.rtl.${machine} PRIVATE OMPT) +endif() + +if(LIBOMP_HAVE_VERSION_SCRIPT_FLAG) + target_link_libraries(omptarget.rtl.${machine} PRIVATE + "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exports") +endif() + +# Install plugin under the lib destination folder. +install(TARGETS omptarget.rtl.${machine} + LIBRARY DESTINATION "${OPENMP_INSTALL_LIBDIR}") +set_target_properties(omptarget.rtl.${machine} PROPERTIES + INSTALL_RPATH "$ORIGIN" BUILD_RPATH "$ORIGIN:${CMAKE_CURRENT_BINARY_DIR}/.." + POSITION_INDEPENDENT_CODE ON + CXX_VISIBILITY_PRESET protected) + +target_include_directories(omptarget.rtl.${machine} PRIVATE + ${LIBOMPTARGET_INCLUDE_DIR}) + +if(LIBOMPTARGET_DEP_LIBFFI_FOUND) + list(APPEND LIBOMPTARGET_TESTED_PLUGINS omptarget.rtl.${machine}) + set(LIBOMPTARGET_TESTED_PLUGINS + "${LIBOMPTARGET_TESTED_PLUGINS}" PARENT_SCOPE) +else() + libomptarget_say("Not generating ${tmachine_name} tests. LibFFI not found.") +endif() + +# Define macro to be used as prefix of the runtime messages for this target. +target_compile_definitions(omptarget.rtl.${machine} PRIVATE TARGET_NAME=${machine}) +# TODO: This should be automatized in Debug.h. +target_compile_definitions(omptarget.rtl.${machine} PRIVATE + DEBUG_PREFIX="TARGET ${machine} RTL") + +# Define the target specific triples and ELF machine values. +if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le$" OR + CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64$") + target_compile_definitions(omptarget.rtl.${machine} PRIVATE TARGET_ELF_ID=EM_PPC64) + target_compile_definitions(omptarget.rtl.${machine} PRIVATE + LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE="powerpc64-ibm-linux-gnu") + list(APPEND LIBOMPTARGET_SYSTEM_TARGETS + "powerpc64-ibm-linux-gnu" "powerpc64-ibm-linux-gnu-LTO") + set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE) +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64$") + target_compile_definitions(omptarget.rtl.${machine} PRIVATE TARGET_ELF_ID=EM_X86_64) + target_compile_definitions(omptarget.rtl.${machine} PRIVATE + LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE="x86_64-pc-linux-gnu") + list(APPEND LIBOMPTARGET_SYSTEM_TARGETS + "x86_64-pc-linux-gnu" "x86_64-pc-linux-gnu-LTO") + set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE) +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64$") + target_compile_definitions(omptarget.rtl.${machine} PRIVATE TARGET_ELF_ID=EM_AARCH64) + target_compile_definitions(omptarget.rtl.${machine} PRIVATE + LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE="aarch64-unknown-linux-gnu") + list(APPEND LIBOMPTARGET_SYSTEM_TARGETS + "aarch64-unknown-linux-gnu" "aarch64-unknown-linux-gnu-LTO") + set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE) +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x$") + target_compile_definitions(omptarget.rtl.${machine} TARGET_ELF_ID=EM_S390) + target_compile_definitions(omptarget.rtl.${machine} PRIVATE + LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE="s390x-ibm-linux-gnu") + list(APPEND LIBOMPTARGET_SYSTEM_TARGETS + "s390x-ibm-linux-gnu" "s390x-ibm-linux-gnu-LTO") + set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE) +endif() diff --git a/openmp/libomptarget/plugins-nextgen/generic-elf-64bit/dynamic_ffi/ffi.cpp b/openmp/libomptarget/plugins-nextgen/host/dynamic_ffi/ffi.cpp similarity index 100% rename from openmp/libomptarget/plugins-nextgen/generic-elf-64bit/dynamic_ffi/ffi.cpp rename to openmp/libomptarget/plugins-nextgen/host/dynamic_ffi/ffi.cpp diff --git a/openmp/libomptarget/plugins-nextgen/generic-elf-64bit/dynamic_ffi/ffi.h b/openmp/libomptarget/plugins-nextgen/host/dynamic_ffi/ffi.h similarity index 100% rename from openmp/libomptarget/plugins-nextgen/generic-elf-64bit/dynamic_ffi/ffi.h rename to openmp/libomptarget/plugins-nextgen/host/dynamic_ffi/ffi.h diff --git a/openmp/libomptarget/plugins-nextgen/generic-elf-64bit/src/rtl.cpp b/openmp/libomptarget/plugins-nextgen/host/src/rtl.cpp similarity index 97% rename from openmp/libomptarget/plugins-nextgen/generic-elf-64bit/src/rtl.cpp rename to openmp/libomptarget/plugins-nextgen/host/src/rtl.cpp index 38fc275804fa..1ef18814a26a 100644 --- a/openmp/libomptarget/plugins-nextgen/generic-elf-64bit/src/rtl.cpp +++ b/openmp/libomptarget/plugins-nextgen/host/src/rtl.cpp @@ -35,7 +35,12 @@ // The ELF ID should be defined at compile-time by the build system. #ifndef TARGET_ELF_ID -#define TARGET_ELF_ID ELF::EM_NONE +#define TARGET_ELF_ID EM_NONE +#endif + +// The target triple should be defined at compile-time by the build system. +#ifndef LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE +#define LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE "" #endif namespace llvm { @@ -395,7 +400,7 @@ struct GenELF64PluginTy final : public GenericPluginTy { Error deinitImpl() override { return Plugin::success(); } /// Get the ELF code to recognize the compatible binary images. - uint16_t getMagicElfBits() const override { return TARGET_ELF_ID; } + uint16_t getMagicElfBits() const override { return ELF::TARGET_ELF_ID; } /// This plugin does not support exchanging data between two devices. bool isDataExchangable(int32_t SrcDeviceId, int32_t DstDeviceId) override { @@ -406,7 +411,7 @@ struct GenELF64PluginTy final : public GenericPluginTy { Expected isELFCompatible(StringRef) const override { return true; } Triple::ArchType getTripleArch() const override { - return Triple::LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE; + return llvm::Triple(LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE).getArch(); } }; diff --git a/openmp/libomptarget/plugins-nextgen/ppc64/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/ppc64/CMakeLists.txt deleted file mode 100644 index 77466c111ee0..000000000000 --- a/openmp/libomptarget/plugins-nextgen/ppc64/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -##===----------------------------------------------------------------------===## -# -# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# -##===----------------------------------------------------------------------===## -# -# Build a plugin for a ppc64 machine if available. -# -##===----------------------------------------------------------------------===## - -if(CMAKE_SYSTEM_NAME MATCHES "Linux") - build_generic_elf64("ppc64" "PPC64" "ppc64" "ppc64" "powerpc64-ibm-linux-gnu" "21") -else() - libomptarget_say("Not building ppc64 NextGen offloading plugin: machine not found in the system.") -endif() diff --git a/openmp/libomptarget/plugins-nextgen/ppc64le/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/ppc64le/CMakeLists.txt deleted file mode 100644 index 91d21627a327..000000000000 --- a/openmp/libomptarget/plugins-nextgen/ppc64le/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -##===----------------------------------------------------------------------===## -# -# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# -##===----------------------------------------------------------------------===## -# -# Build a plugin for a ppc64le machine if available. -# -##===----------------------------------------------------------------------===## - -if(CMAKE_SYSTEM_NAME MATCHES "Linux") - build_generic_elf64("ppc64le" "PPC64le" "ppc64" "ppc64le" "powerpc64le-ibm-linux-gnu" "21") -else() - libomptarget_say("Not building ppc64le NextGen offloading plugin: machine not found in the system.") -endif() diff --git a/openmp/libomptarget/plugins-nextgen/s390x/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/s390x/CMakeLists.txt deleted file mode 100644 index 0388a235d289..000000000000 --- a/openmp/libomptarget/plugins-nextgen/s390x/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -##===----------------------------------------------------------------------===## -# -# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# -##===----------------------------------------------------------------------===## -# -# Build a plugin for a s390x machine if available. -# -##===----------------------------------------------------------------------===## - -if(CMAKE_SYSTEM_NAME MATCHES "Linux") - build_generic_elf64("s390x" "S390X" "s390x" "systemz" "s390x-ibm-linux-gnu" "22") -else() - libomptarget_say("Not building s390x NextGen offloading plugin: machine not found in the system.") -endif() diff --git a/openmp/libomptarget/plugins-nextgen/x86_64/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/x86_64/CMakeLists.txt deleted file mode 100644 index 27cf3e069a37..000000000000 --- a/openmp/libomptarget/plugins-nextgen/x86_64/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -##===----------------------------------------------------------------------===## -# -# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# -##===----------------------------------------------------------------------===## -# -# Build a plugin for a x86_64 machine if available. -# -##===----------------------------------------------------------------------===## - -if(CMAKE_SYSTEM_NAME MATCHES "Linux") - build_generic_elf64("x86_64" "x86_64" "x86_64" "x86_64" "x86_64-pc-linux-gnu" "62") -else() - libomptarget_say("Not building x86_64 NextGen offloading plugin: machine not found in the system.") -endif() -- GitLab From 1918d4bcb21af6a7e4de32073455ac51f2f9673f Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Wed, 20 Mar 2024 16:00:11 -0700 Subject: [PATCH 079/296] [BOLT][CMake] Build rt library despite unreadable map_files (#77876) Emit a warning and print a suggested workaround. Fixes https://github.com/llvm/llvm-project/issues/77822. --- bolt/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bolt/CMakeLists.txt b/bolt/CMakeLists.txt index f163d4534287..cc3a70fa35e0 100644 --- a/bolt/CMakeLists.txt +++ b/bolt/CMakeLists.txt @@ -45,9 +45,9 @@ if (BOLT_ENABLE_RUNTIME) execute_process(COMMAND ls /proc/self/map_files RESULT_VARIABLE LS OUTPUT_QUIET ERROR_QUIET) if (LS) - set(BOLT_ENABLE_RUNTIME OFF) message(WARNING - "BOLT runtime is disabled as /proc/self/map_files is unreadable.") + "BOLT runtime may not be able to read /proc/self/map_files. Please use + `--instrumentation-binpath ` option.") endif() endif() -- GitLab From ad00e7e5ed5ab050151c115b627e11a8e3960e25 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Wed, 20 Mar 2024 16:24:21 -0700 Subject: [PATCH 080/296] [BOLT] Write and parse BF/BB hashes in BAT This increases BAT section size to: - large binary: 34832976 bytes (0.90x original), - medium binary: 3586800 bytes (0.60x original), - small binary: 816 bytes (0.57x original). Test Plan: Updated bolt/test/X86/bolt-address-translation.test Reviewers: rafaelauler, dcci, ayermolo, maksfb Reviewed By: rafaelauler Pull Request: https://github.com/llvm/llvm-project/pull/76907 --- bolt/docs/BAT.md | 2 + .../bolt/Profile/BoltAddressTranslation.h | 10 +++ bolt/lib/Profile/BoltAddressTranslation.cpp | 86 +++++++++++++++++-- bolt/test/X86/bolt-address-translation.test | 11 +-- 4 files changed, 95 insertions(+), 14 deletions(-) diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md index d1cab984d148..060fc632f686 100644 --- a/bolt/docs/BAT.md +++ b/bolt/docs/BAT.md @@ -79,6 +79,7 @@ Hot indices are delta encoded, implicitly starting at zero. | ------ | ------| ----------- | | `Address` | Continuous, Delta, ULEB128 | Function address in the output binary | | `HotIndex` | Delta, ULEB128 | Cold functions only: index of corresponding hot function in hot functions table | +| `FuncHash` | 8b | Hot functions only: function hash for input function | | `NumEntries` | ULEB128 | Number of address translation entries for a function | | `EqualElems` | ULEB128 | Hot functions only: number of equal offsets in the beginning of a function | | `BranchEntries` | Bitmask, `alignTo(EqualElems, 8)` bits | Hot functions only: if `EqualElems` is non-zero, bitmask denoting entries with `BRANCHENTRY` bit | @@ -94,6 +95,7 @@ entry is encoded. Input offsets implicitly start at zero. | ------ | ------| ----------- | | `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | | `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | +| `BBHash` | Optional, 8b | Basic block entries only: basic block hash in input binary | `BRANCHENTRY` bit denotes whether a given offset pair is a control flow source (branch or call instruction). If not set, it signifies a control flow target diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index 844f0c54e68f..5f2f0959d93f 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -115,6 +115,13 @@ public: /// Save function and basic block hashes used for metadata dump. void saveMetadata(BinaryContext &BC); + /// Returns BB hash by function output address (after BOLT) and basic block + /// input offset. + size_t getBBHash(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const; + + /// Returns BF hash by function output address (after BOLT). + size_t getBFHash(uint64_t OutputAddress) const; + private: /// Helper to update \p Map by inserting one or more BAT entries reflecting /// \p BB for function located at \p FuncAddress. At least one entry will be @@ -150,6 +157,9 @@ private: /// Links outlined cold bocks to their original function std::map ColdPartSource; + /// Links output address of a main fragment back to input address. + std::unordered_map ReverseMap; + /// Identifies the address of a control-flow changing instructions in a /// translation map entry const static uint32_t BRANCHENTRY = 0x1; diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index 5477d3b7d1c3..e27985251775 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -23,6 +23,9 @@ const char *BoltAddressTranslation::SECTION_NAME = ".note.bolt_bat"; void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, const BinaryBasicBlock &BB, uint64_t FuncAddress) { + uint64_t HotFuncAddress = ColdPartSource.count(FuncAddress) + ? ColdPartSource[FuncAddress] + : FuncAddress; const uint64_t BBOutputOffset = BB.getOutputAddressRange().first - FuncAddress; const uint32_t BBInputOffset = BB.getInputOffset(); @@ -39,6 +42,8 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, LLVM_DEBUG(dbgs() << "BB " << BB.getName() << "\n"); LLVM_DEBUG(dbgs() << " Key: " << Twine::utohexstr(BBOutputOffset) << " Val: " << Twine::utohexstr(BBInputOffset) << "\n"); + LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", + getBBHash(HotFuncAddress, BBInputOffset))); // In case of conflicts (same Key mapping to different Vals), the last // update takes precedence. Of course it is not ideal to have conflicts and // those happen when we have an empty BB that either contained only @@ -72,20 +77,28 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) { LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Writing BOLT Address Translation Tables\n"); for (auto &BFI : BC.getBinaryFunctions()) { const BinaryFunction &Function = BFI.second; + const uint64_t InputAddress = Function.getAddress(); + const uint64_t OutputAddress = Function.getOutputAddress(); // We don't need a translation table if the body of the function hasn't // changed if (Function.isIgnored() || (!BC.HasRelocations && !Function.isSimple())) continue; + // TBD: handle BAT functions w/multiple entry points. + if (Function.isMultiEntry()) + continue; + LLVM_DEBUG(dbgs() << "Function name: " << Function.getPrintName() << "\n"); LLVM_DEBUG(dbgs() << " Address reference: 0x" << Twine::utohexstr(Function.getOutputAddress()) << "\n"); + LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", getBFHash(OutputAddress))); MapTy Map; for (const BinaryBasicBlock *const BB : Function.getLayout().getMainFragment()) writeEntriesForBB(Map, *BB, Function.getOutputAddress()); Maps.emplace(Function.getOutputAddress(), std::move(Map)); + ReverseMap.emplace(OutputAddress, InputAddress); if (!Function.isSplit()) continue; @@ -94,12 +107,12 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) { LLVM_DEBUG(dbgs() << " Cold part\n"); for (const FunctionFragment &FF : Function.getLayout().getSplitFragments()) { + ColdPartSource.emplace(FF.getAddress(), Function.getOutputAddress()); Map.clear(); for (const BinaryBasicBlock *const BB : FF) writeEntriesForBB(Map, *BB, FF.getAddress()); Maps.emplace(FF.getAddress(), std::move(Map)); - ColdPartSource.emplace(FF.getAddress(), Function.getOutputAddress()); } } @@ -109,6 +122,11 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) { writeMaps(Maps, PrevAddress, OS); BC.outs() << "BOLT-INFO: Wrote " << Maps.size() << " BAT maps\n"; + const uint64_t NumBBHashes = std::accumulate( + FuncHashes.begin(), FuncHashes.end(), 0ull, + [](size_t Acc, const auto &B) { return Acc + B.second.second.size(); }); + BC.outs() << "BOLT-INFO: Wrote " << FuncHashes.size() << " function and " + << NumBBHashes << " basic block hashes\n"; } APInt BoltAddressTranslation::calculateBranchEntriesBitMask(MapTy &Map, @@ -155,6 +173,11 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, // Only process cold fragments in cold mode, and vice versa. if (Cold != ColdPartSource.count(Address)) continue; + // NB: here we use the input address because hashes are saved early (in + // `saveMetadata`) before output addresses are assigned. + const uint64_t HotInputAddress = + ReverseMap[Cold ? ColdPartSource[Address] : Address]; + std::pair &FuncHashPair = FuncHashes[HotInputAddress]; MapTy &Map = MapEntry.second; const uint32_t NumEntries = Map.size(); LLVM_DEBUG(dbgs() << "Writing " << NumEntries << " entries for 0x" @@ -166,6 +189,10 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, std::distance(ColdPartSource.begin(), ColdPartSource.find(Address)); encodeULEB128(HotIndex - PrevIndex, OS); PrevIndex = HotIndex; + } else { + // Function hash + LLVM_DEBUG(dbgs() << "Hash: " << formatv("{0:x}\n", FuncHashPair.first)); + OS.write(reinterpret_cast(&FuncHashPair.first), 8); } encodeULEB128(NumEntries, OS); // For hot fragments only: encode the number of equal offsets @@ -197,6 +224,13 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, if (Index++ >= EqualElems) encodeSLEB128(KeyVal.second - InOffset, OS); InOffset = KeyVal.second; // Keeping InOffset as if BRANCHENTRY is encoded + if ((InOffset & BRANCHENTRY) == 0) { + // Basic block hash + size_t BBHash = FuncHashPair.second[InOffset >> 1]; + OS.write(reinterpret_cast(&BBHash), 8); + LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x}\n", KeyVal.first, + InOffset >> 1, BBHash)); + } } } } @@ -239,12 +273,18 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, size_t HotIndex = 0; for (uint32_t I = 0; I < NumFunctions; ++I) { const uint64_t Address = PrevAddress + DE.getULEB128(&Offset, &Err); + uint64_t HotAddress = Cold ? 0 : Address; PrevAddress = Address; if (Cold) { HotIndex += DE.getULEB128(&Offset, &Err); - ColdPartSource.emplace(Address, HotFuncs[HotIndex]); + HotAddress = HotFuncs[HotIndex]; + ColdPartSource.emplace(Address, HotAddress); } else { HotFuncs.push_back(Address); + // Function hash + const size_t FuncHash = DE.getU64(&Offset, &Err); + FuncHashes[Address].first = FuncHash; + LLVM_DEBUG(dbgs() << formatv("{0:x}: hash {1:x}\n", Address, FuncHash)); } const uint32_t NumEntries = DE.getULEB128(&Offset, &Err); // Equal offsets, hot fragments only. @@ -288,12 +328,22 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, InputOffset += InputDelta; } Map.insert(std::pair(OutputOffset, InputOffset)); - LLVM_DEBUG( - dbgs() << formatv("{0:x} -> {1:x} ({2}/{3}b -> {4}/{5}b), {6:x}\n", - OutputOffset, InputOffset, OutputDelta, - getULEB128Size(OutputDelta), InputDelta, - (J < EqualElems) ? 0 : getSLEB128Size(InputDelta), - OutputAddress)); + size_t BBHash = 0; + const bool IsBranchEntry = InputOffset & BRANCHENTRY; + if (!IsBranchEntry) { + BBHash = DE.getU64(&Offset, &Err); + // Map basic block hash to hot fragment by input offset + FuncHashes[HotAddress].second.emplace(InputOffset >> 1, BBHash); + } + LLVM_DEBUG({ + dbgs() << formatv( + "{0:x} -> {1:x} ({2}/{3}b -> {4}/{5}b), {6:x}", OutputOffset, + InputOffset, OutputDelta, getULEB128Size(OutputDelta), InputDelta, + (J < EqualElems) ? 0 : getSLEB128Size(InputDelta), OutputAddress); + if (BBHash) + dbgs() << formatv(" {0:x}", BBHash); + dbgs() << '\n'; + }); } Maps.insert(std::pair(Address, Map)); } @@ -303,7 +353,12 @@ void BoltAddressTranslation::dump(raw_ostream &OS) { const size_t NumTables = Maps.size(); OS << "BAT tables for " << NumTables << " functions:\n"; for (const auto &MapEntry : Maps) { - OS << "Function Address: 0x" << Twine::utohexstr(MapEntry.first) << "\n"; + const uint64_t Address = MapEntry.first; + const uint64_t HotAddress = fetchParentAddress(Address); + OS << "Function Address: 0x" << Twine::utohexstr(Address); + if (HotAddress == 0) + OS << formatv(", hash: {0:x}", getBFHash(Address)); + OS << "\n"; OS << "BB mappings:\n"; for (const auto &Entry : MapEntry.second) { const bool IsBranch = Entry.second & BRANCHENTRY; @@ -312,6 +367,9 @@ void BoltAddressTranslation::dump(raw_ostream &OS) { << "0x" << Twine::utohexstr(Val); if (IsBranch) OS << " (branch)"; + else + OS << formatv(" hash: {0:x}", + getBBHash(HotAddress ? HotAddress : Address, Val)); OS << "\n"; } OS << "\n"; @@ -439,5 +497,15 @@ void BoltAddressTranslation::saveMetadata(BinaryContext &BC) { BB.getHash()); } } + +size_t BoltAddressTranslation::getBBHash(uint64_t FuncOutputAddress, + uint32_t BBInputOffset) const { + return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset); +} + +size_t BoltAddressTranslation::getBFHash(uint64_t OutputAddress) const { + return FuncHashes.at(OutputAddress).first; +} + } // namespace bolt } // namespace llvm diff --git a/bolt/test/X86/bolt-address-translation.test b/bolt/test/X86/bolt-address-translation.test index f2020af2edeb..4277b4e0d0fe 100644 --- a/bolt/test/X86/bolt-address-translation.test +++ b/bolt/test/X86/bolt-address-translation.test @@ -36,7 +36,8 @@ # # CHECK: BOLT: 3 out of 7 functions were overwritten. # CHECK: BOLT-INFO: Wrote 6 BAT maps -# CHECK: BOLT-INFO: BAT section size (bytes): 336 +# CHECK: BOLT-INFO: Wrote 3 function and 58 basic block hashes +# CHECK: BOLT-INFO: BAT section size (bytes): 816 # # usqrt mappings (hot part). We match against any key (left side containing # the bolted binary offsets) because BOLT may change where it puts instructions @@ -44,13 +45,13 @@ # binary offsets (right side) should be the same because these addresses are # hardcoded in the blarge.yaml file. # -# CHECK-BAT-DUMP: Function Address: 0x401170 +# CHECK-BAT-DUMP: Function Address: 0x401170, hash: 0xace6cbc638b31983 # CHECK-BAT-DUMP-NEXT: BB mappings: -# CHECK-BAT-DUMP-NEXT: 0x0 -> 0x0 +# CHECK-BAT-DUMP-NEXT: 0x0 -> 0x0 hash: 0x36007ba1d80c0000 # CHECK-BAT-DUMP-NEXT: 0x8 -> 0x8 (branch) -# CHECK-BAT-DUMP-NEXT: 0x{{.*}} -> 0x39 +# CHECK-BAT-DUMP-NEXT: 0x{{.*}} -> 0x39 hash: 0x5c06705524800039 # CHECK-BAT-DUMP-NEXT: 0x{{.*}} -> 0x3d (branch) -# CHECK-BAT-DUMP-NEXT: 0x{{.*}} -> 0x10 +# CHECK-BAT-DUMP-NEXT: 0x{{.*}} -> 0x10 hash: 0xd70d7a64320e0010 # CHECK-BAT-DUMP-NEXT: 0x{{.*}} -> 0x30 (branch) # # CHECK-BAT-DUMP: 3 cold mappings -- GitLab From 5a6c69132fb427d9ba71a72274c66ddb76ae66d5 Mon Sep 17 00:00:00 2001 From: Caroline Tice Date: Wed, 20 Mar 2024 16:34:16 -0700 Subject: [PATCH 081/296] [LLVM][DebugInfo] Add accessor for NameIndex header. This is needed for pending LLD work to create a single unified .debug_names index (rather than just appending all the indices from each .o file). --- llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h index d368c7e0ece8..f1d4fc72d5a7 100644 --- a/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h +++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h @@ -605,6 +605,9 @@ public: NameIndex(const DWARFDebugNames &Section, uint64_t Base) : Section(Section), Base(Base) {} + /// Returns Hdr field + Header getHeader() const { return Hdr; } + /// Reads offset of compilation unit CU. CU is 0-based. uint64_t getCUOffset(uint32_t CU) const; uint32_t getCUCount() const { return Hdr.CompUnitCount; } -- GitLab From 258091e76df69072e7088a5e251a2db7f8e3d0a9 Mon Sep 17 00:00:00 2001 From: Jie Fu Date: Thu, 21 Mar 2024 07:51:56 +0800 Subject: [PATCH 082/296] [mlir] Fix -Wunused-variable in XeGPUOps.cpp (NFC) llvm-project/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp:47:8: error: unused variable 'ty' [-Werror,-Wunused-variable] auto ty = source.getType(); ^ 1 error generated. --- mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp index a0bed513567d..02106f221f32 100644 --- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp +++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp @@ -44,7 +44,7 @@ static std::string makeString(T array, bool breakline = false) { void CreateNdDescOp::build(OpBuilder &builder, OperationState &state, Type tdesc, TypedValue source, llvm::ArrayRef offsets) { - auto ty = source.getType(); + [[maybe_unused]] auto ty = source.getType(); assert(ty.hasStaticShape() && offsets.size() == (size_t)ty.getRank()); llvm::SmallVector staticOffsets; -- GitLab From af90e1975c15edc9195f1a1a87269bed4c83887a Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Wed, 20 Mar 2024 17:38:47 -0700 Subject: [PATCH 083/296] [dfsan] Use non-existent file in test for real --- compiler-rt/test/dfsan/custom.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler-rt/test/dfsan/custom.cpp b/compiler-rt/test/dfsan/custom.cpp index 4bb818813cf7..f544e481b726 100644 --- a/compiler-rt/test/dfsan/custom.cpp +++ b/compiler-rt/test/dfsan/custom.cpp @@ -175,7 +175,7 @@ void test_stat() { s.st_dev = i; SAVE_ORIGINS(s) - ret = stat("/nonexistent", &s); + ret = stat("/nonexistent_581cb021aba7", &s); assert(-1 == ret); ASSERT_ZERO_LABEL(ret); ASSERT_LABEL(s.st_dev, i_label); -- GitLab From aa7e4ba3cad0e00dd37d4baca680ed1633bbdb70 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Wed, 20 Mar 2024 17:46:02 -0700 Subject: [PATCH 084/296] [BOLT] Fix an unused variable warning This patch fixes: bolt/lib/Profile/BoltAddressTranslation.cpp:26:12: error: unused variable 'HotFuncAddress' [-Werror,-Wunused-variable] --- bolt/lib/Profile/BoltAddressTranslation.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index e27985251775..1d61a1b735b4 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -44,6 +44,7 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, << " Val: " << Twine::utohexstr(BBInputOffset) << "\n"); LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", getBBHash(HotFuncAddress, BBInputOffset))); + (void)HotFuncAddress; // In case of conflicts (same Key mapping to different Vals), the last // update takes precedence. Of course it is not ideal to have conflicts and // those happen when we have an empty BB that either contained only -- GitLab From 893717446bbc8f31427b581af5fbaf4154b63402 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Wed, 20 Mar 2024 19:58:53 -0500 Subject: [PATCH 085/296] [libc] Add an option to always build GPU loader utilities (#86040) Summary: Right now it's difficult to build these utilities standalone becayse they're keyed off of the other GPU handling. if someone wants to *just* build these utilities it's not possible without setting up the runtimes build. Since we can't just build these by default add an option to enable it. We can't just use the handling like LIBC_HDRGEN does because this is only for the GPU build, which isn't fully set up until way later. So this is probably the easiest way to just allow people to build these tools even without a GPU build setup. --- libc/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libc/CMakeLists.txt b/libc/CMakeLists.txt index 7afb3c5f0faa..a0d79858a896 100644 --- a/libc/CMakeLists.txt +++ b/libc/CMakeLists.txt @@ -61,7 +61,8 @@ if(LLVM_LIBC_FULL_BUILD OR LLVM_LIBC_GPU_BUILD) endif() endif() # We will build the GPU utilities if we are not doing a runtimes build. -if(LLVM_LIBC_GPU_BUILD AND NOT LLVM_RUNTIMES_BUILD) +option(LIBC_BUILD_GPU_LOADER "Always build the GPU loader utilities" OFF) +if(LIBC_BUILD_GPU_LOADER OR (LLVM_LIBC_GPU_BUILD AND NOT LLVM_RUNTIMES_BUILD)) add_subdirectory(utils/gpu) endif() -- GitLab From 3cd988914e53c4c94e48edd4b6bc7b97b2dd4b49 Mon Sep 17 00:00:00 2001 From: Alexander Yermolovich <43973793+ayermolo@users.noreply.github.com> Date: Wed, 20 Mar 2024 18:11:32 -0700 Subject: [PATCH 086/296] [BOLT][DWARF] Fix Test (#86042) Test was not actually checking bolt binary, and had extra POSTCHECK-NEXT lines. --- bolt/test/X86/dwarf5-label-low-pc.s | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bolt/test/X86/dwarf5-label-low-pc.s b/bolt/test/X86/dwarf5-label-low-pc.s index b71309716334..890d9e024d1a 100644 --- a/bolt/test/X86/dwarf5-label-low-pc.s +++ b/bolt/test/X86/dwarf5-label-low-pc.s @@ -8,6 +8,7 @@ # RUN: llvm-dwarfdump --show-form --verbose --debug-addr %t.bolt > %t.txt # RUN: llvm-dwarfdump --show-form --verbose --debug-info %t.bolt >> %t.txt +# RUN: cat %t.txt | FileCheck --check-prefix=POSTCHECK %s # This test checks that we correctly handle DW_AT_low_pc [DW_FORM_addrx] that is part of DW_TAG_label. @@ -35,16 +36,14 @@ # POSTCHECK-NEXT: DW_AT_name # POSTCHECK-NEXT: DW_AT_decl_file # POSTCHECK-NEXT: DW_AT_decl_line -# POSTCHECK-NEXT: # POSTCHECK-NEXT:DW_AT_low_pc [DW_FORM_addrx] (indexed (00000002) -# POSTCHECK-SAME: [0x[[#ADDR]] +# POSTCHECK-SAME: 0x[[#ADDR]] # POSTCHECK: DW_TAG_label # POSTCHECK-NEXT: DW_AT_name # POSTCHECK-NEXT: DW_AT_decl_file # POSTCHECK-NEXT: DW_AT_decl_line -# POSTCHECK-NEXT: # POSTCHECK-NEXT:DW_AT_low_pc [DW_FORM_addrx] (indexed (00000003) -# POSTCHECK-SAME: [0x[[#ADDR2]] +# POSTCHECK-SAME: 0x[[#ADDR2]] # clang++ main.cpp -g -S # int main() { -- GitLab From 71defe40b7df18508d63fb1b1233324e8a28688f Mon Sep 17 00:00:00 2001 From: Thurston Dang Date: Wed, 20 Mar 2024 18:17:33 -0700 Subject: [PATCH 087/296] [sanitizer_common] Suppress warning of cast from SignalHandlerType to sa_sigaction_t (#86046) Some buildbots (e.g., https://lab.llvm.org/buildbot/#/builders/18/builds/16061/steps/10/logs/stdio) have recently started complaining about ``` cast from 'SignalHandlerType' (aka 'void (*)(int, void *, void *)') to 'sa_sigaction_t' (aka 'void (*)(int, siginfo_t *, void *)') converts to incompatible function type [-Werror,-Wcast-function-type-strict] 219 | sigact.sa_sigaction = (sa_sigaction_t)handler; ``` This patch does an intermediate cast to `'(void (*) (void))'` to suppress the warning. N.B. SignalHandlerType has `'void*'` instead of `'siginfo_t*'` because it is typedef'ed in sanitizer_common/sanitizer_common.h, which does not have access to the header (signal.h) that defines siginfo_t; we therefore cannot fix SignalHandlerType. --- compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp index ece2d7d63dd6..48daa2ed25c1 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp @@ -216,7 +216,7 @@ static void MaybeInstallSigaction(int signum, struct sigaction sigact; internal_memset(&sigact, 0, sizeof(sigact)); - sigact.sa_sigaction = (sa_sigaction_t)handler; + sigact.sa_sigaction = (sa_sigaction_t)(void (*)(void))handler; // Do not block the signal from being received in that signal's handler. // Clients are responsible for handling this correctly. sigact.sa_flags = SA_SIGINFO | SA_NODEFER; -- GitLab From 631248dcd26fdec772cedb569be94ff8f12d0901 Mon Sep 17 00:00:00 2001 From: hstk30-hw Date: Thu, 21 Mar 2024 09:25:24 +0800 Subject: [PATCH 088/296] [X86_64] fix empty structure vaarg in c++ (#77907) SizeInBytes of empty structure is 0 in C, while 1 in C++. And empty structure argument of the function is ignored in X86_64 backend.As a result, the value of variable arguments in C++ is incorrect. fix #77036 Co-authored-by: Longsheng Mou --- clang/lib/CodeGen/Targets/X86.cpp | 4 ++++ clang/test/CodeGenCXX/x86_64-vaarg.cpp | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 clang/test/CodeGenCXX/x86_64-vaarg.cpp diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp index 2291c991fb11..1ec0f159ebcb 100644 --- a/clang/lib/CodeGen/Targets/X86.cpp +++ b/clang/lib/CodeGen/Targets/X86.cpp @@ -3019,6 +3019,10 @@ Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, ABIArgInfo AI = classifyArgumentType(Ty, 0, neededInt, neededSSE, /*isNamedArg*/false); + // Empty records are ignored for parameter passing purposes. + if (AI.isIgnore()) + return CGF.CreateMemTemp(Ty); + // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed // in the registers. If not go to step 7. if (!neededInt && !neededSSE) diff --git a/clang/test/CodeGenCXX/x86_64-vaarg.cpp b/clang/test/CodeGenCXX/x86_64-vaarg.cpp new file mode 100644 index 000000000000..f0177906a09a --- /dev/null +++ b/clang/test/CodeGenCXX/x86_64-vaarg.cpp @@ -0,0 +1,23 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -x c -o - %s | FileCheck %s + +typedef struct { struct {} a; } empty; + +// CHECK-LABEL: @{{.*}}empty_record_test +// CHECK-NEXT: entry: +// CHECK-NEXT: [[RETVAL:%.*]] = alloca [[STRUCT_EMPTY:%.*]], align 1 +// CHECK-NEXT: [[Z_ADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: [[LIST:%.*]] = alloca [1 x %struct.__va_list_tag], align 16 +// CHECK-NEXT: [[TMP:%.*]] = alloca [[STRUCT_EMPTY]], align 1 +// CHECK-NEXT: store i32 [[Z:%.*]], ptr [[Z_ADDR]], align 4 +// CHECK-NEXT: [[ARRAYDECAY:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[LIST]], i64 0, i64 0 +// CHECK-NEXT: call void @llvm.va_start(ptr [[ARRAYDECAY]]) +// CHECK-NEXT: [[ARRAYDECAY1:%.*]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr [[LIST]], i64 0, i64 0 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[RETVAL]], ptr align 1 [[TMP]], i64 {{.*}}, i1 false) +// CHECK-NEXT: ret void +empty empty_record_test(int z, ...) { + __builtin_va_list list; + __builtin_va_start(list, z); + return __builtin_va_arg(list, empty); +} -- GitLab From 35a66f965c0ea3b806b2b1736bfe4e6eb61d3613 Mon Sep 17 00:00:00 2001 From: Freddy Ye Date: Thu, 21 Mar 2024 10:19:28 +0800 Subject: [PATCH 089/296] Precommit test for #85737 (#86056) Copied from llvm/test/CodeGen/X86/domain-reassignment.mir --- .../CodeGen/X86/domain-reassignment-ndd.mir | 929 ++++++++++++++++++ 1 file changed, 929 insertions(+) create mode 100644 llvm/test/CodeGen/X86/domain-reassignment-ndd.mir diff --git a/llvm/test/CodeGen/X86/domain-reassignment-ndd.mir b/llvm/test/CodeGen/X86/domain-reassignment-ndd.mir new file mode 100644 index 000000000000..dcd435619990 --- /dev/null +++ b/llvm/test/CodeGen/X86/domain-reassignment-ndd.mir @@ -0,0 +1,929 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -run-pass x86-domain-reassignment -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512dq -o - %s | FileCheck %s +--- | + ; ModuleID = '../test/CodeGen/X86/gpr-to-mask.ll' + source_filename = "../test/CodeGen/X86/gpr-to-mask.ll" + target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" + target triple = "x86_64-unknown-unknown" + + define void @test_fcmp_storefloat(i1 %cond, ptr %fptr, float %f1, float %f2, float %f3, float %f4, float %f5, float %f6) #0 { + entry: + br i1 %cond, label %if, label %else + + if: ; preds = %entry + %cmp1 = fcmp oeq float %f3, %f4 + br label %exit + + else: ; preds = %entry + %cmp2 = fcmp oeq float %f5, %f6 + br label %exit + + exit: ; preds = %else, %if + %val = phi i1 [ %cmp1, %if ], [ %cmp2, %else ] + %selected = select i1 %val, float %f1, float %f2 + store float %selected, ptr %fptr + ret void + } + + define void @test_8bitops() #0 { + ret void + } + define void @test_16bitops() #0 { + ret void + } + define void @test_32bitops() #0 { + ret void + } + define void @test_64bitops() #0 { + ret void + } + define void @test_16bitext() #0 { + ret void + } + define void @test_32bitext() #0 { + ret void + } + define void @test_64bitext() #0 { + ret void + } + ; Note that this function need to be compiled with -global-isel + ; to obtain testable MIR + define void @test_unused(i64 %0) #0 { + %unused = lshr i64 %0, 7 + ret void + } +... +--- +name: test_fcmp_storefloat +alignment: 16 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +registers: + - { id: 0, class: gr8, preferred-register: '' } + - { id: 1, class: gr8, preferred-register: '' } + - { id: 2, class: gr8, preferred-register: '' } + - { id: 3, class: gr32, preferred-register: '' } + - { id: 4, class: gr64, preferred-register: '' } + - { id: 5, class: vr128x, preferred-register: '' } + - { id: 6, class: fr32x, preferred-register: '' } + - { id: 7, class: fr32x, preferred-register: '' } + - { id: 8, class: fr32x, preferred-register: '' } + - { id: 9, class: fr32x, preferred-register: '' } + - { id: 10, class: fr32x, preferred-register: '' } + - { id: 11, class: gr8, preferred-register: '' } + - { id: 12, class: vk1, preferred-register: '' } + - { id: 13, class: gr32, preferred-register: '' } + - { id: 14, class: vk1, preferred-register: '' } + - { id: 15, class: gr32, preferred-register: '' } + - { id: 16, class: gr32, preferred-register: '' } + - { id: 17, class: gr32, preferred-register: '' } + - { id: 18, class: vk1wm, preferred-register: '' } + - { id: 19, class: vr128x, preferred-register: '' } + - { id: 20, class: vr128, preferred-register: '' } + - { id: 21, class: vr128, preferred-register: '' } + - { id: 22, class: fr32x, preferred-register: '' } +liveins: + - { reg: '$edi', virtual-reg: '%3' } + - { reg: '$rsi', virtual-reg: '%4' } + - { reg: '$xmm0', virtual-reg: '%5' } + - { reg: '$xmm1', virtual-reg: '%6' } + - { reg: '$xmm2', virtual-reg: '%7' } + - { reg: '$xmm3', virtual-reg: '%8' } + - { reg: '$xmm4', virtual-reg: '%9' } + - { reg: '$xmm5', virtual-reg: '%10' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + stackProtector: '' + maxCallFrameSize: 4294967295 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false + savePoint: '' + restorePoint: '' +fixedStack: +stack: +constants: +body: | + ; CHECK-LABEL: name: test_fcmp_storefloat + ; CHECK: bb.0.entry: + ; CHECK: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK: liveins: $edi, $rsi, $xmm0, $xmm1, $xmm2, $xmm3, $xmm4, $xmm5 + ; CHECK: [[COPY:%[0-9]+]]:fr32x = COPY $xmm5 + ; CHECK: [[COPY1:%[0-9]+]]:fr32x = COPY $xmm4 + ; CHECK: [[COPY2:%[0-9]+]]:fr32x = COPY $xmm3 + ; CHECK: [[COPY3:%[0-9]+]]:fr32x = COPY $xmm2 + ; CHECK: [[COPY4:%[0-9]+]]:fr32x = COPY $xmm1 + ; CHECK: [[COPY5:%[0-9]+]]:vr128x = COPY $xmm0 + ; CHECK: [[COPY6:%[0-9]+]]:gr64 = COPY $rsi + ; CHECK: [[COPY7:%[0-9]+]]:gr32 = COPY $edi + ; CHECK: [[COPY8:%[0-9]+]]:gr8 = COPY [[COPY7]].sub_8bit + ; CHECK: TEST8ri killed [[COPY8]], 1, implicit-def $eflags + ; CHECK: JCC_1 %bb.2, 4, implicit $eflags + ; CHECK: JMP_1 %bb.1 + ; CHECK: bb.1.if: + ; CHECK: successors: %bb.3(0x80000000) + ; CHECK: [[VCMPSSZrri:%[0-9]+]]:vk1 = VCMPSSZrri [[COPY3]], [[COPY2]], 0 + ; CHECK: [[COPY9:%[0-9]+]]:vk32 = COPY [[VCMPSSZrri]] + ; CHECK: [[COPY10:%[0-9]+]]:vk8 = COPY [[COPY9]] + ; CHECK: JMP_1 %bb.3 + ; CHECK: bb.2.else: + ; CHECK: successors: %bb.3(0x80000000) + ; CHECK: [[VCMPSSZrri1:%[0-9]+]]:vk1 = VCMPSSZrri [[COPY1]], [[COPY]], 0 + ; CHECK: [[COPY11:%[0-9]+]]:vk32 = COPY [[VCMPSSZrri1]] + ; CHECK: [[COPY12:%[0-9]+]]:vk8 = COPY [[COPY11]] + ; CHECK: bb.3.exit: + ; CHECK: [[PHI:%[0-9]+]]:vk8 = PHI [[COPY12]], %bb.2, [[COPY10]], %bb.1 + ; CHECK: [[DEF:%[0-9]+]]:vk32 = IMPLICIT_DEF + ; CHECK: [[COPY13:%[0-9]+]]:vk32 = COPY [[PHI]] + ; CHECK: [[COPY14:%[0-9]+]]:vk1wm = COPY [[COPY13]] + ; CHECK: [[COPY15:%[0-9]+]]:vr128x = COPY [[COPY4]] + ; CHECK: [[DEF1:%[0-9]+]]:vr128 = IMPLICIT_DEF + ; CHECK: [[VMOVSSZrrk:%[0-9]+]]:vr128 = VMOVSSZrrk [[COPY15]], killed [[COPY14]], killed [[DEF1]], [[COPY5]] + ; CHECK: [[COPY16:%[0-9]+]]:fr32x = COPY [[VMOVSSZrrk]] + ; CHECK: VMOVSSZmr [[COPY6]], 1, $noreg, 0, $noreg, killed [[COPY16]] :: (store (s32) into %ir.fptr) + ; CHECK: RET 0 + bb.0.entry: + successors: %bb.1(0x40000000), %bb.2(0x40000000) + liveins: $edi, $rsi, $xmm0, $xmm1, $xmm2, $xmm3, $xmm4, $xmm5 + + %10 = COPY $xmm5 + %9 = COPY $xmm4 + %8 = COPY $xmm3 + %7 = COPY $xmm2 + %6 = COPY $xmm1 + %5 = COPY $xmm0 + %4 = COPY $rsi + %3 = COPY $edi + %11 = COPY %3.sub_8bit + TEST8ri killed %11, 1, implicit-def $eflags + JCC_1 %bb.2, 4, implicit $eflags + JMP_1 %bb.1 + + bb.1.if: + successors: %bb.3(0x80000000) + + %14 = VCMPSSZrri %7, %8, 0, implicit $mxcsr + + ; check that cross domain copies are replaced with same domain copies. + + %15 = COPY %14 + %0 = COPY %15.sub_8bit + JMP_1 %bb.3 + + bb.2.else: + successors: %bb.3(0x80000000) + %12 = VCMPSSZrri %9, %10, 0, implicit $mxcsr + + ; check that cross domain copies are replaced with same domain copies. + + %13 = COPY %12 + %1 = COPY %13.sub_8bit + + bb.3.exit: + + ; check PHI, IMPLICIT_DEF, and INSERT_SUBREG replacers. + + %2 = PHI %1, %bb.2, %0, %bb.1 + %17 = IMPLICIT_DEF + %16 = INSERT_SUBREG %17, %2, %subreg.sub_8bit_hi + %18 = COPY %16 + %19 = COPY %6 + %21 = IMPLICIT_DEF + %20 = VMOVSSZrrk %19, killed %18, killed %21, %5 + %22 = COPY %20 + VMOVSSZmr %4, 1, $noreg, 0, $noreg, killed %22 :: (store (s32) into %ir.fptr) + RET 0 + +... +--- +name: test_8bitops +alignment: 16 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +registers: + - { id: 0, class: gr64, preferred-register: '' } + - { id: 1, class: vr512, preferred-register: '' } + - { id: 2, class: vr512, preferred-register: '' } + - { id: 3, class: vr512, preferred-register: '' } + - { id: 4, class: vr512, preferred-register: '' } + - { id: 5, class: vk8, preferred-register: '' } + - { id: 6, class: gr32, preferred-register: '' } + - { id: 7, class: gr8, preferred-register: '' } + - { id: 8, class: gr32, preferred-register: '' } + - { id: 9, class: gr32, preferred-register: '' } + - { id: 10, class: vk8wm, preferred-register: '' } + - { id: 11, class: vr512, preferred-register: '' } + - { id: 12, class: gr8, preferred-register: '' } + - { id: 13, class: gr8, preferred-register: '' } + - { id: 14, class: gr8, preferred-register: '' } + - { id: 15, class: gr8, preferred-register: '' } + - { id: 16, class: gr8, preferred-register: '' } + - { id: 17, class: gr8, preferred-register: '' } + - { id: 18, class: gr8, preferred-register: '' } +liveins: + - { reg: '$rdi', virtual-reg: '%0' } + - { reg: '$zmm0', virtual-reg: '%1' } + - { reg: '$zmm1', virtual-reg: '%2' } + - { reg: '$zmm2', virtual-reg: '%3' } + - { reg: '$zmm3', virtual-reg: '%4' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + stackProtector: '' + maxCallFrameSize: 4294967295 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false + savePoint: '' + restorePoint: '' +fixedStack: +stack: +constants: +body: | + ; CHECK-LABEL: name: test_8bitops + ; CHECK: bb.0: + ; CHECK: successors: %bb.1(0x80000000) + ; CHECK: liveins: $rdi, $zmm0, $zmm1, $zmm2, $zmm3 + ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rdi + ; CHECK: [[COPY1:%[0-9]+]]:vr512 = COPY $zmm0 + ; CHECK: [[COPY2:%[0-9]+]]:vr512 = COPY $zmm1 + ; CHECK: [[COPY3:%[0-9]+]]:vr512 = COPY $zmm2 + ; CHECK: [[COPY4:%[0-9]+]]:vr512 = COPY $zmm3 + ; CHECK: [[VCMPPDZrri:%[0-9]+]]:vk8 = VCMPPDZrri [[COPY3]], [[COPY4]], 0 + ; CHECK: [[COPY5:%[0-9]+]]:vk32 = COPY [[VCMPPDZrri]] + ; CHECK: [[COPY6:%[0-9]+]]:vk8 = COPY [[COPY5]] + ; CHECK: [[KSHIFTRBri:%[0-9]+]]:vk8 = KSHIFTRBri [[COPY6]], 2 + ; CHECK: [[KSHIFTLBri:%[0-9]+]]:vk8 = KSHIFTLBri [[KSHIFTRBri]], 1 + ; CHECK: [[KNOTBrr:%[0-9]+]]:vk8 = KNOTBrr [[KSHIFTLBri]] + ; CHECK: [[KORBrr:%[0-9]+]]:vk8 = KORBrr [[KNOTBrr]], [[KSHIFTRBri]] + ; CHECK: [[KANDBrr:%[0-9]+]]:vk8 = KANDBrr [[KORBrr]], [[KSHIFTLBri]] + ; CHECK: [[KXORBrr:%[0-9]+]]:vk8 = KXORBrr [[KANDBrr]], [[KSHIFTRBri]] + ; CHECK: [[KADDBrr:%[0-9]+]]:vk8 = KADDBrr [[KXORBrr]], [[KNOTBrr]] + ; CHECK: [[DEF:%[0-9]+]]:vk32 = IMPLICIT_DEF + ; CHECK: [[COPY7:%[0-9]+]]:vk32 = COPY [[KADDBrr]] + ; CHECK: [[COPY8:%[0-9]+]]:vk8wm = COPY [[COPY7]] + ; CHECK: [[VMOVAPDZrrk:%[0-9]+]]:vr512 = VMOVAPDZrrk [[COPY2]], killed [[COPY8]], [[COPY1]] + ; CHECK: VMOVAPDZmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVAPDZrrk]] + ; CHECK: bb.1: + ; CHECK: successors: %bb.2(0x80000000) + ; CHECK: bb.2: + ; CHECK: RET 0 + bb.0: + liveins: $rdi, $zmm0, $zmm1, $zmm2, $zmm3 + + %0 = COPY $rdi + %1 = COPY $zmm0 + %2 = COPY $zmm1 + %3 = COPY $zmm2 + %4 = COPY $zmm3 + + %5 = VCMPPDZrri %3, %4, 0, implicit $mxcsr + %6 = COPY %5 + %7 = COPY %6.sub_8bit + + %12 = SHR8ri %7, 2, implicit-def dead $eflags + %13 = SHL8ri %12, 1, implicit-def dead $eflags + %14 = NOT8r %13 + %15 = OR8rr %14, %12, implicit-def dead $eflags + %16 = AND8rr %15, %13, implicit-def dead $eflags + %17 = XOR8rr %16, %12, implicit-def dead $eflags + %18 = ADD8rr %17, %14, implicit-def dead $eflags + + %8 = IMPLICIT_DEF + %9 = INSERT_SUBREG %8, %18, %subreg.sub_8bit_hi + %10 = COPY %9 + %11 = VMOVAPDZrrk %2, killed %10, %1 + VMOVAPDZmr %0, 1, $noreg, 0, $noreg, killed %11 + + ; FIXME We can't replace TEST with KTEST due to flag differences + ; TEST8rr %18, %18, implicit-def $eflags + ; JCC_1 %bb.1, 4, implicit $eflags + ; JMP_1 %bb.2 + + bb.1: + + bb.2: + RET 0 + +... +--- +name: test_16bitops +alignment: 16 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +registers: + - { id: 0, class: gr64, preferred-register: '' } + - { id: 1, class: vr512, preferred-register: '' } + - { id: 2, class: vr512, preferred-register: '' } + - { id: 3, class: vr512, preferred-register: '' } + - { id: 4, class: vr512, preferred-register: '' } + - { id: 5, class: vk16, preferred-register: '' } + - { id: 6, class: gr32, preferred-register: '' } + - { id: 7, class: gr16, preferred-register: '' } + - { id: 8, class: gr32, preferred-register: '' } + - { id: 9, class: gr32, preferred-register: '' } + - { id: 10, class: vk16wm, preferred-register: '' } + - { id: 11, class: vr512, preferred-register: '' } + - { id: 12, class: gr16, preferred-register: '' } + - { id: 13, class: gr16, preferred-register: '' } + - { id: 14, class: gr16, preferred-register: '' } + - { id: 15, class: gr16, preferred-register: '' } + - { id: 16, class: gr16, preferred-register: '' } + - { id: 17, class: gr16, preferred-register: '' } +liveins: + - { reg: '$rdi', virtual-reg: '%0' } + - { reg: '$zmm0', virtual-reg: '%1' } + - { reg: '$zmm1', virtual-reg: '%2' } + - { reg: '$zmm2', virtual-reg: '%3' } + - { reg: '$zmm3', virtual-reg: '%4' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + stackProtector: '' + maxCallFrameSize: 4294967295 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false + savePoint: '' + restorePoint: '' +fixedStack: +stack: +constants: +body: | + ; CHECK-LABEL: name: test_16bitops + ; CHECK: bb.0: + ; CHECK: successors: %bb.1(0x80000000) + ; CHECK: liveins: $rdi, $zmm0, $zmm1, $zmm2, $zmm3 + ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rdi + ; CHECK: [[COPY1:%[0-9]+]]:vr512 = COPY $zmm0 + ; CHECK: [[COPY2:%[0-9]+]]:vr512 = COPY $zmm1 + ; CHECK: [[COPY3:%[0-9]+]]:vr512 = COPY $zmm2 + ; CHECK: [[COPY4:%[0-9]+]]:vr512 = COPY $zmm3 + ; CHECK: [[VCMPPSZrri:%[0-9]+]]:vk16 = VCMPPSZrri [[COPY3]], [[COPY4]], 0 + ; CHECK: [[COPY5:%[0-9]+]]:vk32 = COPY [[VCMPPSZrri]] + ; CHECK: [[COPY6:%[0-9]+]]:vk16 = COPY [[COPY5]] + ; CHECK: [[KSHIFTRWri:%[0-9]+]]:vk16 = KSHIFTRWri [[COPY6]], 2 + ; CHECK: [[KSHIFTLWri:%[0-9]+]]:vk16 = KSHIFTLWri [[KSHIFTRWri]], 1 + ; CHECK: [[KNOTWrr:%[0-9]+]]:vk16 = KNOTWrr [[KSHIFTLWri]] + ; CHECK: [[KORWrr:%[0-9]+]]:vk16 = KORWrr [[KNOTWrr]], [[KSHIFTRWri]] + ; CHECK: [[KANDWrr:%[0-9]+]]:vk16 = KANDWrr [[KORWrr]], [[KSHIFTLWri]] + ; CHECK: [[KXORWrr:%[0-9]+]]:vk16 = KXORWrr [[KANDWrr]], [[KSHIFTRWri]] + ; CHECK: [[DEF:%[0-9]+]]:vk32 = IMPLICIT_DEF + ; CHECK: [[COPY7:%[0-9]+]]:vk32 = COPY [[KXORWrr]] + ; CHECK: [[COPY8:%[0-9]+]]:vk16wm = COPY [[COPY7]] + ; CHECK: [[VMOVAPSZrrk:%[0-9]+]]:vr512 = VMOVAPSZrrk [[COPY2]], killed [[COPY8]], [[COPY1]] + ; CHECK: VMOVAPSZmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVAPSZrrk]] + ; CHECK: bb.1: + ; CHECK: successors: %bb.2(0x80000000) + ; CHECK: bb.2: + ; CHECK: RET 0 + bb.0: + liveins: $rdi, $zmm0, $zmm1, $zmm2, $zmm3 + + %0 = COPY $rdi + %1 = COPY $zmm0 + %2 = COPY $zmm1 + %3 = COPY $zmm2 + %4 = COPY $zmm3 + + %5 = VCMPPSZrri %3, %4, 0, implicit $mxcsr + %6 = COPY %5 + %7 = COPY %6.sub_16bit + + %12 = SHR16ri %7, 2, implicit-def dead $eflags + %13 = SHL16ri %12, 1, implicit-def dead $eflags + %14 = NOT16r %13 + %15 = OR16rr %14, %12, implicit-def dead $eflags + %16 = AND16rr %15, %13, implicit-def dead $eflags + %17 = XOR16rr %16, %12, implicit-def dead $eflags + + %8 = IMPLICIT_DEF + %9 = INSERT_SUBREG %8, %17, %subreg.sub_16bit + %10 = COPY %9 + %11 = VMOVAPSZrrk %2, killed %10, %1 + VMOVAPSZmr %0, 1, $noreg, 0, $noreg, killed %11 + + ; FIXME We can't replace TEST with KTEST due to flag differences + ; FIXME TEST16rr %17, %17, implicit-def $eflags + ; FIXME JCC_1 %bb.1, 4, implicit $eflags + ; FIXME JMP_1 %bb.2 + + bb.1: + + bb.2: + RET 0 + +... +--- +name: test_32bitops +alignment: 16 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +registers: + - { id: 0, class: gr64, preferred-register: '' } + - { id: 1, class: vr512, preferred-register: '' } + - { id: 2, class: vr512, preferred-register: '' } + - { id: 3, class: vk32wm, preferred-register: '' } + - { id: 4, class: vr512, preferred-register: '' } + - { id: 5, class: gr32, preferred-register: '' } + - { id: 6, class: gr32, preferred-register: '' } + - { id: 7, class: gr32, preferred-register: '' } + - { id: 8, class: gr32, preferred-register: '' } + - { id: 9, class: gr32, preferred-register: '' } + - { id: 10, class: gr32, preferred-register: '' } + - { id: 11, class: gr32, preferred-register: '' } + - { id: 12, class: gr32, preferred-register: '' } + - { id: 13, class: gr32, preferred-register: '' } +liveins: + - { reg: '$rdi', virtual-reg: '%0' } + - { reg: '$zmm0', virtual-reg: '%1' } + - { reg: '$zmm1', virtual-reg: '%2' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + stackProtector: '' + maxCallFrameSize: 4294967295 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false + savePoint: '' + restorePoint: '' +fixedStack: +stack: +constants: +body: | + ; CHECK-LABEL: name: test_32bitops + ; CHECK: bb.0: + ; CHECK: successors: %bb.1(0x80000000) + ; CHECK: liveins: $rdi, $zmm0, $zmm1 + ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rdi + ; CHECK: [[COPY1:%[0-9]+]]:vr512 = COPY $zmm0 + ; CHECK: [[COPY2:%[0-9]+]]:vr512 = COPY $zmm1 + ; CHECK: [[KMOVDkm:%[0-9]+]]:vk32 = KMOVDkm [[COPY]], 1, $noreg, 0, $noreg + ; CHECK: [[KSHIFTRDri:%[0-9]+]]:vk32 = KSHIFTRDri [[KMOVDkm]], 2 + ; CHECK: [[KSHIFTLDri:%[0-9]+]]:vk32 = KSHIFTLDri [[KSHIFTRDri]], 1 + ; CHECK: [[KNOTDrr:%[0-9]+]]:vk32 = KNOTDrr [[KSHIFTLDri]] + ; CHECK: [[KORDrr:%[0-9]+]]:vk32 = KORDrr [[KNOTDrr]], [[KSHIFTRDri]] + ; CHECK: [[KANDDrr:%[0-9]+]]:vk32 = KANDDrr [[KORDrr]], [[KSHIFTLDri]] + ; CHECK: [[KXORDrr:%[0-9]+]]:vk32 = KXORDrr [[KANDDrr]], [[KSHIFTRDri]] + ; CHECK: [[KANDNDrr:%[0-9]+]]:vk32 = KANDNDrr [[KXORDrr]], [[KORDrr]] + ; CHECK: [[KADDDrr:%[0-9]+]]:vk32 = KADDDrr [[KANDNDrr]], [[KXORDrr]] + ; CHECK: [[COPY3:%[0-9]+]]:vk32wm = COPY [[KADDDrr]] + ; CHECK: [[VMOVDQU16Zrrk:%[0-9]+]]:vr512 = VMOVDQU16Zrrk [[COPY2]], killed [[COPY3]], [[COPY1]] + ; CHECK: VMOVDQA32Zmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVDQU16Zrrk]] + ; CHECK: bb.1: + ; CHECK: successors: %bb.2(0x80000000) + ; CHECK: bb.2: + ; CHECK: RET 0 + bb.0: + liveins: $rdi, $zmm0, $zmm1 + + %0 = COPY $rdi + %1 = COPY $zmm0 + %2 = COPY $zmm1 + + %5 = MOV32rm %0, 1, $noreg, 0, $noreg + %6 = SHR32ri %5, 2, implicit-def dead $eflags + %7 = SHL32ri %6, 1, implicit-def dead $eflags + %8 = NOT32r %7 + %9 = OR32rr %8, %6, implicit-def dead $eflags + %10 = AND32rr %9, %7, implicit-def dead $eflags + %11 = XOR32rr %10, %6, implicit-def dead $eflags + %12 = ANDN32rr %11, %9, implicit-def dead $eflags + %13 = ADD32rr %12, %11, implicit-def dead $eflags + + %3 = COPY %13 + %4 = VMOVDQU16Zrrk %2, killed %3, %1 + VMOVDQA32Zmr %0, 1, $noreg, 0, $noreg, killed %4 + + ; FIXME We can't replace TEST with KTEST due to flag differences + ; FIXME TEST32rr %13, %13, implicit-def $eflags + ; FIXME JCC_1 %bb.1, 4, implicit $eflags + ; FIXME JMP_1 %bb.2 + + bb.1: + + bb.2: + RET 0 + +... +--- +name: test_64bitops +alignment: 16 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +registers: + - { id: 0, class: gr64, preferred-register: '' } + - { id: 1, class: vr512, preferred-register: '' } + - { id: 2, class: vr512, preferred-register: '' } + - { id: 3, class: vk64wm, preferred-register: '' } + - { id: 4, class: vr512, preferred-register: '' } + - { id: 5, class: gr64, preferred-register: '' } + - { id: 6, class: gr64, preferred-register: '' } + - { id: 7, class: gr64, preferred-register: '' } + - { id: 8, class: gr64, preferred-register: '' } + - { id: 9, class: gr64, preferred-register: '' } + - { id: 10, class: gr64, preferred-register: '' } + - { id: 11, class: gr64, preferred-register: '' } + - { id: 12, class: gr64, preferred-register: '' } + - { id: 13, class: gr64, preferred-register: '' } +liveins: + - { reg: '$rdi', virtual-reg: '%0' } + - { reg: '$zmm0', virtual-reg: '%1' } + - { reg: '$zmm1', virtual-reg: '%2' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + stackProtector: '' + maxCallFrameSize: 4294967295 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false + savePoint: '' + restorePoint: '' +fixedStack: +stack: +constants: +body: | + ; CHECK-LABEL: name: test_64bitops + ; CHECK: bb.0: + ; CHECK: successors: %bb.1(0x80000000) + ; CHECK: liveins: $rdi, $zmm0, $zmm1 + ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rdi + ; CHECK: [[COPY1:%[0-9]+]]:vr512 = COPY $zmm0 + ; CHECK: [[COPY2:%[0-9]+]]:vr512 = COPY $zmm1 + ; CHECK: [[KMOVQkm:%[0-9]+]]:vk64 = KMOVQkm [[COPY]], 1, $noreg, 0, $noreg + ; CHECK: [[KSHIFTRQri:%[0-9]+]]:vk64 = KSHIFTRQri [[KMOVQkm]], 2 + ; CHECK: [[KSHIFTLQri:%[0-9]+]]:vk64 = KSHIFTLQri [[KSHIFTRQri]], 1 + ; CHECK: [[KNOTQrr:%[0-9]+]]:vk64 = KNOTQrr [[KSHIFTLQri]] + ; CHECK: [[KORQrr:%[0-9]+]]:vk64 = KORQrr [[KNOTQrr]], [[KSHIFTRQri]] + ; CHECK: [[KANDQrr:%[0-9]+]]:vk64 = KANDQrr [[KORQrr]], [[KSHIFTLQri]] + ; CHECK: [[KXORQrr:%[0-9]+]]:vk64 = KXORQrr [[KANDQrr]], [[KSHIFTRQri]] + ; CHECK: [[KANDNQrr:%[0-9]+]]:vk64 = KANDNQrr [[KXORQrr]], [[KORQrr]] + ; CHECK: [[KADDQrr:%[0-9]+]]:vk64 = KADDQrr [[KANDNQrr]], [[KXORQrr]] + ; CHECK: [[COPY3:%[0-9]+]]:vk64wm = COPY [[KADDQrr]] + ; CHECK: [[VMOVDQU8Zrrk:%[0-9]+]]:vr512 = VMOVDQU8Zrrk [[COPY2]], killed [[COPY3]], [[COPY1]] + ; CHECK: VMOVDQA32Zmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVDQU8Zrrk]] + ; CHECK: bb.1: + ; CHECK: successors: %bb.2(0x80000000) + ; CHECK: bb.2: + ; CHECK: RET 0 + bb.0: + liveins: $rdi, $zmm0, $zmm1 + + %0 = COPY $rdi + %1 = COPY $zmm0 + %2 = COPY $zmm1 + + %5 = MOV64rm %0, 1, $noreg, 0, $noreg + %6 = SHR64ri %5, 2, implicit-def dead $eflags + %7 = SHL64ri %6, 1, implicit-def dead $eflags + %8 = NOT64r %7 + %9 = OR64rr %8, %6, implicit-def dead $eflags + %10 = AND64rr %9, %7, implicit-def dead $eflags + %11 = XOR64rr %10, %6, implicit-def dead $eflags + %12 = ANDN64rr %11, %9, implicit-def dead $eflags + %13 = ADD64rr %12, %11, implicit-def dead $eflags + + %3 = COPY %13 + %4 = VMOVDQU8Zrrk %2, killed %3, %1 + VMOVDQA32Zmr %0, 1, $noreg, 0, $noreg, killed %4 + + ; FIXME We can't replace TEST with KTEST due to flag differences + ; FIXME TEST64rr %13, %13, implicit-def $eflags + ; FIXME JCC_1 %bb.1, 4, implicit $eflags + ; FIXME JMP_1 %bb.2 + + bb.1: + + bb.2: + RET 0 + +... +--- +name: test_16bitext +alignment: 16 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +registers: + - { id: 0, class: gr64, preferred-register: '' } + - { id: 1, class: vr512, preferred-register: '' } + - { id: 2, class: vr512, preferred-register: '' } + - { id: 3, class: vk16wm, preferred-register: '' } + - { id: 4, class: vr512, preferred-register: '' } + - { id: 5, class: gr16, preferred-register: '' } + - { id: 6, class: gr16, preferred-register: '' } +liveins: + - { reg: '$rdi', virtual-reg: '%0' } + - { reg: '$zmm0', virtual-reg: '%1' } + - { reg: '$zmm1', virtual-reg: '%2' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + stackProtector: '' + maxCallFrameSize: 4294967295 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false + savePoint: '' + restorePoint: '' +fixedStack: +stack: +constants: +body: | + bb.0: + liveins: $rdi, $zmm0, $zmm1 + + ; CHECK-LABEL: name: test_16bitext + ; CHECK: liveins: $rdi, $zmm0, $zmm1 + ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rdi + ; CHECK: [[COPY1:%[0-9]+]]:vr512 = COPY $zmm0 + ; CHECK: [[COPY2:%[0-9]+]]:vr512 = COPY $zmm1 + ; CHECK: [[KMOVBkm:%[0-9]+]]:vk8 = KMOVBkm [[COPY]], 1, $noreg, 0, $noreg + ; CHECK: [[COPY3:%[0-9]+]]:vk16 = COPY [[KMOVBkm]] + ; CHECK: [[KNOTWrr:%[0-9]+]]:vk16 = KNOTWrr [[COPY3]] + ; CHECK: [[COPY4:%[0-9]+]]:vk16wm = COPY [[KNOTWrr]] + ; CHECK: [[VMOVAPSZrrk:%[0-9]+]]:vr512 = VMOVAPSZrrk [[COPY2]], killed [[COPY4]], [[COPY1]] + ; CHECK: VMOVAPSZmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVAPSZrrk]] + ; CHECK: RET 0 + %0 = COPY $rdi + %1 = COPY $zmm0 + %2 = COPY $zmm1 + + %5 = MOVZX16rm8 %0, 1, $noreg, 0, $noreg + %6 = NOT16r %5 + + %3 = COPY %6 + %4 = VMOVAPSZrrk %2, killed %3, %1 + VMOVAPSZmr %0, 1, $noreg, 0, $noreg, killed %4 + RET 0 + +... +--- +name: test_32bitext +alignment: 16 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +registers: + - { id: 0, class: gr64, preferred-register: '' } + - { id: 1, class: vr512, preferred-register: '' } + - { id: 2, class: vr512, preferred-register: '' } + - { id: 3, class: vk64wm, preferred-register: '' } + - { id: 4, class: vr512, preferred-register: '' } + - { id: 5, class: gr32, preferred-register: '' } + - { id: 6, class: gr32, preferred-register: '' } + - { id: 7, class: gr32, preferred-register: '' } +liveins: + - { reg: '$rdi', virtual-reg: '%0' } + - { reg: '$zmm0', virtual-reg: '%1' } + - { reg: '$zmm1', virtual-reg: '%2' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + stackProtector: '' + maxCallFrameSize: 4294967295 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false + savePoint: '' + restorePoint: '' +fixedStack: +stack: +constants: +body: | + bb.0: + liveins: $rdi, $zmm0, $zmm1 + + ; CHECK-LABEL: name: test_32bitext + ; CHECK: liveins: $rdi, $zmm0, $zmm1 + ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rdi + ; CHECK: [[COPY1:%[0-9]+]]:vr512 = COPY $zmm0 + ; CHECK: [[COPY2:%[0-9]+]]:vr512 = COPY $zmm1 + ; CHECK: [[KMOVBkm:%[0-9]+]]:vk8 = KMOVBkm [[COPY]], 1, $noreg, 0, $noreg + ; CHECK: [[COPY3:%[0-9]+]]:vk32 = COPY [[KMOVBkm]] + ; CHECK: [[KMOVWkm:%[0-9]+]]:vk16 = KMOVWkm [[COPY]], 1, $noreg, 0, $noreg + ; CHECK: [[COPY4:%[0-9]+]]:vk32 = COPY [[KMOVWkm]] + ; CHECK: [[KADDDrr:%[0-9]+]]:vk32 = KADDDrr [[COPY3]], [[COPY4]] + ; CHECK: [[COPY5:%[0-9]+]]:vk64wm = COPY [[KADDDrr]] + ; CHECK: [[VMOVDQU16Zrrk:%[0-9]+]]:vr512 = VMOVDQU16Zrrk [[COPY2]], killed [[COPY5]], [[COPY1]] + ; CHECK: VMOVDQA32Zmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVDQU16Zrrk]] + ; CHECK: RET 0 + %0 = COPY $rdi + %1 = COPY $zmm0 + %2 = COPY $zmm1 + + %5 = MOVZX32rm8 %0, 1, $noreg, 0, $noreg + %6 = MOVZX32rm16 %0, 1, $noreg, 0, $noreg + %7 = ADD32rr %5, %6, implicit-def dead $eflags + + %3 = COPY %7 + %4 = VMOVDQU16Zrrk %2, killed %3, %1 + VMOVDQA32Zmr %0, 1, $noreg, 0, $noreg, killed %4 + RET 0 + +... +--- +name: test_64bitext +alignment: 16 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +registers: + - { id: 0, class: gr64, preferred-register: '' } + - { id: 1, class: vr512, preferred-register: '' } + - { id: 2, class: vr512, preferred-register: '' } + - { id: 3, class: vk64wm, preferred-register: '' } + - { id: 4, class: vr512, preferred-register: '' } + - { id: 5, class: gr64, preferred-register: '' } + - { id: 6, class: gr64, preferred-register: '' } + - { id: 7, class: gr64, preferred-register: '' } +liveins: + - { reg: '$rdi', virtual-reg: '%0' } + - { reg: '$zmm0', virtual-reg: '%1' } + - { reg: '$zmm1', virtual-reg: '%2' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + stackProtector: '' + maxCallFrameSize: 4294967295 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false + savePoint: '' + restorePoint: '' +fixedStack: +stack: +constants: +body: | + bb.0: + liveins: $rdi, $zmm0, $zmm1 + + ; CHECK-LABEL: name: test_64bitext + ; CHECK: liveins: $rdi, $zmm0, $zmm1 + ; CHECK: [[COPY:%[0-9]+]]:gr64 = COPY $rdi + ; CHECK: [[COPY1:%[0-9]+]]:vr512 = COPY $zmm0 + ; CHECK: [[COPY2:%[0-9]+]]:vr512 = COPY $zmm1 + ; CHECK: [[KMOVBkm:%[0-9]+]]:vk8 = KMOVBkm [[COPY]], 1, $noreg, 0, $noreg + ; CHECK: [[COPY3:%[0-9]+]]:vk64 = COPY [[KMOVBkm]] + ; CHECK: [[KMOVWkm:%[0-9]+]]:vk16 = KMOVWkm [[COPY]], 1, $noreg, 0, $noreg + ; CHECK: [[COPY4:%[0-9]+]]:vk64 = COPY [[KMOVWkm]] + ; CHECK: [[KADDQrr:%[0-9]+]]:vk64 = KADDQrr [[COPY3]], [[COPY4]] + ; CHECK: [[COPY5:%[0-9]+]]:vk64wm = COPY [[KADDQrr]] + ; CHECK: [[VMOVDQU8Zrrk:%[0-9]+]]:vr512 = VMOVDQU8Zrrk [[COPY2]], killed [[COPY5]], [[COPY1]] + ; CHECK: VMOVDQA32Zmr [[COPY]], 1, $noreg, 0, $noreg, killed [[VMOVDQU8Zrrk]] + ; CHECK: RET 0 + %0 = COPY $rdi + %1 = COPY $zmm0 + %2 = COPY $zmm1 + + %5 = MOVZX64rm8 %0, 1, $noreg, 0, $noreg + %6 = MOVZX64rm16 %0, 1, $noreg, 0, $noreg + %7 = ADD64rr %5, %6, implicit-def dead $eflags + + %3 = COPY %7 + %4 = VMOVDQU8Zrrk %2, killed %3, %1 + VMOVDQA32Zmr %0, 1, $noreg, 0, $noreg, killed %4 + RET 0 + +... +--- +name: test_unused +alignment: 16 +exposesReturnsTwice: false +legalized: true +regBankSelected: true +selected: true +failedISel: false +tracksRegLiveness: true +hasWinCFI: false +callsEHReturn: false +callsUnwindInit: false +hasEHCatchret: false +hasEHScopes: false +hasEHFunclets: false +isOutlined: false +debugInstrRef: false +failsVerification: false +tracksDebugUserValues: false +registers: +# Note that this test is supposed to have registers without classes + - { id: 0, class: _, preferred-register: '' } + - { id: 1, class: _, preferred-register: '' } + - { id: 2, class: _, preferred-register: '' } +liveins: + - { reg: '$rdi', virtual-reg: '' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 1 + adjustsStack: false + hasCalls: false + stackProtector: '' + functionContext: '' + maxCallFrameSize: 4294967295 + cvBytesOfCalleeSavedRegisters: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false + hasTailCall: false + localFrameSize: 0 + savePoint: '' + restorePoint: '' +fixedStack: [] +stack: [] +entry_values: [] +callSites: [] +debugValueSubstitutions: [] +constants: [] +machineFunctionInfo: {} +body: | + bb.1 (%ir-block.1): + liveins: $rdi + + RET 0 + +... -- GitLab From 44a81af510801edce842e9574ec4d52cc7bd0ae9 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Thu, 21 Mar 2024 10:26:58 +0800 Subject: [PATCH 090/296] [AArch64] Run LoopSimplifyPass in byte-compare-index.ll (#86053) Make this test case work on both new and legacy pass manager. See also #85215 --- .../LoopIdiom/AArch64/byte-compare-index.ll | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/llvm/test/Transforms/LoopIdiom/AArch64/byte-compare-index.ll b/llvm/test/Transforms/LoopIdiom/AArch64/byte-compare-index.ll index e6a0c5f45375..92ff099afb1c 100644 --- a/llvm/test/Transforms/LoopIdiom/AArch64/byte-compare-index.ll +++ b/llvm/test/Transforms/LoopIdiom/AArch64/byte-compare-index.ll @@ -780,7 +780,7 @@ define i32 @compare_bytes_extra_cmp(ptr %a, ptr %b, i32 %len, i32 %n, i32 %x) { ; CHECK-NEXT: [[LEN_ADDR:%.*]] = phi i32 [ [[LEN]], [[MISMATCH_END]] ], [ [[MISMATCH_RESULT]], [[WHILE_BODY:%.*]] ] ; CHECK-NEXT: [[INC:%.*]] = add i32 [[LEN_ADDR]], 1 ; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[MISMATCH_RESULT]], [[N]] -; CHECK-NEXT: br i1 [[CMP_NOT]], label [[WHILE_END]], label [[WHILE_BODY]] +; CHECK-NEXT: br i1 [[CMP_NOT]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]] ; CHECK: while.body: ; CHECK-NEXT: [[IDXPROM:%.*]] = zext i32 [[MISMATCH_RESULT]] to i64 ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IDXPROM]] @@ -788,11 +788,14 @@ define i32 @compare_bytes_extra_cmp(ptr %a, ptr %b, i32 %len, i32 %n, i32 %x) { ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IDXPROM]] ; CHECK-NEXT: [[TMP46:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1 ; CHECK-NEXT: [[CMP_NOT2:%.*]] = icmp eq i8 [[TMP45]], [[TMP46]] -; CHECK-NEXT: br i1 [[CMP_NOT2]], label [[WHILE_COND]], label [[WHILE_END]] +; CHECK-NEXT: br i1 [[CMP_NOT2]], label [[WHILE_COND]], label [[WHILE_END_LOOPEXIT]] ; CHECK: byte.compare: +; CHECK-NEXT: br label [[WHILE_END_LOOPEXIT]] +; CHECK: while.end.loopexit: +; CHECK-NEXT: [[INC_LCSSA1:%.*]] = phi i32 [ [[MISMATCH_RESULT]], [[WHILE_COND]] ], [ [[MISMATCH_RESULT]], [[WHILE_BODY]] ], [ [[MISMATCH_RESULT]], [[BYTE_COMPARE]] ] ; CHECK-NEXT: br label [[WHILE_END]] ; CHECK: while.end: -; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[MISMATCH_RESULT]], [[WHILE_BODY]] ], [ [[MISMATCH_RESULT]], [[WHILE_COND]] ], [ [[X]], [[ENTRY:%.*]] ], [ [[MISMATCH_RESULT]], [[BYTE_COMPARE]] ] +; CHECK-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[X]], [[ENTRY:%.*]] ], [ [[INC_LCSSA1]], [[WHILE_END_LOOPEXIT]] ] ; CHECK-NEXT: ret i32 [[INC_LCSSA]] ; ; LOOP-DEL-LABEL: define i32 @compare_bytes_extra_cmp( @@ -884,7 +887,7 @@ define i32 @compare_bytes_extra_cmp(ptr %a, ptr %b, i32 %len, i32 %n, i32 %x) { ; NO-TRANSFORM-NEXT: [[LEN_ADDR:%.*]] = phi i32 [ [[LEN]], [[PH]] ], [ [[INC:%.*]], [[WHILE_BODY:%.*]] ] ; NO-TRANSFORM-NEXT: [[INC]] = add i32 [[LEN_ADDR]], 1 ; NO-TRANSFORM-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[INC]], [[N]] -; NO-TRANSFORM-NEXT: br i1 [[CMP_NOT]], label [[WHILE_END]], label [[WHILE_BODY]] +; NO-TRANSFORM-NEXT: br i1 [[CMP_NOT]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]] ; NO-TRANSFORM: while.body: ; NO-TRANSFORM-NEXT: [[IDXPROM:%.*]] = zext i32 [[INC]] to i64 ; NO-TRANSFORM-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IDXPROM]] @@ -892,9 +895,12 @@ define i32 @compare_bytes_extra_cmp(ptr %a, ptr %b, i32 %len, i32 %n, i32 %x) { ; NO-TRANSFORM-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IDXPROM]] ; NO-TRANSFORM-NEXT: [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1 ; NO-TRANSFORM-NEXT: [[CMP_NOT2:%.*]] = icmp eq i8 [[TMP0]], [[TMP1]] -; NO-TRANSFORM-NEXT: br i1 [[CMP_NOT2]], label [[WHILE_COND]], label [[WHILE_END]] +; NO-TRANSFORM-NEXT: br i1 [[CMP_NOT2]], label [[WHILE_COND]], label [[WHILE_END_LOOPEXIT]] +; NO-TRANSFORM: while.end.loopexit: +; NO-TRANSFORM-NEXT: [[INC_LCSSA1:%.*]] = phi i32 [ [[INC]], [[WHILE_COND]] ], [ [[INC]], [[WHILE_BODY]] ] +; NO-TRANSFORM-NEXT: br label [[WHILE_END]] ; NO-TRANSFORM: while.end: -; NO-TRANSFORM-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], [[WHILE_BODY]] ], [ [[INC]], [[WHILE_COND]] ], [ [[X]], [[ENTRY:%.*]] ] +; NO-TRANSFORM-NEXT: [[INC_LCSSA:%.*]] = phi i32 [ [[X]], [[ENTRY:%.*]] ], [ [[INC_LCSSA1]], [[WHILE_END_LOOPEXIT]] ] ; NO-TRANSFORM-NEXT: ret i32 [[INC_LCSSA]] ; entry: @@ -908,7 +914,7 @@ while.cond: %len.addr = phi i32 [ %len, %ph ], [ %inc, %while.body ] %inc = add i32 %len.addr, 1 %cmp.not = icmp eq i32 %inc, %n - br i1 %cmp.not, label %while.end, label %while.body + br i1 %cmp.not, label %while.end.loopexit, label %while.body while.body: %idxprom = zext i32 %inc to i64 @@ -917,10 +923,14 @@ while.body: %arrayidx2 = getelementptr inbounds i8, ptr %b, i64 %idxprom %1 = load i8, ptr %arrayidx2 %cmp.not2 = icmp eq i8 %0, %1 - br i1 %cmp.not2, label %while.cond, label %while.end + br i1 %cmp.not2, label %while.cond, label %while.end.loopexit + +while.end.loopexit: + %inc.lcssa1 = phi i32 [ %inc, %while.cond ], [ %inc, %while.body ] + br label %while.end while.end: - %inc.lcssa = phi i32 [ %inc, %while.body ], [ %inc, %while.cond ], [ %x, %entry ] + %inc.lcssa = phi i32 [ %x, %entry ], [ %inc.lcssa1, %while.end.loopexit ] ret i32 %inc.lcssa } -- GitLab From 0d08282310e4007dfb748132e5c196765b1ffcd2 Mon Sep 17 00:00:00 2001 From: Michael Liao Date: Wed, 20 Mar 2024 22:39:23 -0400 Subject: [PATCH 091/296] [MLIR][XeGPU] Fix shared build. NFC --- mlir/lib/Dialect/XeGPU/IR/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mlir/lib/Dialect/XeGPU/IR/CMakeLists.txt b/mlir/lib/Dialect/XeGPU/IR/CMakeLists.txt index 2e99f39ed86d..617c89a84ee0 100644 --- a/mlir/lib/Dialect/XeGPU/IR/CMakeLists.txt +++ b/mlir/lib/Dialect/XeGPU/IR/CMakeLists.txt @@ -11,5 +11,7 @@ add_mlir_dialect_library(MLIRXeGPUDialect MLIRXeGPUEnumsIncGen LINK_LIBS PUBLIC + MLIRDialectUtils MLIRIR + MLIRViewLikeInterface ) -- GitLab From 0e3fbfd1e106dd027aab9ea4a9a6f116d05a0987 Mon Sep 17 00:00:00 2001 From: Thurston Dang Date: Thu, 21 Mar 2024 02:44:23 +0000 Subject: [PATCH 092/296] Revert "[sanitizer_common] Suppress warning of cast from SignalHandlerType to sa_sigaction_t" This reverts commit 9d79589e7c8b728a592a4b6b3dee53ac471d7946 because it failed to suppress the warning. --- compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp index 48daa2ed25c1..ece2d7d63dd6 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp @@ -216,7 +216,7 @@ static void MaybeInstallSigaction(int signum, struct sigaction sigact; internal_memset(&sigact, 0, sizeof(sigact)); - sigact.sa_sigaction = (sa_sigaction_t)(void (*)(void))handler; + sigact.sa_sigaction = (sa_sigaction_t)handler; // Do not block the signal from being received in that signal's handler. // Clients are responsible for handling this correctly. sigact.sa_flags = SA_SIGINFO | SA_NODEFER; -- GitLab From 5c95484061a58250de7e5abe150c6ebb25898523 Mon Sep 17 00:00:00 2001 From: Vitaly Buka Date: Wed, 20 Mar 2024 18:55:26 -0700 Subject: [PATCH 093/296] [Analysis] Use implicit-check-not in test --- llvm/test/Analysis/AliasSet/intrinsics.ll | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/llvm/test/Analysis/AliasSet/intrinsics.ll b/llvm/test/Analysis/AliasSet/intrinsics.ll index aeb5424ca919..678d6d246e63 100644 --- a/llvm/test/Analysis/AliasSet/intrinsics.ll +++ b/llvm/test/Analysis/AliasSet/intrinsics.ll @@ -1,9 +1,8 @@ -; RUN: opt -passes=print-alias-sets -S -o - < %s 2>&1 | FileCheck %s +; RUN: opt -passes=print-alias-sets -S -o - < %s 2>&1 | FileCheck %s --implicit-check-not="Unknown instructions" ; CHECK: Alias sets for function 'test1': ; CHECK: Alias Set Tracker: 2 alias sets for 2 pointer values. ; CHECK: AliasSet[0x{{[0-9a-f]+}}, 1] must alias, Mod Memory locations: (ptr %a, LocationSize::precise(1)) -; CHECK-NOT: 1 Unknown instruction ; CHECK: AliasSet[0x{{[0-9a-f]+}}, 1] must alias, Mod Memory locations: (ptr %b, LocationSize::precise(1)) define void @test1(i32 %c) { entry: @@ -64,7 +63,6 @@ entry: ; CHECK: Alias sets for function 'test5': ; CHECK: Alias Set Tracker: 2 alias sets for 2 pointer values. ; CHECK: AliasSet[0x{{[0-9a-f]+}}, 1] must alias, Mod Memory locations: (ptr %a, LocationSize::precise(1)) -; CHECK-NOT: 1 Unknown instruction ; CHECK: AliasSet[0x{{[0-9a-f]+}}, 1] must alias, Mod Memory locations: (ptr %b, LocationSize::precise(1)) define void @test5() { entry: -- GitLab From 07a5e31cb3836bf1f00d2f56f03db70145f536c1 Mon Sep 17 00:00:00 2001 From: Freddy Ye Date: Thu, 21 Mar 2024 10:55:26 +0800 Subject: [PATCH 094/296] Move pre-commit test for #85737 (#86062) --- .../{domain-reassignment-ndd.mir => apx/domain-reassignment.mir} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename llvm/test/CodeGen/X86/{domain-reassignment-ndd.mir => apx/domain-reassignment.mir} (100%) diff --git a/llvm/test/CodeGen/X86/domain-reassignment-ndd.mir b/llvm/test/CodeGen/X86/apx/domain-reassignment.mir similarity index 100% rename from llvm/test/CodeGen/X86/domain-reassignment-ndd.mir rename to llvm/test/CodeGen/X86/apx/domain-reassignment.mir -- GitLab From deefe3fbc93b3bdc77fbaf718403a45dae983d12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= Date: Thu, 21 Mar 2024 03:56:40 +0100 Subject: [PATCH 095/296] [GlobalIsel] Post-review combine ADDO (#85961) https://github.com/llvm/llvm-project/pull/82927 --- .../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 10 +-- .../AArch64/GlobalISel/combine-overflow.mir | 84 +++++++++++++++++++ llvm/test/CodeGen/AArch64/arm64-xaluo.ll | 3 +- llvm/test/CodeGen/AArch64/overflow.ll | 39 ++------- llvm/test/CodeGen/AMDGPU/fptoi.i128.ll | 8 +- 5 files changed, 98 insertions(+), 46 deletions(-) diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp index d3f86af1e290..2a521b6b068a 100644 --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -6945,10 +6945,6 @@ bool CombinerHelper::matchAddOverflow(MachineInstr &MI, BuildFnTy &MatchInfo) { LLT DstTy = MRI.getType(Dst); LLT CarryTy = MRI.getType(Carry); - // We want do fold the [u|s]addo. - if (!MRI.hasOneNonDBGUse(Dst)) - return false; - // Fold addo, if the carry is dead -> add, undef. if (MRI.use_nodbg_empty(Carry) && isLegalOrBeforeLegalizer({TargetOpcode::G_ADD, {DstTy}})) { @@ -6959,10 +6955,6 @@ bool CombinerHelper::matchAddOverflow(MachineInstr &MI, BuildFnTy &MatchInfo) { return true; } - // We want do fold the [u|s]addo. - if (!MRI.hasOneNonDBGUse(Carry)) - return false; - // Canonicalize constant to RHS. if (isConstantOrConstantVectorI(LHS) && !isConstantOrConstantVectorI(RHS)) { if (IsSigned) { @@ -6994,7 +6986,7 @@ bool CombinerHelper::matchAddOverflow(MachineInstr &MI, BuildFnTy &MatchInfo) { return true; } - // Fold (addo x, 0) -> x, no borrow + // Fold (addo x, 0) -> x, no carry if (MaybeRHS && *MaybeRHS == 0 && isConstantLegalOrBeforeLegalizer(CarryTy)) { MatchInfo = [=](MachineIRBuilder &B) { B.buildCopy(Dst, LHS); diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir index 6fced31a622d..ec66892b98fc 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir @@ -92,3 +92,87 @@ body: | $w1 = COPY %o_wide RET_ReallyLR implicit $w0 ... +--- +name: add_multiuse +body: | + bb.0: + liveins: $w0, $w1 + ; CHECK-LABEL: name: add_multiuse + ; CHECK: liveins: $w0, $w1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 + ; CHECK-NEXT: %const:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: $w0 = COPY [[COPY]](s32) + ; CHECK-NEXT: $w1 = COPY [[COPY]](s32) + ; CHECK-NEXT: $w2 = COPY %const(s32) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %0:_(s32) = COPY $w0 + %const:_(s32) = G_CONSTANT i32 0 + %add:_(s32), %o:_(s1) = G_SADDO %0, %const + %o_wide:_(s32) = G_ZEXT %o(s1) + $w0 = COPY %add(s32) + $w1 = COPY %add(s32) + $w2 = COPY %o_wide + RET_ReallyLR implicit $w0 +... +--- +name: add_vector +body: | + bb.0: + liveins: $w0, $w1 + ; CHECK-LABEL: name: add_vector + ; CHECK: liveins: $w0, $w1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1 + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $w3 + ; CHECK-NEXT: %bv0:_(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY]](s32), [[COPY1]](s32) + ; CHECK-NEXT: %bv1:_(<4 x s32>) = G_BUILD_VECTOR [[COPY2]](s32), [[COPY3]](s32), [[COPY2]](s32), [[COPY3]](s32) + ; CHECK-NEXT: %add:_(<4 x s32>), %o:_(<4 x s1>) = G_UADDO %bv0, %bv1 + ; CHECK-NEXT: %o_wide:_(<4 x s32>) = G_ZEXT %o(<4 x s1>) + ; CHECK-NEXT: $q0 = COPY %add(<4 x s32>) + ; CHECK-NEXT: $q1 = COPY %o_wide(<4 x s32>) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %0:_(s32) = COPY $w0 + %1:_(s32) = COPY $w1 + %2:_(s32) = COPY $w2 + %3:_(s32) = COPY $w3 + %bv0:_(<4 x s32>) = G_BUILD_VECTOR %0:_(s32), %1:_(s32), %0:_(s32), %1:_(s32) + %bv1:_(<4 x s32>) = G_BUILD_VECTOR %2:_(s32), %3:_(s32), %2:_(s32), %3:_(s32) + %add:_(<4 x s32>), %o:_(<4 x s1>) = G_UADDO %bv0, %bv1 + %o_wide:_(<4 x s32>) = G_ZEXT %o(<4 x s1>) + $q0 = COPY %add(<4 x s32>) + $q1 = COPY %o_wide + RET_ReallyLR implicit $w0 +... +--- +name: add_splat_vector +body: | + bb.0: + liveins: $w0, $w1 + ; CHECK-LABEL: name: add_splat_vector + ; CHECK: liveins: $w0, $w1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1 + ; CHECK-NEXT: %bv0:_(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY]](s32), [[COPY1]](s32) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 false + ; CHECK-NEXT: %o:_(<4 x s1>) = G_BUILD_VECTOR [[C]](s1), [[C]](s1), [[C]](s1), [[C]](s1) + ; CHECK-NEXT: %o_wide:_(<4 x s32>) = G_ZEXT %o(<4 x s1>) + ; CHECK-NEXT: $q0 = COPY %bv0(<4 x s32>) + ; CHECK-NEXT: $q1 = COPY %o_wide(<4 x s32>) + ; CHECK-NEXT: RET_ReallyLR implicit $w0 + %0:_(s32) = COPY $w0 + %1:_(s32) = COPY $w1 + %2:_(s32) = COPY $w2 + %3:_(s32) = COPY $w3 + %const:_(s32) = G_CONSTANT i32 0 + %bv0:_(<4 x s32>) = G_BUILD_VECTOR %0:_(s32), %1:_(s32), %0:_(s32), %1:_(s32) + %bv1:_(<4 x s32>) = G_BUILD_VECTOR %const:_(s32), %const:_(s32), %const:_(s32), %const:_(s32) + %add:_(<4 x s32>), %o:_(<4 x s1>) = G_SADDO %bv0, %bv1 + %o_wide:_(<4 x s32>) = G_ZEXT %o(<4 x s1>) + $q0 = COPY %add(<4 x s32>) + $q1 = COPY %o_wide + RET_ReallyLR implicit $w0 +... diff --git a/llvm/test/CodeGen/AArch64/arm64-xaluo.ll b/llvm/test/CodeGen/AArch64/arm64-xaluo.ll index 77c70668b65a..0ec2d763685e 100644 --- a/llvm/test/CodeGen/AArch64/arm64-xaluo.ll +++ b/llvm/test/CodeGen/AArch64/arm64-xaluo.ll @@ -2643,8 +2643,7 @@ define i8 @pr60530() { ; ; GISEL-LABEL: pr60530: ; GISEL: // %bb.0: -; GISEL-NEXT: mov w8, #1 // =0x1 -; GISEL-NEXT: sbfx w0, w8, #0, #1 +; GISEL-NEXT: mov w0, #255 // =0xff ; GISEL-NEXT: ret %1 = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 0, i8 1) %2 = extractvalue { i8, i1 } %1, 1 diff --git a/llvm/test/CodeGen/AArch64/overflow.ll b/llvm/test/CodeGen/AArch64/overflow.ll index 1fd60c030979..977141f2b84f 100644 --- a/llvm/test/CodeGen/AArch64/overflow.ll +++ b/llvm/test/CodeGen/AArch64/overflow.ll @@ -64,21 +64,10 @@ entry: } define i32 @saddo.select.i64(i32 %v1, i32 %v2, i1 %v3, i64 %v4, i64 %v5) { -; SDAG-LABEL: saddo.select.i64: -; SDAG: // %bb.0: // %entry -; SDAG-NEXT: mov w0, w1 -; SDAG-NEXT: ret -; -; GISEL-LABEL: saddo.select.i64: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: mov w8, #13 // =0xd -; GISEL-NEXT: and x9, x3, #0xc -; GISEL-NEXT: and x8, x4, x8 -; GISEL-NEXT: cmn x9, x8 -; GISEL-NEXT: cset w8, vs -; GISEL-NEXT: tst w8, #0x1 -; GISEL-NEXT: csel w0, w0, w1, ne -; GISEL-NEXT: ret +; CHECK-LABEL: saddo.select.i64: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: mov w0, w1 +; CHECK-NEXT: ret entry: %lhs = and i64 %v4, 12 %rhs = and i64 %v5, 13 @@ -89,22 +78,10 @@ entry: } define i32 @uaddo.select.i64(i32 %v1, i32 %v2, i1 %v3, i64 %v4, i64 %v5) { -; SDAG-LABEL: uaddo.select.i64: -; SDAG: // %bb.0: // %entry -; SDAG-NEXT: mov w0, w1 -; SDAG-NEXT: ret -; -; GISEL-LABEL: uaddo.select.i64: -; GISEL: // %bb.0: // %entry -; GISEL-NEXT: mov w8, #9 // =0x9 -; GISEL-NEXT: mov w9, #10 // =0xa -; GISEL-NEXT: and x8, x3, x8 -; GISEL-NEXT: and x9, x4, x9 -; GISEL-NEXT: cmn x8, x9 -; GISEL-NEXT: cset w8, hs -; GISEL-NEXT: tst w8, #0x1 -; GISEL-NEXT: csel w0, w0, w1, ne -; GISEL-NEXT: ret +; CHECK-LABEL: uaddo.select.i64: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: mov w0, w1 +; CHECK-NEXT: ret entry: %lhs = and i64 %v4, 9 %rhs = and i64 %v5, 10 diff --git a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll index b2311a87059c..a69418d43641 100644 --- a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll +++ b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll @@ -238,7 +238,7 @@ define i128 @fptosi_f64_to_i128(double %x) { ; GISEL-NEXT: s_xor_b64 s[16:17], exec, s[6:7] ; GISEL-NEXT: s_cbranch_execz .LBB0_4 ; GISEL-NEXT: ; %bb.3: ; %fp-to-i-if-else -; GISEL-NEXT: v_add_co_u32_e32 v6, vcc, 0xfffffbcd, v6 +; GISEL-NEXT: v_add_u32_e32 v6, 0xfffffbcd, v6 ; GISEL-NEXT: v_lshlrev_b64 v[0:1], v6, v[4:5] ; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v6 ; GISEL-NEXT: v_cndmask_b32_e32 v11, 0, v0, vcc @@ -612,7 +612,7 @@ define i128 @fptoui_f64_to_i128(double %x) { ; GISEL-NEXT: s_xor_b64 s[16:17], exec, s[6:7] ; GISEL-NEXT: s_cbranch_execz .LBB1_4 ; GISEL-NEXT: ; %bb.3: ; %fp-to-i-if-else -; GISEL-NEXT: v_add_co_u32_e32 v6, vcc, 0xfffffbcd, v6 +; GISEL-NEXT: v_add_u32_e32 v6, 0xfffffbcd, v6 ; GISEL-NEXT: v_lshlrev_b64 v[0:1], v6, v[4:5] ; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v6 ; GISEL-NEXT: v_cndmask_b32_e32 v11, 0, v0, vcc @@ -978,7 +978,7 @@ define i128 @fptosi_f32_to_i128(float %x) { ; GISEL-NEXT: s_xor_b64 s[16:17], exec, s[6:7] ; GISEL-NEXT: s_cbranch_execz .LBB2_4 ; GISEL-NEXT: ; %bb.3: ; %fp-to-i-if-else -; GISEL-NEXT: v_add_co_u32_e32 v6, vcc, 0xffffff6a, v6 +; GISEL-NEXT: v_add_u32_e32 v6, 0xffffff6a, v6 ; GISEL-NEXT: v_lshlrev_b64 v[0:1], v6, v[4:5] ; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v6 ; GISEL-NEXT: v_cndmask_b32_e32 v11, 0, v0, vcc @@ -1338,7 +1338,7 @@ define i128 @fptoui_f32_to_i128(float %x) { ; GISEL-NEXT: s_xor_b64 s[16:17], exec, s[6:7] ; GISEL-NEXT: s_cbranch_execz .LBB3_4 ; GISEL-NEXT: ; %bb.3: ; %fp-to-i-if-else -; GISEL-NEXT: v_add_co_u32_e32 v6, vcc, 0xffffff6a, v6 +; GISEL-NEXT: v_add_u32_e32 v6, 0xffffff6a, v6 ; GISEL-NEXT: v_lshlrev_b64 v[0:1], v6, v[4:5] ; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v6 ; GISEL-NEXT: v_cndmask_b32_e32 v11, 0, v0, vcc -- GitLab From 29bf32efbb646b2ab3dec25f100419fc75635878 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Thu, 21 Mar 2024 10:57:51 +0800 Subject: [PATCH 096/296] [NewPM][AArch64] Add AArch64PassRegistry.def (#85215) PR #83567 ports `SelectionDAGISel` to the new pass manager, then each backend should provide `DagToDagISel()` in new pass manager style. Then each target should provide `PassRegistry.def` to register backend passes in `registerPassBuilderCallbacks` to reduce duplicate code. This PR adds `AArch64PassRegistry.def` to AArch64 backend and boilerplate code in `registerPassBuilderCallbacks`. --- .../llvm/Passes/TargetPassRegistry.inc | 192 ++++++++++++++++++ .../Target/AArch64/AArch64PassRegistry.def | 20 ++ .../Target/AArch64/AArch64TargetMachine.cpp | 4 + .../LoopIdiom/AArch64/byte-compare-index.ll | 3 + 4 files changed, 219 insertions(+) create mode 100644 llvm/include/llvm/Passes/TargetPassRegistry.inc create mode 100644 llvm/lib/Target/AArch64/AArch64PassRegistry.def diff --git a/llvm/include/llvm/Passes/TargetPassRegistry.inc b/llvm/include/llvm/Passes/TargetPassRegistry.inc new file mode 100644 index 000000000000..50766a99f6a7 --- /dev/null +++ b/llvm/include/llvm/Passes/TargetPassRegistry.inc @@ -0,0 +1,192 @@ +//===- TargetPassRegistry.inc - Registry of passes --------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is used as the registry of passes in registerPassBuilderCallbacks +// Just put the following lines in the body of registerPassBuilderCallbacks: +// #define GET_PASS_REGISTRY "PassRegistry.def" +// #include "llvm/Passes/TargetPassRegistry.inc" +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifdef GET_PASS_REGISTRY + +#if !__has_include(GET_PASS_REGISTRY) +#error "must provide PassRegistry.def" +#endif + +if (PopulateClassToPassNames) { + auto *PIC = PB.getPassInstrumentationCallbacks(); + +#define ADD_CLASS_PASS_TO_PASS_NAME(NAME, CREATE_PASS) \ + PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME); +#define ADD_CLASS_PASS_TO_PASS_NAME_WITH_PARAMS(NAME, CLASS) \ + PIC->addClassToPassName(CLASS, NAME); + +#define MODULE_ANALYSIS(NAME, CREATE_PASS) \ + ADD_CLASS_PASS_TO_PASS_NAME(NAME, CREATE_PASS) +#define MODULE_PASS(NAME, CREATE_PASS) \ + ADD_CLASS_PASS_TO_PASS_NAME(NAME, CREATE_PASS) +#define MODULE_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \ + ADD_CLASS_PASS_TO_PASS_NAME_WITH_PARAMS(NAME, CLASS) +#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ + ADD_CLASS_PASS_TO_PASS_NAME(NAME, CREATE_PASS) +#define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ + ADD_CLASS_PASS_TO_PASS_NAME(NAME, CREATE_PASS) +#define FUNCTION_PASS(NAME, CREATE_PASS) \ + ADD_CLASS_PASS_TO_PASS_NAME(NAME, CREATE_PASS) +#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \ + ADD_CLASS_PASS_TO_PASS_NAME_WITH_PARAMS(NAME, CLASS) +#define LOOP_ANALYSIS(NAME, CREATE_PASS) \ + ADD_CLASS_PASS_TO_PASS_NAME(NAME, CREATE_PASS) +#define LOOP_PASS(NAME, CREATE_PASS) \ + ADD_CLASS_PASS_TO_PASS_NAME(NAME, CREATE_PASS) +#define MACHINE_FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ + ADD_CLASS_PASS_TO_PASS_NAME(NAME, CREATE_PASS) +#define MACHINE_FUNCTION_PASS(NAME, CREATE_PASS) \ + ADD_CLASS_PASS_TO_PASS_NAME(NAME, CREATE_PASS) +#define MACHINE_FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, \ + PARAMS) \ + ADD_CLASS_PASS_TO_PASS_NAME_WITH_PARAMS(NAME, CLASS) +#include GET_PASS_REGISTRY +#undef MODULE_ANALYSIS +#undef MODULE_PASS +#undef MODULE_PASS_WITH_PARAMS +#undef FUNCTION_ANALYSIS +#undef FUNCTION_ALIAS_ANALYSIS +#undef FUNCTION_PASS +#undef FUNCTION_PASS_WITH_PARAMS +#undef LOOP_ANALYSIS +#undef LOOP_PASS +#undef MACHINE_FUNCTION_ANALYSIS +#undef MACHINE_FUNCTION_PASS +#undef MACHINE_FUNCTION_PASS_WITH_PARAMS +#undef ADD_CLASS_PASS_TO_PASS_NAME +#undef ADD_CLASS_PASS_TO_PASS_NAME_WITH_PARAMS +} + +#define ADD_PASS(NAME, CREATE_PASS) \ + if (Name == NAME) { \ + PM.addPass(CREATE_PASS); \ + return true; \ + } + +#define ADD_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ + if (PassBuilder::checkParametrizedPassName(Name, NAME)) { \ + auto Params = PassBuilder::parsePassParameters(PARSER, Name, NAME); \ + if (!Params) \ + return false; \ + PM.addPass(CREATE_PASS(Params.get())); \ + return true; \ + } + +PB.registerPipelineParsingCallback([=](StringRef Name, ModulePassManager &PM, + ArrayRef) { +#define MODULE_PASS(NAME, CREATE_PASS) ADD_PASS(NAME, CREATE_PASS) +#include GET_PASS_REGISTRY +#undef MODULE_PASS + return false; +}); + +PB.registerPipelineParsingCallback([=](StringRef Name, ModulePassManager &PM, + ArrayRef) { +#define MODULE_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \ + ADD_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) +#include GET_PASS_REGISTRY +#undef MODULE_PASS_WITH_PARAMS + return false; +}); + +PB.registerPipelineParsingCallback([=](StringRef Name, FunctionPassManager &PM, + ArrayRef) { +#define FUNCTION_PASS(NAME, CREATE_PASS) ADD_PASS(NAME, CREATE_PASS) +#include GET_PASS_REGISTRY +#undef FUNCTION_PASS + return false; +}); + +PB.registerPipelineParsingCallback([=](StringRef Name, FunctionPassManager &PM, + ArrayRef) { +#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \ + ADD_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) +#include GET_PASS_REGISTRY +#undef FUNCTION_PASS_WITH_PARAMS + return false; +}); + +PB.registerPipelineParsingCallback([=](StringRef Name, LoopPassManager &PM, + ArrayRef) { +#define LOOP_PASS(NAME, CREATE_PASS) ADD_PASS(NAME, CREATE_PASS) +#include GET_PASS_REGISTRY + return false; +}); + +PB.registerPipelineParsingCallback([=](StringRef Name, + MachineFunctionPassManager &PM, + ArrayRef) { +#define MACHINE_FUNCTION_PASS(NAME, CREATE_PASS) ADD_PASS(NAME, CREATE_PASS) +#include GET_PASS_REGISTRY + return false; +}); + +PB.registerPipelineParsingCallback([=](StringRef Name, FunctionPassManager &PM, + ArrayRef) { +#define MACHINE_FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, \ + PARAMS) \ + ADD_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) +#include GET_PASS_REGISTRY +#undef MACHINE_FUNCTION_PASS_WITH_PARAMS + return false; +}); + +#undef ADD_PASS +#undef ADD_PASS_WITH_PARAMS + +PB.registerAnalysisRegistrationCallback([](ModuleAnalysisManager &AM) { +#define MODULE_ANALYSIS(NAME, CREATE_PASS) \ + AM.registerPass([&] { return CREATE_PASS; }); +#include GET_PASS_REGISTRY +#undef MODULE_ANALYSIS +}); + +PB.registerAnalysisRegistrationCallback([](FunctionAnalysisManager &AM) { +#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ + AM.registerPass([&] { return CREATE_PASS; }); +#include GET_PASS_REGISTRY +#undef FUNCTION_ANALYSIS +}); + +PB.registerParseAACallback([](StringRef Name, AAManager &AM) { +#define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ + if (Name == NAME) { \ + AM.registerFunctionAnalysis< \ + std::remove_reference_t>(); \ + return true; \ + } +#include GET_PASS_REGISTRY +#undef FUNCTION_ALIAS_ANALYSIS + return false; +}); + +PB.registerAnalysisRegistrationCallback([](LoopAnalysisManager &AM) { +#define LOOP_ANALYSIS(NAME, CREATE_PASS) \ + AM.registerPass([&] { return CREATE_PASS; }); +#include GET_PASS_REGISTRY +#undef LOOP_ANALYSIS +}); + +PB.registerAnalysisRegistrationCallback([](MachineFunctionAnalysisManager &AM) { +#define MACHINE_FUNCTION_ANALYSIS(NAME, CREATE_PASS) \ + AM.registerPass([&] { return CREATE_PASS; }); +#include GET_PASS_REGISTRY +#undef MACHINE_FUNCTION_ANALYSIS +}); + +#undef GET_PASS_REGISTRY +#endif // GET_PASS_REGISTRY diff --git a/llvm/lib/Target/AArch64/AArch64PassRegistry.def b/llvm/lib/Target/AArch64/AArch64PassRegistry.def new file mode 100644 index 000000000000..ca944579f93a --- /dev/null +++ b/llvm/lib/Target/AArch64/AArch64PassRegistry.def @@ -0,0 +1,20 @@ +//===- AArch64PassRegistry.def - Registry of AArch64 passes -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is used as the registry of passes that are part of the +// AArch64 backend. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef LOOP_PASS +#define LOOP_PASS(NAME, CREATE_PASS) +#endif +LOOP_PASS("aarch64-lit", AArch64LoopIdiomTransformPass()) +#undef LOOP_PASS diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp index e5e60459e814..08238fdf167b 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -547,6 +547,10 @@ public: void AArch64TargetMachine::registerPassBuilderCallbacks( PassBuilder &PB, bool PopulateClassToPassNames) { + +#define GET_PASS_REGISTRY "AArch64PassRegistry.def" +#include "llvm/Passes/TargetPassRegistry.inc" + PB.registerLateLoopOptimizationsEPCallback( [=](LoopPassManager &LPM, OptimizationLevel Level) { LPM.addPass(AArch64LoopIdiomTransformPass()); diff --git a/llvm/test/Transforms/LoopIdiom/AArch64/byte-compare-index.ll b/llvm/test/Transforms/LoopIdiom/AArch64/byte-compare-index.ll index 92ff099afb1c..daa64f2e2ea7 100644 --- a/llvm/test/Transforms/LoopIdiom/AArch64/byte-compare-index.ll +++ b/llvm/test/Transforms/LoopIdiom/AArch64/byte-compare-index.ll @@ -2,6 +2,9 @@ ; RUN: opt -aarch64-lit -aarch64-lit-verify -verify-dom-info -mtriple aarch64-unknown-linux-gnu -mattr=+sve -S < %s | FileCheck %s ; RUN: opt -aarch64-lit -simplifycfg -mtriple aarch64-unknown-linux-gnu -mattr=+sve -S < %s | FileCheck %s --check-prefix=LOOP-DEL ; RUN: opt -aarch64-lit -mtriple aarch64-unknown-linux-gnu -S < %s | FileCheck %s --check-prefix=NO-TRANSFORM +; RUN: opt -p aarch64-lit -aarch64-lit-verify -verify-dom-info -mtriple aarch64-unknown-linux-gnu -mattr=+sve -S < %s | FileCheck %s +; RUN: opt -passes='function(loop(aarch64-lit)),simplifycfg' -mtriple aarch64-unknown-linux-gnu -mattr=+sve -S < %s | FileCheck %s --check-prefix=LOOP-DEL +; RUN: opt -p aarch64-lit -mtriple aarch64-unknown-linux-gnu -S < %s | FileCheck %s --check-prefix=NO-TRANSFORM define i32 @compare_bytes_simple(ptr %a, ptr %b, i32 %len, i32 %extra, i32 %n) { ; CHECK-LABEL: define i32 @compare_bytes_simple( -- GitLab From a5d7fc1d1000ffb1d21796f5d587f277c2957d66 Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Thu, 21 Mar 2024 12:30:48 +0900 Subject: [PATCH 097/296] [mlir][sparse] Fix typos in comments (#86074) --- .../test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir | 3 +-- .../Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir index 34d450c2403f..7ecccad212cd 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir @@ -285,8 +285,7 @@ module { %has_runtime = sparse_tensor.has_runtime_library scf.if %has_runtime { // sparse_tensor.assemble copies buffers when running with the runtime - // library. Deallocations are needed not needed when running in codgen - // mode. + // library. Deallocations are not needed when running in codegen mode. bufferization.dealloc_tensor %s4 : tensor<10x10xf64, #SortedCOO> bufferization.dealloc_tensor %s5 : tensor<10x10xf64, #SortedCOOI32> bufferization.dealloc_tensor %csr : tensor<2x2xf64, #CSR> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir index fe8836266a47..20ae7e86285c 100755 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir @@ -146,8 +146,7 @@ module { %has_runtime = sparse_tensor.has_runtime_library scf.if %has_runtime { // sparse_tensor.assemble copies buffers when running with the runtime - // library. Deallocations are needed not needed when running in codgen - // mode. + // library. Deallocations are not needed when running in codegen mode. bufferization.dealloc_tensor %s0 : tensor<4x3x2xf32, #CCC> bufferization.dealloc_tensor %s1 : tensor<4x3x2xf32, #BatchedCSR> bufferization.dealloc_tensor %s2 : tensor<4x3x2xf32, #CSRDense> -- GitLab From 7bb87d533891c2bcfa1c9132605f0d3e8227d444 Mon Sep 17 00:00:00 2001 From: Madhur Amilkanthwar Date: Thu, 21 Mar 2024 09:54:03 +0530 Subject: [PATCH 098/296] [AArch64][GlobalISel] Take abs scalar codegen closer to SDAG (#84886) This patch improves codegen for scalar (<128bits) version of llvm.abs intrinsic by using the existing non-XOR based lowering. This takes the generated code closer to SDAG. codegen with GISel for > 128 bit types is not very good with these method so not doing so. --- .../llvm/CodeGen/GlobalISel/LegalizerHelper.h | 1 + .../CodeGen/GlobalISel/LegalizerHelper.cpp | 15 +++++++++- .../AArch64/GISel/AArch64LegalizerInfo.cpp | 7 +++++ .../AArch64/GlobalISel/legalize-abs.mir | 28 ++++++++++++------- llvm/test/CodeGen/AArch64/abs.ll | 26 ++++++++--------- 5 files changed, 51 insertions(+), 26 deletions(-) diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index 5bb3692f0a46..284f434fbb9b 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -429,6 +429,7 @@ public: LegalizeResult lowerDIVREM(MachineInstr &MI); LegalizeResult lowerAbsToAddXor(MachineInstr &MI); LegalizeResult lowerAbsToMaxNeg(MachineInstr &MI); + LegalizeResult lowerAbsToCNeg(MachineInstr &MI); LegalizeResult lowerVectorReduction(MachineInstr &MI); LegalizeResult lowerMemcpyInline(MachineInstr &MI); LegalizeResult lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen = 0); diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index abe23af00a78..8d608f6ac5e4 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -8215,9 +8215,22 @@ LegalizerHelper::lowerAbsToMaxNeg(MachineInstr &MI) { // %res = G_SMAX %a, %v2 Register SrcReg = MI.getOperand(1).getReg(); LLT Ty = MRI.getType(SrcReg); + auto Zero = MIRBuilder.buildConstant(Ty, 0); + auto Sub = MIRBuilder.buildSub(Ty, Zero, SrcReg); + MIRBuilder.buildSMax(MI.getOperand(0), SrcReg, Sub); + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::lowerAbsToCNeg(MachineInstr &MI) { + Register SrcReg = MI.getOperand(1).getReg(); + Register DestReg = MI.getOperand(0).getReg(); + LLT Ty = MRI.getType(SrcReg), IType = LLT::scalar(1); auto Zero = MIRBuilder.buildConstant(Ty, 0).getReg(0); auto Sub = MIRBuilder.buildSub(Ty, Zero, SrcReg).getReg(0); - MIRBuilder.buildSMax(MI.getOperand(0), SrcReg, Sub); + auto ICmp = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, IType, SrcReg, Zero); + MIRBuilder.buildSelect(DestReg, ICmp, SrcReg, Sub); MI.eraseFromParent(); return Legalized; } diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index 996abe8e4739..34e2c1d9c8e2 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -1012,6 +1012,11 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) ABSActions .legalFor({s32, s64}); ABSActions.legalFor(PackedVectorAllTypeList) + .customIf([=](const LegalityQuery &Q) { + // TODO: Fix suboptimal codegen for 128+ bit types. + LLT SrcTy = Q.Types[0]; + return SrcTy.isScalar() && SrcTy.getSizeInBits() < 128; + }) .widenScalarIf( [=](const LegalityQuery &Query) { return Query.Types[0] == v4s8; }, [=](const LegalityQuery &Query) { return std::make_pair(0, v4s16); }) @@ -1264,6 +1269,8 @@ bool AArch64LegalizerInfo::legalizeCustom( return legalizeDynStackAlloc(MI, Helper); case TargetOpcode::G_PREFETCH: return legalizePrefetch(MI, Helper); + case TargetOpcode::G_ABS: + return Helper.lowerAbsToCNeg(MI); } llvm_unreachable("expected switch to return"); diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-abs.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-abs.mir index 3123e304116f..0d429ae38402 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-abs.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-abs.mir @@ -8,11 +8,12 @@ body: | bb.0: ; CHECK-LABEL: name: abs_s32 ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 - ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 31 - ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[COPY]], [[C]](s64) - ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[ASHR]] - ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[ADD]], [[ASHR]] - ; CHECK-NEXT: $w0 = COPY [[XOR]](s32) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0 + ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[C]], [[COPY]] + ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sgt), [[COPY]](s32), [[C]] + ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s32), [[COPY]], [[SUB]] + ; CHECK-NEXT: $w0 = COPY [[SELECT]](s32) + ; ; CHECK-CSSC-LABEL: name: abs_s32 ; CHECK-CSSC: [[COPY:%[0-9]+]]:_(s32) = COPY $w0 ; CHECK-CSSC-NEXT: [[ABS:%[0-9]+]]:_(s32) = G_ABS [[COPY]] @@ -28,11 +29,12 @@ body: | bb.0: ; CHECK-LABEL: name: abs_s64 ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 - ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63 - ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[C]](s64) - ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[COPY]], [[ASHR]] - ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[ADD]], [[ASHR]] - ; CHECK-NEXT: $x0 = COPY [[XOR]](s64) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s64) = G_SUB [[C]], [[COPY]] + ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sgt), [[COPY]](s64), [[C]] + ; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s32), [[COPY]], [[SUB]] + ; CHECK-NEXT: $x0 = COPY [[SELECT]](s64) + ; ; CHECK-CSSC-LABEL: name: abs_s64 ; CHECK-CSSC: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 ; CHECK-CSSC-NEXT: [[ABS:%[0-9]+]]:_(s64) = G_ABS [[COPY]] @@ -55,6 +57,7 @@ body: | ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(<4 x s16>) = G_ABS [[COPY]] ; CHECK-NEXT: $d0 = COPY [[ABS]](<4 x s16>) ; CHECK-NEXT: RET_ReallyLR implicit $d0 + ; ; CHECK-CSSC-LABEL: name: abs_v4s16 ; CHECK-CSSC: liveins: $d0 ; CHECK-CSSC-NEXT: {{ $}} @@ -82,6 +85,7 @@ body: | ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(<8 x s16>) = G_ABS [[COPY]] ; CHECK-NEXT: $q0 = COPY [[ABS]](<8 x s16>) ; CHECK-NEXT: RET_ReallyLR implicit $q0 + ; ; CHECK-CSSC-LABEL: name: abs_v8s16 ; CHECK-CSSC: liveins: $q0 ; CHECK-CSSC-NEXT: {{ $}} @@ -109,6 +113,7 @@ body: | ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(<2 x s32>) = G_ABS [[COPY]] ; CHECK-NEXT: $d0 = COPY [[ABS]](<2 x s32>) ; CHECK-NEXT: RET_ReallyLR implicit $d0 + ; ; CHECK-CSSC-LABEL: name: abs_v2s32 ; CHECK-CSSC: liveins: $d0 ; CHECK-CSSC-NEXT: {{ $}} @@ -136,6 +141,7 @@ body: | ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(<4 x s32>) = G_ABS [[COPY]] ; CHECK-NEXT: $q0 = COPY [[ABS]](<4 x s32>) ; CHECK-NEXT: RET_ReallyLR implicit $q0 + ; ; CHECK-CSSC-LABEL: name: abs_v4s32 ; CHECK-CSSC: liveins: $q0 ; CHECK-CSSC-NEXT: {{ $}} @@ -163,6 +169,7 @@ body: | ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(<8 x s8>) = G_ABS [[COPY]] ; CHECK-NEXT: $d0 = COPY [[ABS]](<8 x s8>) ; CHECK-NEXT: RET_ReallyLR implicit $d0 + ; ; CHECK-CSSC-LABEL: name: abs_v4s8 ; CHECK-CSSC: liveins: $d0 ; CHECK-CSSC-NEXT: {{ $}} @@ -190,6 +197,7 @@ body: | ; CHECK-NEXT: [[ABS:%[0-9]+]]:_(<16 x s8>) = G_ABS [[COPY]] ; CHECK-NEXT: $q0 = COPY [[ABS]](<16 x s8>) ; CHECK-NEXT: RET_ReallyLR implicit $q0 + ; ; CHECK-CSSC-LABEL: name: abs_v16s8 ; CHECK-CSSC: liveins: $q0 ; CHECK-CSSC-NEXT: {{ $}} diff --git a/llvm/test/CodeGen/AArch64/abs.ll b/llvm/test/CodeGen/AArch64/abs.ll index e00f70b94e3b..78c1ff7b9937 100644 --- a/llvm/test/CodeGen/AArch64/abs.ll +++ b/llvm/test/CodeGen/AArch64/abs.ll @@ -15,9 +15,8 @@ define i8 @abs_i8(i8 %a){ ; CHECK-GI-LABEL: abs_i8: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: sxtb w8, w0 -; CHECK-GI-NEXT: asr w8, w8, #7 -; CHECK-GI-NEXT: add w9, w0, w8 -; CHECK-GI-NEXT: eor w0, w9, w8 +; CHECK-GI-NEXT: cmp w8, #0 +; CHECK-GI-NEXT: cneg w0, w0, le ; CHECK-GI-NEXT: ret entry: %res = call i8 @llvm.abs.i8(i8 %a, i1 0) @@ -36,9 +35,8 @@ define i16 @abs_i16(i16 %a){ ; CHECK-GI-LABEL: abs_i16: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: sxth w8, w0 -; CHECK-GI-NEXT: asr w8, w8, #15 -; CHECK-GI-NEXT: add w9, w0, w8 -; CHECK-GI-NEXT: eor w0, w9, w8 +; CHECK-GI-NEXT: cmp w8, #0 +; CHECK-GI-NEXT: cneg w0, w0, le ; CHECK-GI-NEXT: ret entry: %res = call i16 @llvm.abs.i16(i16 %a, i1 0) @@ -55,9 +53,8 @@ define i32 @abs_i32(i32 %a){ ; ; CHECK-GI-LABEL: abs_i32: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: asr w8, w0, #31 -; CHECK-GI-NEXT: add w9, w0, w8 -; CHECK-GI-NEXT: eor w0, w9, w8 +; CHECK-GI-NEXT: cmp w0, #0 +; CHECK-GI-NEXT: cneg w0, w0, le ; CHECK-GI-NEXT: ret entry: %res = call i32 @llvm.abs.i32(i32 %a, i1 0) @@ -74,9 +71,8 @@ define i64 @abs_i64(i64 %a){ ; ; CHECK-GI-LABEL: abs_i64: ; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: asr x8, x0, #63 -; CHECK-GI-NEXT: add x9, x0, x8 -; CHECK-GI-NEXT: eor x0, x9, x8 +; CHECK-GI-NEXT: cmp x0, #0 +; CHECK-GI-NEXT: cneg x0, x0, le ; CHECK-GI-NEXT: ret entry: %res = call i64 @llvm.abs.i64(i64 %a, i1 0) @@ -248,9 +244,9 @@ define <1 x i32> @abs_v1i32(<1 x i32> %a){ ; CHECK-GI-LABEL: abs_v1i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: asr w9, w8, #31 -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: eor w8, w8, w9 +; CHECK-GI-NEXT: fmov w9, s0 +; CHECK-GI-NEXT: cmp w8, #0 +; CHECK-GI-NEXT: cneg w8, w9, le ; CHECK-GI-NEXT: fmov s0, w8 ; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0 ; CHECK-GI-NEXT: ret -- GitLab From 35d3b3430eff16403d004d9f0b0369f0814cf140 Mon Sep 17 00:00:00 2001 From: Matthias Springer Date: Thu, 21 Mar 2024 14:16:02 +0900 Subject: [PATCH 099/296] [mlir][bufferization] Add "bottom-up from terminators" analysis heuristic (#83964) One-Shot Bufferize currently does not support loops where a yielded value bufferizes to a buffer that is different from the buffer of the region iter_arg. In such a case, the bufferization fails with an error such as: ``` Yield operand #0 is not equivalent to the corresponding iter bbArg scf.yield %0 : tensor<5xf32> ``` One common reason for non-equivalent buffers is that an op on the path from the region iter_arg to the terminator bufferizes out-of-place. Ops that are analyzed earlier are more likely to bufferize in-place. This commit adds a new heuristic that gives preference to ops that are reachable on the reverse SSA use-def chain from a region terminator and are within the parent region of the terminator. This is expected to work better than the existing heuristics for loops where an iter_arg is written to multiple times within a loop, but only one write is fed into the terminator. Current users of One-Shot Bufferize are not affected by this change. "Bottom-up" is still the default heuristic. Users can switch to the new heuristic manually. This commit also turns the "fuzzer" pass option into a heuristic, cleaning up the code a bit. --- .../IR/BufferizableOpInterface.h | 4 - .../Transforms/OneShotAnalysis.h | 12 +- .../Bufferization/Transforms/Passes.td | 18 +++ .../Bufferization/Transforms/Bufferize.cpp | 5 + .../Transforms/OneShotAnalysis.cpp | 116 ++++++++++++++---- .../Dialect/Arith/one-shot-bufferize.mlir | 6 +- ...ne-shot-bufferize-allow-return-allocs.mlir | 6 +- ...e-analysis-bottom-up-from-terminators.mlir | 36 ++++++ .../one-shot-bufferize-partial.mlir | 6 +- .../Transforms/one-shot-bufferize.mlir | 6 +- ...-module-bufferize-allow-return-allocs.mlir | 6 +- .../one-shot-module-bufferize-analysis.mlir | 11 +- .../Transforms/one-shot-module-bufferize.mlir | 6 +- .../Dialect/Linalg/one-shot-bufferize.mlir | 6 +- .../SCF/one-shot-bufferize-analysis.mlir | 6 +- mlir/test/Dialect/SCF/one-shot-bufferize.mlir | 6 +- .../Dialect/Tensor/one-shot-bufferize.mlir | 6 +- 17 files changed, 198 insertions(+), 64 deletions(-) create mode 100644 mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-analysis-bottom-up-from-terminators.mlir diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h index 3a61a4b34765..2d8add82383b 100644 --- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h +++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h @@ -366,10 +366,6 @@ struct BufferizationOptions { DefaultMemorySpaceFn defaultMemorySpaceFn = [](TensorType t) -> std::optional { return Attribute(); }; - /// Seed for the analysis fuzzer. If set to `0`, the fuzzer is deactivated. - /// Should be used only with `testAnalysisOnly = true`. - unsigned analysisFuzzerSeed = 0; - /// If set to `true`, the analysis is skipped. A buffer is copied before every /// write. This flag cannot be used together with `testAnalysisOnly = true`. bool copyBeforeWrite = false; diff --git a/mlir/include/mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h b/mlir/include/mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h index a29af853eb21..d50a3042aeea 100644 --- a/mlir/include/mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h +++ b/mlir/include/mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h @@ -24,7 +24,12 @@ class OneShotAnalysisState; /// Options for analysis-enabled bufferization. struct OneShotBufferizationOptions : public BufferizationOptions { - enum class AnalysisHeuristic { BottomUp, TopDown }; + enum class AnalysisHeuristic { + BottomUp, + TopDown, + BottomUpFromTerminators, + Fuzzer + }; OneShotBufferizationOptions() = default; @@ -42,6 +47,11 @@ struct OneShotBufferizationOptions : public BufferizationOptions { /// Specify the functions that should not be analyzed. copyBeforeWrite will be /// set to true when bufferizing them. llvm::ArrayRef noAnalysisFuncFilter; + + /// Seed for the analysis fuzzer. Used only if the heuristic is set to + /// `AnalysisHeuristic::Fuzzer`. The fuzzer should be used only with + /// `testAnalysisOnly = true`. + unsigned analysisFuzzerSeed = 0; }; /// State for analysis-enabled bufferization. This class keeps track of alias diff --git a/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.td b/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.td index 1c3cdec81a39..1303dc2c9ae1 100644 --- a/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.td +++ b/mlir/include/mlir/Dialect/Bufferization/Transforms/Passes.td @@ -459,6 +459,24 @@ def OneShotBufferize : Pass<"one-shot-bufferize", "ModuleOp"> { argument is read/written and which returned values are aliasing/equivalent. For debugging purposes, such information can be printed with `test-analysis-only`. + + The order in which ops are analyzed is important. The analysis is greedy and + ops that are analyzed earlier are more likely to bufferize in-place. The + heuristic can be set with `analysis-heuristic`. At the moment, the following + heuristics are available: + + * `bottom-up` (default): Analyze ops from bottom to top. + * `top-down`: Analyze ops from top to bottom. + * `fuzzer`: Randomize the ordering of ops with `analysis-fuzzer-seed`. + * `bottom-up-from-terminators`: Traverse the reverse use-def chains of + tensor IR, starting from region branch terminators (bottom-up). Nested + regions are traversed before enclosing regions. Analyze the traversed ops + first, then analyze the remaining ops bottom-up. This heuristic is useful + for bufferizing loop constructs. One-Shot Bufferize currently supports + only such IR where yielded tensor values bufferize to equivalent region + iter_args, and first analyzing all ops on the path from the "yielding" op + to the beginning of the loop body makes it more likely for the region + iter_args and yielded values to bufferize to equivalent buffers. }]; let options = [ Option<"allowReturnAllocsFromLoops", "allow-return-allocs-from-loops", diff --git a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp index 8dbf70162012..32f4e6a0fe89 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp @@ -182,6 +182,11 @@ parseHeuristicOption(const std::string &s) { return OneShotBufferizationOptions::AnalysisHeuristic::BottomUp; if (s == "top-down") return OneShotBufferizationOptions::AnalysisHeuristic::TopDown; + if (s == "bottom-up-from-terminators") + return OneShotBufferizationOptions::AnalysisHeuristic:: + BottomUpFromTerminators; + if (s == "fuzzer") + return OneShotBufferizationOptions::AnalysisHeuristic::Fuzzer; llvm_unreachable("invalid analysisheuristic option"); } diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp index fba9cd873063..531016130d1d 100644 --- a/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp +++ b/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp @@ -51,6 +51,7 @@ #include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/IR/AsmState.h" #include "mlir/IR/Dominance.h" +#include "mlir/IR/Iterators.h" #include "mlir/IR/Operation.h" #include "mlir/IR/TypeUtilities.h" #include "mlir/Interfaces/ControlFlowInterfaces.h" @@ -1094,41 +1095,104 @@ static void equivalenceAnalysis(Operation *op, OneShotAnalysisState &state) { equivalenceAnalysis(ops, state); } -LogicalResult OneShotAnalysisState::analyzeOp(Operation *op, - const DominanceInfo &domInfo) { - // Collect ops so we can build our own reverse traversal. - SmallVector ops; - op->walk([&](Operation *op) { - // No tensors => no buffers. - if (!hasTensorSemantics(op)) +/// "Bottom-up from terminators" heuristic. +static SmallVector +bottomUpFromTerminatorsHeuristic(Operation *op, + const OneShotAnalysisState &state) { + SetVector traversedOps; + + // Find region terminators. + op->walk([&](RegionBranchTerminatorOpInterface term) { + if (!traversedOps.insert(term)) return; - ops.push_back(op); + // Follow the reverse SSA use-def chain from each yielded value as long as + // we stay within the same region. + SmallVector worklist; + for (Value v : term->getOperands()) { + if (!isa(v.getType())) + continue; + auto opResult = dyn_cast(v); + if (!opResult) + continue; + worklist.push_back(opResult); + } + while (!worklist.empty()) { + OpResult opResult = worklist.pop_back_val(); + Operation *defOp = opResult.getDefiningOp(); + if (!traversedOps.insert(defOp)) + continue; + if (!term->getParentRegion()->findAncestorOpInRegion(*defOp)) + continue; + AliasingOpOperandList aliases = state.getAliasingOpOperands(opResult); + for (auto alias : aliases) { + Value v = alias.opOperand->get(); + if (!isa(v.getType())) + continue; + auto opResult = dyn_cast(v); + if (!opResult) + continue; + worklist.push_back(opResult); + } + } }); - if (getOptions().analysisFuzzerSeed) { - // This is a fuzzer. For testing purposes only. Randomize the order in which - // operations are analyzed. The bufferization quality is likely worse, but - // we want to make sure that no assertions are triggered anywhere. - std::mt19937 g(getOptions().analysisFuzzerSeed); - llvm::shuffle(ops.begin(), ops.end(), g); - } + // Analyze traversed ops, then all remaining ops. + SmallVector result(traversedOps.begin(), traversedOps.end()); + op->walk([&](Operation *op) { + if (!traversedOps.contains(op) && hasTensorSemantics(op)) + result.push_back(op); + }); + return result; +} +LogicalResult OneShotAnalysisState::analyzeOp(Operation *op, + const DominanceInfo &domInfo) { OneShotBufferizationOptions::AnalysisHeuristic heuristic = getOptions().analysisHeuristic; - if (heuristic == OneShotBufferizationOptions::AnalysisHeuristic::BottomUp) { - // Default: Walk ops in reverse for better interference analysis. - for (Operation *op : reverse(ops)) - if (failed(analyzeSingleOp(op, domInfo))) - return failure(); - } else if (heuristic == - OneShotBufferizationOptions::AnalysisHeuristic::TopDown) { - for (Operation *op : ops) - if (failed(analyzeSingleOp(op, domInfo))) - return failure(); + + SmallVector orderedOps; + if (heuristic == + OneShotBufferizationOptions::AnalysisHeuristic::BottomUpFromTerminators) { + orderedOps = bottomUpFromTerminatorsHeuristic(op, *this); } else { - llvm_unreachable("unsupported heuristic"); + op->walk([&](Operation *op) { + // No tensors => no buffers. + if (!hasTensorSemantics(op)) + return; + orderedOps.push_back(op); + }); + switch (heuristic) { + case OneShotBufferizationOptions::AnalysisHeuristic::BottomUp: { + // Default: Walk ops in reverse for better interference analysis. + std::reverse(orderedOps.begin(), orderedOps.end()); + break; + } + case OneShotBufferizationOptions::AnalysisHeuristic::TopDown: { + // Ops are already sorted top-down in `orderedOps`. + break; + } + case OneShotBufferizationOptions::AnalysisHeuristic::Fuzzer: { + assert(getOptions().analysisFuzzerSeed && + "expected that fuzzer seed it set"); + // This is a fuzzer. For testing purposes only. Randomize the order in + // which operations are analyzed. The bufferization quality is likely + // worse, but we want to make sure that no assertions are triggered + // anywhere. + std::mt19937 g(getOptions().analysisFuzzerSeed); + llvm::shuffle(orderedOps.begin(), orderedOps.end(), g); + break; + } + default: { + llvm_unreachable("unsupported heuristic"); + } + } } + // Analyze ops in the computed order. + for (Operation *op : orderedOps) + if (failed(analyzeSingleOp(op, domInfo))) + return failure(); + equivalenceAnalysis(op, *this); return success(); } diff --git a/mlir/test/Dialect/Arith/one-shot-bufferize.mlir b/mlir/test/Dialect/Arith/one-shot-bufferize.mlir index 174bf2fc8e4b..f6bdca7f4d9e 100644 --- a/mlir/test/Dialect/Arith/one-shot-bufferize.mlir +++ b/mlir/test/Dialect/Arith/one-shot-bufferize.mlir @@ -1,9 +1,9 @@ // RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries" -split-input-file | FileCheck %s // Run fuzzer with different seeds. -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=23 bufferize-function-boundaries" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=59 bufferize-function-boundaries" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=91 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91 bufferize-function-boundaries" -split-input-file -o /dev/null // Test bufferization using memref types that have no layout map. // RUN: mlir-opt %s -one-shot-bufferize="unknown-type-conversion=identity-layout-map function-boundary-type-conversion=identity-layout-map bufferize-function-boundaries" -split-input-file -o /dev/null diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-allow-return-allocs.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-allow-return-allocs.mlir index e4375950d336..8f0170b17381 100644 --- a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-allow-return-allocs.mlir +++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-allow-return-allocs.mlir @@ -1,9 +1,9 @@ // RUN: mlir-opt %s -one-shot-bufferize="allow-unknown-ops" -canonicalize -split-input-file | FileCheck %s // Run fuzzer with different seeds. -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=23" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=59" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=91" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91" -split-input-file -o /dev/null // CHECK-LABEL: func @buffer_not_deallocated( // CHECK-SAME: %[[t:.*]]: tensor diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-analysis-bottom-up-from-terminators.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-analysis-bottom-up-from-terminators.mlir new file mode 100644 index 000000000000..1b75edc4c157 --- /dev/null +++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-analysis-bottom-up-from-terminators.mlir @@ -0,0 +1,36 @@ +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=bottom-up-from-terminators" -split-input-file | FileCheck %s + +// CHECK-LABEL: func @simple_test( +func.func @simple_test(%lb: index, %ub: index, %step: index, %f1: f32, %f2: f32) -> (tensor<5xf32>, tensor<5xf32>) { + %c0 = arith.constant 0 : index + %p = arith.constant 0.0 : f32 + + // Make sure that ops that feed into region terminators bufferize in-place + // (if possible). + // Note: This test case fails to bufferize with a "top-down" or "bottom-up" + // heuristic. + + %0 = tensor.empty() : tensor<5xf32> + %1 = scf.for %iv = %lb to %ub step %step iter_args(%t = %0) -> (tensor<5xf32>) { + // CHECK: linalg.fill {__inplace_operands_attr__ = ["none", "false"]} + %2 = linalg.fill ins(%f1 : f32) outs(%t : tensor<5xf32>) -> tensor<5xf32> + // CHECK: linalg.fill {__inplace_operands_attr__ = ["none", "true"]} + %3 = linalg.fill ins(%f2 : f32) outs(%t : tensor<5xf32>) -> tensor<5xf32> + %4 = vector.transfer_read %2[%c0], %p : tensor<5xf32>, vector<5xf32> + vector.print %4 : vector<5xf32> + scf.yield %3 : tensor<5xf32> + } + + %5 = tensor.empty() : tensor<5xf32> + %6 = scf.for %iv = %lb to %ub step %step iter_args(%t = %0) -> (tensor<5xf32>) { + // CHECK: linalg.fill {__inplace_operands_attr__ = ["none", "true"]} + %7 = linalg.fill ins(%f1 : f32) outs(%t : tensor<5xf32>) -> tensor<5xf32> + // CHECK: linalg.fill {__inplace_operands_attr__ = ["none", "false"]} + %8 = linalg.fill ins(%f2 : f32) outs(%t : tensor<5xf32>) -> tensor<5xf32> + %9 = vector.transfer_read %8[%c0], %p : tensor<5xf32>, vector<5xf32> + vector.print %9 : vector<5xf32> + scf.yield %7 : tensor<5xf32> + } + + return %1, %6 : tensor<5xf32>, tensor<5xf32> +} diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-partial.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-partial.mlir index 2c5f2083f589..9380c81ce235 100644 --- a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-partial.mlir +++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize-partial.mlir @@ -4,9 +4,9 @@ // RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="allow-unknown-ops unknown-type-conversion=identity-layout-map" -split-input-file | FileCheck %s --check-prefix=CHECK-NO-LAYOUT-MAP // Run fuzzer with different seeds. -// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=23" -split-input-file -o /dev/null -// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=59" -split-input-file -o /dev/null -// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=91" -split-input-file -o /dev/null +// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23" -split-input-file -o /dev/null +// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59" -split-input-file -o /dev/null +// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91" -split-input-file -o /dev/null // RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="dialect-filter=tensor,bufferization allow-unknown-ops" -canonicalize -split-input-file | FileCheck %s --check-prefix=CHECK-TENSOR // RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="dialect-filter=scf,bufferization allow-unknown-ops" -canonicalize -split-input-file | FileCheck %s --check-prefix=CHECK-SCF diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir index 611b67e198c0..0ed3a9f077ce 100644 --- a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir +++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir @@ -1,9 +1,9 @@ // RUN: mlir-opt %s -one-shot-bufferize="allow-unknown-ops" -verify-diagnostics -split-input-file | FileCheck %s // Run fuzzer with different seeds. -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=23" -verify-diagnostics -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=59" -verify-diagnostics -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=91" -verify-diagnostics -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23" -verify-diagnostics -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59" -verify-diagnostics -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91" -verify-diagnostics -split-input-file -o /dev/null // Run with top-down analysis. // RUN: mlir-opt %s -one-shot-bufferize="allow-unknown-ops analysis-heuristic=top-down" -verify-diagnostics -split-input-file | FileCheck %s --check-prefix=CHECK-TOP-DOWN-ANALYSIS diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-allow-return-allocs.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-allow-return-allocs.mlir index 9319ac61d928..c58b153d438c 100644 --- a/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-allow-return-allocs.mlir +++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-allow-return-allocs.mlir @@ -2,9 +2,9 @@ // RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 " -split-input-file | FileCheck %s --check-prefix=NO-DROP // Run fuzzer with different seeds. -// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-fuzzer-seed=23" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-fuzzer-seed=59" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-fuzzer-seed=91" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91" -split-input-file -o /dev/null // Test bufferization using memref types that have no layout map. // RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 unknown-type-conversion=identity-layout-map function-boundary-type-conversion=identity-layout-map" -split-input-file -o /dev/null diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-analysis.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-analysis.mlir index 6e7b113aa35c..42d9cc00d3ff 100644 --- a/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-analysis.mlir +++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize-analysis.mlir @@ -1,9 +1,14 @@ // RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only" -split-input-file | FileCheck %s // Run fuzzer with different seeds. -// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only analysis-fuzzer-seed=23" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only analysis-fuzzer-seed=59" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only analysis-fuzzer-seed=91" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91" -split-input-file -o /dev/null + +// Try different heuristics. Not checking the result, just make sure that we do +// not crash. +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only analysis-heuristic=bottom-up-from-terminators" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only analysis-heuristic=top-down" -split-input-file -o /dev/null // TODO: Extract op-specific test cases and move them to their respective // dialects. diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize.mlir index 39f4835b28ff..429c9e4dea9e 100644 --- a/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize.mlir +++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-module-bufferize.mlir @@ -2,9 +2,9 @@ // RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1" -canonicalize -drop-equivalent-buffer-results -split-input-file | FileCheck %s // Run fuzzer with different seeds. -// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-fuzzer-seed=23" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-fuzzer-seed=59" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-fuzzer-seed=91" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91" -split-input-file -o /dev/null // Test bufferization using memref types that have no layout map. // RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries=1 unknown-type-conversion=identity-layout-map function-boundary-type-conversion=identity-layout-map" -split-input-file | FileCheck %s --check-prefix=CHECK-NO-LAYOUT-MAP diff --git a/mlir/test/Dialect/Linalg/one-shot-bufferize.mlir b/mlir/test/Dialect/Linalg/one-shot-bufferize.mlir index c69701b65e20..9616a3e32a06 100644 --- a/mlir/test/Dialect/Linalg/one-shot-bufferize.mlir +++ b/mlir/test/Dialect/Linalg/one-shot-bufferize.mlir @@ -1,9 +1,9 @@ // RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries" -canonicalize -buffer-loop-hoisting -drop-equivalent-buffer-results -split-input-file | FileCheck %s // Run fuzzer with different seeds. -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=23 bufferize-function-boundaries" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=59 bufferize-function-boundaries" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=91 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91 bufferize-function-boundaries" -split-input-file -o /dev/null // Test bufferization using memref types that have no layout map. // RUN: mlir-opt %s -one-shot-bufferize="unknown-type-conversion=identity-layout-map function-boundary-type-conversion=identity-layout-map bufferize-function-boundaries" -drop-equivalent-buffer-results -split-input-file | FileCheck %s --check-prefix=CHECK-NO-LAYOUT-MAP diff --git a/mlir/test/Dialect/SCF/one-shot-bufferize-analysis.mlir b/mlir/test/Dialect/SCF/one-shot-bufferize-analysis.mlir index 7d23498f32e1..4d82021e86f5 100644 --- a/mlir/test/Dialect/SCF/one-shot-bufferize-analysis.mlir +++ b/mlir/test/Dialect/SCF/one-shot-bufferize-analysis.mlir @@ -1,9 +1,9 @@ // RUN: mlir-opt %s -one-shot-bufferize="allow-return-allocs-from-loops bufferize-function-boundaries test-analysis-only" -split-input-file | FileCheck %s // Run fuzzer with different seeds. -// RUN: mlir-opt %s -one-shot-bufferize="allow-return-allocs-from-loops bufferize-function-boundaries test-analysis-only analysis-fuzzer-seed=23" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="allow-return-allocs-from-loops bufferize-function-boundaries test-analysis-only analysis-fuzzer-seed=59" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="allow-return-allocs-from-loops bufferize-function-boundaries test-analysis-only analysis-fuzzer-seed=91" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="allow-return-allocs-from-loops bufferize-function-boundaries test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="allow-return-allocs-from-loops bufferize-function-boundaries test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="allow-return-allocs-from-loops bufferize-function-boundaries test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91" -split-input-file -o /dev/null // CHECK-LABEL: func @scf_for_yield_only func.func @scf_for_yield_only( diff --git a/mlir/test/Dialect/SCF/one-shot-bufferize.mlir b/mlir/test/Dialect/SCF/one-shot-bufferize.mlir index 24da8d84b18e..485fdd9b0e59 100644 --- a/mlir/test/Dialect/SCF/one-shot-bufferize.mlir +++ b/mlir/test/Dialect/SCF/one-shot-bufferize.mlir @@ -1,9 +1,9 @@ // RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="allow-return-allocs-from-loops bufferize-function-boundaries" -cse -canonicalize -drop-equivalent-buffer-results -split-input-file | FileCheck %s // Run fuzzer with different seeds. -// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="allow-return-allocs-from-loops test-analysis-only analysis-fuzzer-seed=23 bufferize-function-boundaries" -split-input-file -o /dev/null -// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="allow-return-allocs-from-loops test-analysis-only analysis-fuzzer-seed=59 bufferize-function-boundaries" -split-input-file -o /dev/null -// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="allow-return-allocs-from-loops test-analysis-only analysis-fuzzer-seed=91 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="allow-return-allocs-from-loops analysis-heuristic=fuzzer test-analysis-only analysis-fuzzer-seed=23 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="allow-return-allocs-from-loops analysis-heuristic=fuzzer test-analysis-only analysis-fuzzer-seed=59 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="allow-return-allocs-from-loops analysis-heuristic=fuzzer test-analysis-only analysis-fuzzer-seed=91 bufferize-function-boundaries" -split-input-file -o /dev/null // Test bufferization using memref types that have no layout map. // RUN: mlir-opt %s -allow-unregistered-dialect -one-shot-bufferize="allow-return-allocs-from-loops unknown-type-conversion=identity-layout-map function-boundary-type-conversion=identity-layout-map bufferize-function-boundaries" -split-input-file -o /dev/null diff --git a/mlir/test/Dialect/Tensor/one-shot-bufferize.mlir b/mlir/test/Dialect/Tensor/one-shot-bufferize.mlir index 38c3bb8af810..e2169fe1404c 100644 --- a/mlir/test/Dialect/Tensor/one-shot-bufferize.mlir +++ b/mlir/test/Dialect/Tensor/one-shot-bufferize.mlir @@ -1,9 +1,9 @@ // RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries" -drop-equivalent-buffer-results -split-input-file | FileCheck %s // Run fuzzer with different seeds. -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=23 bufferize-function-boundaries" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=59 bufferize-function-boundaries" -split-input-file -o /dev/null -// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-fuzzer-seed=91 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=23 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=59 bufferize-function-boundaries" -split-input-file -o /dev/null +// RUN: mlir-opt %s -one-shot-bufferize="test-analysis-only analysis-heuristic=fuzzer analysis-fuzzer-seed=91 bufferize-function-boundaries" -split-input-file -o /dev/null // Test bufferization using memref types that have no layout map. // RUN: mlir-opt %s -one-shot-bufferize="unknown-type-conversion=identity-layout-map bufferize-function-boundaries" -split-input-file -o /dev/null -- GitLab From 733640d29ede70585e0e3e1dcc47b935981f791e Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Thu, 21 Mar 2024 10:54:03 +0530 Subject: [PATCH 100/296] Attributor: Handle inferring align from use by atomics (#85762) --- .../Transforms/IPO/AttributorAttributes.cpp | 6 + llvm/test/Transforms/Attributor/align.ll | 163 +++++++++++++++--- .../test/Transforms/Attributor/nocapture-1.ll | 12 +- llvm/test/Transforms/Attributor/nofpclass.ll | 2 +- 4 files changed, 151 insertions(+), 32 deletions(-) diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp index f98833bd1198..ff680e998e71 100644 --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -5190,6 +5190,12 @@ static unsigned getKnownAlignForUse(Attributor &A, AAAlign &QueryingAA, } else if (auto *LI = dyn_cast(I)) { if (LI->getPointerOperand() == UseV) MA = LI->getAlign(); + } else if (auto *AI = dyn_cast(I)) { + if (AI->getPointerOperand() == UseV) + MA = AI->getAlign(); + } else if (auto *AI = dyn_cast(I)) { + if (AI->getPointerOperand() == UseV) + MA = AI->getAlign(); } if (!MA || *MA <= QueryingAA.getKnownAlign()) diff --git a/llvm/test/Transforms/Attributor/align.ll b/llvm/test/Transforms/Attributor/align.ll index 5103b6f1f1e9..9880e53fd43a 100644 --- a/llvm/test/Transforms/Attributor/align.ll +++ b/llvm/test/Transforms/Attributor/align.ll @@ -11,10 +11,10 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" ; TEST 1 ;; ;. -; CHECK: @[[A1:[a-zA-Z0-9_$"\\.-]+]] = common global i8 0, align 8 -; CHECK: @[[A2:[a-zA-Z0-9_$"\\.-]+]] = common global i8 0, align 16 -; CHECK: @[[CND:[a-zA-Z0-9_$"\\.-]+]] = external global i1 -; CHECK: @[[G:[a-zA-Z0-9_$"\\.-]+]] = global i8 0, align 32 +; CHECK: @a1 = common global i8 0, align 8 +; CHECK: @a2 = common global i8 0, align 16 +; CHECK: @cnd = external global i1 +; CHECK: @G = global i8 0, align 32 ;. define ptr @test1(ptr align 8 %0) #0 { ; CHECK: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(none) uwtable @@ -158,18 +158,31 @@ define internal ptr @f1(ptr readnone %0) local_unnamed_addr #0 { ; Function Attrs: nounwind readnone ssp uwtable define ptr @f2(ptr readnone %0) local_unnamed_addr #0 { -; CHECK: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(none) uwtable -; CHECK-LABEL: define {{[^@]+}}@f2 -; CHECK-SAME: (ptr nofree readnone [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] { -; CHECK-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null -; CHECK-NEXT: br i1 [[TMP2]], label [[TMP4:%.*]], label [[TMP3:%.*]] -; CHECK: 3: -; CHECK-NEXT: br label [[TMP5:%.*]] -; CHECK: 4: -; CHECK-NEXT: br label [[TMP5]] -; CHECK: 5: -; CHECK-NEXT: [[TMP6:%.*]] = phi ptr [ [[TMP0]], [[TMP3]] ], [ @a1, [[TMP4]] ] -; CHECK-NEXT: ret ptr [[TMP6]] +; TUNIT: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(none) uwtable +; TUNIT-LABEL: define {{[^@]+}}@f2 +; TUNIT-SAME: (ptr nofree readnone [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] { +; TUNIT-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null +; TUNIT-NEXT: br i1 [[TMP2]], label [[TMP4:%.*]], label [[TMP3:%.*]] +; TUNIT: 3: +; TUNIT-NEXT: br label [[TMP5:%.*]] +; TUNIT: 4: +; TUNIT-NEXT: br label [[TMP5]] +; TUNIT: 5: +; TUNIT-NEXT: [[TMP6:%.*]] = phi ptr [ [[TMP0]], [[TMP3]] ], [ @a1, [[TMP4]] ] +; TUNIT-NEXT: ret ptr [[TMP6]] +; +; CGSCC: Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(none) uwtable +; CGSCC-LABEL: define {{[^@]+}}@f2 +; CGSCC-SAME: (ptr nofree readnone [[TMP0:%.*]]) local_unnamed_addr #[[ATTR0]] { +; CGSCC-NEXT: [[TMP2:%.*]] = icmp eq ptr [[TMP0]], null +; CGSCC-NEXT: br i1 [[TMP2]], label [[TMP4:%.*]], label [[TMP3:%.*]] +; CGSCC: 3: +; CGSCC-NEXT: br label [[TMP5:%.*]] +; CGSCC: 4: +; CGSCC-NEXT: br label [[TMP5]] +; CGSCC: 5: +; CGSCC-NEXT: [[TMP6:%.*]] = phi ptr [ [[TMP0]], [[TMP3]] ], [ @a1, [[TMP4]] ] +; CGSCC-NEXT: ret ptr [[TMP6]] ; %2 = icmp eq ptr %0, null br i1 %2, label %5, label %3 @@ -222,7 +235,7 @@ define align 4 ptr @test7() #0 { ; CGSCC: Function Attrs: mustprogress nofree noinline nosync nounwind willreturn memory(none) uwtable ; CGSCC-LABEL: define {{[^@]+}}@test7 ; CGSCC-SAME: () #[[ATTR1:[0-9]+]] { -; CGSCC-NEXT: [[C:%.*]] = tail call noundef nonnull align 8 dereferenceable(1) ptr @f1() #[[ATTR14:[0-9]+]] +; CGSCC-NEXT: [[C:%.*]] = tail call noundef nonnull align 8 dereferenceable(1) ptr @f1() #[[ATTR15:[0-9]+]] ; CGSCC-NEXT: ret ptr [[C]] ; %c = tail call ptr @f1(ptr align 8 dereferenceable(1) @a1) @@ -933,7 +946,7 @@ define i32 @musttail_caller_1(ptr %p) { ; TUNIT-NEXT: [[C:%.*]] = load i1, ptr @cnd, align 1 ; TUNIT-NEXT: br i1 [[C]], label [[MT:%.*]], label [[EXIT:%.*]] ; TUNIT: mt: -; TUNIT-NEXT: [[V:%.*]] = musttail call i32 @musttail_callee_1(ptr nocapture nofree noundef readonly [[P]]) #[[ATTR12:[0-9]+]] +; TUNIT-NEXT: [[V:%.*]] = musttail call i32 @musttail_callee_1(ptr nocapture nofree noundef readonly [[P]]) #[[ATTR13:[0-9]+]] ; TUNIT-NEXT: ret i32 [[V]] ; TUNIT: exit: ; TUNIT-NEXT: ret i32 0 @@ -944,7 +957,7 @@ define i32 @musttail_caller_1(ptr %p) { ; CGSCC-NEXT: [[C:%.*]] = load i1, ptr @cnd, align 1 ; CGSCC-NEXT: br i1 [[C]], label [[MT:%.*]], label [[EXIT:%.*]] ; CGSCC: mt: -; CGSCC-NEXT: [[V:%.*]] = musttail call i32 @musttail_callee_1(ptr nocapture nofree noundef nonnull readonly dereferenceable(4) [[P]]) #[[ATTR15:[0-9]+]] +; CGSCC-NEXT: [[V:%.*]] = musttail call i32 @musttail_callee_1(ptr nocapture nofree noundef nonnull readonly dereferenceable(4) [[P]]) #[[ATTR16:[0-9]+]] ; CGSCC-NEXT: ret i32 [[V]] ; CGSCC: exit: ; CGSCC-NEXT: ret i32 0 @@ -1076,13 +1089,13 @@ define ptr @aligned_8_return_caller(ptr align(16) %a, i1 %c1, i1 %c2) { ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none) ; TUNIT-LABEL: define {{[^@]+}}@aligned_8_return_caller ; TUNIT-SAME: (ptr nofree readnone align 16 "no-capture-maybe-returned" [[A:%.*]], i1 [[C1:%.*]], i1 [[C2:%.*]]) #[[ATTR10]] { -; TUNIT-NEXT: [[R:%.*]] = call align 8 ptr @aligned_8_return(ptr noalias nofree readnone align 16 "no-capture-maybe-returned" [[A]], i1 noundef [[C1]], i1 [[C2]]) #[[ATTR13:[0-9]+]] +; TUNIT-NEXT: [[R:%.*]] = call align 8 ptr @aligned_8_return(ptr noalias nofree readnone align 16 "no-capture-maybe-returned" [[A]], i1 noundef [[C1]], i1 [[C2]]) #[[ATTR14:[0-9]+]] ; TUNIT-NEXT: ret ptr [[R]] ; ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none) ; CGSCC-LABEL: define {{[^@]+}}@aligned_8_return_caller ; CGSCC-SAME: (ptr nofree readnone align 16 [[A:%.*]], i1 noundef [[C1:%.*]], i1 [[C2:%.*]]) #[[ATTR13:[0-9]+]] { -; CGSCC-NEXT: [[R:%.*]] = call align 8 ptr @aligned_8_return(ptr noalias nofree readnone align 16 [[A]], i1 noundef [[C1]], i1 [[C2]]) #[[ATTR14]] +; CGSCC-NEXT: [[R:%.*]] = call align 8 ptr @aligned_8_return(ptr noalias nofree readnone align 16 [[A]], i1 noundef [[C1]], i1 [[C2]]) #[[ATTR15]] ; CGSCC-NEXT: ret ptr [[R]] ; %r = call ptr @aligned_8_return(ptr %a, i1 %c1, i1 %c2) @@ -1101,6 +1114,104 @@ entry: ret i32 0 } +define i64 @infer_align_atomicrmw(ptr align 4 %p) { +; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) +; TUNIT-LABEL: define {{[^@]+}}@infer_align_atomicrmw +; TUNIT-SAME: (ptr nocapture nofree align 16 [[P:%.*]]) #[[ATTR12:[0-9]+]] { +; TUNIT-NEXT: [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1 +; TUNIT-NEXT: [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3 +; TUNIT-NEXT: [[RET:%.*]] = atomicrmw add ptr [[ARRAYIDX1]], i64 4 seq_cst, align 16 +; TUNIT-NEXT: ret i64 [[RET]] +; +; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) +; CGSCC-LABEL: define {{[^@]+}}@infer_align_atomicrmw +; CGSCC-SAME: (ptr nocapture nofree align 16 [[P:%.*]]) #[[ATTR14:[0-9]+]] { +; CGSCC-NEXT: [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1 +; CGSCC-NEXT: [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3 +; CGSCC-NEXT: [[RET:%.*]] = atomicrmw add ptr [[ARRAYIDX1]], i64 4 seq_cst, align 16 +; CGSCC-NEXT: ret i64 [[RET]] +; + %arrayidx0 = getelementptr i64, ptr %p, i64 1 + %arrayidx1 = getelementptr i64, ptr %arrayidx0, i64 3 + %ret = atomicrmw add ptr %arrayidx1, i64 4 seq_cst, align 16 + ret i64 %ret +} + +define ptr @infer_align_atomicrmw_ptr(ptr align 4 %p, ptr %val) { +; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) +; TUNIT-LABEL: define {{[^@]+}}@infer_align_atomicrmw_ptr +; TUNIT-SAME: (ptr nocapture nofree align 16 [[P:%.*]], ptr nofree [[VAL:%.*]]) #[[ATTR12]] { +; TUNIT-NEXT: [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1 +; TUNIT-NEXT: [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3 +; TUNIT-NEXT: [[RET:%.*]] = atomicrmw xchg ptr [[ARRAYIDX1]], ptr [[VAL]] seq_cst, align 16 +; TUNIT-NEXT: ret ptr [[RET]] +; +; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) +; CGSCC-LABEL: define {{[^@]+}}@infer_align_atomicrmw_ptr +; CGSCC-SAME: (ptr nocapture nofree align 16 [[P:%.*]], ptr nofree [[VAL:%.*]]) #[[ATTR14]] { +; CGSCC-NEXT: [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1 +; CGSCC-NEXT: [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3 +; CGSCC-NEXT: [[RET:%.*]] = atomicrmw xchg ptr [[ARRAYIDX1]], ptr [[VAL]] seq_cst, align 16 +; CGSCC-NEXT: ret ptr [[RET]] +; + %arrayidx0 = getelementptr i64, ptr %p, i64 1 + %arrayidx1 = getelementptr i64, ptr %arrayidx0, i64 3 + %ret = atomicrmw xchg ptr %arrayidx1, ptr %val seq_cst, align 16 + ret ptr %ret +} + +define i64 @infer_align_cmpxchg(ptr align 4 %p) { +; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) +; TUNIT-LABEL: define {{[^@]+}}@infer_align_cmpxchg +; TUNIT-SAME: (ptr nocapture nofree align 16 [[P:%.*]]) #[[ATTR12]] { +; TUNIT-NEXT: [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1 +; TUNIT-NEXT: [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3 +; TUNIT-NEXT: [[CMPX:%.*]] = cmpxchg ptr [[ARRAYIDX1]], i64 4, i64 1 seq_cst seq_cst, align 16 +; TUNIT-NEXT: [[RET:%.*]] = extractvalue { i64, i1 } [[CMPX]], 0 +; TUNIT-NEXT: ret i64 [[RET]] +; +; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) +; CGSCC-LABEL: define {{[^@]+}}@infer_align_cmpxchg +; CGSCC-SAME: (ptr nocapture nofree align 16 [[P:%.*]]) #[[ATTR14]] { +; CGSCC-NEXT: [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1 +; CGSCC-NEXT: [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3 +; CGSCC-NEXT: [[CMPX:%.*]] = cmpxchg ptr [[ARRAYIDX1]], i64 4, i64 1 seq_cst seq_cst, align 16 +; CGSCC-NEXT: [[RET:%.*]] = extractvalue { i64, i1 } [[CMPX]], 0 +; CGSCC-NEXT: ret i64 [[RET]] +; + %arrayidx0 = getelementptr i64, ptr %p, i64 1 + %arrayidx1 = getelementptr i64, ptr %arrayidx0, i64 3 + %cmpx = cmpxchg ptr %arrayidx1, i64 4, i64 1 seq_cst seq_cst, align 16 + %ret = extractvalue { i64, i1 } %cmpx, 0 + ret i64 %ret +} + +define ptr @infer_align_cmpxchg_ptr(ptr align 4 %p, ptr %cmp0, ptr %cmp1) { +; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) +; TUNIT-LABEL: define {{[^@]+}}@infer_align_cmpxchg_ptr +; TUNIT-SAME: (ptr nocapture nofree align 16 [[P:%.*]], ptr nofree [[CMP0:%.*]], ptr nofree [[CMP1:%.*]]) #[[ATTR12]] { +; TUNIT-NEXT: [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1 +; TUNIT-NEXT: [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3 +; TUNIT-NEXT: [[CMPX:%.*]] = cmpxchg ptr [[ARRAYIDX1]], ptr [[CMP0]], ptr [[CMP1]] seq_cst seq_cst, align 16 +; TUNIT-NEXT: [[RET:%.*]] = extractvalue { ptr, i1 } [[CMPX]], 0 +; TUNIT-NEXT: ret ptr [[RET]] +; +; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) +; CGSCC-LABEL: define {{[^@]+}}@infer_align_cmpxchg_ptr +; CGSCC-SAME: (ptr nocapture nofree align 16 [[P:%.*]], ptr nofree [[CMP0:%.*]], ptr nofree [[CMP1:%.*]]) #[[ATTR14]] { +; CGSCC-NEXT: [[ARRAYIDX0:%.*]] = getelementptr i64, ptr [[P]], i64 1 +; CGSCC-NEXT: [[ARRAYIDX1:%.*]] = getelementptr i64, ptr [[ARRAYIDX0]], i64 3 +; CGSCC-NEXT: [[CMPX:%.*]] = cmpxchg ptr [[ARRAYIDX1]], ptr [[CMP0]], ptr [[CMP1]] seq_cst seq_cst, align 16 +; CGSCC-NEXT: [[RET:%.*]] = extractvalue { ptr, i1 } [[CMPX]], 0 +; CGSCC-NEXT: ret ptr [[RET]] +; + %arrayidx0 = getelementptr i64, ptr %p, i64 1 + %arrayidx1 = getelementptr i64, ptr %arrayidx0, i64 3 + %cmpx = cmpxchg ptr %arrayidx1, ptr %cmp0, ptr %cmp1 seq_cst seq_cst, align 16 + %ret = extractvalue { ptr, i1 } %cmpx, 0 + ret ptr %ret +} + declare void @implicit_cast_callee(i64) attributes #0 = { nounwind uwtable noinline } @@ -1119,8 +1230,9 @@ attributes #2 = { null_pointer_is_valid } ; TUNIT: attributes #[[ATTR9]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(write) } ; TUNIT: attributes #[[ATTR10]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) } ; TUNIT: attributes #[[ATTR11]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(read) } -; TUNIT: attributes #[[ATTR12]] = { nofree nosync nounwind willreturn memory(read) } -; TUNIT: attributes #[[ATTR13]] = { nofree nosync nounwind willreturn } +; TUNIT: attributes #[[ATTR12]] = { mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) } +; TUNIT: attributes #[[ATTR13]] = { nofree nosync nounwind willreturn memory(read) } +; TUNIT: attributes #[[ATTR14]] = { nofree nosync nounwind willreturn } ;. ; CGSCC: attributes #[[ATTR0]] = { mustprogress nofree noinline norecurse nosync nounwind willreturn memory(none) uwtable } ; CGSCC: attributes #[[ATTR1]] = { mustprogress nofree noinline nosync nounwind willreturn memory(none) uwtable } @@ -1136,6 +1248,7 @@ attributes #2 = { null_pointer_is_valid } ; CGSCC: attributes #[[ATTR11]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) } ; CGSCC: attributes #[[ATTR12]] = { mustprogress nofree nosync nounwind willreturn memory(read) } ; CGSCC: attributes #[[ATTR13]] = { mustprogress nofree nosync nounwind willreturn memory(none) } -; CGSCC: attributes #[[ATTR14]] = { nofree nosync willreturn } -; CGSCC: attributes #[[ATTR15]] = { nofree willreturn memory(read) } +; CGSCC: attributes #[[ATTR14]] = { mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) } +; CGSCC: attributes #[[ATTR15]] = { nofree nosync willreturn } +; CGSCC: attributes #[[ATTR16]] = { nofree willreturn memory(read) } ;. diff --git a/llvm/test/Transforms/Attributor/nocapture-1.ll b/llvm/test/Transforms/Attributor/nocapture-1.ll index 7d2f0a1351a4..f61388f71c46 100644 --- a/llvm/test/Transforms/Attributor/nocapture-1.ll +++ b/llvm/test/Transforms/Attributor/nocapture-1.ll @@ -524,13 +524,13 @@ define void @test6_2(ptr %x6_2, ptr %y6_2, ptr %z6_2) { define void @test_cmpxchg(ptr %p) { ; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) ; TUNIT-LABEL: define {{[^@]+}}@test_cmpxchg -; TUNIT-SAME: (ptr nocapture nofree noundef nonnull dereferenceable(4) [[P:%.*]]) #[[ATTR8:[0-9]+]] { +; TUNIT-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR8:[0-9]+]] { ; TUNIT-NEXT: [[TMP1:%.*]] = cmpxchg ptr [[P]], i32 0, i32 1 acquire monotonic, align 4 ; TUNIT-NEXT: ret void ; ; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) ; CGSCC-LABEL: define {{[^@]+}}@test_cmpxchg -; CGSCC-SAME: (ptr nocapture nofree noundef nonnull dereferenceable(4) [[P:%.*]]) #[[ATTR11:[0-9]+]] { +; CGSCC-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR11:[0-9]+]] { ; CGSCC-NEXT: [[TMP1:%.*]] = cmpxchg ptr [[P]], i32 0, i32 1 acquire monotonic, align 4 ; CGSCC-NEXT: ret void ; @@ -541,13 +541,13 @@ define void @test_cmpxchg(ptr %p) { define void @test_cmpxchg_ptr(ptr %p, ptr %q) { ; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) ; TUNIT-LABEL: define {{[^@]+}}@test_cmpxchg_ptr -; TUNIT-SAME: (ptr nocapture nofree noundef nonnull dereferenceable(8) [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR8]] { +; TUNIT-SAME: (ptr nocapture nofree noundef nonnull align 8 dereferenceable(8) [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR8]] { ; TUNIT-NEXT: [[TMP1:%.*]] = cmpxchg ptr [[P]], ptr null, ptr [[Q]] acquire monotonic, align 8 ; TUNIT-NEXT: ret void ; ; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) ; CGSCC-LABEL: define {{[^@]+}}@test_cmpxchg_ptr -; CGSCC-SAME: (ptr nocapture nofree noundef nonnull dereferenceable(8) [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR11]] { +; CGSCC-SAME: (ptr nocapture nofree noundef nonnull align 8 dereferenceable(8) [[P:%.*]], ptr nofree [[Q:%.*]]) #[[ATTR11]] { ; CGSCC-NEXT: [[TMP1:%.*]] = cmpxchg ptr [[P]], ptr null, ptr [[Q]] acquire monotonic, align 8 ; CGSCC-NEXT: ret void ; @@ -558,13 +558,13 @@ define void @test_cmpxchg_ptr(ptr %p, ptr %q) { define void @test_atomicrmw(ptr %p) { ; TUNIT: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) ; TUNIT-LABEL: define {{[^@]+}}@test_atomicrmw -; TUNIT-SAME: (ptr nocapture nofree noundef nonnull dereferenceable(4) [[P:%.*]]) #[[ATTR8]] { +; TUNIT-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR8]] { ; TUNIT-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[P]], i32 1 seq_cst, align 4 ; TUNIT-NEXT: ret void ; ; CGSCC: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) ; CGSCC-LABEL: define {{[^@]+}}@test_atomicrmw -; CGSCC-SAME: (ptr nocapture nofree noundef nonnull dereferenceable(4) [[P:%.*]]) #[[ATTR11]] { +; CGSCC-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[P:%.*]]) #[[ATTR11]] { ; CGSCC-NEXT: [[TMP1:%.*]] = atomicrmw add ptr [[P]], i32 1 seq_cst, align 4 ; CGSCC-NEXT: ret void ; diff --git a/llvm/test/Transforms/Attributor/nofpclass.ll b/llvm/test/Transforms/Attributor/nofpclass.ll index 442464cde389..4df647cf3bb5 100644 --- a/llvm/test/Transforms/Attributor/nofpclass.ll +++ b/llvm/test/Transforms/Attributor/nofpclass.ll @@ -1813,7 +1813,7 @@ define double @fpext(float nofpclass(inf nan) %arg) { define float @atomicrmw_fadd(ptr %ptr, float nofpclass(inf nan) %val) { ; CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) ; CHECK-LABEL: define float @atomicrmw_fadd -; CHECK-SAME: (ptr nocapture nofree noundef nonnull dereferenceable(4) [[PTR:%.*]], float nofpclass(nan inf) [[VAL:%.*]]) #[[ATTR6:[0-9]+]] { +; CHECK-SAME: (ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[PTR:%.*]], float nofpclass(nan inf) [[VAL:%.*]]) #[[ATTR6:[0-9]+]] { ; CHECK-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr [[PTR]], float [[VAL]] seq_cst, align 4 ; CHECK-NEXT: ret float [[RESULT]] ; -- GitLab From df9ed9cf52f82aed023adc968ca2a0e7f7cccc69 Mon Sep 17 00:00:00 2001 From: srcarroll <50210727+srcarroll@users.noreply.github.com> Date: Thu, 21 Mar 2024 00:25:07 -0500 Subject: [PATCH 101/296] [mlir][transform] Fix failure in flattening already flattened linalg ops (#86037) The previous implementation was doing an early successful return on `rank <= 1` without adding the original op to transform results. This resulted in errors about number of returns. This patch fixes this by adding the original op to results. Additionally, we first check if op is elementwise and return a slienceable failure early if not. --- .../TransformOps/LinalgTransformOps.cpp | 15 ++++++++----- .../Dialect/Linalg/flatten-elementwise.mlir | 21 +++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp index d82a6beb1086..ecf998312482 100644 --- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp +++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp @@ -3269,15 +3269,20 @@ DiagnosedSilenceableFailure transform::FlattenElementwiseLinalgOp::applyToOne( transform::ApplyToEachResultList &results, transform::TransformState &state) { rewriter.setInsertionPoint(target); - if (target.getNumLoops() <= 1) + if (!isElementwise(target)) { + failed(rewriter.notifyMatchFailure( + target, "only elementwise flattening is supported")); + return emitDefaultSilenceableFailure(target); + } + // If rank <= 1, do nothing + if (target.getNumLoops() <= 1) { + results.push_back(target); return DiagnosedSilenceableFailure::success(); + } ReassociationIndices reassociation(target.getNumLoops()); std::iota(reassociation.begin(), reassociation.end(), 0); auto maybeFlattened = - (isElementwise(target)) - ? collapseOpIterationDims(target, reassociation, rewriter) - : FailureOr(rewriter.notifyMatchFailure( - target, "only elementwise flattening is supported")); + collapseOpIterationDims(target, reassociation, rewriter); if (failed(maybeFlattened)) return emitDefaultSilenceableFailure(target); results.push_back(maybeFlattened->collapsedOp); diff --git a/mlir/test/Dialect/Linalg/flatten-elementwise.mlir b/mlir/test/Dialect/Linalg/flatten-elementwise.mlir index 858c133dd536..5a27fe76b134 100644 --- a/mlir/test/Dialect/Linalg/flatten-elementwise.mlir +++ b/mlir/test/Dialect/Linalg/flatten-elementwise.mlir @@ -67,6 +67,27 @@ module attributes {transform.with_named_sequence} { // ----- +// CHECK-LABEL: func.func @map_already_flat( +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9_]*]]: memref<32xf32> +// CHECK-SAME: %[[ARG1:[a-zA-Z0-9_]*]]: memref<32xf32> +// CHECK-SAME: %[[ARG2:[a-zA-Z0-9_]*]]: memref<32xf32> +// CHECK-NEXT: linalg.map { arith.addf } ins(%[[ARG0]], %[[ARG1]] : memref<32xf32>, memref<32xf32>) outs(%[[ARG2]] : memref<32xf32>) +func.func @map_already_flat(%arg0: memref<32xf32>, %arg1: memref<32xf32>, %arg2: memref<32xf32>) { + linalg.map {arith.addf} ins(%arg0, %arg1: memref<32xf32>, memref<32xf32>) outs(%arg2: memref<32xf32>) + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match interface{LinalgOp} in %arg1 : (!transform.any_op) -> !transform.any_op + %flattened = transform.structured.flatten_elementwise %0 + : (!transform.any_op) -> !transform.any_op + transform.yield + } +} + +// ----- + // CHECK: #[[$MAP0:.*]] = affine_map<(d0) -> (d0)> // CHECK-LABEL: func.func @generic // CHECK-SAME: %[[ARG0:[a-zA-Z0-9_]*]]: memref<32x7xf32> -- GitLab From 26c290b46ac6b4a81feb28ae1862fac961138a24 Mon Sep 17 00:00:00 2001 From: Nathan Lanza Date: Thu, 21 Mar 2024 01:42:59 -0400 Subject: [PATCH 102/296] [cmake] Place clang behind mlir in the list of external projects (#86050) In preparation for the initial ClangIR upstreaming process, move clang behind MLIR in the list of external projects. Otherwise, cmake will attempt to build clang before MLIR. --- llvm/tools/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/tools/CMakeLists.txt b/llvm/tools/CMakeLists.txt index c6116ac81d12..2969877b9ee7 100644 --- a/llvm/tools/CMakeLists.txt +++ b/llvm/tools/CMakeLists.txt @@ -37,11 +37,11 @@ add_llvm_tool_subdirectory(llvm-profdata) # Projects supported via LLVM_EXTERNAL_*_SOURCE_DIR need to be explicitly # specified. -add_llvm_external_project(clang) add_llvm_external_project(lld) add_llvm_external_project(lldb) add_llvm_external_project(mlir) -# Flang depends on mlir, so place it afterward +# ClangIR and Flang depends on mlir, so place them afterwards +add_llvm_external_project(clang) add_llvm_external_project(flang) add_llvm_external_project(bolt) -- GitLab From d59730d7060f33dd1607be1fd7813be78759a953 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 20 Mar 2024 22:45:38 -0700 Subject: [PATCH 103/296] [CMake] Change GCC_INSTALL_PREFIX from warning to fatal error (#85891) unless USE_DEPRECATED_GCC_INSTALL_PREFIX (temporary escape hatch) is set. Setting GCC_INSTALL_PREFIX leads to a warning for Clang 18.1 (#77537) and will be completely removed for Clang 20. Link: discourse.llvm.org/t/add-gcc-install-dir-deprecate-gcc-toolchain-and-remove-gcc-install-prefix/65091 Link: discourse.llvm.org/t/correct-cmake-parameters-for-building-clang-and-lld-for-riscv/72833 --- clang/CMakeLists.txt | 5 +++-- clang/docs/ReleaseNotes.rst | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt index 47fc2e4886cf..761dab8c28c1 100644 --- a/clang/CMakeLists.txt +++ b/clang/CMakeLists.txt @@ -190,11 +190,12 @@ set(CLANG_RESOURCE_DIR "" CACHE STRING set(C_INCLUDE_DIRS "" CACHE STRING "Colon separated list of directories clang will search for headers.") +set(USE_DEPRECATED_GCC_INSTALL_PREFIX OFF CACHE BOOL "Temporary workaround before GCC_INSTALL_PREFIX is completely removed") set(GCC_INSTALL_PREFIX "" CACHE PATH "Directory where gcc is installed." ) set(DEFAULT_SYSROOT "" CACHE STRING "Default to all compiler invocations for --sysroot=." ) -if(GCC_INSTALL_PREFIX) - message(WARNING "GCC_INSTALL_PREFIX is deprecated and will be removed. Use " +if(GCC_INSTALL_PREFIX AND NOT USE_DEPRECATED_GCC_INSTALL_PREFIX) + message(FATAL_ERROR "GCC_INSTALL_PREFIX is deprecated and will be removed. Use " "configuration files (https://clang.llvm.org/docs/UsersManual.html#configuration-files)" "to specify the default --gcc-install-dir= or --gcc-triple=. --gcc-toolchain= is discouraged. " "See https://github.com/llvm/llvm-project/pull/77537 for detail.") diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index c0b0c8a8a3ea..a9c55ef662a0 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -37,6 +37,9 @@ These changes are ones which we think may surprise users when upgrading to Clang |release| because of the opportunity they pose for disruption to existing code bases. +- Setting the deprecated CMake variable ``GCC_INSTALL_PREFIX`` (which sets the + default ``--gcc-toolchain=``) now leads to a fatal error. + C/C++ Language Potentially Breaking Changes ------------------------------------------- -- GitLab From ce8e86971036cb34c3d32cf0b70169379c85ae2f Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 20 Mar 2024 22:53:51 -0700 Subject: [PATCH 104/296] [RISCV] Convert an assertion to an if condition in getRegAllocationHints (#85998) With GPR pairs from Zdinx, we can't guarantee there are no subregisters on integer instruction operands. I've been able to get these assertions to fire after some other recent PRs. I've added a FIXME to support this properly. I just wanted to prevent the assertion failure for now. No test case because my other patch #85982 that allowed me to fail the assert hasn't been approved yet, and I don't know for that that patch is required to hit this assert. It's just what exposed it for me. So I think this patch is a good precaution regardless. --- llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp index 10bf1e88d741..952d17468da5 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp @@ -741,8 +741,11 @@ bool RISCVRegisterInfo::getRegAllocationHints( bool NeedGPRC) -> void { Register Reg = MO.getReg(); Register PhysReg = Reg.isPhysical() ? Reg : Register(VRM->getPhys(Reg)); - if (PhysReg && (!NeedGPRC || RISCV::GPRCRegClass.contains(PhysReg))) { - assert(!MO.getSubReg() && !VRRegMO.getSubReg() && "Unexpected subreg!"); + // TODO: Support GPRPair subregisters? Need to be careful with even/odd + // registers. If the virtual register is an odd register of a pair and the + // physical register is even (or vice versa), we should not add the hint. + if (PhysReg && (!NeedGPRC || RISCV::GPRCRegClass.contains(PhysReg)) && + !MO.getSubReg() && !VRRegMO.getSubReg()) { if (!MRI->isReserved(PhysReg) && !is_contained(Hints, PhysReg)) TwoAddrHints.insert(PhysReg); } -- GitLab From cbcdf126ccc774c063b5d5140c1393ff5305dded Mon Sep 17 00:00:00 2001 From: Nathan Lanza Date: Wed, 20 Mar 2024 23:50:47 -0700 Subject: [PATCH 105/296] Revert "[cmake] Place clang behind mlir in the list of external projects (#86050)" This reverts commit 26c290b46ac6b4a81feb28ae1862fac961138a24. --- llvm/tools/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/tools/CMakeLists.txt b/llvm/tools/CMakeLists.txt index 2969877b9ee7..c6116ac81d12 100644 --- a/llvm/tools/CMakeLists.txt +++ b/llvm/tools/CMakeLists.txt @@ -37,11 +37,11 @@ add_llvm_tool_subdirectory(llvm-profdata) # Projects supported via LLVM_EXTERNAL_*_SOURCE_DIR need to be explicitly # specified. +add_llvm_external_project(clang) add_llvm_external_project(lld) add_llvm_external_project(lldb) add_llvm_external_project(mlir) -# ClangIR and Flang depends on mlir, so place them afterwards -add_llvm_external_project(clang) +# Flang depends on mlir, so place it afterward add_llvm_external_project(flang) add_llvm_external_project(bolt) -- GitLab From 8fb2160a76b5f051f4cc8f5c8c097830bc91c22c Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Thu, 21 Mar 2024 15:38:43 +0800 Subject: [PATCH 106/296] [RISCV] Use DenseMap to track V0 definition. NFC (#84465) Reviving some of the progress on #71764. To recap, we explored removing the V0 register copies to simplify the pass, but hit a limitation with the register allocator due to our use of the vmv0 singleton reg class and early-clobber constraints. So since we will have to continue to track the definition of V0 ourselves, this patch simplifies it by storing it in a map. It will allow us to move about copies to V0 in #71764 without having to do extra bookkeeping. --- llvm/lib/Target/RISCV/RISCVFoldMasks.cpp | 48 +++++++++++++----------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp b/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp index fddbaa97d063..2089f5dda6fe 100644 --- a/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp +++ b/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp @@ -47,10 +47,13 @@ public: StringRef getPassName() const override { return "RISC-V Fold Masks"; } private: - bool convertToUnmasked(MachineInstr &MI, MachineInstr *MaskDef) const; - bool convertVMergeToVMv(MachineInstr &MI, MachineInstr *MaskDef) const; + bool convertToUnmasked(MachineInstr &MI) const; + bool convertVMergeToVMv(MachineInstr &MI) const; - bool isAllOnesMask(MachineInstr *MaskDef) const; + bool isAllOnesMask(const MachineInstr *MaskDef) const; + + /// Maps uses of V0 to the corresponding def of V0. + DenseMap V0Defs; }; } // namespace @@ -59,10 +62,9 @@ char RISCVFoldMasks::ID = 0; INITIALIZE_PASS(RISCVFoldMasks, DEBUG_TYPE, "RISC-V Fold Masks", false, false) -bool RISCVFoldMasks::isAllOnesMask(MachineInstr *MaskDef) const { - if (!MaskDef) - return false; - assert(MaskDef->isCopy() && MaskDef->getOperand(0).getReg() == RISCV::V0); +bool RISCVFoldMasks::isAllOnesMask(const MachineInstr *MaskDef) const { + assert(MaskDef && MaskDef->isCopy() && + MaskDef->getOperand(0).getReg() == RISCV::V0); Register SrcReg = TRI->lookThruCopyLike(MaskDef->getOperand(1).getReg(), MRI); if (!SrcReg.isVirtual()) return false; @@ -89,8 +91,7 @@ bool RISCVFoldMasks::isAllOnesMask(MachineInstr *MaskDef) const { // Transform (VMERGE_VVM_ false, false, true, allones, vl, sew) to // (VMV_V_V_ false, true, vl, sew). It may decrease uses of VMSET. -bool RISCVFoldMasks::convertVMergeToVMv(MachineInstr &MI, - MachineInstr *V0Def) const { +bool RISCVFoldMasks::convertVMergeToVMv(MachineInstr &MI) const { #define CASE_VMERGE_TO_VMV(lmul) \ case RISCV::PseudoVMERGE_VVM_##lmul: \ NewOpc = RISCV::PseudoVMV_V_V_##lmul; \ @@ -116,7 +117,7 @@ bool RISCVFoldMasks::convertVMergeToVMv(MachineInstr &MI, return false; assert(MI.getOperand(4).isReg() && MI.getOperand(4).getReg() == RISCV::V0); - if (!isAllOnesMask(V0Def)) + if (!isAllOnesMask(V0Defs.lookup(&MI))) return false; MI.setDesc(TII->get(NewOpc)); @@ -133,14 +134,13 @@ bool RISCVFoldMasks::convertVMergeToVMv(MachineInstr &MI, return true; } -bool RISCVFoldMasks::convertToUnmasked(MachineInstr &MI, - MachineInstr *MaskDef) const { +bool RISCVFoldMasks::convertToUnmasked(MachineInstr &MI) const { const RISCV::RISCVMaskedPseudoInfo *I = RISCV::getMaskedPseudoInfo(MI.getOpcode()); if (!I) return false; - if (!isAllOnesMask(MaskDef)) + if (!isAllOnesMask(V0Defs.lookup(&MI))) return false; // There are two classes of pseudos in the table - compares and @@ -198,20 +198,26 @@ bool RISCVFoldMasks::runOnMachineFunction(MachineFunction &MF) { // $v0:vr = COPY %mask:vr // %x:vr = Pseudo_MASK %a:vr, %b:br, $v0:vr // - // Because $v0 isn't in SSA, keep track of it so we can check the mask operand - // on each pseudo. - MachineInstr *CurrentV0Def; - for (MachineBasicBlock &MBB : MF) { - CurrentV0Def = nullptr; - for (MachineInstr &MI : MBB) { - Changed |= convertToUnmasked(MI, CurrentV0Def); - Changed |= convertVMergeToVMv(MI, CurrentV0Def); + // Because $v0 isn't in SSA, keep track of its definition at each use so we + // can check mask operands. + for (const MachineBasicBlock &MBB : MF) { + const MachineInstr *CurrentV0Def = nullptr; + for (const MachineInstr &MI : MBB) { + if (MI.readsRegister(RISCV::V0, TRI)) + V0Defs[&MI] = CurrentV0Def; if (MI.definesRegister(RISCV::V0, TRI)) CurrentV0Def = &MI; } } + for (MachineBasicBlock &MBB : MF) { + for (MachineInstr &MI : MBB) { + Changed |= convertToUnmasked(MI); + Changed |= convertVMergeToVMv(MI); + } + } + return Changed; } -- GitLab From adda597388dc148ac235e755b3e8bbd0a12a3e15 Mon Sep 17 00:00:00 2001 From: Tobias Gysi Date: Thu, 21 Mar 2024 09:07:57 +0100 Subject: [PATCH 107/296] [MLIR] Add index bitwidth to the DataLayout (#85927) When importing from LLVM IR the data layout of all pointer types contains an index bitwidth that should be used for index computations. This revision adds a getter to the DataLayout that provides access to the already stored bitwidth. The function returns an optional since only pointer-like types have an index bitwidth. Querying the bitwidth of a non-pointer type returns std::nullopt. The new function works for the built-in Index type and, using a type interface, for the LLVMPointerType. --- mlir/docs/DataLayout.md | 4 +- mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td | 2 +- .../mlir/Interfaces/DataLayoutInterfaces.h | 13 +++++ .../mlir/Interfaces/DataLayoutInterfaces.td | 28 ++++++++++ mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp | 25 +++++++-- mlir/lib/Interfaces/DataLayoutInterfaces.cpp | 30 ++++++++++- mlir/lib/Target/LLVMIR/ModuleTranslation.cpp | 11 ++-- mlir/test/Dialect/LLVMIR/layout.mlir | 30 +++++++---- .../DataLayoutInterfaces/module.mlir | 4 +- .../DataLayoutInterfaces/query.mlir | 51 ++++++++++++++++--- .../DataLayoutInterfaces/types.mlir | 1 + mlir/test/Target/LLVMIR/data-layout.mlir | 2 +- .../lib/Dialect/DLTI/TestDataLayoutQuery.cpp | 12 +++-- mlir/test/lib/Dialect/Test/TestTypeDefs.td | 3 +- mlir/test/lib/Dialect/Test/TestTypes.cpp | 8 ++- .../Interfaces/DataLayoutInterfacesTest.cpp | 7 +++ 16 files changed, 193 insertions(+), 38 deletions(-) diff --git a/mlir/docs/DataLayout.md b/mlir/docs/DataLayout.md index b9dde30519d6..86ad51a517ae 100644 --- a/mlir/docs/DataLayout.md +++ b/mlir/docs/DataLayout.md @@ -77,6 +77,7 @@ public: llvm::TypeSize getTypeSizeInBits(Type type) const; uint64_t getTypeABIAlignment(Type type) const; uint64_t getTypePreferredAlignment(Type type) const; + std::optional getTypeIndexBitwidth(Type type) const; }; ``` @@ -267,7 +268,8 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< >} {} ``` -specifies that `index` has 32 bits. All other layout properties of `index` match +specifies that `index` has 32 bits and index computations should be performed +using 32-bit precision as well. All other layout properties of `index` match those of the integer type with the same bitwidth defined above. In absence of the corresponding entry, `index` is assumed to be a 64-bit diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td index 96cdbf01b4bd..b7176aa93ff1 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMTypes.td @@ -123,7 +123,7 @@ def LLVMFunctionType : LLVMType<"LLVMFunction", "func"> { def LLVMPointerType : LLVMType<"LLVMPointer", "ptr", [ DeclareTypeInterfaceMethods]> { + "getIndexBitwidth", "areCompatible", "verifyEntries"]>]> { let summary = "LLVM pointer type"; let description = [{ The `!llvm.ptr` type is an LLVM pointer type. This type typically represents diff --git a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h index 4a21f76dfc5d..046354677e6a 100644 --- a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h +++ b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h @@ -57,6 +57,13 @@ uint64_t getDefaultPreferredAlignment(Type type, const DataLayout &dataLayout, ArrayRef params); +/// Default handler for the index bitwidth request. Computes the result for +/// the built-in index type and dispatches to the DataLayoutTypeInterface for +/// other types. +std::optional +getDefaultIndexBitwidth(Type type, const DataLayout &dataLayout, + ArrayRef params); + /// Default handler for alloca memory space request. Dispatches to the /// DataLayoutInterface if specified, otherwise returns the default. Attribute getDefaultAllocaMemorySpace(DataLayoutEntryInterface entry); @@ -180,6 +187,11 @@ public: /// Returns the preferred of the given type in the current scope. uint64_t getTypePreferredAlignment(Type t) const; + /// Returns the bitwidth that should be used when performing index + /// computations for the given pointer-like type in the current scope. If the + /// type is not a pointer-like type, it returns std::nullopt. + std::optional getTypeIndexBitwidth(Type t) const; + /// Returns the memory space used for AllocaOps. Attribute getAllocaMemorySpace() const; @@ -216,6 +228,7 @@ private: mutable DenseMap bitsizes; mutable DenseMap abiAlignments; mutable DenseMap preferredAlignments; + mutable DenseMap> indexBitwidths; /// Cache for alloca, global, and program memory spaces. mutable std::optional allocaMemorySpace; diff --git a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td index a8def967fffc..0ee7a116d114 100644 --- a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td +++ b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td @@ -280,6 +280,22 @@ def DataLayoutOpInterface : OpInterface<"DataLayoutOpInterface"> { params); }] >, + StaticInterfaceMethod< + /*description=*/"Returns the bitwidth that should be used when " + "performing index computations for the type computed " + "using the relevant entries. The data layout object can " + "be used for recursive queries.", + /*retTy=*/"std::optional", + /*methodName=*/"getIndexBitwidth", + /*args=*/(ins "::mlir::Type":$type, + "const ::mlir::DataLayout &":$dataLayout, + "::mlir::DataLayoutEntryListRef":$params), + /*methodBody=*/"", + /*defaultImplementation=*/[{ + return ::mlir::detail::getDefaultIndexBitwidth(type, dataLayout, + params); + }] + >, StaticInterfaceMethod< /*description=*/"Returns the memory space used by the ABI computed " "using the relevant entries. The data layout object " @@ -400,6 +416,18 @@ def DataLayoutTypeInterface : TypeInterface<"DataLayoutTypeInterface"> { /*args=*/(ins "const ::mlir::DataLayout &":$dataLayout, "::mlir::DataLayoutEntryListRef":$params) >, + InterfaceMethod< + /*description=*/"Returns the bitwidth that should be used when " + "performing index computations for the given " + "pointer-like type. If the type is not a pointer-like " + "type, returns std::nullopt.", + /*retTy=*/"std::optional", + /*methodName=*/"getIndexBitwidth", + /*args=*/(ins "const ::mlir::DataLayout &":$dataLayout, + "::mlir::DataLayoutEntryListRef":$params), + /*methodBody=*/"", + /*defaultImplementation=*/[{ return std::nullopt; }] + >, InterfaceMethod< /*desc=*/"Returns true if the two lists of entries are compatible, that " "is, that `newLayout` spec entries can be nested in an op with " diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp index 443e245887ea..630187f220a4 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMTypes.cpp @@ -287,15 +287,22 @@ getPointerDataLayoutEntry(DataLayoutEntryListRef params, LLVMPointerType type, } } if (currentEntry) { - return *extractPointerSpecValue(currentEntry, pos) / - (pos == PtrDLEntryPos::Size ? 1 : kBitsInByte); + std::optional value = extractPointerSpecValue(currentEntry, pos); + // If the optional `PtrDLEntryPos::Index` entry is not available, use the + // pointer size as the index bitwidth. + if (!value && pos == PtrDLEntryPos::Index) + value = extractPointerSpecValue(currentEntry, PtrDLEntryPos::Size); + bool isSizeOrIndex = + pos == PtrDLEntryPos::Size || pos == PtrDLEntryPos::Index; + return *value / (isSizeOrIndex ? 1 : kBitsInByte); } // If not found, and this is the pointer to the default memory space, assume // 64-bit pointers. if (type.getAddressSpace() == 0) { - return pos == PtrDLEntryPos::Size ? kDefaultPointerSizeBits - : kDefaultPointerAlignment; + bool isSizeOrIndex = + pos == PtrDLEntryPos::Size || pos == PtrDLEntryPos::Index; + return isSizeOrIndex ? kDefaultPointerSizeBits : kDefaultPointerAlignment; } return std::nullopt; @@ -332,6 +339,16 @@ LLVMPointerType::getPreferredAlignment(const DataLayout &dataLayout, return dataLayout.getTypePreferredAlignment(get(getContext())); } +std::optional +LLVMPointerType::getIndexBitwidth(const DataLayout &dataLayout, + DataLayoutEntryListRef params) const { + if (std::optional indexBitwidth = + getPointerDataLayoutEntry(params, *this, PtrDLEntryPos::Index)) + return *indexBitwidth; + + return dataLayout.getTypeIndexBitwidth(get(getContext())); +} + bool LLVMPointerType::areCompatible(DataLayoutEntryListRef oldLayout, DataLayoutEntryListRef newLayout) const { for (DataLayoutEntryInterface newEntry : newLayout) { diff --git a/mlir/lib/Interfaces/DataLayoutInterfaces.cpp b/mlir/lib/Interfaces/DataLayoutInterfaces.cpp index 65c41f44192a..b5b7d78cfeff 100644 --- a/mlir/lib/Interfaces/DataLayoutInterfaces.cpp +++ b/mlir/lib/Interfaces/DataLayoutInterfaces.cpp @@ -218,7 +218,23 @@ uint64_t mlir::detail::getDefaultPreferredAlignment( reportMissingDataLayout(type); } -// Returns the memory space used for allocal operations if specified in the +std::optional mlir::detail::getDefaultIndexBitwidth( + Type type, const DataLayout &dataLayout, + ArrayRef params) { + if (isa(type)) + return getIndexBitwidth(params); + + if (auto typeInterface = dyn_cast(type)) + if (std::optional indexBitwidth = + typeInterface.getIndexBitwidth(dataLayout, params)) + return *indexBitwidth; + + // Return std::nullopt for all other types, which are assumed to be non + // pointer-like types. + return std::nullopt; +} + +// Returns the memory space used for alloca operations if specified in the // given entry. If the entry is empty the default memory space represented by // an empty attribute is returned. Attribute @@ -520,6 +536,18 @@ uint64_t mlir::DataLayout::getTypePreferredAlignment(Type t) const { }); } +std::optional mlir::DataLayout::getTypeIndexBitwidth(Type t) const { + checkValid(); + return cachedLookup>(t, indexBitwidths, [&](Type ty) { + DataLayoutEntryList list; + if (originalLayout) + list = originalLayout.getSpecForType(ty.getTypeID()); + if (auto iface = dyn_cast_or_null(scope)) + return iface.getIndexBitwidth(ty, *this, list); + return detail::getDefaultIndexBitwidth(ty, *this, list); + }); +} + mlir::Attribute mlir::DataLayout::getAllocaMemorySpace() const { checkValid(); if (allocaMemorySpace) diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp index 995544238e4a..f90495d407fd 100644 --- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp @@ -274,16 +274,15 @@ translateDataLayout(DataLayoutSpecInterface attribute, layoutStream << ":" << preferred; return success(); }) - .Case([&](LLVMPointerType ptrType) { - layoutStream << "p" << ptrType.getAddressSpace() << ":"; + .Case([&](LLVMPointerType type) { + layoutStream << "p" << type.getAddressSpace() << ":"; uint64_t size = dataLayout.getTypeSizeInBits(type); uint64_t abi = dataLayout.getTypeABIAlignment(type) * 8u; uint64_t preferred = dataLayout.getTypePreferredAlignment(type) * 8u; - layoutStream << size << ":" << abi << ":" << preferred; - if (std::optional index = extractPointerSpecValue( - entry.getValue(), PtrDLEntryPos::Index)) - layoutStream << ":" << *index; + uint64_t index = *dataLayout.getTypeIndexBitwidth(type); + layoutStream << size << ":" << abi << ":" << preferred << ":" + << index; return success(); }) .Default([loc](Type type) { diff --git a/mlir/test/Dialect/LLVMIR/layout.mlir b/mlir/test/Dialect/LLVMIR/layout.mlir index 2868e1740f86..a78fb771242e 100644 --- a/mlir/test/Dialect/LLVMIR/layout.mlir +++ b/mlir/test/Dialect/LLVMIR/layout.mlir @@ -7,6 +7,7 @@ module { // CHECK: alloca_memory_space = 0 // CHECK: bitsize = 64 // CHECK: global_memory_space = 0 + // CHECK: index = 64 // CHECK: preferred = 8 // CHECK: program_memory_space = 0 // CHECK: size = 8 @@ -16,6 +17,7 @@ module { // CHECK: alloca_memory_space = 0 // CHECK: bitsize = 64 // CHECK: global_memory_space = 0 + // CHECK: index = 64 // CHECK: preferred = 8 // CHECK: program_memory_space = 0 // CHECK: size = 8 @@ -25,6 +27,7 @@ module { // CHECK: alloca_memory_space = 0 // CHECK: bitsize = 64 // CHECK: global_memory_space = 0 + // CHECK: index = 64 // CHECK: preferred = 8 // CHECK: program_memory_space = 0 // CHECK: size = 8 @@ -39,7 +42,7 @@ module { module attributes { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry : vector<3xi64>>, #dlti.dl_entry, dense<[64, 64, 64]> : vector<3xi64>>, - #dlti.dl_entry, dense<[32, 64, 64]> : vector<3xi64>>, + #dlti.dl_entry, dense<[32, 64, 64, 24]> : vector<4xi64>>, #dlti.dl_entry<"dlti.alloca_memory_space", 5 : ui64>, #dlti.dl_entry<"dlti.global_memory_space", 2 : ui64>, #dlti.dl_entry<"dlti.program_memory_space", 3 : ui64>, @@ -51,6 +54,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< // CHECK: alloca_memory_space = 5 // CHECK: bitsize = 32 // CHECK: global_memory_space = 2 + // CHECK: index = 32 // CHECK: preferred = 8 // CHECK: program_memory_space = 3 // CHECK: size = 4 @@ -60,6 +64,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< // CHECK: alloca_memory_space = 5 // CHECK: bitsize = 32 // CHECK: global_memory_space = 2 + // CHECK: index = 32 // CHECK: preferred = 8 // CHECK: program_memory_space = 3 // CHECK: size = 4 @@ -69,24 +74,17 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< // CHECK: alloca_memory_space = 5 // CHECK: bitsize = 64 // CHECK: global_memory_space = 2 + // CHECK: index = 64 // CHECK: preferred = 8 // CHECK: program_memory_space = 3 // CHECK: size = 8 // CHECK: stack_alignment = 128 "test.data_layout_query"() : () -> !llvm.ptr<5> - // CHECK: alignment = 4 - // CHECK: alloca_memory_space = 5 - // CHECK: bitsize = 32 - // CHECK: global_memory_space = 2 - // CHECK: preferred = 8 - // CHECK: program_memory_space = 3 - // CHECK: size = 4 - // CHECK: stack_alignment = 128 - "test.data_layout_query"() : () -> !llvm.ptr<3> // CHECK: alignment = 8 // CHECK: alloca_memory_space = 5 // CHECK: bitsize = 32 // CHECK: global_memory_space = 2 + // CHECK: index = 24 // CHECK: preferred = 8 // CHECK: program_memory_space = 3 // CHECK: size = 4 @@ -134,6 +132,7 @@ module { // simple case // CHECK: alignment = 4 // CHECK: bitsize = 32 + // CHECK: index = 0 // CHECK: preferred = 4 // CHECK: size = 4 "test.data_layout_query"() : () -> !llvm.struct<(i32)> @@ -141,6 +140,7 @@ module { // padding inbetween // CHECK: alignment = 8 // CHECK: bitsize = 128 + // CHECK: index = 0 // CHECK: preferred = 8 // CHECK: size = 16 "test.data_layout_query"() : () -> !llvm.struct<(i32, f64)> @@ -148,6 +148,7 @@ module { // padding at end of struct // CHECK: alignment = 8 // CHECK: bitsize = 128 + // CHECK: index = 0 // CHECK: preferred = 8 // CHECK: size = 16 "test.data_layout_query"() : () -> !llvm.struct<(f64, i32)> @@ -155,6 +156,7 @@ module { // packed // CHECK: alignment = 1 // CHECK: bitsize = 96 + // CHECK: index = 0 // CHECK: preferred = 8 // CHECK: size = 12 "test.data_layout_query"() : () -> !llvm.struct @@ -162,6 +164,7 @@ module { // empty // CHECK: alignment = 1 // CHECK: bitsize = 0 + // CHECK: index = 0 // CHECK: preferred = 1 // CHECK: size = 0 "test.data_layout_query"() : () -> !llvm.struct<()> @@ -179,6 +182,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< // Strict alignment is applied // CHECK: alignment = 4 // CHECK: bitsize = 16 + // CHECK: index = 0 // CHECK: preferred = 4 // CHECK: size = 2 "test.data_layout_query"() : () -> !llvm.struct<(i16)> @@ -186,6 +190,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< // No impact on structs that have stricter requirements // CHECK: alignment = 8 // CHECK: bitsize = 128 + // CHECK: index = 0 // CHECK: preferred = 8 // CHECK: size = 16 "test.data_layout_query"() : () -> !llvm.struct<(i32, f64)> @@ -193,6 +198,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< // Only the preferred alignment of structs is affected // CHECK: alignment = 1 // CHECK: bitsize = 32 + // CHECK: index = 0 // CHECK: preferred = 4 // CHECK: size = 4 "test.data_layout_query"() : () -> !llvm.struct @@ -200,6 +206,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< // empty // CHECK: alignment = 4 // CHECK: bitsize = 0 + // CHECK: index = 0 // CHECK: preferred = 4 // CHECK: size = 0 "test.data_layout_query"() : () -> !llvm.struct<()> @@ -265,6 +272,7 @@ module { // simple case // CHECK: alignment = 4 // CHECK: bitsize = 64 + // CHECK: index = 0 // CHECK: preferred = 4 // CHECK: size = 8 "test.data_layout_query"() : () -> !llvm.array<2 x i32> @@ -272,6 +280,7 @@ module { // size 0 // CHECK: alignment = 8 // CHECK: bitsize = 0 + // CHECK: index = 0 // CHECK: preferred = 8 // CHECK: size = 0 "test.data_layout_query"() : () -> !llvm.array<0 x f64> @@ -279,6 +288,7 @@ module { // alignment info matches element type // CHECK: alignment = 4 // CHECK: bitsize = 64 + // CHECK: index = 0 // CHECK: preferred = 8 // CHECK: size = 8 "test.data_layout_query"() : () -> !llvm.array<1 x i64> diff --git a/mlir/test/Interfaces/DataLayoutInterfaces/module.mlir b/mlir/test/Interfaces/DataLayoutInterfaces/module.mlir index 096e7ceb3cbc..97286ce75806 100644 --- a/mlir/test/Interfaces/DataLayoutInterfaces/module.mlir +++ b/mlir/test/Interfaces/DataLayoutInterfaces/module.mlir @@ -2,11 +2,13 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry, ["size", 12]>, - #dlti.dl_entry, ["alignment", 32]>>} { + #dlti.dl_entry, ["alignment", 32]>, + #dlti.dl_entry, ["index", 7]>>} { // CHECK-LABEL: @module_level_layout func.func @module_level_layout() { // CHECK: alignment = 32 // CHECK: bitsize = 12 + // CHECK: index = 7 // CHECK: preferred = 1 // CHECK: size = 2 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> diff --git a/mlir/test/Interfaces/DataLayoutInterfaces/query.mlir b/mlir/test/Interfaces/DataLayoutInterfaces/query.mlir index 9f9240ac6f8c..d3bc91339d16 100644 --- a/mlir/test/Interfaces/DataLayoutInterfaces/query.mlir +++ b/mlir/test/Interfaces/DataLayoutInterfaces/query.mlir @@ -4,24 +4,34 @@ func.func @no_layout_builtin() { // CHECK: alignment = 4 // CHECK: bitsize = 32 + // CHECK: index = 0 // CHECK: preferred = 4 // CHECK: size = 4 "test.data_layout_query"() : () -> i32 // CHECK: alignment = 8 // CHECK: bitsize = 64 + // CHECK: index = 0 // CHECK: preferred = 8 // CHECK: size = 8 "test.data_layout_query"() : () -> f64 // CHECK: alignment = 4 // CHECK: bitsize = 64 + // CHECK: index = 0 // CHECK: preferred = 4 // CHECK: size = 8 "test.data_layout_query"() : () -> complex // CHECK: alignment = 1 // CHECK: bitsize = 14 + // CHECK: index = 0 // CHECK: preferred = 1 // CHECK: size = 2 "test.data_layout_query"() : () -> complex + // CHECK: alignment = 4 + // CHECK: bitsize = 64 + // CHECK: index = 64 + // CHECK: preferred = 8 + // CHECK: size = 8 + "test.data_layout_query"() : () -> index return } @@ -30,6 +40,7 @@ func.func @no_layout_builtin() { func.func @no_layout_custom() { // CHECK: alignment = 1 // CHECK: bitsize = 1 + // CHECK: index = 1 // CHECK: preferred = 1 // CHECK: size = 1 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> @@ -41,6 +52,7 @@ func.func @layout_op_no_layout() { "test.op_with_data_layout"() ({ // CHECK: alignment = 1 // CHECK: bitsize = 1 + // CHECK: index = 1 // CHECK: preferred = 1 // CHECK: size = 1 "test.data_layout_query"() : () -> !test.test_type_with_layout<1000> @@ -54,13 +66,15 @@ func.func @layout_op() { "test.op_with_data_layout"() ({ // CHECK: alignment = 20 // CHECK: bitsize = 10 + // CHECK: index = 30 // CHECK: preferred = 1 // CHECK: size = 2 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> "test.maybe_terminator"() : () -> () }) { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry, ["size", 10]>, - #dlti.dl_entry, ["alignment", 20]> + #dlti.dl_entry, ["alignment", 20]>, + #dlti.dl_entry, ["index", 30]> >} : () -> () return } @@ -72,13 +86,15 @@ func.func @nested_inner_only() { "test.op_with_data_layout"() ({ // CHECK: alignment = 20 // CHECK: bitsize = 10 + // CHECK: index = 30 // CHECK: preferred = 1 // CHECK: size = 2 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> "test.maybe_terminator"() : () -> () }) { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry, ["size", 10]>, - #dlti.dl_entry, ["alignment", 20]> + #dlti.dl_entry, ["alignment", 20]>, + #dlti.dl_entry, ["index", 30]> >} : () -> () "test.maybe_terminator"() : () -> () }) : () -> () @@ -92,6 +108,7 @@ func.func @nested_outer_only() { "test.op_with_data_layout"() ({ // CHECK: alignment = 20 // CHECK: bitsize = 10 + // CHECK: index = 30 // CHECK: preferred = 1 // CHECK: size = 2 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> @@ -100,7 +117,8 @@ func.func @nested_outer_only() { "test.maybe_terminator"() : () -> () }) { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry, ["size", 10]>, - #dlti.dl_entry, ["alignment", 20]> + #dlti.dl_entry, ["alignment", 20]>, + #dlti.dl_entry, ["index", 30]> >} : () -> () return } @@ -112,6 +130,7 @@ func.func @nested_middle_only() { "test.op_with_data_layout"() ({ // CHECK: alignment = 20 // CHECK: bitsize = 10 + // CHECK: index = 30 // CHECK: preferred = 1 // CHECK: size = 2 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> @@ -120,7 +139,8 @@ func.func @nested_middle_only() { "test.maybe_terminator"() : () -> () }) { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry, ["size", 10]>, - #dlti.dl_entry, ["alignment", 20]> + #dlti.dl_entry, ["alignment", 20]>, + #dlti.dl_entry, ["index", 30]> >} : () -> () "test.maybe_terminator"() : () -> () }) : () -> () @@ -134,6 +154,7 @@ func.func @nested_combine_with_missing() { "test.op_with_data_layout"() ({ // CHECK: alignment = 20 // CHECK: bitsize = 10 + // CHECK: index = 21 // CHECK: preferred = 30 // CHECK: size = 2 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> @@ -146,13 +167,15 @@ func.func @nested_combine_with_missing() { >} : () -> () // CHECK: alignment = 1 // CHECK: bitsize = 42 + // CHECK: index = 21 // CHECK: preferred = 30 // CHECK: size = 6 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> "test.maybe_terminator"() : () -> () }) { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry, ["size", 42]>, - #dlti.dl_entry, ["preferred", 30]> + #dlti.dl_entry, ["preferred", 30]>, + #dlti.dl_entry, ["index", 21]> >}: () -> () return } @@ -164,6 +187,7 @@ func.func @nested_combine_all() { "test.op_with_data_layout"() ({ // CHECK: alignment = 20 // CHECK: bitsize = 3 + // CHECK: index = 40 // CHECK: preferred = 30 // CHECK: size = 1 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> @@ -174,16 +198,19 @@ func.func @nested_combine_all() { >} : () -> () // CHECK: alignment = 20 // CHECK: bitsize = 10 + // CHECK: index = 40 // CHECK: preferred = 30 // CHECK: size = 2 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> "test.maybe_terminator"() : () -> () }) { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry, ["size", 10]>, - #dlti.dl_entry, ["alignment", 20]> + #dlti.dl_entry, ["alignment", 20]>, + #dlti.dl_entry, ["index", 40]> >} : () -> () // CHECK: alignment = 1 // CHECK: bitsize = 42 + // CHECK: index = 1 // CHECK: preferred = 30 // CHECK: size = 6 "test.data_layout_query"() : () -> !test.test_type_with_layout<10> @@ -200,18 +227,22 @@ func.func @integers() { "test.op_with_data_layout"() ({ // CHECK: alignment = 8 // CHECK: bitsize = 32 + // CHECK: index = 0 // CHECK: preferred = 8 "test.data_layout_query"() : () -> i32 // CHECK: alignment = 16 // CHECK: bitsize = 56 + // CHECK: index = 0 // CHECK: preferred = 16 "test.data_layout_query"() : () -> i56 // CHECK: alignment = 16 // CHECK: bitsize = 64 + // CHECK: index = 0 // CHECK: preferred = 16 "test.data_layout_query"() : () -> i64 // CHECK: alignment = 16 // CHECK: bitsize = 128 + // CHECK: index = 0 // CHECK: preferred = 16 "test.data_layout_query"() : () -> i128 "test.maybe_terminator"() : () -> () @@ -222,18 +253,22 @@ func.func @integers() { "test.op_with_data_layout"() ({ // CHECK: alignment = 8 // CHECK: bitsize = 32 + // CHECK: index = 0 // CHECK: preferred = 16 "test.data_layout_query"() : () -> i32 // CHECK: alignment = 16 // CHECK: bitsize = 56 + // CHECK: index = 0 // CHECK: preferred = 32 "test.data_layout_query"() : () -> i56 // CHECK: alignment = 16 // CHECK: bitsize = 64 + // CHECK: index = 0 // CHECK: preferred = 32 "test.data_layout_query"() : () -> i64 // CHECK: alignment = 16 // CHECK: bitsize = 128 + // CHECK: index = 0 // CHECK: preferred = 32 "test.data_layout_query"() : () -> i128 "test.maybe_terminator"() : () -> () @@ -248,10 +283,12 @@ func.func @floats() { "test.op_with_data_layout"() ({ // CHECK: alignment = 8 // CHECK: bitsize = 32 + // CHECK: index = 0 // CHECK: preferred = 8 "test.data_layout_query"() : () -> f32 // CHECK: alignment = 16 // CHECK: bitsize = 80 + // CHECK: index = 0 // CHECK: preferred = 16 "test.data_layout_query"() : () -> f80 "test.maybe_terminator"() : () -> () @@ -262,10 +299,12 @@ func.func @floats() { "test.op_with_data_layout"() ({ // CHECK: alignment = 8 // CHECK: bitsize = 32 + // CHECK: index = 0 // CHECK: preferred = 16 "test.data_layout_query"() : () -> f32 // CHECK: alignment = 16 // CHECK: bitsize = 80 + // CHECK: index = 0 // CHECK: preferred = 32 "test.data_layout_query"() : () -> f80 "test.maybe_terminator"() : () -> () diff --git a/mlir/test/Interfaces/DataLayoutInterfaces/types.mlir b/mlir/test/Interfaces/DataLayoutInterfaces/types.mlir index 55bb1d2eac91..82ae02cf92ad 100644 --- a/mlir/test/Interfaces/DataLayoutInterfaces/types.mlir +++ b/mlir/test/Interfaces/DataLayoutInterfaces/types.mlir @@ -40,6 +40,7 @@ module @index attributes { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry>} { func.func @query() { // CHECK: bitsize = 32 + // CHECK: index = 32 "test.data_layout_query"() : () -> index return } diff --git a/mlir/test/Target/LLVMIR/data-layout.mlir b/mlir/test/Target/LLVMIR/data-layout.mlir index e61972a0dd97..881d6727e2a1 100644 --- a/mlir/test/Target/LLVMIR/data-layout.mlir +++ b/mlir/test/Target/LLVMIR/data-layout.mlir @@ -6,7 +6,7 @@ // CHECK: S128- // CHECK: i64:64:128 // CHECK: f80:128:256 -// CHECK: p0:32:64:128 +// CHECK: p0:32:64:128:32 // CHECK: p1:32:32:32:16 module attributes {dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry<"dlti.endianness", "big">, diff --git a/mlir/test/lib/Dialect/DLTI/TestDataLayoutQuery.cpp b/mlir/test/lib/Dialect/DLTI/TestDataLayoutQuery.cpp index 740562e77830..3da48ffa403e 100644 --- a/mlir/test/lib/Dialect/DLTI/TestDataLayoutQuery.cpp +++ b/mlir/test/lib/Dialect/DLTI/TestDataLayoutQuery.cpp @@ -36,19 +36,21 @@ struct TestDataLayoutQuery return; const DataLayout &layout = layouts.getAbove(op); - unsigned size = layout.getTypeSize(op.getType()); - unsigned bitsize = layout.getTypeSizeInBits(op.getType()); - unsigned alignment = layout.getTypeABIAlignment(op.getType()); - unsigned preferred = layout.getTypePreferredAlignment(op.getType()); + llvm::TypeSize size = layout.getTypeSize(op.getType()); + llvm::TypeSize bitsize = layout.getTypeSizeInBits(op.getType()); + uint64_t alignment = layout.getTypeABIAlignment(op.getType()); + uint64_t preferred = layout.getTypePreferredAlignment(op.getType()); + uint64_t index = layout.getTypeIndexBitwidth(op.getType()).value_or(0); Attribute allocaMemorySpace = layout.getAllocaMemorySpace(); Attribute programMemorySpace = layout.getProgramMemorySpace(); Attribute globalMemorySpace = layout.getGlobalMemorySpace(); - unsigned stackAlignment = layout.getStackAlignment(); + uint64_t stackAlignment = layout.getStackAlignment(); op->setAttrs( {builder.getNamedAttr("size", builder.getIndexAttr(size)), builder.getNamedAttr("bitsize", builder.getIndexAttr(bitsize)), builder.getNamedAttr("alignment", builder.getIndexAttr(alignment)), builder.getNamedAttr("preferred", builder.getIndexAttr(preferred)), + builder.getNamedAttr("index", builder.getIndexAttr(index)), builder.getNamedAttr("alloca_memory_space", allocaMemorySpace == Attribute() ? builder.getUI32IntegerAttr(0) diff --git a/mlir/test/lib/Dialect/Test/TestTypeDefs.td b/mlir/test/lib/Dialect/Test/TestTypeDefs.td index 1957845c842f..492642b711e0 100644 --- a/mlir/test/lib/Dialect/Test/TestTypeDefs.td +++ b/mlir/test/lib/Dialect/Test/TestTypeDefs.td @@ -148,7 +148,8 @@ def TestType : Test_Type<"Test", [ } def TestTypeWithLayoutType : Test_Type<"TestTypeWithLayout", [ - DeclareTypeInterfaceMethods + DeclareTypeInterfaceMethods ]> { let mnemonic = "test_type_with_layout"; let parameters = (ins "unsigned":$key); diff --git a/mlir/test/lib/Dialect/Test/TestTypes.cpp b/mlir/test/lib/Dialect/Test/TestTypes.cpp index 2f4c9b689069..7a195eb25a3b 100644 --- a/mlir/test/lib/Dialect/Test/TestTypes.cpp +++ b/mlir/test/lib/Dialect/Test/TestTypes.cpp @@ -276,6 +276,12 @@ uint64_t TestTypeWithLayoutType::getPreferredAlignment( return extractKind(params, "preferred"); } +std::optional +TestTypeWithLayoutType::getIndexBitwidth(const DataLayout &dataLayout, + DataLayoutEntryListRef params) const { + return extractKind(params, "index"); +} + bool TestTypeWithLayoutType::areCompatible( DataLayoutEntryListRef oldLayout, DataLayoutEntryListRef newLayout) const { unsigned old = extractKind(oldLayout, "alignment"); @@ -297,7 +303,7 @@ TestTypeWithLayoutType::verifyEntries(DataLayoutEntryListRef params, (void)kind; assert(kind && (kind.getValue() == "size" || kind.getValue() == "alignment" || - kind.getValue() == "preferred") && + kind.getValue() == "preferred" || kind.getValue() == "index") && "unexpected kind"); assert(llvm::isa(array.getValue().back())); } diff --git a/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp b/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp index 794e19710fad..d6b8d7392f32 100644 --- a/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp +++ b/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp @@ -345,6 +345,8 @@ TEST(DataLayout, NullSpec) { EXPECT_EQ(layout.getTypeABIAlignment(Float16Type::get(&ctx)), 16u); EXPECT_EQ(layout.getTypePreferredAlignment(IntegerType::get(&ctx, 42)), 128u); EXPECT_EQ(layout.getTypePreferredAlignment(Float16Type::get(&ctx)), 32u); + EXPECT_EQ(layout.getTypeIndexBitwidth(Float16Type::get(&ctx)), std::nullopt); + EXPECT_EQ(layout.getTypeIndexBitwidth(IndexType::get(&ctx)), 64u); EXPECT_EQ(layout.getAllocaMemorySpace(), Attribute()); EXPECT_EQ(layout.getProgramMemorySpace(), Attribute()); @@ -373,6 +375,8 @@ TEST(DataLayout, EmptySpec) { EXPECT_EQ(layout.getTypeABIAlignment(Float16Type::get(&ctx)), 16u); EXPECT_EQ(layout.getTypePreferredAlignment(IntegerType::get(&ctx, 42)), 128u); EXPECT_EQ(layout.getTypePreferredAlignment(Float16Type::get(&ctx)), 32u); + EXPECT_EQ(layout.getTypeIndexBitwidth(Float16Type::get(&ctx)), std::nullopt); + EXPECT_EQ(layout.getTypeIndexBitwidth(IndexType::get(&ctx)), 64u); EXPECT_EQ(layout.getAllocaMemorySpace(), Attribute()); EXPECT_EQ(layout.getProgramMemorySpace(), Attribute()); @@ -385,6 +389,7 @@ TEST(DataLayout, SpecWithEntries) { "dltest.op_with_layout"() { dltest.layout = #dltest.spec< #dlti.dl_entry, #dlti.dl_entry, + #dlti.dl_entry, #dlti.dl_entry<"dltest.alloca_memory_space", 5 : i32>, #dlti.dl_entry<"dltest.program_memory_space", 3 : i32>, #dlti.dl_entry<"dltest.global_memory_space", 2 : i32>, @@ -408,6 +413,8 @@ TEST(DataLayout, SpecWithEntries) { EXPECT_EQ(layout.getTypeABIAlignment(Float16Type::get(&ctx)), 8u); EXPECT_EQ(layout.getTypePreferredAlignment(IntegerType::get(&ctx, 42)), 16u); EXPECT_EQ(layout.getTypePreferredAlignment(Float16Type::get(&ctx)), 16u); + EXPECT_EQ(layout.getTypeIndexBitwidth(Float16Type::get(&ctx)), std::nullopt); + EXPECT_EQ(layout.getTypeIndexBitwidth(IndexType::get(&ctx)), 42u); EXPECT_EQ(layout.getTypeSize(IntegerType::get(&ctx, 32)), 32u); EXPECT_EQ(layout.getTypeSize(Float32Type::get(&ctx)), 32u); -- GitLab From a29e9e32c50273abffc53e3700bbc23985f0a7af Mon Sep 17 00:00:00 2001 From: Roberto Bampi Date: Thu, 21 Mar 2024 09:29:25 +0100 Subject: [PATCH 108/296] [clang-format] Add --fail-on-incomplete-format. (#84346) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the moment clang-format will return exit code 0 on incomplete results. In scripts it would sometimes be useful if clang-format would instead fail in those cases, signalling that there was something wrong with the code being formatted. --------- Co-authored-by: Björn Schäpers Co-authored-by: Owen Pan --- clang/docs/ClangFormat.rst | 1 + clang/test/Format/fail-on-incomplete.cpp | 4 ++++ clang/tools/clang-format/ClangFormat.cpp | 13 +++++++++---- 3 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 clang/test/Format/fail-on-incomplete.cpp diff --git a/clang/docs/ClangFormat.rst b/clang/docs/ClangFormat.rst index 819d9ee9f9cd..80dc38a075c8 100644 --- a/clang/docs/ClangFormat.rst +++ b/clang/docs/ClangFormat.rst @@ -61,6 +61,7 @@ to format C/C++/Java/JavaScript/JSON/Objective-C/Protobuf/C# code. --dry-run - If set, do not actually make the formatting changes --dump-config - Dump configuration options to stdout and exit. Can be used with -style option. + --fail-on-incomplete-format - If set, fail with exit code 1 on incomplete format. --fallback-style= - The name of the predefined style used as a fallback in case clang-format is invoked with -style=file, but can not find the .clang-format diff --git a/clang/test/Format/fail-on-incomplete.cpp b/clang/test/Format/fail-on-incomplete.cpp new file mode 100644 index 000000000000..ccd77af4d599 --- /dev/null +++ b/clang/test/Format/fail-on-incomplete.cpp @@ -0,0 +1,4 @@ +// RUN: not clang-format -style=LLVM -fail-on-incomplete-format %s +// RUN: clang-format -style=LLVM %s + +int a( diff --git a/clang/tools/clang-format/ClangFormat.cpp b/clang/tools/clang-format/ClangFormat.cpp index e122cea50f72..ed401135ad84 100644 --- a/clang/tools/clang-format/ClangFormat.cpp +++ b/clang/tools/clang-format/ClangFormat.cpp @@ -205,6 +205,11 @@ static cl::list FileNames(cl::Positional, cl::desc("[@] [ ...]"), cl::cat(ClangFormatCategory)); +static cl::opt FailOnIncompleteFormat( + "fail-on-incomplete-format", + cl::desc("If set, fail with exit code 1 on incomplete format."), + cl::init(false), cl::cat(ClangFormatCategory)); + namespace clang { namespace format { @@ -399,7 +404,7 @@ class ClangFormatDiagConsumer : public DiagnosticConsumer { }; // Returns true on error. -static bool format(StringRef FileName) { +static bool format(StringRef FileName, bool ErrorOnIncompleteFormat = false) { const bool IsSTDIN = FileName == "-"; if (!OutputXML && Inplace && IsSTDIN) { errs() << "error: cannot use -i when reading from stdin.\n"; @@ -535,7 +540,7 @@ static bool format(StringRef FileName) { Rewrite.getEditBuffer(ID).write(outs()); } } - return false; + return ErrorOnIncompleteFormat && !Status.FormatComplete; } } // namespace format @@ -699,7 +704,7 @@ int main(int argc, const char **argv) { } if (FileNames.empty()) - return clang::format::format("-"); + return clang::format::format("-", FailOnIncompleteFormat); if (FileNames.size() > 1 && (!Offsets.empty() || !Lengths.empty() || !LineRanges.empty())) { @@ -717,7 +722,7 @@ int main(int argc, const char **argv) { errs() << "Formatting [" << FileNo++ << "/" << FileNames.size() << "] " << FileName << "\n"; } - Error |= clang::format::format(FileName); + Error |= clang::format::format(FileName, FailOnIncompleteFormat); } return Error ? 1 : 0; } -- GitLab From 1404640533fdeda5fb381019a436e4ee84bb2174 Mon Sep 17 00:00:00 2001 From: Adrian Kuegel Date: Thu, 21 Mar 2024 08:32:05 +0000 Subject: [PATCH 109/296] [mlir][Bazel] Add target for index dialect python bindings --- .../mlir/python/BUILD.bazel | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel index 0c3ed22e7360..d6b0832f4c1a 100644 --- a/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel @@ -492,6 +492,44 @@ filegroup( ], ) +##---------------------------------------------------------------------------## +# Index dialect. +##---------------------------------------------------------------------------## + +gentbl_filegroup( + name = "IndexOpsPyGen", + tbl_outs = [ + ( + [ + "-gen-python-enum-bindings", + "-bind-dialect=index", + ], + "mlir/dialects/_index_enum_gen.py", + ), + ( + [ + "-gen-python-op-bindings", + "-bind-dialect=index", + ], + "mlir/dialects/_index_ops_gen.py", + ), + ], + tblgen = "//mlir:mlir-tblgen", + td_file = "mlir/dialects/IndexOps.td", + deps = [ + "//mlir:IndexOpsTdFiles", + "//mlir:OpBaseTdFiles", + ], +) + +filegroup( + name = "IndexOpsPyFiles", + srcs = [ + "mlir/dialects/index.py", + ":IndexOpsPyGen", + ], +) + ##---------------------------------------------------------------------------## # Math dialect. ##---------------------------------------------------------------------------## -- GitLab From b6b703b2dfc1d1ba45ebc64ed6b53a3a46f531f5 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Thu, 21 Mar 2024 14:24:06 +0530 Subject: [PATCH 110/296] AMDGPU: Infer no-agpr usage in AMDGPUAttributor (#85948) SIMachineFunctionInfo has a scan of the function body for inline asm which may use AGPRs, or callees in SIMachineFunctionInfo. Move this into the attributor, so it actually works interprocedurally. Could probably avoid most of the test churn if this bothered to avoid adding this on subtargets without AGPRs. We should also probably try to delete the MIR scan in usesAGPRs but it seems to be trickier to eliminate. --- llvm/docs/AMDGPUUsage.rst | 5 + llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp | 96 ++++++- .../Target/AMDGPU/SIMachineFunctionInfo.cpp | 30 +-- .../AMDGPU/addrspacecast-constantexpr.ll | 6 +- .../AMDGPU/amdgpu-attributor-no-agpr.ll | 255 ++++++++++++++++++ .../annotate-kernel-features-hsa-call.ll | 44 +-- .../AMDGPU/annotate-kernel-features-hsa.ll | 26 +- .../AMDGPU/annotate-kernel-features.ll | 19 +- .../AMDGPU/copy-vgpr-clobber-spill-vgpr.mir | 2 +- .../CodeGen/AMDGPU/direct-indirect-call.ll | 2 +- .../AMDGPU/duplicate-attribute-indirect.ll | 2 +- .../AMDGPU/implicitarg-offset-attributes.ll | 30 +-- .../AMDGPU/preload-kernargs-inreg-hints.ll | 20 +- .../AMDGPU/propagate-flat-work-group-size.ll | 18 +- .../CodeGen/AMDGPU/propagate-waves-per-eu.ll | 44 +-- .../AMDGPU/recursive_global_initializer.ll | 2 +- .../AMDGPU/remove-no-kernel-id-attribute.ll | 8 +- .../CodeGen/AMDGPU/simple-indirect-call.ll | 2 +- .../uniform-work-group-attribute-missing.ll | 4 +- .../AMDGPU/uniform-work-group-multistep.ll | 4 +- ...niform-work-group-nested-function-calls.ll | 4 +- ...ork-group-prevent-attribute-propagation.ll | 6 +- .../uniform-work-group-propagate-attribute.ll | 4 +- .../uniform-work-group-recursion-test.ll | 6 +- .../CodeGen/AMDGPU/uniform-work-group-test.ll | 4 +- .../CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll | 1 + 26 files changed, 485 insertions(+), 159 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/amdgpu-attributor-no-agpr.ll diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst index 29ea5005c0c4..6e6d6b157148 100644 --- a/llvm/docs/AMDGPUUsage.rst +++ b/llvm/docs/AMDGPUUsage.rst @@ -1454,6 +1454,11 @@ The AMDGPU backend supports the following LLVM IR attributes. CLANG attribute [CLANG-ATTR]_. Clang only emits this attribute when all the three numbers are >= 1. + "amdgpu-no-agpr" Indicates the function will not require allocating AGPRs. This is only + relevant on subtargets with AGPRs. The behavior is undefined if a + function which requires AGPRs is reached through any function marked + with this attribute. + ======================================= ========================================================== Calling Conventions diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp index d7f5110427ec..9bd30458bc0a 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp @@ -918,6 +918,96 @@ AAAMDWavesPerEU &AAAMDWavesPerEU::createForPosition(const IRPosition &IRP, llvm_unreachable("AAAMDWavesPerEU is only valid for function position"); } +static bool inlineAsmUsesAGPRs(const InlineAsm *IA) { + for (const auto &CI : IA->ParseConstraints()) { + for (StringRef Code : CI.Codes) { + Code.consume_front("{"); + if (Code.starts_with("a")) + return true; + } + } + + return false; +} + +struct AAAMDGPUNoAGPR + : public IRAttribute, + AAAMDGPUNoAGPR> { + AAAMDGPUNoAGPR(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {} + + static AAAMDGPUNoAGPR &createForPosition(const IRPosition &IRP, + Attributor &A) { + if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION) + return *new (A.Allocator) AAAMDGPUNoAGPR(IRP, A); + llvm_unreachable("AAAMDGPUNoAGPR is only valid for function position"); + } + + void initialize(Attributor &A) override { + Function *F = getAssociatedFunction(); + if (F->hasFnAttribute("amdgpu-no-agpr")) + indicateOptimisticFixpoint(); + } + + const std::string getAsStr(Attributor *A) const override { + return getAssumed() ? "amdgpu-no-agpr" : "amdgpu-maybe-agpr"; + } + + void trackStatistics() const override {} + + ChangeStatus updateImpl(Attributor &A) override { + // TODO: Use AACallEdges, but then we need a way to inspect asm edges. + + auto CheckForNoAGPRs = [&](Instruction &I) { + const auto &CB = cast(I); + const Value *CalleeOp = CB.getCalledOperand(); + const Function *Callee = dyn_cast(CalleeOp); + if (!Callee) { + if (const InlineAsm *IA = dyn_cast(CalleeOp)) + return !inlineAsmUsesAGPRs(IA); + return false; + } + + // Some intrinsics may use AGPRs, but if we have a choice, we are not + // required to use AGPRs. + if (Callee->isIntrinsic()) + return true; + + // TODO: Handle callsite attributes + const auto *CalleeInfo = A.getAAFor( + *this, IRPosition::function(*Callee), DepClassTy::REQUIRED); + return CalleeInfo && CalleeInfo->getAssumed(); + }; + + bool UsedAssumedInformation = false; + if (!A.checkForAllCallLikeInstructions(CheckForNoAGPRs, *this, + UsedAssumedInformation)) + return indicatePessimisticFixpoint(); + return ChangeStatus::UNCHANGED; + } + + ChangeStatus manifest(Attributor &A) override { + if (!getAssumed()) + return ChangeStatus::UNCHANGED; + LLVMContext &Ctx = getAssociatedFunction()->getContext(); + return A.manifestAttrs(getIRPosition(), + {Attribute::get(Ctx, "amdgpu-no-agpr")}); + } + + const std::string getName() const override { return "AAAMDGPUNoAGPR"; } + const char *getIdAddr() const override { return &ID; } + + /// This function should return true if the type of the \p AA is + /// AAAMDGPUNoAGPRs + static bool classof(const AbstractAttribute *AA) { + return (AA->getIdAddr() == &ID); + } + + static const char ID; +}; + +const char AAAMDGPUNoAGPR::ID = 0; + static void addPreloadKernArgHint(Function &F, TargetMachine &TM) { const GCNSubtarget &ST = TM.getSubtarget(F); for (unsigned I = 0; @@ -946,8 +1036,9 @@ static bool runImpl(Module &M, AnalysisGetter &AG, TargetMachine &TM) { DenseSet Allowed( {&AAAMDAttributes::ID, &AAUniformWorkGroupSize::ID, &AAPotentialValues::ID, &AAAMDFlatWorkGroupSize::ID, - &AAAMDWavesPerEU::ID, &AACallEdges::ID, &AAPointerInfo::ID, - &AAPotentialConstantValues::ID, &AAUnderlyingObjects::ID}); + &AAAMDWavesPerEU::ID, &AAAMDGPUNoAGPR::ID, &AACallEdges::ID, + &AAPointerInfo::ID, &AAPotentialConstantValues::ID, + &AAUnderlyingObjects::ID}); AttributorConfig AC(CGUpdater); AC.Allowed = &Allowed; @@ -963,6 +1054,7 @@ static bool runImpl(Module &M, AnalysisGetter &AG, TargetMachine &TM) { if (!F.isIntrinsic()) { A.getOrCreateAAFor(IRPosition::function(F)); A.getOrCreateAAFor(IRPosition::function(F)); + A.getOrCreateAAFor(IRPosition::function(F)); CallingConv::ID CC = F.getCallingConv(); if (!AMDGPU::isEntryFunctionCC(CC)) { A.getOrCreateAAFor(IRPosition::function(F)); diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp index 2569f40fec0e..12433dc83c48 100644 --- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp @@ -748,35 +748,7 @@ bool SIMachineFunctionInfo::initializeBaseYamlFields( } bool SIMachineFunctionInfo::mayUseAGPRs(const Function &F) const { - for (const BasicBlock &BB : F) { - for (const Instruction &I : BB) { - const auto *CB = dyn_cast(&I); - if (!CB) - continue; - - if (CB->isInlineAsm()) { - const InlineAsm *IA = dyn_cast(CB->getCalledOperand()); - for (const auto &CI : IA->ParseConstraints()) { - for (StringRef Code : CI.Codes) { - Code.consume_front("{"); - if (Code.starts_with("a")) - return true; - } - } - continue; - } - - const Function *Callee = - dyn_cast(CB->getCalledOperand()->stripPointerCasts()); - if (!Callee) - return true; - - if (Callee->getIntrinsicID() == Intrinsic::not_intrinsic) - return true; - } - } - - return false; + return !F.hasFnAttribute("amdgpu-no-agpr"); } bool SIMachineFunctionInfo::usesAGPRs(const MachineFunction &MF) const { diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll index 66034af5c351..cff9ce050667 100644 --- a/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll +++ b/llvm/test/CodeGen/AMDGPU/addrspacecast-constantexpr.ll @@ -233,9 +233,9 @@ attributes #1 = { nounwind } ; AKF_HSA: attributes #[[ATTR1]] = { nounwind } ;. ; ATTRIBUTOR_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: readwrite) } -; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ;. ; AKF_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500} ;. diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-attributor-no-agpr.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-attributor-no-agpr.ll new file mode 100644 index 000000000000..33b1cc65dc56 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-attributor-no-agpr.ll @@ -0,0 +1,255 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-attributes --check-globals all --version 4 +; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx90a -passes=amdgpu-attributor %s | FileCheck %s + +define amdgpu_kernel void @kernel_uses_asm_virtreg() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_uses_asm_virtreg( +; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: call void asm sideeffect " +; CHECK-NEXT: ret void +; + call void asm sideeffect "; use $0", "a"(i32 poison) + ret void +} + +define amdgpu_kernel void @kernel_uses_asm_virtreg_def() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_uses_asm_virtreg_def( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: [[DEF:%.*]] = call i32 asm sideeffect " +; CHECK-NEXT: ret void +; + %def = call i32 asm sideeffect "; def $0", "=a"() + ret void +} + +define amdgpu_kernel void @kernel_uses_asm_physreg_def_tuple() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_uses_asm_physreg_def_tuple( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: [[DEF:%.*]] = call i64 asm sideeffect " +; CHECK-NEXT: ret void +; + %def = call i64 asm sideeffect "; def $0", "={a[0:1]}"() + ret void +} + +define amdgpu_kernel void @kernel_uses_asm_virtreg_second_arg() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_uses_asm_virtreg_second_arg( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: call void asm sideeffect " +; CHECK-NEXT: ret void +; + call void asm sideeffect "; use $0", "v,a"(i32 poison, i32 poison) + ret void +} + +define amdgpu_kernel void @kernel_uses_non_agpr_asm() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_uses_non_agpr_asm( +; CHECK-SAME: ) #[[ATTR1:[0-9]+]] { +; CHECK-NEXT: call void asm sideeffect " +; CHECK-NEXT: ret void +; + call void asm sideeffect "; use $0", "v"(i32 poison) + ret void +} + +define amdgpu_kernel void @kernel_uses_asm_physreg() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_uses_asm_physreg( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: call void asm sideeffect " +; CHECK-NEXT: ret void +; + call void asm sideeffect "; use $0", "{a0}"(i32 poison) + ret void +} + +define amdgpu_kernel void @kernel_uses_asm_physreg_tuple() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_uses_asm_physreg_tuple( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: call void asm sideeffect " +; CHECK-NEXT: ret void +; + call void asm sideeffect "; use $0", "{a[0:1]}"(i64 poison) + ret void +} + +define void @func_uses_asm_virtreg_agpr() { +; CHECK-LABEL: define void @func_uses_asm_virtreg_agpr( +; CHECK-SAME: ) #[[ATTR2:[0-9]+]] { +; CHECK-NEXT: call void asm sideeffect " +; CHECK-NEXT: ret void +; + call void asm sideeffect "; use $0", "a"(i32 poison) + ret void +} + +define void @func_uses_asm_physreg_agpr() { +; CHECK-LABEL: define void @func_uses_asm_physreg_agpr( +; CHECK-SAME: ) #[[ATTR2]] { +; CHECK-NEXT: call void asm sideeffect " +; CHECK-NEXT: ret void +; + call void asm sideeffect "; use $0", "{a0}"(i32 poison) + ret void +} + +define void @func_uses_asm_physreg_agpr_tuple() { +; CHECK-LABEL: define void @func_uses_asm_physreg_agpr_tuple( +; CHECK-SAME: ) #[[ATTR2]] { +; CHECK-NEXT: call void asm sideeffect " +; CHECK-NEXT: ret void +; + call void asm sideeffect "; use $0", "{a[0:1]}"(i64 poison) + ret void +} + +declare void @unknown() + +define amdgpu_kernel void @kernel_calls_extern() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_extern( +; CHECK-SAME: ) #[[ATTR4:[0-9]+]] { +; CHECK-NEXT: call void @unknown() +; CHECK-NEXT: ret void +; + call void @unknown() + ret void +} + +define amdgpu_kernel void @kernel_calls_extern_marked_callsite() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_extern_marked_callsite( +; CHECK-SAME: ) #[[ATTR4]] { +; CHECK-NEXT: call void @unknown() #[[ATTR9:[0-9]+]] +; CHECK-NEXT: ret void +; + call void @unknown() #0 + ret void +} + +define amdgpu_kernel void @kernel_calls_indirect(ptr %indirect) { +; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_indirect( +; CHECK-SAME: ptr [[INDIRECT:%.*]]) #[[ATTR4]] { +; CHECK-NEXT: call void [[INDIRECT]]() +; CHECK-NEXT: ret void +; + call void %indirect() + ret void +} + +define amdgpu_kernel void @kernel_calls_indirect_marked_callsite(ptr %indirect) { +; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_indirect_marked_callsite( +; CHECK-SAME: ptr [[INDIRECT:%.*]]) #[[ATTR4]] { +; CHECK-NEXT: call void [[INDIRECT]]() #[[ATTR9]] +; CHECK-NEXT: ret void +; + call void %indirect() #0 + ret void +} + +define amdgpu_kernel void @kernel_transitively_uses_agpr_asm() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_transitively_uses_agpr_asm( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: call void @func_uses_asm_physreg_agpr() +; CHECK-NEXT: ret void +; + call void @func_uses_asm_physreg_agpr() + ret void +} + +define void @empty() { +; CHECK-LABEL: define void @empty( +; CHECK-SAME: ) #[[ATTR5:[0-9]+]] { +; CHECK-NEXT: ret void +; + ret void +} + +define void @also_empty() { +; CHECK-LABEL: define void @also_empty( +; CHECK-SAME: ) #[[ATTR5]] { +; CHECK-NEXT: ret void +; + ret void +} + +define amdgpu_kernel void @kernel_calls_empty() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_empty( +; CHECK-SAME: ) #[[ATTR1]] { +; CHECK-NEXT: call void @empty() +; CHECK-NEXT: ret void +; + call void @empty() + ret void +} + +define amdgpu_kernel void @kernel_calls_non_agpr_and_agpr() { +; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_non_agpr_and_agpr( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: call void @empty() +; CHECK-NEXT: call void @func_uses_asm_physreg_agpr() +; CHECK-NEXT: ret void +; + call void @empty() + call void @func_uses_asm_physreg_agpr() + ret void +} + +define amdgpu_kernel void @kernel_calls_generic_intrinsic(ptr %ptr0, ptr %ptr1, i64 %size) { +; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_generic_intrinsic( +; CHECK-SAME: ptr [[PTR0:%.*]], ptr [[PTR1:%.*]], i64 [[SIZE:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[PTR0]], ptr [[PTR1]], i64 [[SIZE]], i1 false) +; CHECK-NEXT: ret void +; + call void @llvm.memcpy.p0.p0.i64(ptr %ptr0, ptr %ptr1, i64 %size, i1 false) + ret void +} + +declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32 immarg, i32 immarg, i32 immarg) + +define amdgpu_kernel void @kernel_calls_mfma.f32.32x32x1f32(ptr addrspace(1) %out, float %a, float %b, <32 x float> %c) { +; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_mfma.f32.32x32x1f32( +; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], float [[A:%.*]], float [[B:%.*]], <32 x float> [[C:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[RESULT:%.*]] = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float [[A]], float [[B]], <32 x float> [[C]], i32 0, i32 0, i32 0) +; CHECK-NEXT: store <32 x float> [[RESULT]], ptr addrspace(1) [[OUT]], align 128 +; CHECK-NEXT: ret void +; + %result = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float %a, float %b, <32 x float> %c, i32 0, i32 0, i32 0) + store <32 x float> %result, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @kernel_calls_workitem_id_x(ptr addrspace(1) %out) { +; CHECK-LABEL: define amdgpu_kernel void @kernel_calls_workitem_id_x( +; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR1]] { +; CHECK-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.workitem.id.x() +; CHECK-NEXT: store i32 [[RESULT]], ptr addrspace(1) [[OUT]], align 4 +; CHECK-NEXT: ret void +; + %result = call i32 @llvm.amdgcn.workitem.id.x() + store i32 %result, ptr addrspace(1) %out + ret void +} + +define amdgpu_kernel void @indirect_calls_none_agpr(i1 %cond) { +; CHECK-LABEL: define amdgpu_kernel void @indirect_calls_none_agpr( +; CHECK-SAME: i1 [[COND:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[FPTR:%.*]] = select i1 [[COND]], ptr @empty, ptr @also_empty +; CHECK-NEXT: call void [[FPTR]]() +; CHECK-NEXT: ret void +; + %fptr = select i1 %cond, ptr @empty, ptr @also_empty + call void %fptr() + ret void +} + + +attributes #0 = { "amdgpu-no-agpr" } +;. +; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx90a" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="gfx90a" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR2]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,8" "target-cpu"="gfx90a" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR3:[0-9]+]] = { "amdgpu-waves-per-eu"="4,8" "target-cpu"="gfx90a" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR4]] = { "target-cpu"="gfx90a" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR5]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,8" "target-cpu"="gfx90a" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR6:[0-9]+]] = { convergent nocallback nofree nosync nounwind willreturn memory(none) "target-cpu"="gfx90a" } +; CHECK: attributes #[[ATTR7:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) "target-cpu"="gfx90a" } +; CHECK: attributes #[[ATTR8:[0-9]+]] = { nocallback nofree nounwind willreturn memory(argmem: readwrite) "target-cpu"="gfx90a" } +; CHECK: attributes #[[ATTR9]] = { "amdgpu-no-agpr" } +;. diff --git a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll index af0eb23d8e99..3d4ae84d9c69 100644 --- a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll +++ b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll @@ -1025,33 +1025,33 @@ attributes #6 = { "enqueued-block" } ; AKF_HSA: attributes #[[ATTR8]] = { "amdgpu-calls" } ;. ; ATTRIBUTOR_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } -; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR4]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR5]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR6]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR7]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR8]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR9]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR10]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR11]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR12]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR13]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="gfx900" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR14]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="gfx900" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR15]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR4]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR5]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR6]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR7]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR8]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR9]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR10]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR11]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR12]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="fiji" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR13]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="gfx900" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR14]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "target-cpu"="gfx900" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR15]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "target-cpu"="fiji" "uniform-work-group-size"="false" } ; ATTRIBUTOR_HSA: attributes #[[ATTR16]] = { nounwind "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR17]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR17]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ; ATTRIBUTOR_HSA: attributes #[[ATTR18]] = { nounwind "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR19]] = { nounwind sanitize_address "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR20]] = { nounwind sanitize_address "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR21]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR22]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR19]] = { nounwind sanitize_address "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR20]] = { nounwind sanitize_address "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR21]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR22]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } ; ATTRIBUTOR_HSA: attributes #[[ATTR23:[0-9]+]] = { nounwind sanitize_address "amdgpu-no-implicitarg-ptr" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ; ATTRIBUTOR_HSA: attributes #[[ATTR24:[0-9]+]] = { "amdgpu-waves-per-eu"="4,10" "enqueued-block" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR25]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "enqueued-block" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR25]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "enqueued-block" "uniform-work-group-size"="false" } ; ATTRIBUTOR_HSA: attributes #[[ATTR26]] = { "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR27]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR27]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ; ATTRIBUTOR_HSA: attributes #[[ATTR28]] = { nounwind } ; ATTRIBUTOR_HSA: attributes #[[ATTR29]] = { "enqueued-block" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll index 9a9c28ac632f..43cdf85ed381 100644 --- a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll +++ b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features-hsa.ll @@ -643,19 +643,19 @@ attributes #1 = { nounwind } ; AKF_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-stack-objects" } ;. ; ATTRIBUTOR_HSA: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } -; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR4]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR5]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR6]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR7]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR8]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR9]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR10]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR11]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR12]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_HSA: attributes #[[ATTR13]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR1]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR2]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR3]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR4]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR5]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR6]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR7]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR8]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR9]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR10]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR11]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR12]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_HSA: attributes #[[ATTR13]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ;. ; AKF_HSA: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500} ;. diff --git a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll index 6c5e58c74033..547ff69592ca 100644 --- a/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll +++ b/llvm/test/CodeGen/AMDGPU/annotate-kernel-features.ll @@ -393,17 +393,18 @@ define amdgpu_kernel void @use_get_local_size_z(ptr addrspace(1) %ptr) #1 { attributes #0 = { nounwind readnone } attributes #1 = { nounwind } +;. ; AKF_CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } ; AKF_CHECK: attributes #[[ATTR1]] = { nounwind } ;. ; ATTRIBUTOR_CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } -; ATTRIBUTOR_CHECK: attributes #[[ATTR1]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_CHECK: attributes #[[ATTR2]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_CHECK: attributes #[[ATTR3]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_CHECK: attributes #[[ATTR4]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_CHECK: attributes #[[ATTR5]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_CHECK: attributes #[[ATTR6]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "uniform-work-group-size"="false" } -; ATTRIBUTOR_CHECK: attributes #[[ATTR7]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; ATTRIBUTOR_CHECK: attributes #[[ATTR8]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" } -; ATTRIBUTOR_CHECK: attributes #[[ATTR9]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" } +; ATTRIBUTOR_CHECK: attributes #[[ATTR1]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_CHECK: attributes #[[ATTR2]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_CHECK: attributes #[[ATTR3]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_CHECK: attributes #[[ATTR4]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_CHECK: attributes #[[ATTR5]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_CHECK: attributes #[[ATTR6]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "uniform-work-group-size"="false" } +; ATTRIBUTOR_CHECK: attributes #[[ATTR7]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; ATTRIBUTOR_CHECK: attributes #[[ATTR8]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" } +; ATTRIBUTOR_CHECK: attributes #[[ATTR9]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workitem-id-x" "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/copy-vgpr-clobber-spill-vgpr.mir b/llvm/test/CodeGen/AMDGPU/copy-vgpr-clobber-spill-vgpr.mir index 895185cb41a3..577d38e65668 100644 --- a/llvm/test/CodeGen/AMDGPU/copy-vgpr-clobber-spill-vgpr.mir +++ b/llvm/test/CodeGen/AMDGPU/copy-vgpr-clobber-spill-vgpr.mir @@ -333,7 +333,7 @@ ret void } - attributes #0 = { "amdgpu-waves-per-eu"="4,4" } + attributes #0 = { "amdgpu-waves-per-eu"="4,4" "amdgpu-no-agpr" } ... --- diff --git a/llvm/test/CodeGen/AMDGPU/direct-indirect-call.ll b/llvm/test/CodeGen/AMDGPU/direct-indirect-call.ll index 0c034192869b..386f9cd3f9ce 100644 --- a/llvm/test/CodeGen/AMDGPU/direct-indirect-call.ll +++ b/llvm/test/CodeGen/AMDGPU/direct-indirect-call.ll @@ -35,6 +35,6 @@ define amdgpu_kernel void @test_direct_indirect_call() { ret void } ;. -; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ; CHECK: attributes #[[ATTR1]] = { "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/duplicate-attribute-indirect.ll b/llvm/test/CodeGen/AMDGPU/duplicate-attribute-indirect.ll index 0069370cc972..05558c555c58 100644 --- a/llvm/test/CodeGen/AMDGPU/duplicate-attribute-indirect.ll +++ b/llvm/test/CodeGen/AMDGPU/duplicate-attribute-indirect.ll @@ -42,6 +42,6 @@ attributes #0 = { "amdgpu-no-dispatch-id" } ;. ; AKF_GCN: attributes #[[ATTR0]] = { "amdgpu-calls" "amdgpu-no-dispatch-id" "amdgpu-stack-objects" } ;. -; ATTRIBUTOR_GCN: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; ATTRIBUTOR_GCN: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ; ATTRIBUTOR_GCN: attributes #[[ATTR1]] = { "amdgpu-no-dispatch-id" "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/implicitarg-offset-attributes.ll b/llvm/test/CodeGen/AMDGPU/implicitarg-offset-attributes.ll index a5792bf29ddc..4c21f8729745 100644 --- a/llvm/test/CodeGen/AMDGPU/implicitarg-offset-attributes.ll +++ b/llvm/test/CodeGen/AMDGPU/implicitarg-offset-attributes.ll @@ -258,25 +258,25 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo ;. ; V4: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } -; V4: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; V4: attributes #[[ATTR2]] = { "amdgpu-no-completion-action" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } -; V4: attributes #[[ATTR3]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } -; V4: attributes #[[ATTR4]] = { "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } -; V4: attributes #[[ATTR5]] = { "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V4: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; V4: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V4: attributes #[[ATTR3]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V4: attributes #[[ATTR4]] = { "amdgpu-no-agpr" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V4: attributes #[[ATTR5]] = { "amdgpu-no-agpr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } ;. ; V5: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } -; V5: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; V5: attributes #[[ATTR2]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } -; V5: attributes #[[ATTR3]] = { "amdgpu-no-completion-action" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } -; V5: attributes #[[ATTR4]] = { "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } -; V5: attributes #[[ATTR5]] = { "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V5: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; V5: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V5: attributes #[[ATTR3]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V5: attributes #[[ATTR4]] = { "amdgpu-no-agpr" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V5: attributes #[[ATTR5]] = { "amdgpu-no-agpr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } ;. ; V6: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } -; V6: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; V6: attributes #[[ATTR2]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } -; V6: attributes #[[ATTR3]] = { "amdgpu-no-completion-action" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } -; V6: attributes #[[ATTR4]] = { "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } -; V6: attributes #[[ATTR5]] = { "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V6: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; V6: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V6: attributes #[[ATTR3]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V6: attributes #[[ATTR4]] = { "amdgpu-no-agpr" "amdgpu-no-default-queue" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } +; V6: attributes #[[ATTR5]] = { "amdgpu-no-agpr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-multigrid-sync-arg" "uniform-work-group-size"="false" } ;. ; V4: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 400} ;. diff --git a/llvm/test/CodeGen/AMDGPU/preload-kernargs-inreg-hints.ll b/llvm/test/CodeGen/AMDGPU/preload-kernargs-inreg-hints.ll index e7488e059ee9..20edbd6c0d0f 100644 --- a/llvm/test/CodeGen/AMDGPU/preload-kernargs-inreg-hints.ll +++ b/llvm/test/CodeGen/AMDGPU/preload-kernargs-inreg-hints.ll @@ -157,27 +157,27 @@ define amdgpu_kernel void @test_preload_hint_kernel_1_call_func(ptr %0) #0 { define amdgpu_kernel void @test_preload_hint_kernel_1_call_intrinsic(i16 %0) #0 { ; NO-PRELOAD-LABEL: define {{[^@]+}}@test_preload_hint_kernel_1_call_intrinsic -; NO-PRELOAD-SAME: (i16 [[TMP0:%.*]]) #[[ATTR2]] { +; NO-PRELOAD-SAME: (i16 [[TMP0:%.*]]) #[[ATTR3:[0-9]+]] { ; NO-PRELOAD-NEXT: call void @llvm.amdgcn.set.prio(i16 [[TMP0]]) ; NO-PRELOAD-NEXT: ret void ; ; PRELOAD-1-LABEL: define {{[^@]+}}@test_preload_hint_kernel_1_call_intrinsic -; PRELOAD-1-SAME: (i16 inreg [[TMP0:%.*]]) #[[ATTR2]] { +; PRELOAD-1-SAME: (i16 inreg [[TMP0:%.*]]) #[[ATTR3:[0-9]+]] { ; PRELOAD-1-NEXT: call void @llvm.amdgcn.set.prio(i16 [[TMP0]]) ; PRELOAD-1-NEXT: ret void ; ; PRELOAD-3-LABEL: define {{[^@]+}}@test_preload_hint_kernel_1_call_intrinsic -; PRELOAD-3-SAME: (i16 inreg [[TMP0:%.*]]) #[[ATTR2]] { +; PRELOAD-3-SAME: (i16 inreg [[TMP0:%.*]]) #[[ATTR3:[0-9]+]] { ; PRELOAD-3-NEXT: call void @llvm.amdgcn.set.prio(i16 [[TMP0]]) ; PRELOAD-3-NEXT: ret void ; ; PRELOAD-16-LABEL: define {{[^@]+}}@test_preload_hint_kernel_1_call_intrinsic -; PRELOAD-16-SAME: (i16 inreg [[TMP0:%.*]]) #[[ATTR2]] { +; PRELOAD-16-SAME: (i16 inreg [[TMP0:%.*]]) #[[ATTR3:[0-9]+]] { ; PRELOAD-16-NEXT: call void @llvm.amdgcn.set.prio(i16 [[TMP0]]) ; PRELOAD-16-NEXT: ret void ; ; PRELOAD-20-LABEL: define {{[^@]+}}@test_preload_hint_kernel_1_call_intrinsic -; PRELOAD-20-SAME: (i16 inreg [[TMP0:%.*]]) #[[ATTR2]] { +; PRELOAD-20-SAME: (i16 inreg [[TMP0:%.*]]) #[[ATTR3:[0-9]+]] { ; PRELOAD-20-NEXT: call void @llvm.amdgcn.set.prio(i16 [[TMP0]]) ; PRELOAD-20-NEXT: ret void ; @@ -235,23 +235,23 @@ define amdgpu_kernel void @test_preload_hint_kernel_2_preexisting(i32 inreg %0, define amdgpu_kernel void @test_preload_hint_kernel_incompatible_attributes(ptr addrspace(4) byref(i32) %0, ptr nest %1) { ; NO-PRELOAD-LABEL: define {{[^@]+}}@test_preload_hint_kernel_incompatible_attributes -; NO-PRELOAD-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR3:[0-9]+]] { +; NO-PRELOAD-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR4:[0-9]+]] { ; NO-PRELOAD-NEXT: ret void ; ; PRELOAD-1-LABEL: define {{[^@]+}}@test_preload_hint_kernel_incompatible_attributes -; PRELOAD-1-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR3:[0-9]+]] { +; PRELOAD-1-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR4:[0-9]+]] { ; PRELOAD-1-NEXT: ret void ; ; PRELOAD-3-LABEL: define {{[^@]+}}@test_preload_hint_kernel_incompatible_attributes -; PRELOAD-3-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR3:[0-9]+]] { +; PRELOAD-3-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR4:[0-9]+]] { ; PRELOAD-3-NEXT: ret void ; ; PRELOAD-16-LABEL: define {{[^@]+}}@test_preload_hint_kernel_incompatible_attributes -; PRELOAD-16-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR3:[0-9]+]] { +; PRELOAD-16-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR4:[0-9]+]] { ; PRELOAD-16-NEXT: ret void ; ; PRELOAD-20-LABEL: define {{[^@]+}}@test_preload_hint_kernel_incompatible_attributes -; PRELOAD-20-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR3:[0-9]+]] { +; PRELOAD-20-SAME: (ptr addrspace(4) byref(i32) [[TMP0:%.*]], ptr nest [[TMP1:%.*]]) #[[ATTR4:[0-9]+]] { ; PRELOAD-20-NEXT: ret void ; ret void diff --git a/llvm/test/CodeGen/AMDGPU/propagate-flat-work-group-size.ll b/llvm/test/CodeGen/AMDGPU/propagate-flat-work-group-size.ll index d92ba7774bd3..d070dc3b770f 100644 --- a/llvm/test/CodeGen/AMDGPU/propagate-flat-work-group-size.ll +++ b/llvm/test/CodeGen/AMDGPU/propagate-flat-work-group-size.ll @@ -203,13 +203,13 @@ attributes #5 = { "amdgpu-flat-work-group-size"="128,512" } attributes #6 = { "amdgpu-flat-work-group-size"="512,512" } attributes #7 = { "amdgpu-flat-work-group-size"="64,256" } ;. -; CHECK: attributes #[[ATTR0]] = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { "amdgpu-flat-work-group-size"="64,128" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR2]] = { "amdgpu-flat-work-group-size"="128,512" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR3]] = { "amdgpu-flat-work-group-size"="64,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR4]] = { "amdgpu-flat-work-group-size"="128,128" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR5]] = { "amdgpu-flat-work-group-size"="512,512" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR6]] = { "amdgpu-flat-work-group-size"="64,256" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR7]] = { "amdgpu-flat-work-group-size"="128,256" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR8]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR0]] = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-flat-work-group-size"="64,128" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR2]] = { "amdgpu-flat-work-group-size"="128,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR3]] = { "amdgpu-flat-work-group-size"="64,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR4]] = { "amdgpu-flat-work-group-size"="128,128" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR5]] = { "amdgpu-flat-work-group-size"="512,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR6]] = { "amdgpu-flat-work-group-size"="64,256" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR7]] = { "amdgpu-flat-work-group-size"="128,256" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR8]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu.ll b/llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu.ll index 2df219bd0401..f62f1d57aec8 100644 --- a/llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu.ll +++ b/llvm/test/CodeGen/AMDGPU/propagate-waves-per-eu.ll @@ -399,26 +399,26 @@ attributes #17 = { "amdgpu-waves-per-eu"="5,8" } attributes #18 = { "amdgpu-waves-per-eu"="9,10" } attributes #19 = { "amdgpu-waves-per-eu"="8,9" } ;. -; CHECK: attributes #[[ATTR0]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,8" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,2" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR2]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,4" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR3]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,9" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR4]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,1" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR5]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,2" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR6]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="9,9" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR7]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,8" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR8]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="3,8" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR9]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR10]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR11]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="0,8" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR12]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,123" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR13]] = { "amdgpu-flat-work-group-size"="1,512" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR14]] = { "amdgpu-flat-work-group-size"="1,512" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="3,6" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR15]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="6,9" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR16]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="6,8" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR17]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="5,5" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR18]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="8,8" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR19]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="9,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR20]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="9,9" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR21]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="8,9" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR0]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,8" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,2" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR2]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,4" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR3]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,9" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR4]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,1" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR5]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,2" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR6]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="9,9" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR7]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,8" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR8]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="3,8" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR9]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR10]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR11]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="0,8" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR12]] = { "amdgpu-flat-work-group-size"="1,64" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="1,123" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR13]] = { "amdgpu-flat-work-group-size"="1,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR14]] = { "amdgpu-flat-work-group-size"="1,512" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="3,6" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR15]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="6,9" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR16]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="6,8" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR17]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="5,5" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR18]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="8,8" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR19]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="9,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR20]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="9,9" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR21]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="8,9" "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/recursive_global_initializer.ll b/llvm/test/CodeGen/AMDGPU/recursive_global_initializer.ll index eaef63bbfc3c..c1d647c5d3b9 100644 --- a/llvm/test/CodeGen/AMDGPU/recursive_global_initializer.ll +++ b/llvm/test/CodeGen/AMDGPU/recursive_global_initializer.ll @@ -19,5 +19,5 @@ define void @hoge() { ret void } ;. -; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll b/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll index 297a056526ca..384a9c4043a1 100644 --- a/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll +++ b/llvm/test/CodeGen/AMDGPU/remove-no-kernel-id-attribute.ll @@ -191,11 +191,11 @@ define amdgpu_kernel void @kernel_lds_recursion() { !1 = !{i32 1, !"amdhsa_code_object_version", i32 400} ;. -; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR2]] = { "amdgpu-lds-size"="2" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR2]] = { "amdgpu-lds-size"="2" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ; CHECK: attributes #[[ATTR3]] = { "amdgpu-lds-size"="4" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR4]] = { "amdgpu-lds-size"="2" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR4]] = { "amdgpu-lds-size"="2" "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } ; CHECK: attributes #[[ATTR5:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(none) } ; CHECK: attributes #[[ATTR6:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } ;. diff --git a/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll b/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll index f229f33664e1..539cfc71a80f 100644 --- a/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll +++ b/llvm/test/CodeGen/AMDGPU/simple-indirect-call.ll @@ -73,7 +73,7 @@ define amdgpu_kernel void @test_simple_indirect_call() { ;. ; AKF_GCN: attributes #[[ATTR0]] = { "amdgpu-calls" "amdgpu-stack-objects" } ;. -; ATTRIBUTOR_GCN: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; ATTRIBUTOR_GCN: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ; ATTRIBUTOR_GCN: attributes #[[ATTR1]] = { "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-attribute-missing.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-attribute-missing.ll index 8d5dc7943164..049db01badac 100644 --- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-attribute-missing.ll +++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-attribute-missing.ll @@ -31,6 +31,6 @@ define amdgpu_kernel void @kernel1() #1 { attributes #0 = { "uniform-work-group-size"="true" } ;. -; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-multistep.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-multistep.ll index 7a6f82d589e6..c9387f196dff 100644 --- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-multistep.ll +++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-multistep.ll @@ -98,7 +98,7 @@ define amdgpu_kernel void @kernel2() #0 { attributes #0 = { "uniform-work-group-size"="true" } ;. ; CHECK: attributes #[[ATTR0]] = { "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ; CHECK: attributes #[[ATTR2]] = { "uniform-work-group-size"="true" } -; CHECK: attributes #[[ATTR3]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="true" } +; CHECK: attributes #[[ATTR3]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="true" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-nested-function-calls.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-nested-function-calls.ll index c04154c7c23f..7183da2c5efc 100644 --- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-nested-function-calls.ll +++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-nested-function-calls.ll @@ -41,6 +41,6 @@ define amdgpu_kernel void @kernel3() #2 { attributes #2 = { "uniform-work-group-size"="true" } ;. -; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="true" } +; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="true" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-prevent-attribute-propagation.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-prevent-attribute-propagation.ll index 2d5ff045d12e..6ed04cf63d20 100644 --- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-prevent-attribute-propagation.ll +++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-prevent-attribute-propagation.ll @@ -41,7 +41,7 @@ define amdgpu_kernel void @kernel2() #2 { attributes #1 = { "uniform-work-group-size"="true" } ;. -; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="true" } -; CHECK: attributes #[[ATTR2]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="true" } +; CHECK: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-propagate-attribute.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-propagate-attribute.ll index e8bf6fc8321b..d5ba2fd617c6 100644 --- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-propagate-attribute.ll +++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-propagate-attribute.ll @@ -52,8 +52,8 @@ attributes #0 = { nounwind } attributes #1 = { "uniform-work-group-size"="false" } attributes #2 = { "uniform-work-group-size"="true" } ;. -; CHECK: attributes #[[ATTR0]] = { nounwind "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR0]] = { nounwind "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } ; CHECK: attributes #[[ATTR2]] = { nounwind "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } ; CHECK: attributes #[[ATTR3]] = { "uniform-work-group-size"="true" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-recursion-test.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-recursion-test.ll index 473eea4eedce..7f0dfeaf75c8 100644 --- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-recursion-test.ll +++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-recursion-test.ll @@ -101,7 +101,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %m) #1 { attributes #0 = { nounwind readnone } attributes #1 = { "uniform-work-group-size"="true" } ;. -; CHECK: attributes #[[ATTR0]] = { nounwind memory(none) "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { nounwind memory(none) "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="true" } -; CHECK: attributes #[[ATTR2]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="true" } +; CHECK: attributes #[[ATTR0]] = { nounwind memory(none) "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { nounwind memory(none) "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="true" } +; CHECK: attributes #[[ATTR2]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="true" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/uniform-work-group-test.ll b/llvm/test/CodeGen/AMDGPU/uniform-work-group-test.ll index 221f1a11676f..8616c73ad51c 100644 --- a/llvm/test/CodeGen/AMDGPU/uniform-work-group-test.ll +++ b/llvm/test/CodeGen/AMDGPU/uniform-work-group-test.ll @@ -61,6 +61,6 @@ define amdgpu_kernel void @kernel3() #0 { attributes #0 = { "uniform-work-group-size"="false" } ;. -; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } -; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR0]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,10" "uniform-work-group-size"="false" } +; CHECK: attributes #[[ATTR1]] = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" } ;. diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll index 717d3d975aaf..040799435db4 100644 --- a/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll +++ b/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll @@ -540,6 +540,7 @@ define internal void @use512vgprs() { } define void @foo() #0 { + call void asm sideeffect "; use $0", "a"(i32 0) ret void } -- GitLab From b433076fcbacba8a3b91446390bbea5843322bcd Mon Sep 17 00:00:00 2001 From: Antonio Frighetto Date: Thu, 7 Mar 2024 07:49:40 +0100 Subject: [PATCH 111/296] [clang][CodeGen] Allow `memcpy` replace with trivial auto var init When emitting the storage (or memory copy operations) for constant initializers, the decision whether to split a constant structure or array store into a sequence of field stores or to use `memcpy` is based upon the optimization level and the size of the initializer. In afe8b93ffdfef5d8879e1894b9d7dda40dee2b8d, we extended this by allowing constants to be split when the array (or struct) type does not match the type of data the address to the object (constant) is expected to contain. This may happen when `emitStoresForConstant` is called by `EmitAutoVarInit`, as the element type of the address gets shrunk. When this occurs, let the initializer be split into a bunch of stores only under `-ftrivial-auto-var-init=pattern`. Fixes: https://github.com/llvm/llvm-project/issues/84178. --- clang/lib/CodeGen/CGDecl.cpp | 43 ++++++++++++++--------- clang/test/CodeGen/aapcs-align.cpp | 4 +-- clang/test/CodeGen/aapcs64-align.cpp | 8 ++--- clang/test/CodeGen/attr-counted-by.c | 26 ++++---------- clang/test/CodeGenCXX/auto-var-init.cpp | 27 +++++++------- clang/test/CodeGenOpenCL/amdgpu-printf.cl | 9 +---- clang/test/OpenMP/bug54082.c | 4 +-- 7 files changed, 56 insertions(+), 65 deletions(-) diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp index dc42faf8dbb9..2ef5ed04af30 100644 --- a/clang/lib/CodeGen/CGDecl.cpp +++ b/clang/lib/CodeGen/CGDecl.cpp @@ -1242,27 +1242,38 @@ static void emitStoresForConstant(CodeGenModule &CGM, const VarDecl &D, return; } - // If the initializer is small, use a handful of stores. + // If the initializer is small or trivialAutoVarInit is set, use a handful of + // stores. + bool IsTrivialAutoVarInitPattern = + CGM.getContext().getLangOpts().getTrivialAutoVarInit() == + LangOptions::TrivialAutoVarInitKind::Pattern; if (shouldSplitConstantStore(CGM, ConstantSize)) { if (auto *STy = dyn_cast(Ty)) { - const llvm::StructLayout *Layout = - CGM.getDataLayout().getStructLayout(STy); - for (unsigned i = 0; i != constant->getNumOperands(); i++) { - CharUnits CurOff = CharUnits::fromQuantity(Layout->getElementOffset(i)); - Address EltPtr = Builder.CreateConstInBoundsByteGEP( - Loc.withElementType(CGM.Int8Ty), CurOff); - emitStoresForConstant(CGM, D, EltPtr, isVolatile, Builder, - constant->getAggregateElement(i), IsAutoInit); + if (STy == Loc.getElementType() || + (STy != Loc.getElementType() && IsTrivialAutoVarInitPattern)) { + const llvm::StructLayout *Layout = + CGM.getDataLayout().getStructLayout(STy); + for (unsigned i = 0; i != constant->getNumOperands(); i++) { + CharUnits CurOff = + CharUnits::fromQuantity(Layout->getElementOffset(i)); + Address EltPtr = Builder.CreateConstInBoundsByteGEP( + Loc.withElementType(CGM.Int8Ty), CurOff); + emitStoresForConstant(CGM, D, EltPtr, isVolatile, Builder, + constant->getAggregateElement(i), IsAutoInit); + } + return; } - return; } else if (auto *ATy = dyn_cast(Ty)) { - for (unsigned i = 0; i != ATy->getNumElements(); i++) { - Address EltPtr = Builder.CreateConstGEP( - Loc.withElementType(ATy->getElementType()), i); - emitStoresForConstant(CGM, D, EltPtr, isVolatile, Builder, - constant->getAggregateElement(i), IsAutoInit); + if (ATy == Loc.getElementType() || + (ATy != Loc.getElementType() && IsTrivialAutoVarInitPattern)) { + for (unsigned i = 0; i != ATy->getNumElements(); i++) { + Address EltPtr = Builder.CreateConstGEP( + Loc.withElementType(ATy->getElementType()), i); + emitStoresForConstant(CGM, D, EltPtr, isVolatile, Builder, + constant->getAggregateElement(i), IsAutoInit); + } + return; } - return; } } diff --git a/clang/test/CodeGen/aapcs-align.cpp b/clang/test/CodeGen/aapcs-align.cpp index 2886a32974b0..4f393d9e6b7f 100644 --- a/clang/test/CodeGen/aapcs-align.cpp +++ b/clang/test/CodeGen/aapcs-align.cpp @@ -134,8 +134,8 @@ void g6() { f6m(1, 2, 3, 4, 5, s); } // CHECK: define{{.*}} void @g6 -// CHECK: call void @f6(i32 noundef 1, [4 x i32] [i32 6, i32 7, i32 0, i32 0]) -// CHECK: call void @f6m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [4 x i32] [i32 6, i32 7, i32 0, i32 0]) +// CHECK: call void @f6(i32 noundef 1, [4 x i32] [i32 6, i32 7, i32 0, i32 undef]) +// CHECK: call void @f6m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [4 x i32] [i32 6, i32 7, i32 0, i32 undef]) // CHECK: declare void @f6(i32 noundef, [4 x i32]) // CHECK: declare void @f6m(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, [4 x i32]) } diff --git a/clang/test/CodeGen/aapcs64-align.cpp b/clang/test/CodeGen/aapcs64-align.cpp index 759413cbc4b5..de231f2123b9 100644 --- a/clang/test/CodeGen/aapcs64-align.cpp +++ b/clang/test/CodeGen/aapcs64-align.cpp @@ -75,8 +75,8 @@ void g4() { f4m(1, 2, 3, 4, 5, s); } // CHECK: define{{.*}} void @g4() -// CHECK: call void @f4(i32 noundef 1, [2 x i64] %{{.*}}) -// CHECK: void @f4m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [2 x i64] %{{.*}}) +// CHECK: call void @f4(i32 noundef 1, [2 x i64] [i64 30064771078, i64 0]) +// CHECK: void @f4m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [2 x i64] [i64 30064771078, i64 0]) // CHECK: declare void @f4(i32 noundef, [2 x i64]) // CHECK: declare void @f4m(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, [2 x i64]) @@ -95,8 +95,8 @@ void f5m(int, int, int, int, int, P16); f5m(1, 2, 3, 4, 5, s); } // CHECK: define{{.*}} void @g5() -// CHECK: call void @f5(i32 noundef 1, [2 x i64] %{{.*}}) -// CHECK: void @f5m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [2 x i64] %{{.*}}) +// CHECK: call void @f5(i32 noundef 1, [2 x i64] [i64 30064771078, i64 0]) +// CHECK: void @f5m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [2 x i64] [i64 30064771078, i64 0]) // CHECK: declare void @f5(i32 noundef, [2 x i64]) // CHECK: declare void @f5m(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, [2 x i64]) diff --git a/clang/test/CodeGen/attr-counted-by.c b/clang/test/CodeGen/attr-counted-by.c index e5685e39173b..1fb39f9a3466 100644 --- a/clang/test/CodeGen/attr-counted-by.c +++ b/clang/test/CodeGen/attr-counted-by.c @@ -1314,17 +1314,10 @@ int test14(int idx) { // NO-SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test15( // NO-SANITIZE-WITH-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR4]] { // NO-SANITIZE-WITH-ATTR-NEXT: entry: -// NO-SANITIZE-WITH-ATTR-NEXT: [[FOO:%.*]] = alloca [[STRUCT_ANON_8:%.*]], align 4 -// NO-SANITIZE-WITH-ATTR-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[FOO]]) #[[ATTR12]] -// NO-SANITIZE-WITH-ATTR-NEXT: store i32 1, ptr [[FOO]], align 4 -// NO-SANITIZE-WITH-ATTR-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[FOO]], i64 4 -// NO-SANITIZE-WITH-ATTR-NEXT: store i32 2, ptr [[TMP0]], align 4 -// NO-SANITIZE-WITH-ATTR-NEXT: [[BLAH:%.*]] = getelementptr inbounds i8, ptr [[FOO]], i64 8 // NO-SANITIZE-WITH-ATTR-NEXT: [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64 -// NO-SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[BLAH]], i64 0, i64 [[IDXPROM]] -// NO-SANITIZE-WITH-ATTR-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]] -// NO-SANITIZE-WITH-ATTR-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[FOO]]) #[[ATTR12]] -// NO-SANITIZE-WITH-ATTR-NEXT: ret i32 [[TMP1]] +// NO-SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr getelementptr inbounds ([[STRUCT_ANON_8:%.*]], ptr @__const.test15.foo, i64 1, i32 0), i64 0, i64 [[IDXPROM]] +// NO-SANITIZE-WITH-ATTR-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]] +// NO-SANITIZE-WITH-ATTR-NEXT: ret i32 [[TMP0]] // // SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test15( // SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR0]] { @@ -1342,17 +1335,10 @@ int test14(int idx) { // NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test15( // NO-SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR1]] { // NO-SANITIZE-WITHOUT-ATTR-NEXT: entry: -// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[FOO:%.*]] = alloca [[STRUCT_ANON_8:%.*]], align 4 -// NO-SANITIZE-WITHOUT-ATTR-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[FOO]]) #[[ATTR9]] -// NO-SANITIZE-WITHOUT-ATTR-NEXT: store i32 1, ptr [[FOO]], align 4 -// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[FOO]], i64 4 -// NO-SANITIZE-WITHOUT-ATTR-NEXT: store i32 2, ptr [[TMP0]], align 4 -// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[BLAH:%.*]] = getelementptr inbounds i8, ptr [[FOO]], i64 8 // NO-SANITIZE-WITHOUT-ATTR-NEXT: [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64 -// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr [[BLAH]], i64 0, i64 [[IDXPROM]] -// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]] -// NO-SANITIZE-WITHOUT-ATTR-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[FOO]]) #[[ATTR9]] -// NO-SANITIZE-WITHOUT-ATTR-NEXT: ret i32 [[TMP1]] +// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr getelementptr inbounds ([[STRUCT_ANON_8:%.*]], ptr @__const.test15.foo, i64 1, i32 0), i64 0, i64 [[IDXPROM]] +// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]] +// NO-SANITIZE-WITHOUT-ATTR-NEXT: ret i32 [[TMP0]] // int test15(int idx) { struct { diff --git a/clang/test/CodeGenCXX/auto-var-init.cpp b/clang/test/CodeGenCXX/auto-var-init.cpp index 991eb73fe45c..7803ed5b633f 100644 --- a/clang/test/CodeGenCXX/auto-var-init.cpp +++ b/clang/test/CodeGenCXX/auto-var-init.cpp @@ -1,8 +1,8 @@ // RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks %s -emit-llvm -o - | FileCheck %s -check-prefixes=CHECK,CHECK-O0 // RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks -ftrivial-auto-var-init=pattern %s -emit-llvm -o - | FileCheck %s -check-prefixes=CHECK-O0,PATTERN,PATTERN-O0 -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks -ftrivial-auto-var-init=pattern %s -O1 -emit-llvm -o - | FileCheck %s -check-prefixes=CHECK-O1,PATTERN,PATTERN-O1 +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks -ftrivial-auto-var-init=pattern %s -O1 -emit-llvm -o - | FileCheck %s -check-prefixes=PATTERN,PATTERN-O1 // RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks -ftrivial-auto-var-init=zero %s -emit-llvm -o - | FileCheck %s -check-prefixes=CHECK-O0,ZERO,ZERO-O0 -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks -ftrivial-auto-var-init=zero %s -O1 -emit-llvm -o - | FileCheck %s -check-prefixes=CHECK-O1,ZERO,ZERO-O1 +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks -ftrivial-auto-var-init=zero %s -O1 -emit-llvm -o - | FileCheck %s -check-prefixes=ZERO,ZERO-O1 // RUN: %clang_cc1 -std=c++14 -triple i386-unknown-unknown -fblocks -ftrivial-auto-var-init=pattern %s -emit-llvm -o - | FileCheck %s -check-prefixes=CHECK-O0,PATTERN,PATTERN-O0 #pragma clang diagnostic ignored "-Winaccessible-base" @@ -1303,9 +1303,10 @@ TEST_CUSTOM(semivolatile, semivolatile, { 0x44444444, 0x44444444 }); // CHECK-O0: call void @llvm.memcpy // CHECK-NOT: !annotation // CHECK-O0: call void @{{.*}}used{{.*}}%custom) -// CHECK-O1: store i32 1145324612, ptr %custom, align 4 -// CHECK-O1-NEXT: %[[I:[^ ]*]] = getelementptr inbounds i8, ptr %custom, i64 4 -// CHECK-O1-NEXT: store i32 1145324612, ptr %[[I]], align 4 +// PATTERN-O1: store i32 1145324612, ptr %custom, align 4 +// PATTERN-O1-NEXT: %[[I:[^ ]*]] = getelementptr inbounds i8, ptr %custom, i64 4 +// PATTERN-O1-NEXT: store i32 1145324612, ptr %[[I]], align 4 +// ZERO-O1: store i64 4919131752989213764, ptr %custom, align 8 // CHECK-NOT: !annotation TEST_UNINIT(semivolatileinit, semivolatileinit); @@ -1418,7 +1419,8 @@ TEST_CUSTOM(matching, matching, { .f = 0xf00f }); // CHECK-O0: call void @llvm.memcpy // CHECK-NOT: !annotation // CHECK-O0: call void @{{.*}}used{{.*}}%custom) -// CHECK-O1: store float 6.145500e+04, ptr {{.*}}, align 4 +// PATTERN-O1: store float 6.145500e+04, ptr {{.*}}, align 4 +// ZERO-O1: store i32 1198526208, ptr %custom, align 4 // CHECK-NOT: !annotation TEST_UNINIT(matchingreverse, matchingreverse); @@ -1445,7 +1447,8 @@ TEST_CUSTOM(matchingreverse, matchingreverse, { .i = 0xf00f }); // CHECK-O0: call void @llvm.memcpy // CHECK-NOT: !annotation // CHECK-O0: call void @{{.*}}used{{.*}}%custom) -// CHECK-O1: store i32 61455, ptr %custom, align 4 +// PATTERN-O1: store i32 61455, ptr %custom, align 4 +// ZERO-O1: store i32 61455, ptr %custom, align 4 // CHECK-NOT: !annotation TEST_UNINIT(unmatched, unmatched); @@ -1471,7 +1474,8 @@ TEST_CUSTOM(unmatched, unmatched, { .i = 0x3badbeef }); // CHECK-O0: call void @llvm.memcpy // CHECK-NOT: !annotation // CHECK-O0: call void @{{.*}}used{{.*}}%custom) -// CHECK-O1: store i32 1001242351, ptr {{.*}}, align 4 +// PATTERN-O1: store i32 1001242351, ptr {{.*}}, align 4 +// ZERO-O1: store i32 1001242351, ptr {{.*}}, align 4 // CHECK-NOT: !annotation TEST_UNINIT(unmatchedreverse, unmatchedreverse); @@ -1504,9 +1508,7 @@ TEST_CUSTOM(unmatchedreverse, unmatchedreverse, { .c = 42 }); // PATTERN-O1-NEXT: store i8 -86, ptr %[[I]], align {{.*}} // PATTERN-O1-NEXT: %[[I:[^ ]*]] = getelementptr inbounds i8, ptr %custom, i64 3 // PATTERN-O1-NEXT: store i8 -86, ptr %[[I]], align {{.*}} -// ZERO-O1: store i8 42, ptr {{.*}}, align 4 -// ZERO-O1-NEXT: %[[I:[^ ]*]] = getelementptr inbounds i8, ptr %custom, i64 1 -// ZERO-O1-NEXT: call void @llvm.memset.{{.*}}({{.*}}, i8 0, i64 3, {{.*}}) +// ZERO-O1: store i32 42, ptr {{.*}}, align 4 TEST_UNINIT(unmatchedfp, unmatchedfp); // CHECK-LABEL: @test_unmatchedfp_uninit() @@ -1531,7 +1533,8 @@ TEST_CUSTOM(unmatchedfp, unmatchedfp, { .d = 3.1415926535897932384626433 }); // CHECK-O0: call void @llvm.memcpy // CHECK-NOT: !annotation // CHECK-O0: call void @{{.*}}used{{.*}}%custom) -// CHECK-O1: store double 0x400921FB54442D18, ptr %custom, align 8 +// PATTERN-O1: store double 0x400921FB54442D18, ptr %custom, align 8 +// ZERO-O1: store i64 4614256656552045848, ptr %custom, align 8 // CHECK-NOT: !annotation TEST_UNINIT(emptyenum, emptyenum); diff --git a/clang/test/CodeGenOpenCL/amdgpu-printf.cl b/clang/test/CodeGenOpenCL/amdgpu-printf.cl index 6c84485b66b4..edf6dbf8657c 100644 --- a/clang/test/CodeGenOpenCL/amdgpu-printf.cl +++ b/clang/test/CodeGenOpenCL/amdgpu-printf.cl @@ -30,14 +30,7 @@ __kernel void test_printf_int(int i) { // CHECK-NEXT: [[S:%.*]] = alloca [4 x i8], align 1, addrspace(5) // CHECK-NEXT: store i32 [[I:%.*]], ptr addrspace(5) [[I_ADDR]], align 4, !tbaa [[TBAA8]] // CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[S]]) #[[ATTR5:[0-9]+]] -// CHECK-NEXT: [[LOC0:%.*]] = getelementptr i8, ptr addrspace(5) [[S]], i64 0 -// CHECK-NEXT: store i8 102, ptr addrspace(5) [[LOC0]], align 1 -// CHECK-NEXT: [[LOC1:%.*]] = getelementptr i8, ptr addrspace(5) [[S]], i64 1 -// CHECK-NEXT: store i8 111, ptr addrspace(5) [[LOC1]], align 1 -// CHECK-NEXT: [[LOC2:%.*]] = getelementptr i8, ptr addrspace(5) [[S]], i64 2 -// CHECK-NEXT: store i8 111, ptr addrspace(5) [[LOC2]], align 1 -// CHECK-NEXT: [[LOC3:%.*]] = getelementptr i8, ptr addrspace(5) [[S]], i64 3 -// CHECK-NEXT: store i8 0, ptr addrspace(5) [[LOC3]], align 1 +// CHECK-NEXT: call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) align 1 [[S]], ptr addrspace(4) align 1 @__const.test_printf_str_int.s, i64 4, i1 false) // CHECK-NEXT: [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i8], ptr addrspace(5) [[S]], i64 0, i64 0 // CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(5) [[I_ADDR]], align 4, !tbaa [[TBAA8]] // CHECK-NEXT: [[CALL:%.*]] = call i32 (ptr addrspace(4), ...) @printf(ptr addrspace(4) noundef @.str.2, ptr addrspace(5) noundef [[ARRAYDECAY]], i32 noundef [[TMP2]]) #[[ATTR4]] diff --git a/clang/test/OpenMP/bug54082.c b/clang/test/OpenMP/bug54082.c index b88b68fd4301..337c120983e0 100644 --- a/clang/test/OpenMP/bug54082.c +++ b/clang/test/OpenMP/bug54082.c @@ -69,9 +69,7 @@ void foo() { // CHECK-NEXT: [[X_TRAITS:%.*]] = alloca [1 x %struct.omp_alloctrait_t], align 16 // CHECK-NEXT: [[X_ALLOC:%.*]] = alloca i64, align 8 // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[X_TRAITS]]) #[[ATTR5:[0-9]+]] -// CHECK-NEXT: store i32 2, ptr [[X_TRAITS]], align 16 -// CHECK-NEXT: [[LOC0:%.*]] = getelementptr inbounds i8, ptr [[X_TRAITS]], i64 8 -// CHECK-NEXT: store i64 64, ptr [[LOC0]], align 8 +// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(16) [[X_TRAITS]], ptr noundef nonnull align 16 dereferenceable(16) @__const.foo.x_traits, i64 16, i1 false) // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[X_ALLOC]]) #[[ATTR5]] // CHECK-NEXT: [[CALL:%.*]] = call i64 @omp_init_allocator(i64 noundef 0, i32 noundef 1, ptr noundef nonnull [[X_TRAITS]]) #[[ATTR5]] // CHECK-NEXT: store i64 [[CALL]], ptr [[X_ALLOC]], align 8, !tbaa [[TBAA3:![0-9]+]] -- GitLab From 9fb85b09946122aa5793b647d7939ac17817c5f5 Mon Sep 17 00:00:00 2001 From: Christian Sigg Date: Thu, 21 Mar 2024 10:07:03 +0100 Subject: [PATCH 112/296] [mlir][bazel] Update BUILD after 29bf32efbb646b2ab3dec25f100419fc75635878. --- utils/bazel/llvm-project-overlay/llvm/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel index 4802daa66286..07c5a00c07d7 100644 --- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel @@ -2585,6 +2585,7 @@ cc_library( hdrs = glob([ "include/llvm/Passes/*.h", "include/llvm/Passes/*.def", + "include/llvm/Passes/*.inc", ]) + ["include/llvm-c/Transforms/PassBuilder.h"], copts = llvm_copts, deps = [ -- GitLab From 7b5a5be2a7216906c20f9bcac2209ea3502a7a73 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 21 Mar 2024 09:17:55 +0000 Subject: [PATCH 113/296] [DAG] visitSUB/visitSUBO - move getAsNonOpaqueConstant into the if() where its used. NFC. Noticed while beginning some cleanup for moving to pattern matchers --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index f199625bf67a..188472219882 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -3725,13 +3725,10 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { if (SDValue NewSel = foldBinOpIntoSelect(N)) return NewSel; - ConstantSDNode *N1C = getAsNonOpaqueConstant(N1); - // fold (sub x, c) -> (add x, -c) - if (N1C) { + if (ConstantSDNode *N1C = getAsNonOpaqueConstant(N1)) return DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getConstant(-N1C->getAPIntValue(), DL, VT)); - } if (isNullOrNullSplat(N0)) { unsigned BitWidth = VT.getScalarSizeInBits(); @@ -4131,13 +4128,11 @@ SDValue DAGCombiner::visitSUBO(SDNode *N) { return CombineTo(N, DAG.getConstant(0, DL, VT), DAG.getConstant(0, DL, CarryVT)); - ConstantSDNode *N1C = getAsNonOpaqueConstant(N1); - // fold (subox, c) -> (addo x, -c) - if (IsSigned && N1C && !N1C->isMinSignedValue()) { - return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, - DAG.getConstant(-N1C->getAPIntValue(), DL, VT)); - } + if (ConstantSDNode *N1C = getAsNonOpaqueConstant(N1)) + if (IsSigned && !N1C->isMinSignedValue()) + return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, + DAG.getConstant(-N1C->getAPIntValue(), DL, VT)); // fold (subo x, 0) -> x + no borrow if (isNullOrNullSplat(N1)) -- GitLab From ee5e027cc64957c0e18b8c38ce10d4b84314511c Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 21 Mar 2024 09:29:39 +0000 Subject: [PATCH 114/296] [X86] getShuffleCost - recognise concat_vector(X,Y) shuffle as InsertSubvector instead of PermuteTwoSrc We don't have a concat_vector shuffle kind and improveShuffleKindFromMask won't alter the base type to match it as InsertSubvector. But since this is how X86 will lower concat_vector anyhow, just recognise it explicitly. Another step for #67803 --- .../lib/Target/X86/X86TargetTransformInfo.cpp | 8 ++++ .../Transforms/PhaseOrdering/X86/pr67803.ll | 43 +++---------------- 2 files changed, 14 insertions(+), 37 deletions(-) diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index d336ab9d309c..e9bdc6ab5bea 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -1480,6 +1480,14 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Kind = improveShuffleKindFromMask(Kind, Mask, BaseTp, Index, SubTp); + // Recognize a basic concat_vector shuffle. + if (Kind == TTI::SK_PermuteTwoSrc && + Mask.size() == (2 * BaseTp->getElementCount().getKnownMinValue()) && + ShuffleVectorInst::isIdentityMask(Mask, Mask.size())) + return getShuffleCost(TTI::SK_InsertSubvector, + VectorType::getDoubleElementsVectorType(BaseTp), Mask, + CostKind, Mask.size() / 2, BaseTp); + // Treat Transpose as 2-op shuffles - there's no difference in lowering. if (Kind == TTI::SK_Transpose) Kind = TTI::SK_PermuteTwoSrc; diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll index e61b254b7a5f..495ec0a63399 100644 --- a/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll +++ b/llvm/test/Transforms/PhaseOrdering/X86/pr67803.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK -; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK -; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX512 +; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s +; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s +; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s define <4 x i64> @PR67803(<4 x i64> %x, <4 x i64> %y, <4 x i64> %a, <4 x i64> %b) { ; CHECK-LABEL: @PR67803( @@ -11,16 +11,14 @@ define <4 x i64> @PR67803(<4 x i64> %x, <4 x i64> %y, <4 x i64> %a, <4 x i64> %b ; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <8 x i32> [[TMP0]], [[TMP1]] ; CHECK-NEXT: [[CMP_I21:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> ; CHECK-NEXT: [[SEXT_I22:%.*]] = sext <4 x i1> [[CMP_I21]] to <4 x i32> -; CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[SEXT_I22]] to <2 x i64> ; CHECK-NEXT: [[CMP_I:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> ; CHECK-NEXT: [[SEXT_I:%.*]] = sext <4 x i1> [[CMP_I]] to <4 x i32> -; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x i32> [[SEXT_I]] to <2 x i64> -; CHECK-NEXT: [[SHUFFLE_I:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> +; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[SEXT_I22]], <4 x i32> [[SEXT_I]], <8 x i32> ; CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8> ; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <32 x i8> [[TMP5]], <32 x i8> poison, <16 x i32> ; CHECK-NEXT: [[TMP7:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8> ; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <32 x i8> [[TMP7]], <32 x i8> poison, <16 x i32> -; CHECK-NEXT: [[TMP9:%.*]] = bitcast <4 x i64> [[SHUFFLE_I]] to <32 x i8> +; CHECK-NEXT: [[TMP9:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8> ; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <32 x i8> [[TMP9]], <32 x i8> poison, <16 x i32> ; CHECK-NEXT: [[TMP11:%.*]] = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[TMP6]], <16 x i8> [[TMP8]], <16 x i8> [[TMP10]]) ; CHECK-NEXT: [[TMP12:%.*]] = bitcast <16 x i8> [[TMP11]] to <2 x i64> @@ -28,42 +26,13 @@ define <4 x i64> @PR67803(<4 x i64> %x, <4 x i64> %y, <4 x i64> %a, <4 x i64> %b ; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <32 x i8> [[TMP13]], <32 x i8> poison, <16 x i32> ; CHECK-NEXT: [[TMP15:%.*]] = bitcast <4 x i64> [[B]] to <32 x i8> ; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <32 x i8> [[TMP15]], <32 x i8> poison, <16 x i32> -; CHECK-NEXT: [[TMP17:%.*]] = bitcast <4 x i64> [[SHUFFLE_I]] to <32 x i8> +; CHECK-NEXT: [[TMP17:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8> ; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <32 x i8> [[TMP17]], <32 x i8> poison, <16 x i32> ; CHECK-NEXT: [[TMP19:%.*]] = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[TMP14]], <16 x i8> [[TMP16]], <16 x i8> [[TMP18]]) ; CHECK-NEXT: [[TMP20:%.*]] = bitcast <16 x i8> [[TMP19]] to <2 x i64> ; CHECK-NEXT: [[SHUFFLE_I23:%.*]] = shufflevector <2 x i64> [[TMP12]], <2 x i64> [[TMP20]], <4 x i32> ; CHECK-NEXT: ret <4 x i64> [[SHUFFLE_I23]] ; -; AVX512-LABEL: @PR67803( -; AVX512-NEXT: entry: -; AVX512-NEXT: [[TMP0:%.*]] = bitcast <4 x i64> [[X:%.*]] to <8 x i32> -; AVX512-NEXT: [[TMP1:%.*]] = bitcast <4 x i64> [[Y:%.*]] to <8 x i32> -; AVX512-NEXT: [[TMP2:%.*]] = icmp sgt <8 x i32> [[TMP0]], [[TMP1]] -; AVX512-NEXT: [[CMP_I21:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> -; AVX512-NEXT: [[SEXT_I22:%.*]] = sext <4 x i1> [[CMP_I21]] to <4 x i32> -; AVX512-NEXT: [[CMP_I:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <4 x i32> -; AVX512-NEXT: [[SEXT_I:%.*]] = sext <4 x i1> [[CMP_I]] to <4 x i32> -; AVX512-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[SEXT_I22]], <4 x i32> [[SEXT_I]], <8 x i32> -; AVX512-NEXT: [[TMP4:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8> -; AVX512-NEXT: [[TMP5:%.*]] = shufflevector <32 x i8> [[TMP4]], <32 x i8> poison, <16 x i32> -; AVX512-NEXT: [[TMP6:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8> -; AVX512-NEXT: [[TMP7:%.*]] = shufflevector <32 x i8> [[TMP6]], <32 x i8> poison, <16 x i32> -; AVX512-NEXT: [[TMP8:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8> -; AVX512-NEXT: [[TMP9:%.*]] = shufflevector <32 x i8> [[TMP8]], <32 x i8> poison, <16 x i32> -; AVX512-NEXT: [[TMP10:%.*]] = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[TMP5]], <16 x i8> [[TMP7]], <16 x i8> [[TMP9]]) -; AVX512-NEXT: [[TMP11:%.*]] = bitcast <16 x i8> [[TMP10]] to <2 x i64> -; AVX512-NEXT: [[TMP12:%.*]] = bitcast <4 x i64> [[A]] to <32 x i8> -; AVX512-NEXT: [[TMP13:%.*]] = shufflevector <32 x i8> [[TMP12]], <32 x i8> poison, <16 x i32> -; AVX512-NEXT: [[TMP14:%.*]] = bitcast <4 x i64> [[B]] to <32 x i8> -; AVX512-NEXT: [[TMP15:%.*]] = shufflevector <32 x i8> [[TMP14]], <32 x i8> poison, <16 x i32> -; AVX512-NEXT: [[TMP16:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8> -; AVX512-NEXT: [[TMP17:%.*]] = shufflevector <32 x i8> [[TMP16]], <32 x i8> poison, <16 x i32> -; AVX512-NEXT: [[TMP18:%.*]] = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[TMP13]], <16 x i8> [[TMP15]], <16 x i8> [[TMP17]]) -; AVX512-NEXT: [[TMP19:%.*]] = bitcast <16 x i8> [[TMP18]] to <2 x i64> -; AVX512-NEXT: [[SHUFFLE_I23:%.*]] = shufflevector <2 x i64> [[TMP11]], <2 x i64> [[TMP19]], <4 x i32> -; AVX512-NEXT: ret <4 x i64> [[SHUFFLE_I23]] -; entry: %0 = bitcast <4 x i64> %x to <8 x i32> %extract = shufflevector <8 x i32> %0, <8 x i32> poison, <4 x i32> -- GitLab From a6a9215b93bcbf901cd11d2dd02cce1a245d3ffe Mon Sep 17 00:00:00 2001 From: Johannes Reifferscheid Date: Thu, 21 Mar 2024 10:33:49 +0100 Subject: [PATCH 115/296] Lower shuffle to single-result form if possible. (#84321) We currently always lower shuffle to the struct-returning variant. I saw some cases where this survived all the way through ptx, resulting in increased register usage. The easiest fix is to simply lower to the single-result version when the predicate is unused. --- .../GPUToNVVM/LowerGpuOpsToNVVMOps.cpp | 23 ++++++++---- .../Conversion/GPUToNVVM/gpu-to-nvvm.mlir | 36 +++++++++++++++++-- 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp index d6a5d8cd74d5..b95fba20a00c 100644 --- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp +++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp @@ -155,8 +155,6 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern { auto valueTy = adaptor.getValue().getType(); auto int32Type = IntegerType::get(rewriter.getContext(), 32); auto predTy = IntegerType::get(rewriter.getContext(), 1); - auto resultTy = LLVM::LLVMStructType::getLiteral(rewriter.getContext(), - {valueTy, predTy}); Value one = rewriter.create(loc, int32Type, 1); Value minusOne = rewriter.create(loc, int32Type, -1); @@ -176,14 +174,25 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern { rewriter.create(loc, int32Type, adaptor.getWidth(), one); } - auto returnValueAndIsValidAttr = rewriter.getUnitAttr(); + bool predIsUsed = !op->getResult(1).use_empty(); + UnitAttr returnValueAndIsValidAttr = nullptr; + Type resultTy = valueTy; + if (predIsUsed) { + returnValueAndIsValidAttr = rewriter.getUnitAttr(); + resultTy = LLVM::LLVMStructType::getLiteral(rewriter.getContext(), + {valueTy, predTy}); + } Value shfl = rewriter.create( loc, resultTy, activeMask, adaptor.getValue(), adaptor.getOffset(), maskAndClamp, convertShflKind(op.getMode()), returnValueAndIsValidAttr); - Value shflValue = rewriter.create(loc, shfl, 0); - Value isActiveSrcLane = rewriter.create(loc, shfl, 1); - - rewriter.replaceOp(op, {shflValue, isActiveSrcLane}); + if (predIsUsed) { + Value shflValue = rewriter.create(loc, shfl, 0); + Value isActiveSrcLane = + rewriter.create(loc, shfl, 1); + rewriter.replaceOp(op, {shflValue, isActiveSrcLane}); + } else { + rewriter.replaceOp(op, {shfl, nullptr}); + } return success(); } }; diff --git a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir index dd3b6c2080aa..8877ee083286 100644 --- a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir +++ b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir @@ -112,7 +112,7 @@ gpu.module @test_module_3 { gpu.module @test_module_4 { // CHECK-LABEL: func @gpu_shuffle() - func.func @gpu_shuffle() -> (f32, f32, f32, f32) { + func.func @gpu_shuffle() -> (f32, f32, f32, f32, i1, i1, i1, i1) { // CHECK: %[[#VALUE:]] = llvm.mlir.constant(1.000000e+00 : f32) : f32 %arg0 = arith.constant 1.0 : f32 // CHECK: %[[#OFFSET:]] = llvm.mlir.constant(4 : i32) : i32 @@ -143,11 +143,41 @@ gpu.module @test_module_4 { // CHECK: nvvm.shfl.sync idx {{.*}} {return_value_and_is_valid} : f32 -> !llvm.struct<(f32, i1)> %shfli, %predi = gpu.shuffle idx %arg0, %arg1, %arg2 : f32 - func.return %shfl, %shflu, %shfld, %shfli : f32, f32,f32, f32 + func.return %shfl, %shflu, %shfld, %shfli, %pred, %predu, %predd, %predi + : f32, f32,f32, f32, i1, i1, i1, i1 } -} + // CHECK-LABEL: func @gpu_shuffle_unused_pred() + func.func @gpu_shuffle_unused_pred() -> (f32, f32, f32, f32) { + // CHECK: %[[#VALUE:]] = llvm.mlir.constant(1.000000e+00 : f32) : f32 + %arg0 = arith.constant 1.0 : f32 + // CHECK: %[[#OFFSET:]] = llvm.mlir.constant(4 : i32) : i32 + %arg1 = arith.constant 4 : i32 + // CHECK: %[[#WIDTH:]] = llvm.mlir.constant(23 : i32) : i32 + %arg2 = arith.constant 23 : i32 + // CHECK: %[[#ONE:]] = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[#MINUS_ONE:]] = llvm.mlir.constant(-1 : i32) : i32 + // CHECK: %[[#THIRTY_TWO:]] = llvm.mlir.constant(32 : i32) : i32 + // CHECK: %[[#NUM_LANES:]] = llvm.sub %[[#THIRTY_TWO]], %[[#WIDTH]] : i32 + // CHECK: %[[#MASK:]] = llvm.lshr %[[#MINUS_ONE]], %[[#NUM_LANES]] : i32 + // CHECK: %[[#CLAMP:]] = llvm.sub %[[#WIDTH]], %[[#ONE]] : i32 + // CHECK: %[[#SHFL:]] = nvvm.shfl.sync bfly %[[#MASK]], %[[#VALUE]], %[[#OFFSET]], %[[#CLAMP]] : f32 -> f32 + %shfl, %pred = gpu.shuffle xor %arg0, %arg1, %arg2 : f32 + // CHECK: %[[#ONE:]] = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[#MINUS_ONE:]] = llvm.mlir.constant(-1 : i32) : i32 + // CHECK: %[[#THIRTY_TWO:]] = llvm.mlir.constant(32 : i32) : i32 + // CHECK: %[[#NUM_LANES:]] = llvm.sub %[[#THIRTY_TWO]], %[[#WIDTH]] : i32 + // CHECK: %[[#MASK:]] = llvm.lshr %[[#MINUS_ONE]], %[[#NUM_LANES]] : i32 + // CHECK: %[[#SHFL:]] = nvvm.shfl.sync up %[[#MASK]], %[[#VALUE]], %[[#OFFSET]], %[[#NUM_LANES]] : f32 -> f32 + %shflu, %predu = gpu.shuffle up %arg0, %arg1, %arg2 : f32 + // CHECK: nvvm.shfl.sync down {{.*}} : f32 -> f32 + %shfld, %predd = gpu.shuffle down %arg0, %arg1, %arg2 : f32 + // CHECK: nvvm.shfl.sync idx {{.*}} : f32 -> f32 + %shfli, %predi = gpu.shuffle idx %arg0, %arg1, %arg2 : f32 + func.return %shfl, %shflu, %shfld, %shfli : f32, f32,f32, f32 + } +} gpu.module @test_module_5 { // CHECK-LABEL: func @gpu_sync() -- GitLab From 11aa95f83b7bf980ea13f1bb75e09af89a733acb Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 21 Mar 2024 09:46:31 +0000 Subject: [PATCH 116/296] [DAG] visitSUB - pull out repeated getScalarSizeInBits() calls. NFC. --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 188472219882..d370c57ce8e3 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -3695,6 +3695,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); EVT VT = N0.getValueType(); + unsigned BitWidth = VT.getScalarSizeInBits(); SDLoc DL(N); auto PeekThroughFreeze = [](SDValue N) { @@ -3731,7 +3732,6 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { DAG.getConstant(-N1C->getAPIntValue(), DL, VT)); if (isNullOrNullSplat(N0)) { - unsigned BitWidth = VT.getScalarSizeInBits(); // Right-shifting everything out but the sign bit followed by negation is // the same as flipping arithmetic/logical shift type without the negation: // -(X >>u 31) -> (X >>s 31) @@ -3931,7 +3931,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { SDValue S0 = N1.getOperand(0); if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0)) if (ConstantSDNode *C = isConstOrConstSplat(N1.getOperand(1))) - if (C->getAPIntValue() == (VT.getScalarSizeInBits() - 1)) + if (C->getAPIntValue() == (BitWidth - 1)) return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0); } } @@ -3974,8 +3974,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { if (!LegalOperations && N1.getOpcode() == ISD::SRL && N1.hasOneUse()) { SDValue ShAmt = N1.getOperand(1); ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt); - if (ShAmtC && - ShAmtC->getAPIntValue() == (N1.getScalarValueSizeInBits() - 1)) { + if (ShAmtC && ShAmtC->getAPIntValue() == (BitWidth - 1)) { SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt); return DAG.getNode(ISD::ADD, DL, VT, N0, SRA); } @@ -3986,7 +3985,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { // N0 - (X << BW-1) --> N0 + (X << BW-1) if (N1.getOpcode() == ISD::SHL) { ConstantSDNode *ShlC = isConstOrConstSplat(N1.getOperand(1)); - if (ShlC && ShlC->getAPIntValue() == VT.getScalarSizeInBits() - 1) + if (ShlC && ShlC->getAPIntValue() == (BitWidth - 1)) return DAG.getNode(ISD::ADD, DL, VT, N1, N0); } -- GitLab From 23de3862dce582ce91c1aa914467d982cb1a73b4 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 21 Mar 2024 09:54:12 +0000 Subject: [PATCH 117/296] [DAG] visitSUB - use sd_match to match SUB(MAX,MIN) -> ABD pattern. NFC. Seriously simplifies the commutation matching logic. --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index d370c57ce8e3..a3f5d433d920 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -4018,23 +4018,17 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { } } - // max(a,b) - min(a,b) --> abd(a,b) - auto MatchSubMaxMin = [&](unsigned Max, unsigned Min, unsigned Abd) { - if (N0.getOpcode() != Max || N1.getOpcode() != Min) - return SDValue(); - if ((N0.getOperand(0) != N1.getOperand(0) || - N0.getOperand(1) != N1.getOperand(1)) && - (N0.getOperand(0) != N1.getOperand(1) || - N0.getOperand(1) != N1.getOperand(0))) - return SDValue(); - if (!hasOperation(Abd, VT)) - return SDValue(); - return DAG.getNode(Abd, DL, VT, N0.getOperand(0), N0.getOperand(1)); - }; - if (SDValue R = MatchSubMaxMin(ISD::SMAX, ISD::SMIN, ISD::ABDS)) - return R; - if (SDValue R = MatchSubMaxMin(ISD::UMAX, ISD::UMIN, ISD::ABDU)) - return R; + // smax(a,b) - smin(a,b) --> abds(a,b) + if (hasOperation(ISD::ABDS, VT) && + sd_match(N0, m_SMax(m_Value(A), m_Value(B))) && + sd_match(N1, m_SMin(m_Specific(A), m_Specific(B)))) + return DAG.getNode(ISD::ABDS, DL, VT, A, B); + + // umax(a,b) - umin(a,b) --> abdu(a,b) + if (hasOperation(ISD::ABDU, VT) && + sd_match(N0, m_UMax(m_Value(A), m_Value(B))) && + sd_match(N1, m_UMin(m_Specific(A), m_Specific(B)))) + return DAG.getNode(ISD::ABDU, DL, VT, A, B); return SDValue(); } -- GitLab From ccb3a8feaa5b132dc829e55e069dde62008df4a8 Mon Sep 17 00:00:00 2001 From: Pierre van Houtryve Date: Thu, 21 Mar 2024 11:28:35 +0100 Subject: [PATCH 118/296] [AMDGPU][LowerModuleLDS] Refactor partially lowered module detection (#85793) Refactor the logic that checks if a module contains mixed absolute/non-lowered LDS GVs. The check now happens latter when the "worklists" are formed. This is because in some cases (OpenMP) we can have non-lowered GVs in a lowered module, and this is normal because those GVs are just unused and removed from the list at some point before the end of `getUsesOfLDSByFunction`. Doing the check later ensures that if a mixed module is spotted, then it's a _real_ mixed module that needs rejection, not a module containing an intentionally ignored GV. --- .../AMDGPU/AMDGPULowerModuleLDSPass.cpp | 40 ++++++++++++------- .../lds-mixed-absolute-addresses-unused.ll | 26 ++++++++++++ .../lds-reject-mixed-absolute-addresses.ll | 2 +- 3 files changed, 52 insertions(+), 16 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/lds-mixed-absolute-addresses-unused.ll diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp index b85cb26fdc95..595f09664c55 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp @@ -340,26 +340,11 @@ public: // Get uses from the current function, excluding uses by called functions // Two output variables to avoid walking the globals list twice - std::optional HasAbsoluteGVs; for (auto &GV : M.globals()) { if (!AMDGPU::isLDSVariableToLower(GV)) { continue; } - // Check if the module is consistent: either all GVs are absolute (happens - // when we run the pass more than once), or none are. - const bool IsAbsolute = GV.isAbsoluteSymbolRef(); - if (HasAbsoluteGVs.has_value()) { - if (*HasAbsoluteGVs != IsAbsolute) { - report_fatal_error( - "Module cannot mix absolute and non-absolute LDS GVs"); - } - } else - HasAbsoluteGVs = IsAbsolute; - - if (IsAbsolute) - continue; - for (User *V : GV.users()) { if (auto *I = dyn_cast(V)) { Function *F = I->getFunction(); @@ -469,6 +454,31 @@ public: } } + // Verify that we fall into one of 2 cases: + // - All variables are absolute: this is a re-run of the pass + // so we don't have anything to do. + // - No variables are absolute. + std::optional HasAbsoluteGVs; + for (auto &Map : {direct_map_kernel, indirect_map_kernel}) { + for (auto &[Fn, GVs] : Map) { + for (auto *GV : GVs) { + bool IsAbsolute = GV->isAbsoluteSymbolRef(); + if (HasAbsoluteGVs.has_value()) { + if (*HasAbsoluteGVs != IsAbsolute) { + report_fatal_error( + "Module cannot mix absolute and non-absolute LDS GVs"); + } + } else + HasAbsoluteGVs = IsAbsolute; + } + } + } + + // If we only had absolute GVs, we have nothing to do, return an empty + // result. + if (HasAbsoluteGVs && *HasAbsoluteGVs) + return {FunctionVariableMap(), FunctionVariableMap()}; + return {std::move(direct_map_kernel), std::move(indirect_map_kernel)}; } diff --git a/llvm/test/CodeGen/AMDGPU/lds-mixed-absolute-addresses-unused.ll b/llvm/test/CodeGen/AMDGPU/lds-mixed-absolute-addresses-unused.ll new file mode 100644 index 000000000000..d101d8da5e0f --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/lds-mixed-absolute-addresses-unused.ll @@ -0,0 +1,26 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -mtriple=amdgcn-- -amdgpu-lower-module-lds < %s 2>&1 | FileCheck %s +; RUN: opt -S -mtriple=amdgcn-- -passes=amdgpu-lower-module-lds < %s 2>&1 | FileCheck %s + +; This looks like a partially lowered module, but the non-lowered GV isn't used by any kernels. +; In such cases, LowerModuleLDS is free to leave it in and ignore it, and we want to make sure +; LowerModuleLDS doesn't crash if it re-runs on such modules. +@notLowered = addrspace(3) global i32 poison +@lowered = addrspace(3) global i32 poison, !absolute_symbol !0 + +@llvm.compiler.used = appending addrspace(1) global [1 x ptr] [ptr addrspacecast (ptr addrspace(3) @notLowered to ptr)], section "llvm.metadata" + +define amdgpu_kernel void @kern(i32 %val0) { +; CHECK-LABEL: define amdgpu_kernel void @kern( +; CHECK-SAME: i32 [[VAL0:%.*]]) { +; CHECK-NEXT: [[VAL1:%.*]] = add i32 [[VAL0]], 4 +; CHECK-NEXT: store i32 [[VAL1]], ptr addrspace(3) @lowered, align 4 +; CHECK-NEXT: ret void +; + %val1 = add i32 %val0, 4 + store i32 %val1, ptr addrspace(3) @lowered + ret void +} + + +!0 = !{i32 0, i32 1} diff --git a/llvm/test/CodeGen/AMDGPU/lds-reject-mixed-absolute-addresses.ll b/llvm/test/CodeGen/AMDGPU/lds-reject-mixed-absolute-addresses.ll index b512a43aa102..b1f4f2ef1ef5 100644 --- a/llvm/test/CodeGen/AMDGPU/lds-reject-mixed-absolute-addresses.ll +++ b/llvm/test/CodeGen/AMDGPU/lds-reject-mixed-absolute-addresses.ll @@ -8,7 +8,7 @@ define amdgpu_kernel void @kern() { %val0 = load i32, ptr addrspace(3) @var1 %val1 = add i32 %val0, 4 - store i32 %val1, ptr addrspace(3) @var1 + store i32 %val1, ptr addrspace(3) @var2 ret void } -- GitLab From 8ecc377c88ba32978ace6a67de895403eeba3a22 Mon Sep 17 00:00:00 2001 From: Jacek Caban Date: Thu, 21 Mar 2024 11:40:46 +0100 Subject: [PATCH 119/296] [llvm-lib] Use ARM64EC machine type for import libraries when -machine:arm64x is used. (#85972) This is compatible with MSVC, `-machine:arm64x` is essentially an alias to `-machine:arm64ec`. To make a type library that exposes both native and EC symbols, an additional `-defArm64Native` argument is needed in both cases. --- llvm/lib/Object/COFFImportFile.cpp | 7 +++++-- llvm/test/tools/llvm-lib/arm64ec-implib.test | 13 ++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Object/COFFImportFile.cpp b/llvm/lib/Object/COFFImportFile.cpp index 46c8e702581e..8224a1492502 100644 --- a/llvm/lib/Object/COFFImportFile.cpp +++ b/llvm/lib/Object/COFFImportFile.cpp @@ -626,8 +626,11 @@ Error writeImportLibrary(StringRef ImportName, StringRef Path, MachineTypes Machine, bool MinGW, ArrayRef NativeExports) { - MachineTypes NativeMachine = - isArm64EC(Machine) ? IMAGE_FILE_MACHINE_ARM64 : Machine; + MachineTypes NativeMachine = Machine; + if (isArm64EC(Machine)) { + NativeMachine = IMAGE_FILE_MACHINE_ARM64; + Machine = IMAGE_FILE_MACHINE_ARM64EC; + } std::vector Members; ObjectFactory OF(llvm::sys::path::filename(ImportName), NativeMachine); diff --git a/llvm/test/tools/llvm-lib/arm64ec-implib.test b/llvm/test/tools/llvm-lib/arm64ec-implib.test index 00eddd2a4752..9ce53fe0fea0 100644 --- a/llvm/test/tools/llvm-lib/arm64ec-implib.test +++ b/llvm/test/tools/llvm-lib/arm64ec-implib.test @@ -34,17 +34,17 @@ ARMAP-NEXT: test_NULL_THUNK_DATA in test.dll RUN: llvm-readobj test.lib | FileCheck -check-prefix=READOBJ %s -READOBJ: File: test.lib(test.dll) +READOBJ: File: test{{.*}}.lib(test.dll) READOBJ-NEXT: Format: COFF-ARM64{{$}} READOBJ-NEXT: Arch: aarch64 READOBJ-NEXT: AddressSize: 64bit READOBJ-EMPTY: -READOBJ-NEXT: File: test.lib(test.dll) +READOBJ-NEXT: File: test{{.*}}.lib(test.dll) READOBJ-NEXT: Format: COFF-ARM64{{$}} READOBJ-NEXT: Arch: aarch64 READOBJ-NEXT: AddressSize: 64bit READOBJ-EMPTY: -READOBJ-NEXT: File: test.lib(test.dll) +READOBJ-NEXT: File: test{{.*}}.lib(test.dll) READOBJ-NEXT: Format: COFF-ARM64{{$}} READOBJ-NEXT: Arch: aarch64 READOBJ-NEXT: AddressSize: 64bit @@ -96,6 +96,11 @@ READOBJ-NEXT: Name type: name READOBJ-NEXT: Export name: dataexp READOBJ-NEXT: Symbol: __imp_dataexp +Using -machine:arm64x gives the same output. +RUN: llvm-lib -machine:arm64x -def:test.def -out:testx.lib +RUN: llvm-nm --print-armap testx.lib | FileCheck -check-prefix=ARMAP %s +RUN: llvm-readobj testx.lib | FileCheck -check-prefix=READOBJ %s + Creating a new lib containing the existing lib: RUN: llvm-lib -machine:arm64ec test.lib -out:test2.lib RUN: llvm-nm --print-armap test2.lib | FileCheck -check-prefix=ARMAP %s @@ -246,7 +251,9 @@ READOBJX-NEXT: Symbol: __imp_dataexp RUN: llvm-lib -machine:arm64ec -def:test.def -defArm64Native:test2.def -out:test2.lib +RUN: llvm-lib -machine:arm64ec -def:test.def -defArm64Native:test2.def -out:test2x.lib RUN: llvm-nm --print-armap test2.lib | FileCheck -check-prefix=ARMAPX2 %s +RUN: llvm-nm --print-armap test2x.lib | FileCheck -check-prefix=ARMAPX2 %s ARMAPX2: Archive map ARMAPX2-NEXT: __IMPORT_DESCRIPTOR_test2 in test2.dll -- GitLab From 0124e0821dd87bb021f31d1d9aecc2e0f3a52514 Mon Sep 17 00:00:00 2001 From: Jacek Caban Date: Thu, 21 Mar 2024 11:43:48 +0100 Subject: [PATCH 120/296] [Object][COFF][NFC] Introduce Arm64ECThunkType enum. (#85936) And use it in EC lowering code. It will be useful for LLD too. --- llvm/include/llvm/BinaryFormat/COFF.h | 6 ++ .../AArch64/AArch64Arm64ECCallLowering.cpp | 57 ++++++++++--------- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/llvm/include/llvm/BinaryFormat/COFF.h b/llvm/include/llvm/BinaryFormat/COFF.h index 72461d0d9c31..4c31cd847bdf 100644 --- a/llvm/include/llvm/BinaryFormat/COFF.h +++ b/llvm/include/llvm/BinaryFormat/COFF.h @@ -806,6 +806,12 @@ enum Feat00Flags : uint32_t { Kernel = 0x40000000, }; +enum class Arm64ECThunkType : uint8_t { + GuestExit = 0, + Entry = 1, + Exit = 4, +}; + inline bool isReservedSectionNumber(int32_t SectionNumber) { return SectionNumber <= 0; } diff --git a/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp b/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp index f147ded2ab70..9b5cbe3de137 100644 --- a/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp @@ -30,6 +30,7 @@ #include "llvm/TargetParser/Triple.h" using namespace llvm; +using namespace llvm::COFF; using namespace llvm::object; using OperandBundleDef = OperandBundleDefT; @@ -45,8 +46,6 @@ static cl::opt GenerateThunks("arm64ec-generate-thunks", cl::Hidden, namespace { -enum class ThunkType { GuestExit, Entry, Exit }; - class AArch64Arm64ECCallLowering : public ModulePass { public: static char ID; @@ -73,15 +72,15 @@ private: Type *I64Ty; Type *VoidTy; - void getThunkType(FunctionType *FT, AttributeList AttrList, ThunkType TT, - raw_ostream &Out, FunctionType *&Arm64Ty, - FunctionType *&X64Ty); + void getThunkType(FunctionType *FT, AttributeList AttrList, + Arm64ECThunkType TT, raw_ostream &Out, + FunctionType *&Arm64Ty, FunctionType *&X64Ty); void getThunkRetType(FunctionType *FT, AttributeList AttrList, raw_ostream &Out, Type *&Arm64RetTy, Type *&X64RetTy, SmallVectorImpl &Arm64ArgTypes, SmallVectorImpl &X64ArgTypes, bool &HasSretPtr); - void getThunkArgTypes(FunctionType *FT, AttributeList AttrList, ThunkType TT, - raw_ostream &Out, + void getThunkArgTypes(FunctionType *FT, AttributeList AttrList, + Arm64ECThunkType TT, raw_ostream &Out, SmallVectorImpl &Arm64ArgTypes, SmallVectorImpl &X64ArgTypes, bool HasSretPtr); void canonicalizeThunkType(Type *T, Align Alignment, bool Ret, @@ -91,13 +90,11 @@ private: } // end anonymous namespace -void AArch64Arm64ECCallLowering::getThunkType(FunctionType *FT, - AttributeList AttrList, - ThunkType TT, raw_ostream &Out, - FunctionType *&Arm64Ty, - FunctionType *&X64Ty) { - Out << (TT == ThunkType::Entry ? "$ientry_thunk$cdecl$" - : "$iexit_thunk$cdecl$"); +void AArch64Arm64ECCallLowering::getThunkType( + FunctionType *FT, AttributeList AttrList, Arm64ECThunkType TT, + raw_ostream &Out, FunctionType *&Arm64Ty, FunctionType *&X64Ty) { + Out << (TT == Arm64ECThunkType::Entry ? "$ientry_thunk$cdecl$" + : "$iexit_thunk$cdecl$"); Type *Arm64RetTy; Type *X64RetTy; @@ -108,7 +105,7 @@ void AArch64Arm64ECCallLowering::getThunkType(FunctionType *FT, // The first argument to a thunk is the called function, stored in x9. // For exit thunks, we pass the called function down to the emulator; // for entry/guest exit thunks, we just call the Arm64 function directly. - if (TT == ThunkType::Exit) + if (TT == Arm64ECThunkType::Exit) Arm64ArgTypes.push_back(PtrTy); X64ArgTypes.push_back(PtrTy); @@ -125,8 +122,8 @@ void AArch64Arm64ECCallLowering::getThunkType(FunctionType *FT, } void AArch64Arm64ECCallLowering::getThunkArgTypes( - FunctionType *FT, AttributeList AttrList, ThunkType TT, raw_ostream &Out, - SmallVectorImpl &Arm64ArgTypes, + FunctionType *FT, AttributeList AttrList, Arm64ECThunkType TT, + raw_ostream &Out, SmallVectorImpl &Arm64ArgTypes, SmallVectorImpl &X64ArgTypes, bool HasSretPtr) { Out << "$"; @@ -163,7 +160,7 @@ void AArch64Arm64ECCallLowering::getThunkArgTypes( X64ArgTypes.push_back(PtrTy); // x5 Arm64ArgTypes.push_back(I64Ty); - if (TT != ThunkType::Entry) { + if (TT != Arm64ECThunkType::Entry) { // FIXME: x5 isn't actually used by the x64 side; revisit once we // have proper isel for varargs X64ArgTypes.push_back(I64Ty); @@ -348,7 +345,8 @@ Function *AArch64Arm64ECCallLowering::buildExitThunk(FunctionType *FT, SmallString<256> ExitThunkName; llvm::raw_svector_ostream ExitThunkStream(ExitThunkName); FunctionType *Arm64Ty, *X64Ty; - getThunkType(FT, Attrs, ThunkType::Exit, ExitThunkStream, Arm64Ty, X64Ty); + getThunkType(FT, Attrs, Arm64ECThunkType::Exit, ExitThunkStream, Arm64Ty, + X64Ty); if (Function *F = M->getFunction(ExitThunkName)) return F; @@ -451,8 +449,8 @@ Function *AArch64Arm64ECCallLowering::buildEntryThunk(Function *F) { SmallString<256> EntryThunkName; llvm::raw_svector_ostream EntryThunkStream(EntryThunkName); FunctionType *Arm64Ty, *X64Ty; - getThunkType(F->getFunctionType(), F->getAttributes(), ThunkType::Entry, - EntryThunkStream, Arm64Ty, X64Ty); + getThunkType(F->getFunctionType(), F->getAttributes(), + Arm64ECThunkType::Entry, EntryThunkStream, Arm64Ty, X64Ty); if (Function *F = M->getFunction(EntryThunkName)) return F; @@ -543,8 +541,8 @@ Function *AArch64Arm64ECCallLowering::buildEntryThunk(Function *F) { Function *AArch64Arm64ECCallLowering::buildGuestExitThunk(Function *F) { llvm::raw_null_ostream NullThunkName; FunctionType *Arm64Ty, *X64Ty; - getThunkType(F->getFunctionType(), F->getAttributes(), ThunkType::GuestExit, - NullThunkName, Arm64Ty, X64Ty); + getThunkType(F->getFunctionType(), F->getAttributes(), + Arm64ECThunkType::GuestExit, NullThunkName, Arm64Ty, X64Ty); auto MangledName = getArm64ECMangledFunctionName(F->getName().str()); assert(MangledName && "Can't guest exit to function that's already native"); std::string ThunkName = *MangledName; @@ -679,7 +677,7 @@ bool AArch64Arm64ECCallLowering::runOnModule(Module &Mod) { struct ThunkInfo { Constant *Src; Constant *Dst; - unsigned Kind; + Arm64ECThunkType Kind; }; SmallVector ThunkMapping; for (Function &F : Mod) { @@ -688,14 +686,17 @@ bool AArch64Arm64ECCallLowering::runOnModule(Module &Mod) { F.getCallingConv() != CallingConv::ARM64EC_Thunk_X64) { if (!F.hasComdat()) F.setComdat(Mod.getOrInsertComdat(F.getName())); - ThunkMapping.push_back({&F, buildEntryThunk(&F), 1}); + ThunkMapping.push_back( + {&F, buildEntryThunk(&F), Arm64ECThunkType::Entry}); } } for (Function *F : DirectCalledFns) { ThunkMapping.push_back( - {F, buildExitThunk(F->getFunctionType(), F->getAttributes()), 4}); + {F, buildExitThunk(F->getFunctionType(), F->getAttributes()), + Arm64ECThunkType::Exit}); if (!F->hasDLLImportStorageClass()) - ThunkMapping.push_back({buildGuestExitThunk(F), F, 0}); + ThunkMapping.push_back( + {buildGuestExitThunk(F), F, Arm64ECThunkType::GuestExit}); } if (!ThunkMapping.empty()) { @@ -704,7 +705,7 @@ bool AArch64Arm64ECCallLowering::runOnModule(Module &Mod) { ThunkMappingArrayElems.push_back(ConstantStruct::getAnon( {ConstantExpr::getBitCast(Thunk.Src, PtrTy), ConstantExpr::getBitCast(Thunk.Dst, PtrTy), - ConstantInt::get(M->getContext(), APInt(32, Thunk.Kind))})); + ConstantInt::get(M->getContext(), APInt(32, uint8_t(Thunk.Kind)))})); } Constant *ThunkMappingArray = ConstantArray::get( llvm::ArrayType::get(ThunkMappingArrayElems[0]->getType(), -- GitLab From 95a834a16c3de0de615d0cfa20a6c8bd973b6a1d Mon Sep 17 00:00:00 2001 From: Pierre van Houtryve Date: Thu, 21 Mar 2024 11:44:47 +0100 Subject: [PATCH 121/296] (Reland) [AMDGPU] Run LowerLDS at the end of the fullLTO pipeline (#85626) Reland of #75333 --- .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 9 ++++ .../CodeGen/AMDGPU/lto-lower-module-lds.ll | 47 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 llvm/test/CodeGen/AMDGPU/lto-lower-module-lds.ll diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index 2b457fe519d9..c96625092a76 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -793,6 +793,15 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks( PM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM))); }); + + PB.registerFullLinkTimeOptimizationLastEPCallback( + [this](ModulePassManager &PM, OptimizationLevel Level) { + // We want to support the -lto-partitions=N option as "best effort". + // For that, we need to lower LDS earlier in the pipeline before the + // module is partitioned for codegen. + if (EnableLowerModuleLDS) + PM.addPass(AMDGPULowerModuleLDSPass(*this)); + }); } int64_t AMDGPUTargetMachine::getNullPointerValue(unsigned AddrSpace) { diff --git a/llvm/test/CodeGen/AMDGPU/lto-lower-module-lds.ll b/llvm/test/CodeGen/AMDGPU/lto-lower-module-lds.ll new file mode 100644 index 000000000000..f1d946376afe --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/lto-lower-module-lds.ll @@ -0,0 +1,47 @@ + +; Default O0 +; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1030 %s -o %t.bc +; RUN: llvm-lto2 run -O0 -cg-opt-level 0 %t.bc -o %t.s -r %t.bc,test,px -debug-pass-manager -debug-pass=Structure 2>&1 | FileCheck %s + +; Unified O0 +; RUN: opt -unified-lto -thinlto-split-lto-unit -thinlto-bc -mtriple=amdgcn-- -mcpu=gfx1030 %s -o %t.bc +; RUN: llvm-lto2 run -unified-lto=full -O0 -cg-opt-level 0 %t.bc -o %t.s -r %t.bc,test,px -debug-pass-manager -debug-pass=Structure 2>&1 | FileCheck %s + +; Default O1 +; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1030 %s -o %t.bc +; RUN: llvm-lto2 run -O1 -cg-opt-level 1 %t.bc -o %t.s -r %t.bc,test,px -debug-pass-manager -debug-pass=Structure 2>&1 | FileCheck %s + +; Unified O1 +; RUN: opt -unified-lto -thinlto-split-lto-unit -thinlto-bc -mtriple=amdgcn-- -mcpu=gfx1030 %s -o %t.bc +; RUN: llvm-lto2 run -unified-lto=full -O1 -cg-opt-level 1 %t.bc -o %t.s -r %t.bc,test,px -debug-pass-manager -debug-pass=Structure 2>&1 | FileCheck %s + +; Default O2 +; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1030 %s -o %t.bc +; RUN: llvm-lto2 run -O2 -cg-opt-level 2 %t.bc -o %t.s -r %t.bc,test,px -debug-pass-manager -debug-pass=Structure 2>&1 | FileCheck %s + +; Unified O2 +; RUN: opt -unified-lto -thinlto-split-lto-unit -thinlto-bc -mtriple=amdgcn-- -mcpu=gfx1030 %s -o %t.bc +; RUN: llvm-lto2 run -unified-lto=full -O2 -cg-opt-level 2 %t.bc -o %t.s -r %t.bc,test,px -debug-pass-manager -debug-pass=Structure 2>&1 | FileCheck %s + +; Default O3 +; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1030 %s -o %t.bc +; RUN: llvm-lto2 run -O3 -cg-opt-level 3 %t.bc -o %t.s -r %t.bc,test,px -debug-pass-manager -debug-pass=Structure 2>&1 | FileCheck %s + +; Unified O3 +; RUN: opt -unified-lto -thinlto-split-lto-unit -thinlto-bc -mtriple=amdgcn-- -mcpu=gfx1030 %s -o %t.bc +; RUN: llvm-lto2 run -unified-lto=full -O3 -cg-opt-level 3 %t.bc -o %t.s -r %t.bc,test,px -debug-pass-manager -debug-pass=Structure 2>&1 | FileCheck %s + +; First print will be from the New PM during the full LTO pipeline. +; Second print will be from the legacy PM during the CG pipeline. + +; CHECK: Running pass: AMDGPULowerModuleLDSPass on [module] +; CHECK: ModulePass Manager +; CHECK: Lower uses of LDS variables from non-kernel functions + +@lds = internal unnamed_addr addrspace(3) global i32 poison, align 4 + +define amdgpu_kernel void @test() { +entry: + store i32 1, ptr addrspace(3) @lds + ret void +} -- GitLab From 2096f37d7a580a4b4ddce2a44abb80ff90f273ee Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Thu, 21 Mar 2024 11:57:07 +0100 Subject: [PATCH 122/296] [libc++][NFC] Use __constexpr_memmove instead of copy_n in <__string/char_traits.h> (#85920) `copy_n` has been used to allow constant evaluation of `char_traits`. We now have `__constexpr_memmove`, which `copy_n` just forwards to. We can call `__constexpr_memmove` directly, avoiding a bunch of instantiations. This reduces the time it takes to include `` from 321ms to 285ms. --- libcxx/include/__string/char_traits.h | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/libcxx/include/__string/char_traits.h b/libcxx/include/__string/char_traits.h index 5880d3a22db2..47ed1057caaa 100644 --- a/libcxx/include/__string/char_traits.h +++ b/libcxx/include/__string/char_traits.h @@ -9,7 +9,6 @@ #ifndef _LIBCPP___STRING_CHAR_TRAITS_H #define _LIBCPP___STRING_CHAR_TRAITS_H -#include <__algorithm/copy_n.h> #include <__algorithm/fill_n.h> #include <__algorithm/find_end.h> #include <__algorithm/find_first_of.h> @@ -144,7 +143,7 @@ struct _LIBCPP_TEMPLATE_VIS char_traits { copy(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT { _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(!std::__is_pointer_in_range(__s1, __s1 + __n, __s2), "char_traits::copy: source and destination ranges overlap"); - std::copy_n(__s2, __n, __s1); + std::__constexpr_memmove(__s1, __s2, __element_count(__n)); return __s1; } @@ -221,7 +220,7 @@ struct _LIBCPP_TEMPLATE_VIS char_traits { copy(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT { _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(!std::__is_pointer_in_range(__s1, __s1 + __n, __s2), "char_traits::copy: source and destination ranges overlap"); - std::copy_n(__s2, __n, __s1); + std::__constexpr_memmove(__s1, __s2, __element_count(__n)); return __s1; } @@ -287,7 +286,7 @@ struct _LIBCPP_TEMPLATE_VIS char_traits { copy(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT { _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(!std::__is_pointer_in_range(__s1, __s1 + __n, __s2), "char_traits::copy: source and destination ranges overlap"); - std::copy_n(__s2, __n, __s1); + std::__constexpr_memmove(__s1, __s2, __element_count(__n)); return __s1; } @@ -366,7 +365,7 @@ struct _LIBCPP_TEMPLATE_VIS char_traits { copy(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT { _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES(!std::__is_pointer_in_range(__s1, __s1 + __n, __s2), "char_traits::copy: source and destination ranges overlap"); - std::copy_n(__s2, __n, __s1); + std::__constexpr_memmove(__s1, __s2, __element_count(__n)); return __s1; } @@ -454,7 +453,7 @@ struct _LIBCPP_TEMPLATE_VIS char_traits { _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static char_type* copy(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT { - std::copy_n(__s2, __n, __s1); + std::__constexpr_memmove(__s1, __s2, __element_count(__n)); return __s1; } -- GitLab From cb071942f881e743b8131688a873dab760c7b88d Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Thu, 21 Mar 2024 12:01:26 +0100 Subject: [PATCH 123/296] [OpenMP] Fix SystemZ build failure Commit a7d5f73a03c81cab8df64dbd099e8acb40f5dfe1 introduced an error in a target_compile_definitions on the SystemZ, causing the build to break. Fixed by adding the missing "PRIVATE". --- openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt index 5ccb20e305e8..58a79898ff80 100644 --- a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt @@ -100,7 +100,7 @@ elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64$") "aarch64-unknown-linux-gnu" "aarch64-unknown-linux-gnu-LTO") set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE) elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x$") - target_compile_definitions(omptarget.rtl.${machine} TARGET_ELF_ID=EM_S390) + target_compile_definitions(omptarget.rtl.${machine} PRIVATE TARGET_ELF_ID=EM_S390) target_compile_definitions(omptarget.rtl.${machine} PRIVATE LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE="s390x-ibm-linux-gnu") list(APPEND LIBOMPTARGET_SYSTEM_TARGETS -- GitLab From e8cf1754988cf281e76ae463b3e1f3c0cda3f230 Mon Sep 17 00:00:00 2001 From: Ulrich Weigand Date: Thu, 21 Mar 2024 12:05:11 +0100 Subject: [PATCH 124/296] [runtimes] Fix OpenMP dependencies (#85977) When building the OpenMP runtime with libomptarget support, the runtimes configure step needs to have a dependency on various tools, in particular opt, so that cmake configure checks yield the correct results. This did not work correctly, as the dependencies were only added if the OPENMP_ENABLE_LIBOMPTARGET was set - but that variable is only set by the openmp/CMakeLists.txt file, which isn't even parsed during the initial cmake run (in fact, it is only parsed when executing the runtimes configure step itself, but then it is too late). Fixed by just adding those dependencies always. In addition, the list of dependencies collected in ${extra_deps}, including those required for OpenMP, was only actually used when configuring runtimes for the default set of targets - when the user specifies a non-default LLVM_RUNTIME_TARGETS, those extra dependencies were ignored (with the exception of ${hdrgen_deps}). Fixed by passing the full ${extra_deps} in this case as well. Fixes: https://github.com/llvm/llvm-project/issues/85933 --- llvm/runtimes/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt index 623c43d564cc..8159d7f8a0a1 100644 --- a/llvm/runtimes/CMakeLists.txt +++ b/llvm/runtimes/CMakeLists.txt @@ -435,7 +435,7 @@ if(runtimes) list(APPEND extra_deps "flang-new") endif() foreach(dep opt llvm-link llvm-extract clang clang-offload-packager) - if(TARGET ${dep} AND OPENMP_ENABLE_LIBOMPTARGET) + if(TARGET ${dep}) list(APPEND extra_deps ${dep}) endif() endforeach() @@ -531,7 +531,7 @@ if(runtimes) check_apple_target(${name} runtime) runtime_register_target(${name} - DEPENDS ${builtins_dep_name} ${hdrgen_deps} + DEPENDS ${builtins_dep_name} ${extra_deps} CMAKE_ARGS -DLLVM_DEFAULT_TARGET_TRIPLE=${name} ${libc_cmake_args} EXTRA_ARGS TARGET_TRIPLE ${name}) endforeach() -- GitLab From 8779edb8b33e8ae7e021c8fa7fff80d77567b28c Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Thu, 21 Mar 2024 12:14:24 +0100 Subject: [PATCH 125/296] [libc++] Deprecates std::errc constants. (#80542) Implements: - LWG3869 Deprecate std::errc constants related to UNIX STREAMS --- libcxx/docs/Status/Cxx23Issues.csv | 2 +- libcxx/include/__system_error/errc.h | 70 +++++++++++++++---- libcxx/include/cerrno | 13 ++++ libcxx/src/random.cpp | 4 +- .../test/std/depr.cerro/cerrno.syn.verify.cpp | 37 ++++++++++ .../depr.cerro/system.error.syn.verify.cpp | 28 ++++++++ .../test/std/diagnostics/syserr/errc.pass.cpp | 2 + 7 files changed, 142 insertions(+), 14 deletions(-) create mode 100644 libcxx/test/std/depr.cerro/cerrno.syn.verify.cpp create mode 100644 libcxx/test/std/depr.cerro/system.error.syn.verify.cpp diff --git a/libcxx/docs/Status/Cxx23Issues.csv b/libcxx/docs/Status/Cxx23Issues.csv index a10319235006..8de265f4d1b6 100644 --- a/libcxx/docs/Status/Cxx23Issues.csv +++ b/libcxx/docs/Status/Cxx23Issues.csv @@ -295,7 +295,7 @@ "`3847 `__","``ranges::to`` can still return views","February 2023","|Complete|","17.0","|ranges|" "`3862 `__","``basic_const_iterator``'s ``common_type`` specialization is underconstrained","February 2023","","","" "`3865 `__","Sorting a range of ``pairs``","February 2023","|Complete|","17.0","|ranges|" -"`3869 `__","Deprecate ``std::errc`` constants related to UNIX STREAMS","February 2023","","","" +"`3869 `__","Deprecate ``std::errc`` constants related to UNIX STREAMS","February 2023","|Complete|","19.0","" "`3870 `__","Remove ``voidify``","February 2023","","","" "`3871 `__","Adjust note about ``terminate``","February 2023","","","" "`3872 `__","``basic_const_iterator`` should have custom ``iter_move``","February 2023","","","" diff --git a/libcxx/include/__system_error/errc.h b/libcxx/include/__system_error/errc.h index f87df86a71e1..e9f3656b7b9c 100644 --- a/libcxx/include/__system_error/errc.h +++ b/libcxx/include/__system_error/errc.h @@ -58,18 +58,18 @@ enum class errc no_child_process, // ECHILD no_link, // ENOLINK no_lock_available, // ENOLCK - no_message_available, // ENODATA + no_message_available, // ENODATA // deprecated no_message, // ENOMSG no_protocol_option, // ENOPROTOOPT no_space_on_device, // ENOSPC - no_stream_resources, // ENOSR + no_stream_resources, // ENOSR // deprecated no_such_device_or_address, // ENXIO no_such_device, // ENODEV no_such_file_or_directory, // ENOENT no_such_process, // ESRCH not_a_directory, // ENOTDIR not_a_socket, // ENOTSOCK - not_a_stream, // ENOSTR + not_a_stream, // ENOSTR // deprecated not_connected, // ENOTCONN not_enough_memory, // ENOMEM not_supported, // ENOTSUP @@ -87,7 +87,7 @@ enum class errc resource_unavailable_try_again, // EAGAIN result_out_of_range, // ERANGE state_not_recoverable, // ENOTRECOVERABLE - stream_timeout, // ETIME + stream_timeout, // ETIME // deprecated text_file_busy, // ETXTBSY timed_out, // ETIMEDOUT too_many_files_open_in_system, // ENFILE @@ -107,12 +107,34 @@ enum class errc # pragma GCC system_header #endif +// The method of pushing and popping the diagnostics fails for GCC. GCC does +// not recognize the pragma's used to generate deprecated diagnostics for +// macros. So GCC does not need the pushing and popping. +// +// TODO Remove this when the deprecated constants are removed. +#if defined(_LIBCPP_COMPILER_CLANG_BASED) +# define _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH _LIBCPP_SUPPRESS_DEPRECATED_PUSH +# define _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP _LIBCPP_SUPPRESS_DEPRECATED_POP +#else +# define _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH +# define _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP +#endif + _LIBCPP_BEGIN_NAMESPACE_STD // Some error codes are not present on all platforms, so we provide equivalents // for them: // enum class errc +// +// LWG3869 deprecates the UNIX STREAMS macros and enum values. +// This makes the code clumbersome: +// - the enum value is deprecated and should show a diagnostic, +// - the macro is deprecated and should _not_ show a diagnostic in this +// context, and +// - the macro is not always available. +// This leads to the odd pushing and popping of the deprecated +// diagnostic. _LIBCPP_DECLARE_STRONG_ENUM(errc){ address_family_not_supported = EAFNOSUPPORT, address_in_use = EADDRINUSE, @@ -154,30 +176,48 @@ _LIBCPP_DECLARE_STRONG_ENUM(errc){ no_child_process = ECHILD, no_link = ENOLINK, no_lock_available = ENOLCK, + // clang-format off + no_message_available _LIBCPP_DEPRECATED = + _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH #ifdef ENODATA - no_message_available = ENODATA, + ENODATA #else - no_message_available = ENOMSG, + ENOMSG #endif + _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP + , + // clang-format on no_message = ENOMSG, no_protocol_option = ENOPROTOOPT, no_space_on_device = ENOSPC, + // clang-format off + no_stream_resources _LIBCPP_DEPRECATED = + _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH #ifdef ENOSR - no_stream_resources = ENOSR, + ENOSR #else - no_stream_resources = ENOMEM, + ENOMEM #endif + _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP + , + // clang-format on no_such_device_or_address = ENXIO, no_such_device = ENODEV, no_such_file_or_directory = ENOENT, no_such_process = ESRCH, not_a_directory = ENOTDIR, not_a_socket = ENOTSOCK, + // clang-format off + not_a_stream _LIBCPP_DEPRECATED = + _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH #ifdef ENOSTR - not_a_stream = ENOSTR, + ENOSTR #else - not_a_stream = EINVAL, + EINVAL #endif + _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP + , + // clang-format on not_connected = ENOTCONN, not_enough_memory = ENOMEM, not_supported = ENOTSUP, @@ -195,11 +235,17 @@ _LIBCPP_DECLARE_STRONG_ENUM(errc){ resource_unavailable_try_again = EAGAIN, result_out_of_range = ERANGE, state_not_recoverable = ENOTRECOVERABLE, + // clang-format off + stream_timeout _LIBCPP_DEPRECATED = + _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH #ifdef ETIME - stream_timeout = ETIME, + ETIME #else - stream_timeout = ETIMEDOUT, + ETIMEDOUT #endif + _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP + , + // clang-format on text_file_busy = ETXTBSY, timed_out = ETIMEDOUT, too_many_files_open_in_system = ENFILE, diff --git a/libcxx/include/cerrno b/libcxx/include/cerrno index d488fa72a54b..6171ae31f184 100644 --- a/libcxx/include/cerrno +++ b/libcxx/include/cerrno @@ -38,4 +38,17 @@ Macros: # pragma GCC system_header #endif +#ifdef ENODATA +# pragma clang deprecated(ENODATA, "ENODATA is deprecated in ISO C++") +#endif +#ifdef ENOSR +# pragma clang deprecated(ENOSR, "ENOSR is deprecated in ISO C++") +#endif +#ifdef ENOSTR +# pragma clang deprecated(ENOSTR, "ENOSTR is deprecated in ISO C++") +#endif +#ifdef ETIME +# pragma clang deprecated(ETIME, "ETIME is deprecated in ISO C++") +#endif + #endif // _LIBCPP_CERRNO diff --git a/libcxx/src/random.cpp b/libcxx/src/random.cpp index c7073c54da6b..93590af310e5 100644 --- a/libcxx/src/random.cpp +++ b/libcxx/src/random.cpp @@ -79,8 +79,10 @@ unsigned random_device::operator()() { char* p = reinterpret_cast(&r); while (n > 0) { ssize_t s = read(__f_, p, n); + _LIBCPP_SUPPRESS_DEPRECATED_PUSH if (s == 0) - __throw_system_error(ENODATA, "random_device got EOF"); + __throw_system_error(ENODATA, "random_device got EOF"); // TODO ENODATA -> ENOMSG + _LIBCPP_SUPPRESS_DEPRECATED_POP if (s == -1) { if (errno != EINTR) __throw_system_error(errno, "random_device got an unexpected error"); diff --git a/libcxx/test/std/depr.cerro/cerrno.syn.verify.cpp b/libcxx/test/std/depr.cerro/cerrno.syn.verify.cpp new file mode 100644 index 000000000000..3a38605570da --- /dev/null +++ b/libcxx/test/std/depr.cerro/cerrno.syn.verify.cpp @@ -0,0 +1,37 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: clang-modules-build +// UNSUPPORTED: apple-clang && c++03 + +// + +// tests LWG 3869 deprecated macros. +// +// Note the macros may not be defined. When they are not defined the +// ifdef XXX does not trigger a deprecated message. So use them in the +// ifdef and test for 2 deprecated messages. + +#include + +#ifdef ENODATA +[[maybe_unused]] int nodata = + ENODATA; // expected-warning@cerrno.syn.verify.cpp:* 2 {{macro 'ENODATA' has been marked as deprecated}} +#endif +#ifdef ENOSR +[[maybe_unused]] int nosr = + ENOSR; // expected-warning@cerrno.syn.verify.cpp:* 2 {{macro 'ENOSR' has been marked as deprecated}} +#endif +#ifdef ENOSTR +[[maybe_unused]] int nostr = + ENOSTR; // expected-warning@cerrno.syn.verify.cpp:* 2 {{macro 'ENOSTR' has been marked as deprecated}} +#endif +#ifdef ETIME +[[maybe_unused]] int timeout = + ETIME; // expected-warning@cerrno.syn.verify.cpp:* 2 {{macro 'ETIME' has been marked as deprecated}} +#endif diff --git a/libcxx/test/std/depr.cerro/system.error.syn.verify.cpp b/libcxx/test/std/depr.cerro/system.error.syn.verify.cpp new file mode 100644 index 000000000000..fab5dd5b5593 --- /dev/null +++ b/libcxx/test/std/depr.cerro/system.error.syn.verify.cpp @@ -0,0 +1,28 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// These macros do not seem to behave as expected on all Apple platforms. +// Since the macros are not provided newer POSIX versions it is expected the +// macros will be retroactively removed from C++. (The deprecation was +// retroactively.) +// UNSUPPORTED: apple-clang && (c++03 || clang-modules-build) + +// + +// enum errc {...} + +// tests LWG 3869 deprecated enum members. + +#include + +[[maybe_unused]] std::errc nodata = + std::errc::no_message_available; // expected-warning {{'no_message_available' is deprecated}} +[[maybe_unused]] std::errc nosr = + std::errc::no_stream_resources; // expected-warning {{'no_stream_resources' is deprecated}} +[[maybe_unused]] std::errc nostr = std::errc::not_a_stream; // expected-warning {{'not_a_stream' is deprecated}} +[[maybe_unused]] std::errc timeout = std::errc::stream_timeout; // expected-warning {{'stream_timeout' is deprecated}} diff --git a/libcxx/test/std/diagnostics/syserr/errc.pass.cpp b/libcxx/test/std/diagnostics/syserr/errc.pass.cpp index e44cb50102e3..4abee08ddc66 100644 --- a/libcxx/test/std/diagnostics/syserr/errc.pass.cpp +++ b/libcxx/test/std/diagnostics/syserr/errc.pass.cpp @@ -6,6 +6,8 @@ // //===----------------------------------------------------------------------===// +// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS + // // enum errc {...} -- GitLab From 7e72cafd68335e45d95ea6b7705bb5b9e7e442c8 Mon Sep 17 00:00:00 2001 From: AtariDreams <83477269+AtariDreams@users.noreply.github.com> Date: Thu, 21 Mar 2024 07:15:17 -0400 Subject: [PATCH 126/296] [SelectionDAG] Add MaskedValueIsZero check to allow folding of zero extended variables we know are safe to extend (#85573) Add ones for every high bit that will cleared. This will allow us to evaluate variables that have their bits known to see if they have no risk of overflow despite the shift amount being greater than the difference between the two types. --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 15 ++- llvm/test/CodeGen/X86/dagcombine-shifts.ll | 127 ++++++++++++++++++ 2 files changed, 139 insertions(+), 3 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index a3f5d433d920..c83793d15b2e 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -13832,11 +13832,20 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) { if (N0.getOpcode() == ISD::SHL) { // If the original shl may be shifting out bits, do not perform this // transformation. - // TODO: Add MaskedValueIsZero check. unsigned KnownZeroBits = ShVal.getValueSizeInBits() - ShVal.getOperand(0).getValueSizeInBits(); - if (ShAmtC->getAPIntValue().ugt(KnownZeroBits)) - return SDValue(); + if (ShAmtC->getAPIntValue().ugt(KnownZeroBits)) { + // If the shift is too large, then see if we can deduce that the + // shift is safe anyway. + // Create a mask that has ones for the bits being shifted out. + APInt ShiftOutMask = + APInt::getHighBitsSet(ShVal.getValueSizeInBits(), + ShAmtC->getAPIntValue().getZExtValue()); + + // Check if the bits being shifted out are known to be zero. + if (!DAG.MaskedValueIsZero(ShVal, ShiftOutMask)) + return SDValue(); + } } // Ensure that the shift amount is wide enough for the shifted value. diff --git a/llvm/test/CodeGen/X86/dagcombine-shifts.ll b/llvm/test/CodeGen/X86/dagcombine-shifts.ll index 42b325dd4c22..734abfe55a4e 100644 --- a/llvm/test/CodeGen/X86/dagcombine-shifts.ll +++ b/llvm/test/CodeGen/X86/dagcombine-shifts.ll @@ -322,5 +322,132 @@ define void @g(i32 %a) nounwind { ret void } +define i32 @shift_zext_shl(i8 zeroext %x) { +; X86-LABEL: shift_zext_shl: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax +; X86-NEXT: andl $64, %eax +; X86-NEXT: shll $9, %eax +; X86-NEXT: retl +; +; X64-LABEL: shift_zext_shl: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: andl $64, %eax +; X64-NEXT: shll $9, %eax +; X64-NEXT: retq + %a = and i8 %x, 64 + %b = zext i8 %a to i16 + %c = shl i16 %b, 9 + %d = zext i16 %c to i32 + ret i32 %d +} + +define i32 @shift_zext_shl2(i8 zeroext %x) { +; X86-LABEL: shift_zext_shl2: +; X86: # %bb.0: +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax +; X86-NEXT: andl $64, %eax +; X86-NEXT: shll $9, %eax +; X86-NEXT: retl +; +; X64-LABEL: shift_zext_shl2: +; X64: # %bb.0: +; X64-NEXT: movl %edi, %eax +; X64-NEXT: andl $64, %eax +; X64-NEXT: shll $9, %eax +; X64-NEXT: retq + %a = and i8 %x, 64 + %b = zext i8 %a to i32 + %c = shl i32 %b, 9 + ret i32 %c +} + +define <4 x i32> @shift_zext_shl_vec(<4 x i8> %x) nounwind { +; X86-LABEL: shift_zext_shl_vec: +; X86: # %bb.0: +; X86-NEXT: pushl %edi +; X86-NEXT: pushl %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edi +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: andl $64, %ecx +; X86-NEXT: shll $9, %ecx +; X86-NEXT: andl $63, %edx +; X86-NEXT: shll $8, %edx +; X86-NEXT: andl $31, %esi +; X86-NEXT: shll $7, %esi +; X86-NEXT: andl $23, %edi +; X86-NEXT: shll $6, %edi +; X86-NEXT: movl %edi, 12(%eax) +; X86-NEXT: movl %esi, 8(%eax) +; X86-NEXT: movl %edx, 4(%eax) +; X86-NEXT: movl %ecx, (%eax) +; X86-NEXT: popl %esi +; X86-NEXT: popl %edi +; X86-NEXT: retl $4 +; +; X64-LABEL: shift_zext_shl_vec: +; X64: # %bb.0: +; X64-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-NEXT: pxor %xmm1, %xmm1 +; X64-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; X64-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; X64-NEXT: retq + %a = and <4 x i8> %x, + %b = zext <4 x i8> %a to <4 x i16> + %c = shl <4 x i16> %b, + %d = zext <4 x i16> %c to <4 x i32> + ret <4 x i32> %d +} + +define <4 x i32> @shift_zext_shl2_vec(<4 x i8> %x) nounwind { +; X86-LABEL: shift_zext_shl2_vec: +; X86: # %bb.0: +; X86-NEXT: pushl %edi +; X86-NEXT: pushl %esi +; X86-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edx +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %esi +; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edi +; X86-NEXT: andl $23, %edi +; X86-NEXT: andl $31, %esi +; X86-NEXT: andl $63, %edx +; X86-NEXT: andl $64, %ecx +; X86-NEXT: shll $9, %ecx +; X86-NEXT: shll $8, %edx +; X86-NEXT: shll $7, %esi +; X86-NEXT: shll $6, %edi +; X86-NEXT: movl %edi, 12(%eax) +; X86-NEXT: movl %esi, 8(%eax) +; X86-NEXT: movl %edx, 4(%eax) +; X86-NEXT: movl %ecx, (%eax) +; X86-NEXT: popl %esi +; X86-NEXT: popl %edi +; X86-NEXT: retl $4 +; +; X64-LABEL: shift_zext_shl2_vec: +; X64: # %bb.0: +; X64-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-NEXT: pxor %xmm1, %xmm1 +; X64-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; X64-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3] +; X64-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; X64-NEXT: pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; X64-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; X64-NEXT: retq + %a = and <4 x i8> %x, + %b = zext <4 x i8> %a to <4 x i32> + %c = shl <4 x i32> %b, + ret <4 x i32> %c +} + declare dso_local void @f(i64) -- GitLab From 83e5a1239242d64110e3dfa96ed3889170ab96b2 Mon Sep 17 00:00:00 2001 From: Christian Sigg Date: Thu, 21 Mar 2024 12:21:44 +0100 Subject: [PATCH 127/296] [mlir][bazel] Don't expose interface headers from //mlir:IR. (#85867) Move 3 interface headers in `//mlir:IR` from `hdrs` to `srcs`. Header files should not be added to multiple targets, but this is hard to avoid because CMake is less strict with headers. But we should at least avoid exposing them as headers by multiple targets because it confuses tooling. --- .../llvm-project-overlay/mlir/BUILD.bazel | 22 ++++++++++++++++--- .../mlir/examples/toy/Ch5/BUILD.bazel | 1 + .../mlir/examples/toy/Ch6/BUILD.bazel | 1 + .../mlir/examples/toy/Ch7/BUILD.bazel | 1 + .../mlir/test/BUILD.bazel | 4 ++++ .../mlir/unittests/BUILD.bazel | 1 + 6 files changed, 27 insertions(+), 3 deletions(-) diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 3951a31bae3e..e20aebe95063 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -330,16 +330,16 @@ cc_library( "lib/Bytecode/*.h", ]) + [ "include/mlir/IR/PDLPatternMatch.h.inc", + "include/mlir/Interfaces/CallInterfaces.h", + "include/mlir/Interfaces/DataLayoutInterfaces.h", + "include/mlir/Interfaces/SideEffectInterfaces.h", "lib/Bytecode/BytecodeOpInterface.cpp", ], hdrs = glob([ "include/mlir/IR/*.h", "include/mlir/Bytecode/*.h", ]) + [ - "include/mlir/Interfaces/CallInterfaces.h", - "include/mlir/Interfaces/DataLayoutInterfaces.h", "include/mlir/Interfaces/FoldInterfaces.h", - "include/mlir/Interfaces/SideEffectInterfaces.h", ], includes = ["include"], deps = [ @@ -1646,6 +1646,7 @@ cc_library( ":IR", ":MemRefDialect", ":Pass", + ":SideEffectInterfaces", ":Support", ":TransformUtils", ":Transforms", @@ -3131,6 +3132,7 @@ cc_library( ":DialectUtils", ":IR", ":InferTypeOpInterface", + ":SideEffectInterfaces", ":SparseTensorAttrDefsIncGen", ":SparseTensorEnums", ":SparseTensorInterfacesIncGen", @@ -4017,6 +4019,7 @@ cc_library( ":AffineDialect", ":Analysis", ":ArithDialect", + ":CallOpInterfaces", ":DialectUtils", ":FuncDialect", ":IR", @@ -4100,6 +4103,7 @@ cc_library( ":Pass", ":SCFDialect", ":SCFUtils", + ":SideEffectInterfaces", ":Support", ":TensorDialect", ":Transforms", @@ -6947,6 +6951,7 @@ cc_library( ]), includes = ["include"], deps = [ + ":CallOpInterfaces", ":CommonFolders", ":ControlFlowInterfaces", ":FunctionInterfaces", @@ -7591,6 +7596,7 @@ cc_library( includes = ["include"], deps = [ ":Analysis", + ":CallOpInterfaces", ":ControlFlowInterfaces", ":FunctionInterfaces", ":IR", @@ -7921,6 +7927,7 @@ cc_library( includes = ["include"], deps = [ ":Analysis", + ":CallOpInterfaces", ":ControlFlowInterfaces", ":FunctionInterfaces", ":IR", @@ -7930,6 +7937,7 @@ cc_library( ":Pass", ":Rewrite", ":RuntimeVerifiableOpInterface", + ":SideEffectInterfaces", ":Support", ":TransformUtils", ":TransformsPassIncGen", @@ -8081,6 +8089,7 @@ cc_library( hdrs = glob(["include/mlir/Conversion/LLVMCommon/*.h"]), includes = ["include"], deps = [ + ":DataLayoutInterfaces", ":IR", ":LLVMDialect", ":Support", @@ -9052,6 +9061,7 @@ cc_library( includes = ["include"], deps = [ ":DLTIDialect", + ":DataLayoutInterfaces", ":IR", ":LLVMConversionIncGen", ":LLVMDialect", @@ -12681,6 +12691,7 @@ cc_library( ":ArithOpsIncGen", ":ArithOpsInterfacesIncGen", ":BufferizationInterfaces", + ":CallOpInterfaces", ":CastInterfaces", ":CommonFolders", ":ControlFlowInterfaces", @@ -12692,6 +12703,7 @@ cc_library( ":InferTypeOpInterface", ":InliningUtils", ":Pass", + ":SideEffectInterfaces", ":Support", ":UBDialect", ":ValueBoundsOpInterfaceIncGen", @@ -12998,6 +13010,7 @@ cc_library( ":ArithDialect", ":ArithUtils", ":BufferizationInterfaces", + ":CallOpInterfaces", ":CastInterfaces", ":ComplexDialect", ":ControlFlowInterfaces", @@ -13012,6 +13025,7 @@ cc_library( ":MemorySlotInterfaces", ":RuntimeVerifiableOpInterface", ":ShapedOpInterfaces", + ":SideEffectInterfaces", ":Support", ":ValueBoundsOpInterface", ":ViewLikeInterface", @@ -13259,6 +13273,7 @@ cc_library( hdrs = glob(["include/mlir/Dialect/MLProgram/IR/*.h"]), includes = ["include"], deps = [ + ":CallOpInterfaces", ":ControlFlowInterfaces", ":FunctionInterfaces", ":IR", @@ -13267,6 +13282,7 @@ cc_library( ":MLProgramOpsIncGen", ":MLProgramTypesIncGen", ":Pass", + ":SideEffectInterfaces", ":Support", ":Transforms", "//llvm:Support", diff --git a/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch5/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch5/BUILD.bazel index ce48e249489d..2c49d52f1ed0 100644 --- a/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch5/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch5/BUILD.bazel @@ -102,6 +102,7 @@ cc_binary( "//mlir:Analysis", "//mlir:ArithDialect", "//mlir:BytecodeReader", + "//mlir:CallOpInterfaces", "//mlir:CastInterfaces", "//mlir:FuncDialect", "//mlir:FuncExtensions", diff --git a/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch6/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch6/BUILD.bazel index 286c08065645..cd7f7f018166 100644 --- a/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch6/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch6/BUILD.bazel @@ -108,6 +108,7 @@ cc_binary( "//mlir:ArithToLLVM", "//mlir:BuiltinToLLVMIRTranslation", "//mlir:BytecodeReader", + "//mlir:CallOpInterfaces", "//mlir:CastInterfaces", "//mlir:ControlFlowToLLVM", "//mlir:ExecutionEngine", diff --git a/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch7/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch7/BUILD.bazel index f4037cab03f6..c03672eb4136 100644 --- a/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch7/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/examples/toy/Ch7/BUILD.bazel @@ -108,6 +108,7 @@ cc_binary( "//mlir:ArithToLLVM", "//mlir:BuiltinToLLVMIRTranslation", "//mlir:BytecodeReader", + "//mlir:CallOpInterfaces", "//mlir:CastInterfaces", "//mlir:ControlFlowToLLVM", "//mlir:ExecutionEngine", diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel index 771cbcc4eea0..df2392fd8c6e 100644 --- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel @@ -94,12 +94,14 @@ cc_library( "//mlir:AffineAnalysis", "//mlir:AffineDialect", "//mlir:Analysis", + "//mlir:CallOpInterfaces", "//mlir:ControlFlowInterfaces", "//mlir:FuncDialect", "//mlir:FunctionInterfaces", "//mlir:IR", "//mlir:MemRefDialect", "//mlir:Pass", + "//mlir:SideEffectInterfaces", "//mlir:Support", ], ) @@ -386,6 +388,7 @@ cc_library( ":TestTypeDefsIncGen", "//llvm:Support", "//mlir:ArithDialect", + "//mlir:CallOpInterfaces", "//mlir:ControlFlowInterfaces", "//mlir:CopyOpInterface", "//mlir:DLTIDialect", @@ -571,6 +574,7 @@ cc_library( "//mlir:Pass", "//mlir:SCFDialect", "//mlir:SPIRVDialect", + "//mlir:SideEffectInterfaces", "//mlir:Support", "//mlir:Transforms", ], diff --git a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel index 252b9ec951f6..c6b630230bb3 100644 --- a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel @@ -40,6 +40,7 @@ cc_test( deps = [ "//llvm:Support", "//mlir:BytecodeReader", + "//mlir:CallOpInterfaces", "//mlir:FunctionInterfaces", "//mlir:IR", "//mlir:Parser", -- GitLab From 3ac243bc0d7922d083af2cf025247b5698556062 Mon Sep 17 00:00:00 2001 From: SahilPatidar Date: Thu, 21 Mar 2024 16:52:08 +0530 Subject: [PATCH 128/296] Update amdgpu_gfx functions to use s0-s3 for inreg SGPR arguments on targets using scratch instructions for stack #78226 (#81394) Resolve #78226 --- llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp | 5 +- llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td | 1 + llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 7 +- .../GlobalISel/irtranslator-call-non-fixed.ll | 10 +- .../AMDGPU/GlobalISel/irtranslator-call.ll | 10 +- llvm/test/CodeGen/AMDGPU/bf16.ll | 2 +- .../CodeGen/AMDGPU/combine_andor_with_cmps.ll | 24 +- .../CodeGen/AMDGPU/function-args-inreg.ll | 133 + .../AMDGPU/gfx-callable-argument-types.ll | 4371 +++++++---------- llvm/test/CodeGen/AMDGPU/indirect-call.ll | 60 +- .../CodeGen/AMDGPU/schedule-addrspaces.ll | 2 +- .../CodeGen/AMDGPU/scratch-pointer-sink.ll | 4 +- 12 files changed, 2069 insertions(+), 2560 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp index 7e1f041fa109..8969b3b5b6ce 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp @@ -715,10 +715,13 @@ bool AMDGPUCallLowering::lowerFormalArguments( if (!IsEntryFunc && !IsGraphics) { // For the fixed ABI, pass workitem IDs in the last argument register. TLI.allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info); + } + if (!IsEntryFunc) { if (!Subtarget.enableFlatScratch()) CCInfo.AllocateReg(Info->getScratchRSrcReg()); - TLI.allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info); + if (!IsGraphics) + TLI.allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info); } IncomingValueAssigner Assigner(AssignFn); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td b/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td index 4be64629ddac..9bd0fd7eca76 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td @@ -23,6 +23,7 @@ def CC_SI_Gfx : CallingConv<[ // 33 is reserved for the frame pointer // 34 is reserved for the base pointer CCIfInRegenableFlatScratch()) CCInfo.AllocateReg(Info->getScratchRSrcReg()); - - allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info); + if (!IsGraphics) + allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info); } if (!IsKernel) { diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll index 5effd24a7520..fad833c0a6ad 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-non-fixed.ll @@ -50,10 +50,10 @@ define amdgpu_gfx void @test_gfx_call_external_void_func_i32_imm_inreg(i32 inreg ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42 ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc ; CHECK-NEXT: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_gfx_void_func_i32_inreg - ; CHECK-NEXT: $sgpr4 = COPY [[C]](s32) + ; CHECK-NEXT: $sgpr0 = COPY [[C]](s32) ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3 ; CHECK-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY1]](<4 x s32>) - ; CHECK-NEXT: $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr4, implicit $sgpr0_sgpr1_sgpr2_sgpr3 + ; CHECK-NEXT: $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3 ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc ; CHECK-NEXT: SI_RETURN call amdgpu_gfx void @external_gfx_void_func_i32_inreg(i32 inreg 42) @@ -99,11 +99,11 @@ define amdgpu_gfx void @test_gfx_call_external_void_func_struct_i8_i32_inreg() # ; CHECK-NEXT: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_gfx_void_func_struct_i8_i32_inreg ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[LOAD1]](s8) ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT]](s16) - ; CHECK-NEXT: $sgpr4 = COPY [[ANYEXT1]](s32) - ; CHECK-NEXT: $sgpr5 = COPY [[LOAD2]](s32) + ; CHECK-NEXT: $sgpr0 = COPY [[ANYEXT1]](s32) + ; CHECK-NEXT: $sgpr1 = COPY [[LOAD2]](s32) ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3 ; CHECK-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY]](<4 x s32>) - ; CHECK-NEXT: $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_struct_i8_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr4, implicit $sgpr5, implicit $sgpr0_sgpr1_sgpr2_sgpr3 + ; CHECK-NEXT: $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_struct_i8_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr0, implicit $sgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3 ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc ; CHECK-NEXT: SI_RETURN %ptr0 = load ptr addrspace(1), ptr addrspace(4) undef diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll index 392b0ae6823e..75670604baa1 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call.ll @@ -942,10 +942,10 @@ define amdgpu_gfx void @test_gfx_call_external_void_func_i32_imm_inreg(i32 inreg ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42 ; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc ; CHECK-NEXT: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_gfx_void_func_i32_inreg - ; CHECK-NEXT: $sgpr4 = COPY [[C]](s32) + ; CHECK-NEXT: $sgpr0 = COPY [[C]](s32) ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3 ; CHECK-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY1]](<4 x s32>) - ; CHECK-NEXT: $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr4, implicit $sgpr0_sgpr1_sgpr2_sgpr3 + ; CHECK-NEXT: $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3 ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc ; CHECK-NEXT: SI_RETURN call amdgpu_gfx void @external_gfx_void_func_i32_inreg(i32 inreg 42) @@ -3984,11 +3984,11 @@ define amdgpu_gfx void @test_gfx_call_external_void_func_struct_i8_i32_inreg() # ; CHECK-NEXT: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_gfx_void_func_struct_i8_i32_inreg ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[LOAD1]](s8) ; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ANYEXT]](s16) - ; CHECK-NEXT: $sgpr4 = COPY [[ANYEXT1]](s32) - ; CHECK-NEXT: $sgpr5 = COPY [[LOAD2]](s32) + ; CHECK-NEXT: $sgpr0 = COPY [[ANYEXT1]](s32) + ; CHECK-NEXT: $sgpr1 = COPY [[LOAD2]](s32) ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3 ; CHECK-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY]](<4 x s32>) - ; CHECK-NEXT: $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_struct_i8_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr4, implicit $sgpr5, implicit $sgpr0_sgpr1_sgpr2_sgpr3 + ; CHECK-NEXT: $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_gfx_void_func_struct_i8_i32_inreg, csr_amdgpu_si_gfx, implicit $sgpr0, implicit $sgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3 ; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc ; CHECK-NEXT: SI_RETURN %ptr0 = load ptr addrspace(1), ptr addrspace(4) undef diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll index 98658834e897..e369f7e3b9a5 100644 --- a/llvm/test/CodeGen/AMDGPU/bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/bf16.ll @@ -3337,7 +3337,7 @@ define amdgpu_gfx void @test_inreg_arg_store(bfloat inreg %in, ptr addrspace(1) ; GFX11-LABEL: test_inreg_arg_store: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: v_mov_b32_e32 v2, s4 +; GFX11-NEXT: v_mov_b32_e32 v2, s0 ; GFX11-NEXT: global_store_b16 v[0:1], v2, off ; GFX11-NEXT: s_setpc_b64 s[30:31] store bfloat %in, ptr addrspace(1) %out diff --git a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll index 10d71a315fbf..e1e3220cc275 100644 --- a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll +++ b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll @@ -472,7 +472,7 @@ define amdgpu_gfx void @test34(i32 inreg %arg1, i32 inreg %arg2) { ; GCN-LABEL: test34: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_min_i32 s0, s4, s5 +; GCN-NEXT: s_min_i32 s0, s0, s1 ; GCN-NEXT: v_mov_b32_e32 v0, 0 ; GCN-NEXT: s_cmpk_lt_i32 s0, 0x3e9 ; GCN-NEXT: v_mov_b32_e32 v1, 0 @@ -492,7 +492,7 @@ define amdgpu_gfx void @test35(i32 inreg %arg1, i32 inreg %arg2) { ; GCN-LABEL: test35: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_max_i32 s0, s4, s5 +; GCN-NEXT: s_max_i32 s0, s0, s1 ; GCN-NEXT: v_mov_b32_e32 v0, 0 ; GCN-NEXT: s_cmpk_gt_i32 s0, 0x3e8 ; GCN-NEXT: v_mov_b32_e32 v1, 0 @@ -512,9 +512,9 @@ define amdgpu_gfx void @test36(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3 ; GCN-LABEL: test36: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_min_u32 s0, s4, s5 +; GCN-NEXT: s_min_u32 s0, s0, s1 ; GCN-NEXT: v_mov_b32_e32 v0, 0 -; GCN-NEXT: s_cmp_lt_u32 s0, s6 +; GCN-NEXT: s_cmp_lt_u32 s0, s2 ; GCN-NEXT: v_mov_b32_e32 v1, 0 ; GCN-NEXT: s_cselect_b32 s0, -1, 0 ; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 @@ -532,9 +532,9 @@ define amdgpu_gfx void @test37(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3 ; GCN-LABEL: test37: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_max_i32 s0, s4, s5 +; GCN-NEXT: s_max_i32 s0, s0, s1 ; GCN-NEXT: v_mov_b32_e32 v0, 0 -; GCN-NEXT: s_cmp_ge_i32 s0, s6 +; GCN-NEXT: s_cmp_ge_i32 s0, s2 ; GCN-NEXT: v_mov_b32_e32 v1, 0 ; GCN-NEXT: s_cselect_b32 s0, -1, 0 ; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 @@ -552,7 +552,7 @@ define amdgpu_gfx void @test38(i32 inreg %arg1, i32 inreg %arg2) { ; GCN-LABEL: test38: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_max_u32 s0, s4, s5 +; GCN-NEXT: s_max_u32 s0, s0, s1 ; GCN-NEXT: v_mov_b32_e32 v0, 0 ; GCN-NEXT: s_cmpk_lt_u32 s0, 0x3e9 ; GCN-NEXT: v_mov_b32_e32 v1, 0 @@ -572,7 +572,7 @@ define amdgpu_gfx void @test39(i32 inreg %arg1, i32 inreg %arg2) { ; GCN-LABEL: test39: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_min_i32 s0, s4, s5 +; GCN-NEXT: s_min_i32 s0, s0, s1 ; GCN-NEXT: v_mov_b32_e32 v0, 0 ; GCN-NEXT: s_cmpk_gt_i32 s0, 0x3e7 ; GCN-NEXT: v_mov_b32_e32 v1, 0 @@ -592,9 +592,9 @@ define amdgpu_gfx void @test40(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3 ; GCN-LABEL: test40: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_max_i32 s0, s4, s5 +; GCN-NEXT: s_max_i32 s0, s0, s1 ; GCN-NEXT: v_mov_b32_e32 v0, 0 -; GCN-NEXT: s_cmp_le_i32 s0, s6 +; GCN-NEXT: s_cmp_le_i32 s0, s2 ; GCN-NEXT: v_mov_b32_e32 v1, 0 ; GCN-NEXT: s_cselect_b32 s0, -1, 0 ; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 @@ -612,9 +612,9 @@ define amdgpu_gfx void @test41(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3 ; GCN-LABEL: test41: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_min_u32 s0, s4, s5 +; GCN-NEXT: s_min_u32 s0, s0, s1 ; GCN-NEXT: v_mov_b32_e32 v0, 0 -; GCN-NEXT: s_cmp_ge_u32 s0, s6 +; GCN-NEXT: s_cmp_ge_u32 s0, s2 ; GCN-NEXT: v_mov_b32_e32 v1, 0 ; GCN-NEXT: s_cselect_b32 s0, -1, 0 ; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0 diff --git a/llvm/test/CodeGen/AMDGPU/function-args-inreg.ll b/llvm/test/CodeGen/AMDGPU/function-args-inreg.ll index 44a9127b4bd0..27845b6b5b2f 100644 --- a/llvm/test/CodeGen/AMDGPU/function-args-inreg.ll +++ b/llvm/test/CodeGen/AMDGPU/function-args-inreg.ll @@ -2176,6 +2176,93 @@ define void @void_func_a5i32_inreg([5 x i32] inreg %arg0, ptr addrspace(1) %ptr) declare void @extern() define void @void_func_a13i32_inreg([13 x i32] inreg %arg0, ptr addrspace(1) %ptr) { +; GFX9-LABEL: void_func_a13i32_inreg: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s27, s33 +; GFX9-NEXT: s_mov_b32 s33, s32 +; GFX9-NEXT: s_or_saveexec_b64 s[28:29], -1 +; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill +; GFX9-NEXT: s_mov_b64 exec, s[28:29] +; GFX9-NEXT: v_mov_b32_e32 v2, s26 +; GFX9-NEXT: global_store_dword v[0:1], v2, off offset:48 +; GFX9-NEXT: v_mov_b32_e32 v5, s25 +; GFX9-NEXT: v_mov_b32_e32 v4, s24 +; GFX9-NEXT: v_mov_b32_e32 v3, s23 +; GFX9-NEXT: v_mov_b32_e32 v2, s22 +; GFX9-NEXT: s_addk_i32 s32, 0x400 +; GFX9-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:32 +; GFX9-NEXT: v_writelane_b32 v40, s27, 2 +; GFX9-NEXT: v_mov_b32_e32 v5, s21 +; GFX9-NEXT: v_mov_b32_e32 v4, s20 +; GFX9-NEXT: v_mov_b32_e32 v3, s19 +; GFX9-NEXT: v_mov_b32_e32 v2, s18 +; GFX9-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:16 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, s17 +; GFX9-NEXT: v_mov_b32_e32 v4, s16 +; GFX9-NEXT: s_getpc_b64 s[16:17] +; GFX9-NEXT: s_add_u32 s16, s16, extern@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s17, s17, extern@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[16:17], 0x0 +; GFX9-NEXT: v_mov_b32_e32 v3, s7 +; GFX9-NEXT: v_mov_b32_e32 v2, s6 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 +; GFX9-NEXT: global_store_dwordx4 v[0:1], v[2:5], off +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s4, v40, 2 +; GFX9-NEXT: s_or_saveexec_b64 s[6:7], -1 +; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload +; GFX9-NEXT: s_mov_b64 exec, s[6:7] +; GFX9-NEXT: s_addk_i32 s32, 0xfc00 +; GFX9-NEXT: s_mov_b32 s33, s4 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: void_func_a13i32_inreg: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: s_mov_b32 s23, s33 +; GFX11-NEXT: s_mov_b32 s33, s32 +; GFX11-NEXT: s_or_saveexec_b32 s24, -1 +; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill +; GFX11-NEXT: s_mov_b32 exec_lo, s24 +; GFX11-NEXT: s_add_i32 s32, s32, 16 +; GFX11-NEXT: v_dual_mov_b32 v4, s20 :: v_dual_mov_b32 v3, s19 +; GFX11-NEXT: v_dual_mov_b32 v2, s18 :: v_dual_mov_b32 v9, s17 +; GFX11-NEXT: s_getpc_b64 s[18:19] +; GFX11-NEXT: s_add_u32 s18, s18, extern@gotpcrel32@lo+4 +; GFX11-NEXT: s_addc_u32 s19, s19, extern@gotpcrel32@hi+12 +; GFX11-NEXT: v_dual_mov_b32 v8, s16 :: v_dual_mov_b32 v7, s7 +; GFX11-NEXT: s_load_b64 s[16:17], s[18:19], 0x0 +; GFX11-NEXT: v_writelane_b32 v40, s23, 2 +; GFX11-NEXT: v_dual_mov_b32 v14, s22 :: v_dual_mov_b32 v5, s21 +; GFX11-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v13, s3 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_dual_mov_b32 v12, s2 :: v_dual_mov_b32 v11, s1 +; GFX11-NEXT: v_mov_b32_e32 v10, s0 +; GFX11-NEXT: s_clause 0x3 +; GFX11-NEXT: global_store_b32 v[0:1], v14, off offset:48 +; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off offset:32 +; GFX11-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16 +; GFX11-NEXT: global_store_b128 v[0:1], v[10:13], off +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 +; GFX11-NEXT: s_or_saveexec_b32 s1, -1 +; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload +; GFX11-NEXT: s_mov_b32 exec_lo, s1 +; GFX11-NEXT: s_add_i32 s32, s32, -16 +; GFX11-NEXT: s_mov_b32 s33, s0 +; GFX11-NEXT: s_waitcnt vmcnt(0) +; GFX11-NEXT: s_setpc_b64 s[30:31] store [13 x i32] %arg0, ptr addrspace(1) %ptr call void @extern() ret void @@ -2203,6 +2290,52 @@ define void @void_func_a13i32_inreg([13 x i32] inreg %arg0, ptr addrspace(1) %p ; FIXME: Should still fail define void @void_func_a16i32_inreg__noimplicit([16 x i32] inreg %arg0, ptr addrspace(1) %ptr) { +; GFX9-LABEL: void_func_a16i32_inreg__noimplicit: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v5, s19 +; GFX9-NEXT: v_mov_b32_e32 v4, s18 +; GFX9-NEXT: v_mov_b32_e32 v3, s17 +; GFX9-NEXT: v_mov_b32_e32 v2, s16 +; GFX9-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:48 +; GFX9-NEXT: s_nop 0 +; GFX9-NEXT: v_mov_b32_e32 v5, s15 +; GFX9-NEXT: v_mov_b32_e32 v4, s14 +; GFX9-NEXT: v_mov_b32_e32 v3, s13 +; GFX9-NEXT: v_mov_b32_e32 v2, s12 +; GFX9-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:32 +; GFX9-NEXT: s_nop 0 +; GFX9-NEXT: v_mov_b32_e32 v5, s11 +; GFX9-NEXT: v_mov_b32_e32 v4, s10 +; GFX9-NEXT: v_mov_b32_e32 v3, s9 +; GFX9-NEXT: v_mov_b32_e32 v2, s8 +; GFX9-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:16 +; GFX9-NEXT: s_nop 0 +; GFX9-NEXT: v_mov_b32_e32 v5, s7 +; GFX9-NEXT: v_mov_b32_e32 v4, s6 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: global_store_dwordx4 v[0:1], v[2:5], off +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: void_func_a16i32_inreg__noimplicit: +; GFX11: ; %bb.0: +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v5, s15 :: v_dual_mov_b32 v4, s14 +; GFX11-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v2, s12 +; GFX11-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v8, s10 +; GFX11-NEXT: v_dual_mov_b32 v7, s9 :: v_dual_mov_b32 v6, s8 +; GFX11-NEXT: v_dual_mov_b32 v13, s7 :: v_dual_mov_b32 v12, s6 +; GFX11-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v10, s4 +; GFX11-NEXT: v_dual_mov_b32 v17, s3 :: v_dual_mov_b32 v16, s2 +; GFX11-NEXT: v_dual_mov_b32 v15, s1 :: v_dual_mov_b32 v14, s0 +; GFX11-NEXT: s_clause 0x3 +; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off offset:48 +; GFX11-NEXT: global_store_b128 v[0:1], v[6:9], off offset:32 +; GFX11-NEXT: global_store_b128 v[0:1], v[10:13], off offset:16 +; GFX11-NEXT: global_store_b128 v[0:1], v[14:17], off +; GFX11-NEXT: s_setpc_b64 s[30:31] store [16 x i32] %arg0, ptr addrspace(1) %ptr ret void } diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll index a118fa388f86..3e1db5fb4e1d 100644 --- a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll +++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll @@ -9567,19 +9567,17 @@ define amdgpu_gfx void @test_call_external_void_func_i8_imm_inreg(i32) #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 3 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 1 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_i8_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_i8_inreg@abs32@lo -; GFX9-NEXT: s_movk_i32 s4, 0x7b +; GFX9-NEXT: s_movk_i32 s0, 0x7b ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 2 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 2 -; GFX9-NEXT: v_readlane_b32 s30, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 3 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -9597,19 +9595,17 @@ define amdgpu_gfx void @test_call_external_void_func_i8_imm_inreg(i32) #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 3 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_i8_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_i8_inreg@abs32@lo +; GFX10-NEXT: s_movk_i32 s0, 0x7b ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_movk_i32 s4, 0x7b -; GFX10-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-NEXT: v_writelane_b32 v40, s31, 2 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 3 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -9627,20 +9623,18 @@ define amdgpu_gfx void @test_call_external_void_func_i8_imm_inreg(i32) #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 3 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_i8_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_i8_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_i8_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_i8_inreg@abs32@lo +; GFX11-NEXT: s_movk_i32 s0, 0x7b ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_movk_i32 s4, 0x7b -; GFX11-NEXT: v_writelane_b32 v40, s30, 1 -; GFX11-NEXT: v_writelane_b32 v40, s31, 2 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 2 -; GFX11-NEXT: v_readlane_b32 s30, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 3 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -9658,19 +9652,17 @@ define amdgpu_gfx void @test_call_external_void_func_i8_imm_inreg(i32) #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_i8_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_i8_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_i8_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_i8_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_movk_i32 s0, 0x7b ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_movk_i32 s4, 0x7b -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 2 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -9692,19 +9684,17 @@ define amdgpu_gfx void @test_call_external_void_func_i16_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 3 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 1 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_i16_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_i16_inreg@abs32@lo -; GFX9-NEXT: s_movk_i32 s4, 0x7b +; GFX9-NEXT: s_movk_i32 s0, 0x7b ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 2 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 2 -; GFX9-NEXT: v_readlane_b32 s30, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 3 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -9722,19 +9712,17 @@ define amdgpu_gfx void @test_call_external_void_func_i16_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 3 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_i16_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_i16_inreg@abs32@lo +; GFX10-NEXT: s_movk_i32 s0, 0x7b ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_movk_i32 s4, 0x7b -; GFX10-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-NEXT: v_writelane_b32 v40, s31, 2 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 3 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -9752,20 +9740,18 @@ define amdgpu_gfx void @test_call_external_void_func_i16_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 3 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_i16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_i16_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_i16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_i16_inreg@abs32@lo +; GFX11-NEXT: s_movk_i32 s0, 0x7b ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_movk_i32 s4, 0x7b -; GFX11-NEXT: v_writelane_b32 v40, s30, 1 -; GFX11-NEXT: v_writelane_b32 v40, s31, 2 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 2 -; GFX11-NEXT: v_readlane_b32 s30, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 3 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -9783,19 +9769,17 @@ define amdgpu_gfx void @test_call_external_void_func_i16_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_i16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_i16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_i16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_i16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_movk_i32 s0, 0x7b ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_movk_i32 s4, 0x7b -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 2 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -9817,19 +9801,17 @@ define amdgpu_gfx void @test_call_external_void_func_i32_imm_inreg(i32) #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 3 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 1 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_i32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 42 +; GFX9-NEXT: s_mov_b32 s0, 42 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 2 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 2 -; GFX9-NEXT: v_readlane_b32 s30, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 3 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -9847,19 +9829,17 @@ define amdgpu_gfx void @test_call_external_void_func_i32_imm_inreg(i32) #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 3 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_i32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_i32_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 42 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 42 -; GFX10-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-NEXT: v_writelane_b32 v40, s31, 2 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 3 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -9877,20 +9857,18 @@ define amdgpu_gfx void @test_call_external_void_func_i32_imm_inreg(i32) #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 3 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_i32_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_i32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 42 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 42 -; GFX11-NEXT: v_writelane_b32 v40, s30, 1 -; GFX11-NEXT: v_writelane_b32 v40, s31, 2 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 2 -; GFX11-NEXT: v_readlane_b32 s30, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 3 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -9908,19 +9886,17 @@ define amdgpu_gfx void @test_call_external_void_func_i32_imm_inreg(i32) #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 42 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 42 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 2 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -9942,22 +9918,18 @@ define amdgpu_gfx void @test_call_external_void_func_i64_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_i64_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_i64_inreg@abs32@lo -; GFX9-NEXT: s_movk_i32 s4, 0x7b -; GFX9-NEXT: s_mov_b32 s5, 0 +; GFX9-NEXT: s_movk_i32 s0, 0x7b +; GFX9-NEXT: s_mov_b32 s1, 0 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -9975,22 +9947,18 @@ define amdgpu_gfx void @test_call_external_void_func_i64_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_i64_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_i64_inreg@abs32@lo +; GFX10-NEXT: s_movk_i32 s0, 0x7b +; GFX10-NEXT: s_mov_b32 s1, 0 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_movk_i32 s4, 0x7b -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 0 -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -10008,23 +9976,19 @@ define amdgpu_gfx void @test_call_external_void_func_i64_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_i64_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_i64_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_i64_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_i64_inreg@abs32@lo +; GFX11-NEXT: s_movk_i32 s0, 0x7b +; GFX11-NEXT: s_mov_b32 s1, 0 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_movk_i32 s4, 0x7b -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 0 -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -10042,22 +10006,18 @@ define amdgpu_gfx void @test_call_external_void_func_i64_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_i64_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_i64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_i64_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_i64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_movk_i32 s0, 0x7b +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_movk_i32 s4, 0x7b -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -10079,26 +10039,23 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 6 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 ; GFX9-NEXT: s_mov_b64 s[34:35], 0 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 4 +; GFX9-NEXT: s_load_dwordx4 s[36:39], s[34:35], 0x0 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 5 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s0, s36 +; GFX9-NEXT: s_mov_b32 s1, s37 +; GFX9-NEXT: s_mov_b32 s2, s38 +; GFX9-NEXT: s_mov_b32 s3, s39 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 5 -; GFX9-NEXT: v_readlane_b32 s30, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 6 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -10116,26 +10073,23 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 6 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b64 s[34:35], 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_load_dwordx4 s[4:7], s[34:35], 0x0 +; GFX10-NEXT: s_load_dwordx4 s[36:39], s[34:35], 0x0 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s0, s36 +; GFX10-NEXT: s_mov_b32 s1, s37 +; GFX10-NEXT: s_mov_b32 s2, s38 +; GFX10-NEXT: s_mov_b32 s3, s39 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 6 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -10153,27 +10107,19 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 6 +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 ; GFX11-NEXT: s_mov_b64 s[0:1], 0 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi +; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_load_b128 s[4:7], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v2i64_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v2i64_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 4 -; GFX11-NEXT: v_writelane_b32 v40, s31, 5 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 5 -; GFX11-NEXT: v_readlane_b32 s30, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 6 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -10191,26 +10137,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 6 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 ; GFX10-SCRATCH-NEXT: s_mov_b64 s[0:1], 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v2i64_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v2i64_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 5 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 6 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -10233,28 +10171,20 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 6 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: v_writelane_b32 v40, s30, 4 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 1 -; GFX9-NEXT: s_mov_b32 s5, 2 -; GFX9-NEXT: s_mov_b32 s6, 3 -; GFX9-NEXT: s_mov_b32 s7, 4 +; GFX9-NEXT: s_mov_b32 s0, 1 +; GFX9-NEXT: s_mov_b32 s1, 2 +; GFX9-NEXT: s_mov_b32 s2, 3 +; GFX9-NEXT: s_mov_b32 s3, 4 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 5 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 5 -; GFX9-NEXT: v_readlane_b32 s30, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 6 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -10272,28 +10202,20 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 6 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 1 +; GFX10-NEXT: s_mov_b32 s1, 2 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: s_mov_b32 s2, 3 +; GFX10-NEXT: s_mov_b32 s3, 4 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 1 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 3 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_mov_b32 s7, 4 -; GFX10-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 6 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -10311,29 +10233,21 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 6 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v2i64_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v2i64_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 1 +; GFX11-NEXT: s_mov_b32 s1, 2 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: s_mov_b32 s2, 3 +; GFX11-NEXT: s_mov_b32 s3, 4 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 1 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 3 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_mov_b32 s7, 4 -; GFX11-NEXT: v_writelane_b32 v40, s30, 4 -; GFX11-NEXT: v_writelane_b32 v40, s31, 5 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 5 -; GFX11-NEXT: v_readlane_b32 s30, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 6 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -10351,28 +10265,20 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 6 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v2i64_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v2i64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v2i64_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v2i64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 1 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 3 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, 4 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s7, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 5 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 6 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -10394,32 +10300,29 @@ define amdgpu_gfx void @test_call_external_void_func_v3i64_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 8 +; GFX9-NEXT: v_writelane_b32 v40, s34, 4 +; GFX9-NEXT: s_mov_b64 s[34:35], 0 +; GFX9-NEXT: s_load_dwordx4 s[36:39], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s4, 0 ; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: s_mov_b64 s[34:35], 0 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s8, 4 -; GFX9-NEXT: v_writelane_b32 v40, s9, 5 -; GFX9-NEXT: v_writelane_b32 v40, s30, 6 +; GFX9-NEXT: v_writelane_b32 v40, s30, 2 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3i64_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3i64_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s8, 1 -; GFX9-NEXT: s_mov_b32 s9, 2 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s0, s36 +; GFX9-NEXT: s_mov_b32 s1, s37 +; GFX9-NEXT: s_mov_b32 s2, s38 +; GFX9-NEXT: s_mov_b32 s3, s39 +; GFX9-NEXT: s_mov_b32 s4, 1 +; GFX9-NEXT: s_mov_b32 s5, 2 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 7 +; GFX9-NEXT: v_writelane_b32 v40, s31, 3 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 7 -; GFX9-NEXT: v_readlane_b32 s30, v40, 6 -; GFX9-NEXT: v_readlane_b32 s9, v40, 5 -; GFX9-NEXT: v_readlane_b32 s8, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 +; GFX9-NEXT: v_readlane_b32 s31, v40, 3 +; GFX9-NEXT: v_readlane_b32 s30, v40, 2 ; GFX9-NEXT: v_readlane_b32 s5, v40, 1 ; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 8 +; GFX9-NEXT: v_readlane_b32 s34, v40, 4 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -10437,32 +10340,29 @@ define amdgpu_gfx void @test_call_external_void_func_v3i64_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 8 +; GFX10-NEXT: v_writelane_b32 v40, s34, 4 ; GFX10-NEXT: s_mov_b64 s[34:35], 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_load_dwordx4 s[4:7], s[34:35], 0x0 +; GFX10-NEXT: s_load_dwordx4 s[36:39], s[34:35], 0x0 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3i64_inreg@abs32@hi +; GFX10-NEXT: v_writelane_b32 v40, s4, 0 ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3i64_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-NEXT: s_mov_b32 s8, 1 -; GFX10-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-NEXT: s_mov_b32 s9, 2 -; GFX10-NEXT: v_writelane_b32 v40, s30, 6 -; GFX10-NEXT: v_writelane_b32 v40, s31, 7 +; GFX10-NEXT: s_mov_b32 s4, 1 +; GFX10-NEXT: v_writelane_b32 v40, s5, 1 +; GFX10-NEXT: s_mov_b32 s5, 2 +; GFX10-NEXT: v_writelane_b32 v40, s30, 2 +; GFX10-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s0, s36 +; GFX10-NEXT: s_mov_b32 s1, s37 +; GFX10-NEXT: s_mov_b32 s2, s38 +; GFX10-NEXT: s_mov_b32 s3, s39 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 7 -; GFX10-NEXT: v_readlane_b32 s30, v40, 6 -; GFX10-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-NEXT: v_readlane_b32 s8, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 +; GFX10-NEXT: v_readlane_b32 s31, v40, 3 +; GFX10-NEXT: v_readlane_b32 s30, v40, 2 ; GFX10-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 8 +; GFX10-NEXT: v_readlane_b32 s34, v40, 4 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -10480,33 +10380,25 @@ define amdgpu_gfx void @test_call_external_void_func_v3i64_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 8 +; GFX11-NEXT: v_writelane_b32 v40, s0, 4 ; GFX11-NEXT: s_mov_b64 s[0:1], 0 -; GFX11-NEXT: s_add_i32 s32, s32, 16 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v3i64_inreg@abs32@hi +; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v3i64_inreg@abs32@lo ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 +; GFX11-NEXT: s_mov_b32 s4, 1 +; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_load_b128 s[4:7], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3i64_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3i64_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s8, 4 -; GFX11-NEXT: s_mov_b32 s8, 1 -; GFX11-NEXT: v_writelane_b32 v40, s9, 5 -; GFX11-NEXT: s_mov_b32 s9, 2 -; GFX11-NEXT: v_writelane_b32 v40, s30, 6 -; GFX11-NEXT: v_writelane_b32 v40, s31, 7 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_mov_b32 s5, 2 +; GFX11-NEXT: v_writelane_b32 v40, s30, 2 +; GFX11-NEXT: v_writelane_b32 v40, s31, 3 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 7 -; GFX11-NEXT: v_readlane_b32 s30, v40, 6 -; GFX11-NEXT: v_readlane_b32 s9, v40, 5 -; GFX11-NEXT: v_readlane_b32 s8, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 +; GFX11-NEXT: v_readlane_b32 s31, v40, 3 +; GFX11-NEXT: v_readlane_b32 s30, v40, 2 ; GFX11-NEXT: v_readlane_b32 s5, v40, 1 ; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 8 +; GFX11-NEXT: v_readlane_b32 s0, v40, 4 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -10524,32 +10416,24 @@ define amdgpu_gfx void @test_call_external_void_func_v3i64_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 8 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 ; GFX10-SCRATCH-NEXT: s_mov_b64 s[0:1], 0 -; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v3i64_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v3i64_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1 +; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3i64_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3i64_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s8, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-SCRATCH-NEXT: s_mov_b32 s9, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 6 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 7 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 7 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 6 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s8, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 8 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -10574,38 +10458,35 @@ define amdgpu_gfx void @test_call_external_void_func_v4i64_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 10 +; GFX9-NEXT: v_writelane_b32 v40, s34, 6 +; GFX9-NEXT: s_mov_b64 s[34:35], 0 +; GFX9-NEXT: s_load_dwordx4 s[36:39], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s4, 0 ; GFX9-NEXT: v_writelane_b32 v40, s5, 1 ; GFX9-NEXT: v_writelane_b32 v40, s6, 2 ; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: s_mov_b64 s[34:35], 0 -; GFX9-NEXT: v_writelane_b32 v40, s8, 4 -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s9, 5 -; GFX9-NEXT: v_writelane_b32 v40, s10, 6 -; GFX9-NEXT: v_writelane_b32 v40, s11, 7 -; GFX9-NEXT: v_writelane_b32 v40, s30, 8 +; GFX9-NEXT: v_writelane_b32 v40, s30, 4 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v4i64_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v4i64_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s8, 1 -; GFX9-NEXT: s_mov_b32 s9, 2 -; GFX9-NEXT: s_mov_b32 s10, 3 -; GFX9-NEXT: s_mov_b32 s11, 4 -; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 9 -; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 9 -; GFX9-NEXT: v_readlane_b32 s30, v40, 8 -; GFX9-NEXT: v_readlane_b32 s11, v40, 7 -; GFX9-NEXT: v_readlane_b32 s10, v40, 6 -; GFX9-NEXT: v_readlane_b32 s9, v40, 5 -; GFX9-NEXT: v_readlane_b32 s8, v40, 4 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s0, s36 +; GFX9-NEXT: s_mov_b32 s1, s37 +; GFX9-NEXT: s_mov_b32 s2, s38 +; GFX9-NEXT: s_mov_b32 s3, s39 +; GFX9-NEXT: s_mov_b32 s4, 1 +; GFX9-NEXT: s_mov_b32 s5, 2 +; GFX9-NEXT: s_mov_b32 s6, 3 +; GFX9-NEXT: s_mov_b32 s7, 4 +; GFX9-NEXT: s_addk_i32 s32, 0x400 +; GFX9-NEXT: v_writelane_b32 v40, s31, 5 +; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX9-NEXT: v_readlane_b32 s31, v40, 5 +; GFX9-NEXT: v_readlane_b32 s30, v40, 4 ; GFX9-NEXT: v_readlane_b32 s7, v40, 3 ; GFX9-NEXT: v_readlane_b32 s6, v40, 2 ; GFX9-NEXT: v_readlane_b32 s5, v40, 1 ; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 10 +; GFX9-NEXT: v_readlane_b32 s34, v40, 6 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -10623,38 +10504,35 @@ define amdgpu_gfx void @test_call_external_void_func_v4i64_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 10 +; GFX10-NEXT: v_writelane_b32 v40, s34, 6 ; GFX10-NEXT: s_mov_b64 s[34:35], 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: s_load_dwordx4 s[36:39], s[34:35], 0x0 +; GFX10-NEXT: s_mov_b32 s35, external_void_func_v4i64_inreg@abs32@hi ; GFX10-NEXT: v_writelane_b32 v40, s4, 0 +; GFX10-NEXT: s_mov_b32 s34, external_void_func_v4i64_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s4, 1 ; GFX10-NEXT: v_writelane_b32 v40, s5, 1 +; GFX10-NEXT: s_mov_b32 s5, 2 ; GFX10-NEXT: v_writelane_b32 v40, s6, 2 +; GFX10-NEXT: s_mov_b32 s6, 3 ; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_load_dwordx4 s[4:7], s[34:35], 0x0 -; GFX10-NEXT: s_mov_b32 s35, external_void_func_v4i64_inreg@abs32@hi -; GFX10-NEXT: s_mov_b32 s34, external_void_func_v4i64_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-NEXT: s_mov_b32 s8, 1 -; GFX10-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-NEXT: s_mov_b32 s9, 2 -; GFX10-NEXT: v_writelane_b32 v40, s10, 6 -; GFX10-NEXT: s_mov_b32 s10, 3 -; GFX10-NEXT: v_writelane_b32 v40, s11, 7 -; GFX10-NEXT: s_mov_b32 s11, 4 -; GFX10-NEXT: v_writelane_b32 v40, s30, 8 -; GFX10-NEXT: v_writelane_b32 v40, s31, 9 +; GFX10-NEXT: s_mov_b32 s7, 4 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s0, s36 +; GFX10-NEXT: s_mov_b32 s1, s37 +; GFX10-NEXT: v_writelane_b32 v40, s30, 4 +; GFX10-NEXT: s_mov_b32 s2, s38 +; GFX10-NEXT: s_mov_b32 s3, s39 +; GFX10-NEXT: v_writelane_b32 v40, s31, 5 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 9 -; GFX10-NEXT: v_readlane_b32 s30, v40, 8 -; GFX10-NEXT: v_readlane_b32 s11, v40, 7 -; GFX10-NEXT: v_readlane_b32 s10, v40, 6 -; GFX10-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-NEXT: v_readlane_b32 s8, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 5 +; GFX10-NEXT: v_readlane_b32 s30, v40, 4 ; GFX10-NEXT: v_readlane_b32 s7, v40, 3 ; GFX10-NEXT: v_readlane_b32 s6, v40, 2 ; GFX10-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 10 +; GFX10-NEXT: v_readlane_b32 s34, v40, 6 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -10672,39 +10550,31 @@ define amdgpu_gfx void @test_call_external_void_func_v4i64_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 10 +; GFX11-NEXT: v_writelane_b32 v40, s0, 6 ; GFX11-NEXT: s_mov_b64 s[0:1], 0 -; GFX11-NEXT: s_add_i32 s32, s32, 16 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v4i64_inreg@abs32@hi +; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v4i64_inreg@abs32@lo ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 +; GFX11-NEXT: s_mov_b32 s4, 1 +; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s5, 1 +; GFX11-NEXT: s_mov_b32 s5, 2 ; GFX11-NEXT: v_writelane_b32 v40, s6, 2 +; GFX11-NEXT: s_mov_b32 s6, 3 ; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_load_b128 s[4:7], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v4i64_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v4i64_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s8, 4 -; GFX11-NEXT: s_mov_b32 s8, 1 -; GFX11-NEXT: v_writelane_b32 v40, s9, 5 -; GFX11-NEXT: s_mov_b32 s9, 2 -; GFX11-NEXT: v_writelane_b32 v40, s10, 6 -; GFX11-NEXT: s_mov_b32 s10, 3 -; GFX11-NEXT: v_writelane_b32 v40, s11, 7 -; GFX11-NEXT: s_mov_b32 s11, 4 -; GFX11-NEXT: v_writelane_b32 v40, s30, 8 -; GFX11-NEXT: v_writelane_b32 v40, s31, 9 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_mov_b32 s7, 4 +; GFX11-NEXT: v_writelane_b32 v40, s30, 4 +; GFX11-NEXT: v_writelane_b32 v40, s31, 5 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 9 -; GFX11-NEXT: v_readlane_b32 s30, v40, 8 -; GFX11-NEXT: v_readlane_b32 s11, v40, 7 -; GFX11-NEXT: v_readlane_b32 s10, v40, 6 -; GFX11-NEXT: v_readlane_b32 s9, v40, 5 -; GFX11-NEXT: v_readlane_b32 s8, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 5 +; GFX11-NEXT: v_readlane_b32 s30, v40, 4 ; GFX11-NEXT: v_readlane_b32 s7, v40, 3 ; GFX11-NEXT: v_readlane_b32 s6, v40, 2 ; GFX11-NEXT: v_readlane_b32 s5, v40, 1 ; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 10 +; GFX11-NEXT: v_readlane_b32 s0, v40, 6 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -10722,38 +10592,30 @@ define amdgpu_gfx void @test_call_external_void_func_v4i64_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 10 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 6 ; GFX10-SCRATCH-NEXT: s_mov_b64 s[0:1], 0 -; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v4i64_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v4i64_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1 +; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 +; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 3 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v4i64_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v4i64_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s8, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-SCRATCH-NEXT: s_mov_b32 s9, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s10, 6 -; GFX10-SCRATCH-NEXT: s_mov_b32 s10, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s11, 7 -; GFX10-SCRATCH-NEXT: s_mov_b32 s11, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 8 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 9 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 9 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 8 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s11, v40, 7 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s10, v40, 6 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s8, v40, 4 +; GFX10-SCRATCH-NEXT: s_mov_b32 s7, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 5 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 4 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 10 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 6 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -10777,19 +10639,17 @@ define amdgpu_gfx void @test_call_external_void_func_f16_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 3 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 1 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_f16_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_f16_inreg@abs32@lo -; GFX9-NEXT: s_movk_i32 s4, 0x4400 +; GFX9-NEXT: s_movk_i32 s0, 0x4400 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 2 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 2 -; GFX9-NEXT: v_readlane_b32 s30, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 3 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -10807,19 +10667,17 @@ define amdgpu_gfx void @test_call_external_void_func_f16_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 3 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_f16_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_f16_inreg@abs32@lo +; GFX10-NEXT: s_movk_i32 s0, 0x4400 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_movk_i32 s4, 0x4400 -; GFX10-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-NEXT: v_writelane_b32 v40, s31, 2 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 3 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -10837,20 +10695,18 @@ define amdgpu_gfx void @test_call_external_void_func_f16_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 3 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_f16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_f16_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_f16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_f16_inreg@abs32@lo +; GFX11-NEXT: s_movk_i32 s0, 0x4400 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_movk_i32 s4, 0x4400 -; GFX11-NEXT: v_writelane_b32 v40, s30, 1 -; GFX11-NEXT: v_writelane_b32 v40, s31, 2 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 2 -; GFX11-NEXT: v_readlane_b32 s30, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 3 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -10868,19 +10724,17 @@ define amdgpu_gfx void @test_call_external_void_func_f16_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_f16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_f16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_f16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_f16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_movk_i32 s0, 0x4400 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_movk_i32 s4, 0x4400 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 2 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -10902,19 +10756,17 @@ define amdgpu_gfx void @test_call_external_void_func_f32_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 3 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 1 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_f32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_f32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 4.0 +; GFX9-NEXT: s_mov_b32 s0, 4.0 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 2 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 2 -; GFX9-NEXT: v_readlane_b32 s30, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 3 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -10932,19 +10784,17 @@ define amdgpu_gfx void @test_call_external_void_func_f32_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 3 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_f32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_f32_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 4.0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 4.0 -; GFX10-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-NEXT: v_writelane_b32 v40, s31, 2 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 3 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -10962,20 +10812,18 @@ define amdgpu_gfx void @test_call_external_void_func_f32_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 3 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_f32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_f32_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_f32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_f32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 4.0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 4.0 -; GFX11-NEXT: v_writelane_b32 v40, s30, 1 -; GFX11-NEXT: v_writelane_b32 v40, s31, 2 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 2 -; GFX11-NEXT: v_readlane_b32 s30, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 3 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -10993,19 +10841,17 @@ define amdgpu_gfx void @test_call_external_void_func_f32_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_f32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_f32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_f32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_f32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 4.0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 4.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 2 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -11027,22 +10873,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2f32_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v2f32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v2f32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 1.0 -; GFX9-NEXT: s_mov_b32 s5, 2.0 +; GFX9-NEXT: s_mov_b32 s0, 1.0 +; GFX9-NEXT: s_mov_b32 s1, 2.0 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -11060,22 +10902,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2f32_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v2f32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v2f32_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 1.0 +; GFX10-NEXT: s_mov_b32 s1, 2.0 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 1.0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2.0 -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -11093,23 +10931,19 @@ define amdgpu_gfx void @test_call_external_void_func_v2f32_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v2f32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v2f32_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v2f32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v2f32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 1.0 +; GFX11-NEXT: s_mov_b32 s1, 2.0 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 1.0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2.0 -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -11127,22 +10961,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2f32_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v2f32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v2f32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v2f32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v2f32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 1.0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2.0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -11164,25 +10994,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3f32_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 5 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s30, 3 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3f32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3f32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 1.0 -; GFX9-NEXT: s_mov_b32 s5, 2.0 -; GFX9-NEXT: s_mov_b32 s6, 4.0 +; GFX9-NEXT: s_mov_b32 s0, 1.0 +; GFX9-NEXT: s_mov_b32 s1, 2.0 +; GFX9-NEXT: s_mov_b32 s2, 4.0 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 4 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 4 -; GFX9-NEXT: v_readlane_b32 s30, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 5 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -11200,25 +11024,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3f32_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 5 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3f32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3f32_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 1.0 +; GFX10-NEXT: s_mov_b32 s1, 2.0 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: s_mov_b32 s2, 4.0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 1.0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2.0 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 4.0 -; GFX10-NEXT: v_writelane_b32 v40, s30, 3 -; GFX10-NEXT: v_writelane_b32 v40, s31, 4 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 4 -; GFX10-NEXT: v_readlane_b32 s30, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 5 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -11236,26 +11054,20 @@ define amdgpu_gfx void @test_call_external_void_func_v3f32_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 5 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3f32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3f32_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v3f32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v3f32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 1.0 +; GFX11-NEXT: s_mov_b32 s1, 2.0 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: s_mov_b32 s2, 4.0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 1.0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2.0 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 4.0 -; GFX11-NEXT: v_writelane_b32 v40, s30, 3 -; GFX11-NEXT: v_writelane_b32 v40, s31, 4 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 4 -; GFX11-NEXT: v_readlane_b32 s30, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 5 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -11273,25 +11085,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3f32_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 5 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3f32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3f32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v3f32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v3f32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 1.0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2.0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 4.0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 4.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 4 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 5 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -11313,31 +11119,23 @@ define amdgpu_gfx void @test_call_external_void_func_v5f32_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 7 +; GFX9-NEXT: v_writelane_b32 v40, s34, 3 ; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: v_writelane_b32 v40, s8, 4 -; GFX9-NEXT: v_writelane_b32 v40, s30, 5 +; GFX9-NEXT: v_writelane_b32 v40, s30, 1 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v5f32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v5f32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 1.0 -; GFX9-NEXT: s_mov_b32 s5, 2.0 -; GFX9-NEXT: s_mov_b32 s6, 4.0 -; GFX9-NEXT: s_mov_b32 s7, -1.0 -; GFX9-NEXT: s_mov_b32 s8, 0.5 +; GFX9-NEXT: s_mov_b32 s0, 1.0 +; GFX9-NEXT: s_mov_b32 s1, 2.0 +; GFX9-NEXT: s_mov_b32 s2, 4.0 +; GFX9-NEXT: s_mov_b32 s3, -1.0 +; GFX9-NEXT: s_mov_b32 s4, 0.5 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 6 +; GFX9-NEXT: v_writelane_b32 v40, s31, 2 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 6 -; GFX9-NEXT: v_readlane_b32 s30, v40, 5 -; GFX9-NEXT: v_readlane_b32 s8, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 +; GFX9-NEXT: v_readlane_b32 s31, v40, 2 +; GFX9-NEXT: v_readlane_b32 s30, v40, 1 ; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 7 +; GFX9-NEXT: v_readlane_b32 s34, v40, 3 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -11355,31 +11153,23 @@ define amdgpu_gfx void @test_call_external_void_func_v5f32_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 7 +; GFX10-NEXT: v_writelane_b32 v40, s34, 3 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v5f32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v5f32_inreg@abs32@lo -; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: s_mov_b32 s0, 1.0 +; GFX10-NEXT: s_mov_b32 s1, 2.0 ; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 1.0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2.0 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 4.0 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_mov_b32 s7, -1.0 -; GFX10-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-NEXT: s_mov_b32 s8, 0.5 -; GFX10-NEXT: v_writelane_b32 v40, s30, 5 -; GFX10-NEXT: v_writelane_b32 v40, s31, 6 +; GFX10-NEXT: s_mov_b32 s2, 4.0 +; GFX10-NEXT: s_mov_b32 s3, -1.0 +; GFX10-NEXT: s_mov_b32 s4, 0.5 +; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: v_writelane_b32 v40, s30, 1 +; GFX10-NEXT: v_writelane_b32 v40, s31, 2 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 6 -; GFX10-NEXT: v_readlane_b32 s30, v40, 5 -; GFX10-NEXT: v_readlane_b32 s8, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 +; GFX10-NEXT: v_readlane_b32 s31, v40, 2 +; GFX10-NEXT: v_readlane_b32 s30, v40, 1 ; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 7 +; GFX10-NEXT: v_readlane_b32 s34, v40, 3 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -11397,32 +11187,24 @@ define amdgpu_gfx void @test_call_external_void_func_v5f32_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 7 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v5f32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v5f32_inreg@abs32@lo -; GFX11-NEXT: s_add_i32 s32, s32, 16 +; GFX11-NEXT: v_writelane_b32 v40, s0, 3 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v5f32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v5f32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 1.0 +; GFX11-NEXT: s_mov_b32 s1, 2.0 ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 1.0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2.0 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 4.0 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_mov_b32 s7, -1.0 -; GFX11-NEXT: v_writelane_b32 v40, s8, 4 -; GFX11-NEXT: s_mov_b32 s8, 0.5 -; GFX11-NEXT: v_writelane_b32 v40, s30, 5 -; GFX11-NEXT: v_writelane_b32 v40, s31, 6 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_mov_b32 s2, 4.0 +; GFX11-NEXT: s_mov_b32 s3, -1.0 +; GFX11-NEXT: s_mov_b32 s4, 0.5 +; GFX11-NEXT: s_add_i32 s32, s32, 16 +; GFX11-NEXT: v_writelane_b32 v40, s30, 1 +; GFX11-NEXT: v_writelane_b32 v40, s31, 2 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 6 -; GFX11-NEXT: v_readlane_b32 s30, v40, 5 -; GFX11-NEXT: v_readlane_b32 s8, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 +; GFX11-NEXT: v_readlane_b32 s31, v40, 2 +; GFX11-NEXT: v_readlane_b32 s30, v40, 1 ; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 7 +; GFX11-NEXT: v_readlane_b32 s0, v40, 3 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -11440,31 +11222,23 @@ define amdgpu_gfx void @test_call_external_void_func_v5f32_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 7 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v5f32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v5f32_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 3 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v5f32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v5f32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 1.0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2.0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 4.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s7, -1.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s8, 0.5 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 5 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 6 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 6 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s8, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 4.0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, -1.0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 0.5 +; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 1 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 2 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 2 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 7 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 3 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -11486,22 +11260,18 @@ define amdgpu_gfx void @test_call_external_void_func_f64_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_f64_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_f64_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 0 -; GFX9-NEXT: s_mov_b32 s5, 0x40100000 +; GFX9-NEXT: s_mov_b32 s0, 0 +; GFX9-NEXT: s_mov_b32 s1, 0x40100000 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -11519,22 +11289,18 @@ define amdgpu_gfx void @test_call_external_void_func_f64_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_f64_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_f64_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 0 +; GFX10-NEXT: s_mov_b32 s1, 0x40100000 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 0x40100000 -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -11552,23 +11318,19 @@ define amdgpu_gfx void @test_call_external_void_func_f64_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_f64_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_f64_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_f64_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_f64_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_mov_b32 s1, 0x40100000 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 0x40100000 -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -11586,22 +11348,18 @@ define amdgpu_gfx void @test_call_external_void_func_f64_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_f64_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_f64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_f64_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_f64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 0x40100000 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 0x40100000 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -11623,28 +11381,20 @@ define amdgpu_gfx void @test_call_external_void_func_v2f64_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 6 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: v_writelane_b32 v40, s30, 4 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v2f64_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v2f64_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 0 -; GFX9-NEXT: s_mov_b32 s5, 2.0 -; GFX9-NEXT: s_mov_b32 s6, 0 -; GFX9-NEXT: s_mov_b32 s7, 0x40100000 +; GFX9-NEXT: s_mov_b32 s2, 0 +; GFX9-NEXT: s_mov_b32 s0, 0 +; GFX9-NEXT: s_mov_b32 s1, 2.0 +; GFX9-NEXT: s_mov_b32 s3, 0x40100000 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 5 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 5 -; GFX9-NEXT: v_readlane_b32 s30, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 6 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -11662,28 +11412,20 @@ define amdgpu_gfx void @test_call_external_void_func_v2f64_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 6 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v2f64_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v2f64_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s2, 0 +; GFX10-NEXT: s_mov_b32 s0, 0 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: s_mov_b32 s1, 2.0 +; GFX10-NEXT: s_mov_b32 s3, 0x40100000 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2.0 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 0 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_mov_b32 s7, 0x40100000 -; GFX10-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 6 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -11701,29 +11443,21 @@ define amdgpu_gfx void @test_call_external_void_func_v2f64_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 6 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v2f64_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v2f64_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v2f64_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v2f64_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_mov_b32 s1, 2.0 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: s_mov_b32 s2, 0 +; GFX11-NEXT: s_mov_b32 s3, 0x40100000 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2.0 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 0 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_mov_b32 s7, 0x40100000 -; GFX11-NEXT: v_writelane_b32 v40, s30, 4 -; GFX11-NEXT: v_writelane_b32 v40, s31, 5 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 5 -; GFX11-NEXT: v_readlane_b32 s30, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 6 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -11741,28 +11475,20 @@ define amdgpu_gfx void @test_call_external_void_func_v2f64_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 6 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v2f64_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v2f64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v2f64_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v2f64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2.0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, 0x40100000 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s7, 0x40100000 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 5 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 6 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -11784,34 +11510,26 @@ define amdgpu_gfx void @test_call_external_void_func_v3f64_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 8 +; GFX9-NEXT: v_writelane_b32 v40, s34, 4 ; GFX9-NEXT: v_writelane_b32 v40, s4, 0 ; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: v_writelane_b32 v40, s8, 4 -; GFX9-NEXT: v_writelane_b32 v40, s9, 5 -; GFX9-NEXT: v_writelane_b32 v40, s30, 6 +; GFX9-NEXT: v_writelane_b32 v40, s30, 2 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3f64_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3f64_inreg@abs32@lo +; GFX9-NEXT: s_mov_b32 s0, 0 +; GFX9-NEXT: s_mov_b32 s1, 2.0 +; GFX9-NEXT: s_mov_b32 s2, 0 +; GFX9-NEXT: s_mov_b32 s3, 0x40100000 ; GFX9-NEXT: s_mov_b32 s4, 0 -; GFX9-NEXT: s_mov_b32 s5, 2.0 -; GFX9-NEXT: s_mov_b32 s6, 0 -; GFX9-NEXT: s_mov_b32 s7, 0x40100000 -; GFX9-NEXT: s_mov_b32 s8, 0 -; GFX9-NEXT: s_mov_b32 s9, 0x40200000 +; GFX9-NEXT: s_mov_b32 s5, 0x40200000 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 7 +; GFX9-NEXT: v_writelane_b32 v40, s31, 3 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 7 -; GFX9-NEXT: v_readlane_b32 s30, v40, 6 -; GFX9-NEXT: v_readlane_b32 s9, v40, 5 -; GFX9-NEXT: v_readlane_b32 s8, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 +; GFX9-NEXT: v_readlane_b32 s31, v40, 3 +; GFX9-NEXT: v_readlane_b32 s30, v40, 2 ; GFX9-NEXT: v_readlane_b32 s5, v40, 1 ; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 8 +; GFX9-NEXT: v_readlane_b32 s34, v40, 4 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -11829,34 +11547,26 @@ define amdgpu_gfx void @test_call_external_void_func_v3f64_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 8 +; GFX10-NEXT: v_writelane_b32 v40, s34, 4 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3f64_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3f64_inreg@abs32@lo -; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: s_mov_b32 s0, 0 +; GFX10-NEXT: s_mov_b32 s1, 2.0 ; GFX10-NEXT: v_writelane_b32 v40, s4, 0 +; GFX10-NEXT: s_mov_b32 s2, 0 +; GFX10-NEXT: s_mov_b32 s3, 0x40100000 ; GFX10-NEXT: s_mov_b32 s4, 0 +; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2.0 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 0 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_mov_b32 s7, 0x40100000 -; GFX10-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-NEXT: s_mov_b32 s8, 0 -; GFX10-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-NEXT: s_mov_b32 s9, 0x40200000 -; GFX10-NEXT: v_writelane_b32 v40, s30, 6 -; GFX10-NEXT: v_writelane_b32 v40, s31, 7 +; GFX10-NEXT: s_mov_b32 s5, 0x40200000 +; GFX10-NEXT: v_writelane_b32 v40, s30, 2 +; GFX10-NEXT: v_writelane_b32 v40, s31, 3 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 7 -; GFX10-NEXT: v_readlane_b32 s30, v40, 6 -; GFX10-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-NEXT: v_readlane_b32 s8, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 +; GFX10-NEXT: v_readlane_b32 s31, v40, 3 +; GFX10-NEXT: v_readlane_b32 s30, v40, 2 ; GFX10-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 8 +; GFX10-NEXT: v_readlane_b32 s34, v40, 4 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -11874,35 +11584,27 @@ define amdgpu_gfx void @test_call_external_void_func_v3f64_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 8 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3f64_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3f64_inreg@abs32@lo -; GFX11-NEXT: s_add_i32 s32, s32, 16 +; GFX11-NEXT: v_writelane_b32 v40, s0, 4 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v3f64_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v3f64_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 0 +; GFX11-NEXT: s_mov_b32 s1, 2.0 ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 +; GFX11-NEXT: s_mov_b32 s2, 0 +; GFX11-NEXT: s_mov_b32 s3, 0x40100000 ; GFX11-NEXT: s_mov_b32 s4, 0 +; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2.0 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 0 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_mov_b32 s7, 0x40100000 -; GFX11-NEXT: v_writelane_b32 v40, s8, 4 -; GFX11-NEXT: s_mov_b32 s8, 0 -; GFX11-NEXT: v_writelane_b32 v40, s9, 5 -; GFX11-NEXT: s_mov_b32 s9, 0x40200000 -; GFX11-NEXT: v_writelane_b32 v40, s30, 6 -; GFX11-NEXT: v_writelane_b32 v40, s31, 7 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_mov_b32 s5, 0x40200000 +; GFX11-NEXT: v_writelane_b32 v40, s30, 2 +; GFX11-NEXT: v_writelane_b32 v40, s31, 3 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 7 -; GFX11-NEXT: v_readlane_b32 s30, v40, 6 -; GFX11-NEXT: v_readlane_b32 s9, v40, 5 -; GFX11-NEXT: v_readlane_b32 s8, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 +; GFX11-NEXT: v_readlane_b32 s31, v40, 3 +; GFX11-NEXT: v_readlane_b32 s30, v40, 2 ; GFX11-NEXT: v_readlane_b32 s5, v40, 1 ; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 8 +; GFX11-NEXT: v_readlane_b32 s0, v40, 4 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -11920,34 +11622,26 @@ define amdgpu_gfx void @test_call_external_void_func_v3f64_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 8 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3f64_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3f64_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v3f64_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v3f64_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2.0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, 0x40100000 ; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 0 +; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2.0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s7, 0x40100000 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s8, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-SCRATCH-NEXT: s_mov_b32 s9, 0x40200000 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 6 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 7 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 7 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 6 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s8, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 0x40200000 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 8 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -11969,19 +11663,17 @@ define amdgpu_gfx void @test_call_external_void_func_v2i16_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 3 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: s_load_dword s4, s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 1 +; GFX9-NEXT: s_load_dword s0, s[34:35], 0x0 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v2i16_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v2i16_inreg@abs32@lo ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 2 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 2 -; GFX9-NEXT: v_readlane_b32 s30, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 3 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -11999,19 +11691,17 @@ define amdgpu_gfx void @test_call_external_void_func_v2i16_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 3 -; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_load_dword s4, s[34:35], 0x0 +; GFX10-NEXT: s_load_dword s0, s[34:35], 0x0 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v2i16_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v2i16_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-NEXT: v_writelane_b32 v40, s31, 2 +; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 3 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -12029,20 +11719,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2i16_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 3 +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v2i16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v2i16_inreg@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_load_b32 s4, s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v2i16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v2i16_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 1 -; GFX11-NEXT: v_writelane_b32 v40, s31, 2 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 2 -; GFX11-NEXT: v_readlane_b32 s30, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 3 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -12060,19 +11748,17 @@ define amdgpu_gfx void @test_call_external_void_func_v2i16_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_load_dword s0, s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v2i16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v2i16_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_load_dword s4, s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v2i16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v2i16_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 2 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -12095,21 +11781,20 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 -; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3i16_inreg@abs32@hi -; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3i16_inreg@abs32@lo +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 +; GFX9-NEXT: s_mov_b32 s37, external_void_func_v3i16_inreg@abs32@hi +; GFX9-NEXT: s_mov_b32 s36, external_void_func_v3i16_inreg@abs32@lo ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 -; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s1, s35 +; GFX9-NEXT: s_mov_b32 s0, s34 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 +; GFX9-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -12127,21 +11812,20 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 +; GFX10-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 +; GFX10-NEXT: s_mov_b32 s37, external_void_func_v3i16_inreg@abs32@hi +; GFX10-NEXT: s_mov_b32 s36, external_void_func_v3i16_inreg@abs32@lo ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x0 -; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3i16_inreg@abs32@hi -; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3i16_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s1, s35 +; GFX10-NEXT: s_mov_b32 s0, s34 +; GFX10-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -12159,22 +11843,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v3i16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v3i16_inreg@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3i16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3i16_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -12192,21 +11872,17 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v3i16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v3i16_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3i16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3i16_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -12229,21 +11905,20 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 -; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3f16_inreg@abs32@hi -; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3f16_inreg@abs32@lo +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 +; GFX9-NEXT: s_mov_b32 s37, external_void_func_v3f16_inreg@abs32@hi +; GFX9-NEXT: s_mov_b32 s36, external_void_func_v3f16_inreg@abs32@lo ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 -; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s1, s35 +; GFX9-NEXT: s_mov_b32 s0, s34 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 +; GFX9-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -12261,21 +11936,20 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 +; GFX10-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 +; GFX10-NEXT: s_mov_b32 s37, external_void_func_v3f16_inreg@abs32@hi +; GFX10-NEXT: s_mov_b32 s36, external_void_func_v3f16_inreg@abs32@lo ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x0 -; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3f16_inreg@abs32@hi -; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3f16_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s1, s35 +; GFX10-NEXT: s_mov_b32 s0, s34 +; GFX10-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -12293,22 +11967,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v3f16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v3f16_inreg@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3f16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3f16_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -12326,21 +11996,17 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v3f16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v3f16_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3f16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3f16_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -12363,22 +12029,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3i16_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3i16_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 0x20001 -; GFX9-NEXT: s_mov_b32 s5, 3 +; GFX9-NEXT: s_mov_b32 s0, 0x20001 +; GFX9-NEXT: s_mov_b32 s1, 3 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -12396,22 +12058,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3i16_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3i16_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 0x20001 +; GFX10-NEXT: s_mov_b32 s1, 3 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 0x20001 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 3 -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -12429,23 +12087,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3i16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3i16_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v3i16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v3i16_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 0x20001 +; GFX11-NEXT: s_mov_b32 s1, 3 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 0x20001 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 3 -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -12463,22 +12117,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3i16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3i16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v3i16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v3i16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 0x20001 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 0x20001 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -12500,22 +12150,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3f16_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3f16_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 0x40003c00 -; GFX9-NEXT: s_movk_i32 s5, 0x4400 +; GFX9-NEXT: s_mov_b32 s0, 0x40003c00 +; GFX9-NEXT: s_movk_i32 s1, 0x4400 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -12533,22 +12179,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3f16_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3f16_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 0x40003c00 +; GFX10-NEXT: s_movk_i32 s1, 0x4400 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 0x40003c00 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_movk_i32 s5, 0x4400 -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -12566,23 +12208,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3f16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3f16_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v3f16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v3f16_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 0x40003c00 +; GFX11-NEXT: s_movk_i32 s1, 0x4400 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 0x40003c00 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_movk_i32 s5, 0x4400 -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -12600,22 +12238,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3f16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3f16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v3f16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v3f16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 0x40003c00 +; GFX10-SCRATCH-NEXT: s_movk_i32 s1, 0x4400 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 0x40003c00 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_movk_i32 s5, 0x4400 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -12637,21 +12271,20 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 -; GFX9-NEXT: s_mov_b32 s35, external_void_func_v4i16_inreg@abs32@hi -; GFX9-NEXT: s_mov_b32 s34, external_void_func_v4i16_inreg@abs32@lo +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 +; GFX9-NEXT: s_mov_b32 s37, external_void_func_v4i16_inreg@abs32@hi +; GFX9-NEXT: s_mov_b32 s36, external_void_func_v4i16_inreg@abs32@lo ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 -; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s0, s34 +; GFX9-NEXT: s_mov_b32 s1, s35 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 +; GFX9-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -12669,21 +12302,20 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 +; GFX10-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 +; GFX10-NEXT: s_mov_b32 s37, external_void_func_v4i16_inreg@abs32@hi +; GFX10-NEXT: s_mov_b32 s36, external_void_func_v4i16_inreg@abs32@lo ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x0 -; GFX10-NEXT: s_mov_b32 s35, external_void_func_v4i16_inreg@abs32@hi -; GFX10-NEXT: s_mov_b32 s34, external_void_func_v4i16_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s0, s34 +; GFX10-NEXT: s_mov_b32 s1, s35 +; GFX10-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -12701,22 +12333,18 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v4i16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v4i16_inreg@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v4i16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v4i16_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -12734,21 +12362,17 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v4i16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v4i16_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v4i16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v4i16_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -12771,22 +12395,18 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v4i16_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v4i16_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 0x20001 -; GFX9-NEXT: s_mov_b32 s5, 0x40003 +; GFX9-NEXT: s_mov_b32 s0, 0x20001 +; GFX9-NEXT: s_mov_b32 s1, 0x40003 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -12804,22 +12424,18 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v4i16_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v4i16_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 0x20001 +; GFX10-NEXT: s_mov_b32 s1, 0x40003 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 0x20001 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 0x40003 -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -12837,23 +12453,19 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v4i16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v4i16_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v4i16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v4i16_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 0x20001 +; GFX11-NEXT: s_mov_b32 s1, 0x40003 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 0x20001 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 0x40003 -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -12871,22 +12483,18 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v4i16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v4i16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v4i16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v4i16_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 0x20001 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 0x40003 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 0x20001 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 0x40003 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -12908,19 +12516,17 @@ define amdgpu_gfx void @test_call_external_void_func_v2f16_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 3 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: s_load_dword s4, s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 1 +; GFX9-NEXT: s_load_dword s0, s[34:35], 0x0 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v2f16_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v2f16_inreg@abs32@lo ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 2 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 2 -; GFX9-NEXT: v_readlane_b32 s30, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 3 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -12938,19 +12544,17 @@ define amdgpu_gfx void @test_call_external_void_func_v2f16_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 3 -; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_load_dword s4, s[34:35], 0x0 +; GFX10-NEXT: s_load_dword s0, s[34:35], 0x0 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v2f16_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v2f16_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-NEXT: v_writelane_b32 v40, s31, 2 +; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 3 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -12968,20 +12572,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2f16_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 3 +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v2f16_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v2f16_inreg@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_load_b32 s4, s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v2f16_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v2f16_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 1 -; GFX11-NEXT: v_writelane_b32 v40, s31, 2 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 2 -; GFX11-NEXT: v_readlane_b32 s30, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 3 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -12999,19 +12601,17 @@ define amdgpu_gfx void @test_call_external_void_func_v2f16_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_load_dword s0, s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v2f16_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v2f16_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_load_dword s4, s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v2f16_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v2f16_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 2 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -13034,21 +12634,20 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 -; GFX9-NEXT: s_mov_b32 s35, external_void_func_v2i32_inreg@abs32@hi -; GFX9-NEXT: s_mov_b32 s34, external_void_func_v2i32_inreg@abs32@lo +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 +; GFX9-NEXT: s_mov_b32 s37, external_void_func_v2i32_inreg@abs32@hi +; GFX9-NEXT: s_mov_b32 s36, external_void_func_v2i32_inreg@abs32@lo ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 -; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s0, s34 +; GFX9-NEXT: s_mov_b32 s1, s35 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 +; GFX9-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -13066,21 +12665,20 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 +; GFX10-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 +; GFX10-NEXT: s_mov_b32 s37, external_void_func_v2i32_inreg@abs32@hi +; GFX10-NEXT: s_mov_b32 s36, external_void_func_v2i32_inreg@abs32@lo ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x0 -; GFX10-NEXT: s_mov_b32 s35, external_void_func_v2i32_inreg@abs32@hi -; GFX10-NEXT: s_mov_b32 s34, external_void_func_v2i32_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s0, s34 +; GFX10-NEXT: s_mov_b32 s1, s35 +; GFX10-NEXT: s_swappc_b64 s[30:31], s[36:37] +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -13098,22 +12696,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v2i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v2i32_inreg@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v2i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v2i32_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -13131,21 +12725,17 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v2i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v2i32_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v2i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v2i32_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -13168,22 +12758,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 4 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s30, 2 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v2i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v2i32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 1 -; GFX9-NEXT: s_mov_b32 s5, 2 +; GFX9-NEXT: s_mov_b32 s0, 1 +; GFX9-NEXT: s_mov_b32 s1, 2 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 3 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 3 -; GFX9-NEXT: v_readlane_b32 s30, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 4 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -13201,22 +12787,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 4 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v2i32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v2i32_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 1 +; GFX10-NEXT: s_mov_b32 s1, 2 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 1 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2 -; GFX10-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-NEXT: v_writelane_b32 v40, s31, 3 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -13234,23 +12816,19 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 4 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v2i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v2i32_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v2i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v2i32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 1 +; GFX11-NEXT: s_mov_b32 s1, 2 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 1 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2 -; GFX11-NEXT: v_writelane_b32 v40, s30, 2 -; GFX11-NEXT: v_writelane_b32 v40, s31, 3 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 3 -; GFX11-NEXT: v_readlane_b32 s30, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -13268,22 +12846,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v2i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v2i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v2i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v2i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 1 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 3 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -13305,25 +12879,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_imm_inreg(i32) #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 5 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s30, 3 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3i32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 3 -; GFX9-NEXT: s_mov_b32 s5, 4 -; GFX9-NEXT: s_mov_b32 s6, 5 +; GFX9-NEXT: s_mov_b32 s0, 3 +; GFX9-NEXT: s_mov_b32 s1, 4 +; GFX9-NEXT: s_mov_b32 s2, 5 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 4 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 4 -; GFX9-NEXT: v_readlane_b32 s30, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 5 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -13341,25 +12909,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_imm_inreg(i32) #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 5 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3i32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3i32_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 3 +; GFX10-NEXT: s_mov_b32 s1, 4 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: s_mov_b32 s2, 5 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 3 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 4 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 5 -; GFX10-NEXT: v_writelane_b32 v40, s30, 3 -; GFX10-NEXT: v_writelane_b32 v40, s31, 4 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 4 -; GFX10-NEXT: v_readlane_b32 s30, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 5 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -13377,26 +12939,20 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_imm_inreg(i32) #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 5 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3i32_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v3i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v3i32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 3 +; GFX11-NEXT: s_mov_b32 s1, 4 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: s_mov_b32 s2, 5 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 3 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 4 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 5 -; GFX11-NEXT: v_writelane_b32 v40, s30, 3 -; GFX11-NEXT: v_writelane_b32 v40, s31, 4 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 4 -; GFX11-NEXT: v_readlane_b32 s30, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 5 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -13414,25 +12970,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_imm_inreg(i32) #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 5 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v3i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v3i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 3 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 5 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 5 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 4 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 5 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -13454,28 +13004,20 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_i32_inreg(i32) #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 6 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: v_writelane_b32 v40, s30, 4 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3i32_i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3i32_i32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 3 -; GFX9-NEXT: s_mov_b32 s5, 4 -; GFX9-NEXT: s_mov_b32 s6, 5 -; GFX9-NEXT: s_mov_b32 s7, 6 +; GFX9-NEXT: s_mov_b32 s0, 3 +; GFX9-NEXT: s_mov_b32 s1, 4 +; GFX9-NEXT: s_mov_b32 s2, 5 +; GFX9-NEXT: s_mov_b32 s3, 6 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 5 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 5 -; GFX9-NEXT: v_readlane_b32 s30, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 6 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -13493,28 +13035,20 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_i32_inreg(i32) #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 6 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3i32_i32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3i32_i32_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 3 +; GFX10-NEXT: s_mov_b32 s1, 4 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: s_mov_b32 s2, 5 +; GFX10-NEXT: s_mov_b32 s3, 6 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 3 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 4 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 5 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_mov_b32 s7, 6 -; GFX10-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 6 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -13532,29 +13066,21 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_i32_inreg(i32) #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 6 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3i32_i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3i32_i32_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v3i32_i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v3i32_i32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 3 +; GFX11-NEXT: s_mov_b32 s1, 4 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: s_mov_b32 s2, 5 +; GFX11-NEXT: s_mov_b32 s3, 6 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 3 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 4 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 5 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_mov_b32 s7, 6 -; GFX11-NEXT: v_writelane_b32 v40, s30, 4 -; GFX11-NEXT: v_writelane_b32 v40, s31, 5 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 5 -; GFX11-NEXT: v_readlane_b32 s30, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 6 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -13572,28 +13098,20 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_i32_inreg(i32) #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 6 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3i32_i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3i32_i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v3i32_i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v3i32_i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 3 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 5 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, 6 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 5 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s7, 6 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 5 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 6 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -13615,25 +13133,22 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 6 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 4 +; GFX9-NEXT: s_load_dwordx4 s[36:39], s[34:35], 0x0 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s0, s36 +; GFX9-NEXT: s_mov_b32 s1, s37 +; GFX9-NEXT: s_mov_b32 s2, s38 +; GFX9-NEXT: s_mov_b32 s3, s39 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 5 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 5 -; GFX9-NEXT: v_readlane_b32 s30, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 6 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -13651,25 +13166,22 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 6 -; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_load_dwordx4 s[4:7], s[34:35], 0x0 +; GFX10-NEXT: s_load_dwordx4 s[36:39], s[34:35], 0x0 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s0, s36 +; GFX10-NEXT: s_mov_b32 s1, s37 +; GFX10-NEXT: s_mov_b32 s2, s38 +; GFX10-NEXT: s_mov_b32 s3, s39 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 6 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -13687,26 +13199,18 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 6 +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_load_b128 s[0:3], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_load_b128 s[4:7], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v4i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v4i32_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 4 -; GFX11-NEXT: v_writelane_b32 v40, s31, 5 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 5 -; GFX11-NEXT: v_readlane_b32 s30, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 6 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -13724,25 +13228,17 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 6 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v4i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v4i32_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 5 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 6 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -13765,28 +13261,20 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 6 -; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: v_writelane_b32 v40, s30, 4 +; GFX9-NEXT: v_writelane_b32 v40, s34, 2 +; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 1 -; GFX9-NEXT: s_mov_b32 s5, 2 -; GFX9-NEXT: s_mov_b32 s6, 3 -; GFX9-NEXT: s_mov_b32 s7, 4 +; GFX9-NEXT: s_mov_b32 s0, 1 +; GFX9-NEXT: s_mov_b32 s1, 2 +; GFX9-NEXT: s_mov_b32 s2, 3 +; GFX9-NEXT: s_mov_b32 s3, 4 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 5 +; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 5 -; GFX9-NEXT: v_readlane_b32 s30, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 -; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 6 +; GFX9-NEXT: v_readlane_b32 s31, v40, 1 +; GFX9-NEXT: v_readlane_b32 s30, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 2 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -13804,28 +13292,20 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 6 +; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 1 +; GFX10-NEXT: s_mov_b32 s1, 2 +; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: s_mov_b32 s2, 3 +; GFX10-NEXT: s_mov_b32 s3, 4 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 1 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 3 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_mov_b32 s7, 4 -; GFX10-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 6 +; GFX10-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 2 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -13843,29 +13323,21 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 6 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v4i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v4i32_inreg@abs32@lo +; GFX11-NEXT: v_writelane_b32 v40, s0, 2 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 1 +; GFX11-NEXT: s_mov_b32 s1, 2 +; GFX11-NEXT: v_writelane_b32 v40, s30, 0 +; GFX11-NEXT: s_mov_b32 s2, 3 +; GFX11-NEXT: s_mov_b32 s3, 4 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 1 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 3 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_mov_b32 s7, 4 -; GFX11-NEXT: v_writelane_b32 v40, s30, 4 -; GFX11-NEXT: v_writelane_b32 v40, s31, 5 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: v_writelane_b32 v40, s31, 1 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 5 -; GFX11-NEXT: v_readlane_b32 s30, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 -; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 6 +; GFX11-NEXT: v_readlane_b32 s31, v40, 1 +; GFX11-NEXT: v_readlane_b32 s30, v40, 0 +; GFX11-NEXT: v_readlane_b32 s0, v40, 2 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -13883,28 +13355,20 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 6 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v4i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v4i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v4i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v4i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 1 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 3 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, 4 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s7, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 5 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 6 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -13926,31 +13390,23 @@ define amdgpu_gfx void @test_call_external_void_func_v5i32_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 7 +; GFX9-NEXT: v_writelane_b32 v40, s34, 3 ; GFX9-NEXT: v_writelane_b32 v40, s4, 0 -; GFX9-NEXT: v_writelane_b32 v40, s5, 1 -; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: v_writelane_b32 v40, s8, 4 -; GFX9-NEXT: v_writelane_b32 v40, s30, 5 +; GFX9-NEXT: v_writelane_b32 v40, s30, 1 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v5i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v5i32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 1 -; GFX9-NEXT: s_mov_b32 s5, 2 -; GFX9-NEXT: s_mov_b32 s6, 3 -; GFX9-NEXT: s_mov_b32 s7, 4 -; GFX9-NEXT: s_mov_b32 s8, 5 +; GFX9-NEXT: s_mov_b32 s0, 1 +; GFX9-NEXT: s_mov_b32 s1, 2 +; GFX9-NEXT: s_mov_b32 s2, 3 +; GFX9-NEXT: s_mov_b32 s3, 4 +; GFX9-NEXT: s_mov_b32 s4, 5 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 6 +; GFX9-NEXT: v_writelane_b32 v40, s31, 2 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 6 -; GFX9-NEXT: v_readlane_b32 s30, v40, 5 -; GFX9-NEXT: v_readlane_b32 s8, v40, 4 -; GFX9-NEXT: v_readlane_b32 s7, v40, 3 -; GFX9-NEXT: v_readlane_b32 s6, v40, 2 -; GFX9-NEXT: v_readlane_b32 s5, v40, 1 +; GFX9-NEXT: v_readlane_b32 s31, v40, 2 +; GFX9-NEXT: v_readlane_b32 s30, v40, 1 ; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 7 +; GFX9-NEXT: v_readlane_b32 s34, v40, 3 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -13965,34 +13421,26 @@ define amdgpu_gfx void @test_call_external_void_func_v5i32_imm_inreg() #0 { ; GFX10-NEXT: s_mov_b32 s34, s33 ; GFX10-NEXT: s_mov_b32 s33, s32 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 -; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill -; GFX10-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 7 -; GFX10-NEXT: s_mov_b32 s35, external_void_func_v5i32_inreg@abs32@hi -; GFX10-NEXT: s_mov_b32 s34, external_void_func_v5i32_inreg@abs32@lo -; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 1 -; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2 -; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 3 -; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_mov_b32 s7, 4 -; GFX10-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-NEXT: s_mov_b32 s8, 5 -; GFX10-NEXT: v_writelane_b32 v40, s30, 5 -; GFX10-NEXT: v_writelane_b32 v40, s31, 6 +; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill +; GFX10-NEXT: s_waitcnt_depctr 0xffe3 +; GFX10-NEXT: s_mov_b32 exec_lo, s35 +; GFX10-NEXT: v_writelane_b32 v40, s34, 3 +; GFX10-NEXT: s_mov_b32 s35, external_void_func_v5i32_inreg@abs32@hi +; GFX10-NEXT: s_mov_b32 s34, external_void_func_v5i32_inreg@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, 1 +; GFX10-NEXT: s_mov_b32 s1, 2 +; GFX10-NEXT: v_writelane_b32 v40, s4, 0 +; GFX10-NEXT: s_mov_b32 s2, 3 +; GFX10-NEXT: s_mov_b32 s3, 4 +; GFX10-NEXT: s_mov_b32 s4, 5 +; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: v_writelane_b32 v40, s30, 1 +; GFX10-NEXT: v_writelane_b32 v40, s31, 2 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 6 -; GFX10-NEXT: v_readlane_b32 s30, v40, 5 -; GFX10-NEXT: v_readlane_b32 s8, v40, 4 -; GFX10-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-NEXT: v_readlane_b32 s5, v40, 1 +; GFX10-NEXT: v_readlane_b32 s31, v40, 2 +; GFX10-NEXT: v_readlane_b32 s30, v40, 1 ; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 7 +; GFX10-NEXT: v_readlane_b32 s34, v40, 3 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -14010,32 +13458,24 @@ define amdgpu_gfx void @test_call_external_void_func_v5i32_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 7 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v5i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v5i32_inreg@abs32@lo -; GFX11-NEXT: s_add_i32 s32, s32, 16 +; GFX11-NEXT: v_writelane_b32 v40, s0, 3 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v5i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v5i32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 1 +; GFX11-NEXT: s_mov_b32 s1, 2 ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 1 -; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2 -; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 3 -; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_mov_b32 s7, 4 -; GFX11-NEXT: v_writelane_b32 v40, s8, 4 -; GFX11-NEXT: s_mov_b32 s8, 5 -; GFX11-NEXT: v_writelane_b32 v40, s30, 5 -; GFX11-NEXT: v_writelane_b32 v40, s31, 6 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_mov_b32 s2, 3 +; GFX11-NEXT: s_mov_b32 s3, 4 +; GFX11-NEXT: s_mov_b32 s4, 5 +; GFX11-NEXT: s_add_i32 s32, s32, 16 +; GFX11-NEXT: v_writelane_b32 v40, s30, 1 +; GFX11-NEXT: v_writelane_b32 v40, s31, 2 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 6 -; GFX11-NEXT: v_readlane_b32 s30, v40, 5 -; GFX11-NEXT: v_readlane_b32 s8, v40, 4 -; GFX11-NEXT: v_readlane_b32 s7, v40, 3 -; GFX11-NEXT: v_readlane_b32 s6, v40, 2 -; GFX11-NEXT: v_readlane_b32 s5, v40, 1 +; GFX11-NEXT: v_readlane_b32 s31, v40, 2 +; GFX11-NEXT: v_readlane_b32 s30, v40, 1 ; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 7 +; GFX11-NEXT: v_readlane_b32 s0, v40, 3 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -14053,31 +13493,23 @@ define amdgpu_gfx void @test_call_external_void_func_v5i32_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 7 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v5i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v5i32_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 3 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v5i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v5i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 1 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s7, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s8, 5 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 5 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 6 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 6 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s8, v40, 4 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 3 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, 4 +; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 5 +; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 1 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 2 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 2 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 7 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 3 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -14099,35 +13531,36 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 10 +; GFX9-NEXT: v_writelane_b32 v40, s34, 6 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s4, 0 ; GFX9-NEXT: v_writelane_b32 v40, s5, 1 ; GFX9-NEXT: v_writelane_b32 v40, s6, 2 -; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: v_writelane_b32 v40, s8, 4 -; GFX9-NEXT: v_writelane_b32 v40, s9, 5 -; GFX9-NEXT: v_writelane_b32 v40, s10, 6 -; GFX9-NEXT: v_writelane_b32 v40, s11, 7 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_load_dwordx8 s[4:11], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 8 +; GFX9-NEXT: s_load_dwordx8 s[36:43], s[34:35], 0x0 +; GFX9-NEXT: v_writelane_b32 v40, s30, 4 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 9 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s0, s36 +; GFX9-NEXT: s_mov_b32 s1, s37 +; GFX9-NEXT: s_mov_b32 s2, s38 +; GFX9-NEXT: s_mov_b32 s3, s39 +; GFX9-NEXT: s_mov_b32 s4, s40 +; GFX9-NEXT: s_mov_b32 s5, s41 +; GFX9-NEXT: s_mov_b32 s6, s42 +; GFX9-NEXT: s_mov_b32 s7, s43 +; GFX9-NEXT: v_writelane_b32 v40, s31, 5 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 9 -; GFX9-NEXT: v_readlane_b32 s30, v40, 8 -; GFX9-NEXT: v_readlane_b32 s11, v40, 7 -; GFX9-NEXT: v_readlane_b32 s10, v40, 6 -; GFX9-NEXT: v_readlane_b32 s9, v40, 5 -; GFX9-NEXT: v_readlane_b32 s8, v40, 4 +; GFX9-NEXT: v_readlane_b32 s31, v40, 5 +; GFX9-NEXT: v_readlane_b32 s30, v40, 4 ; GFX9-NEXT: v_readlane_b32 s7, v40, 3 ; GFX9-NEXT: v_readlane_b32 s6, v40, 2 ; GFX9-NEXT: v_readlane_b32 s5, v40, 1 ; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 10 +; GFX9-NEXT: v_readlane_b32 s34, v40, 6 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -14145,35 +13578,36 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 10 +; GFX10-NEXT: v_writelane_b32 v40, s34, 6 ; GFX10-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s4, 0 ; GFX10-NEXT: v_writelane_b32 v40, s5, 1 ; GFX10-NEXT: v_writelane_b32 v40, s6, 2 ; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-NEXT: v_writelane_b32 v40, s10, 6 -; GFX10-NEXT: v_writelane_b32 v40, s11, 7 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_load_dwordx8 s[4:11], s[34:35], 0x0 +; GFX10-NEXT: s_load_dwordx8 s[36:43], s[34:35], 0x0 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 8 -; GFX10-NEXT: v_writelane_b32 v40, s31, 9 +; GFX10-NEXT: v_writelane_b32 v40, s30, 4 +; GFX10-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s0, s36 +; GFX10-NEXT: s_mov_b32 s1, s37 +; GFX10-NEXT: s_mov_b32 s2, s38 +; GFX10-NEXT: s_mov_b32 s3, s39 +; GFX10-NEXT: s_mov_b32 s4, s40 +; GFX10-NEXT: s_mov_b32 s5, s41 +; GFX10-NEXT: s_mov_b32 s6, s42 +; GFX10-NEXT: s_mov_b32 s7, s43 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 9 -; GFX10-NEXT: v_readlane_b32 s30, v40, 8 -; GFX10-NEXT: v_readlane_b32 s11, v40, 7 -; GFX10-NEXT: v_readlane_b32 s10, v40, 6 -; GFX10-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-NEXT: v_readlane_b32 s8, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 5 +; GFX10-NEXT: v_readlane_b32 s30, v40, 4 ; GFX10-NEXT: v_readlane_b32 s7, v40, 3 ; GFX10-NEXT: v_readlane_b32 s6, v40, 2 ; GFX10-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 10 +; GFX10-NEXT: v_readlane_b32 s34, v40, 6 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -14191,36 +13625,28 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 10 +; GFX11-NEXT: v_writelane_b32 v40, s0, 6 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 ; GFX11-NEXT: v_writelane_b32 v40, s5, 1 ; GFX11-NEXT: v_writelane_b32 v40, s6, 2 ; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: v_writelane_b32 v40, s8, 4 -; GFX11-NEXT: v_writelane_b32 v40, s9, 5 -; GFX11-NEXT: v_writelane_b32 v40, s10, 6 -; GFX11-NEXT: v_writelane_b32 v40, s11, 7 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_load_b256 s[4:11], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v8i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v8i32_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 8 -; GFX11-NEXT: v_writelane_b32 v40, s31, 9 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_load_b256 s[0:7], s[0:1], 0x0 +; GFX11-NEXT: v_writelane_b32 v40, s30, 4 +; GFX11-NEXT: v_writelane_b32 v40, s31, 5 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 9 -; GFX11-NEXT: v_readlane_b32 s30, v40, 8 -; GFX11-NEXT: v_readlane_b32 s11, v40, 7 -; GFX11-NEXT: v_readlane_b32 s10, v40, 6 -; GFX11-NEXT: v_readlane_b32 s9, v40, 5 -; GFX11-NEXT: v_readlane_b32 s8, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 5 +; GFX11-NEXT: v_readlane_b32 s30, v40, 4 ; GFX11-NEXT: v_readlane_b32 s7, v40, 3 ; GFX11-NEXT: v_readlane_b32 s6, v40, 2 ; GFX11-NEXT: v_readlane_b32 s5, v40, 1 ; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 10 +; GFX11-NEXT: v_readlane_b32 s0, v40, 6 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -14238,35 +13664,27 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 10 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 6 ; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s10, 6 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s11, 7 ; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v8i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v8i32_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 8 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 9 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 9 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 8 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s11, v40, 7 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s10, v40, 6 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s8, v40, 4 +; GFX10-SCRATCH-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 5 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 4 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 10 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 6 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -14290,40 +13708,32 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_imm_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 10 +; GFX9-NEXT: v_writelane_b32 v40, s34, 6 ; GFX9-NEXT: v_writelane_b32 v40, s4, 0 ; GFX9-NEXT: v_writelane_b32 v40, s5, 1 ; GFX9-NEXT: v_writelane_b32 v40, s6, 2 ; GFX9-NEXT: v_writelane_b32 v40, s7, 3 -; GFX9-NEXT: v_writelane_b32 v40, s8, 4 -; GFX9-NEXT: v_writelane_b32 v40, s9, 5 -; GFX9-NEXT: v_writelane_b32 v40, s10, 6 -; GFX9-NEXT: v_writelane_b32 v40, s11, 7 -; GFX9-NEXT: v_writelane_b32 v40, s30, 8 +; GFX9-NEXT: v_writelane_b32 v40, s30, 4 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s4, 1 -; GFX9-NEXT: s_mov_b32 s5, 2 -; GFX9-NEXT: s_mov_b32 s6, 3 -; GFX9-NEXT: s_mov_b32 s7, 4 -; GFX9-NEXT: s_mov_b32 s8, 5 -; GFX9-NEXT: s_mov_b32 s9, 6 -; GFX9-NEXT: s_mov_b32 s10, 7 -; GFX9-NEXT: s_mov_b32 s11, 8 +; GFX9-NEXT: s_mov_b32 s0, 1 +; GFX9-NEXT: s_mov_b32 s1, 2 +; GFX9-NEXT: s_mov_b32 s2, 3 +; GFX9-NEXT: s_mov_b32 s3, 4 +; GFX9-NEXT: s_mov_b32 s4, 5 +; GFX9-NEXT: s_mov_b32 s5, 6 +; GFX9-NEXT: s_mov_b32 s6, 7 +; GFX9-NEXT: s_mov_b32 s7, 8 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 9 +; GFX9-NEXT: v_writelane_b32 v40, s31, 5 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 9 -; GFX9-NEXT: v_readlane_b32 s30, v40, 8 -; GFX9-NEXT: v_readlane_b32 s11, v40, 7 -; GFX9-NEXT: v_readlane_b32 s10, v40, 6 -; GFX9-NEXT: v_readlane_b32 s9, v40, 5 -; GFX9-NEXT: v_readlane_b32 s8, v40, 4 +; GFX9-NEXT: v_readlane_b32 s31, v40, 5 +; GFX9-NEXT: v_readlane_b32 s30, v40, 4 ; GFX9-NEXT: v_readlane_b32 s7, v40, 3 ; GFX9-NEXT: v_readlane_b32 s6, v40, 2 ; GFX9-NEXT: v_readlane_b32 s5, v40, 1 ; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 10 +; GFX9-NEXT: v_readlane_b32 s34, v40, 6 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -14341,40 +13751,32 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_imm_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 10 +; GFX10-NEXT: v_writelane_b32 v40, s34, 6 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo -; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: s_mov_b32 s0, 1 +; GFX10-NEXT: s_mov_b32 s1, 2 ; GFX10-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-NEXT: s_mov_b32 s4, 1 +; GFX10-NEXT: s_mov_b32 s2, 3 +; GFX10-NEXT: s_mov_b32 s3, 4 +; GFX10-NEXT: s_mov_b32 s4, 5 +; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-NEXT: s_mov_b32 s5, 2 +; GFX10-NEXT: s_mov_b32 s5, 6 ; GFX10-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-NEXT: s_mov_b32 s6, 3 +; GFX10-NEXT: s_mov_b32 s6, 7 ; GFX10-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-NEXT: s_mov_b32 s7, 4 -; GFX10-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-NEXT: s_mov_b32 s8, 5 -; GFX10-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-NEXT: s_mov_b32 s9, 6 -; GFX10-NEXT: v_writelane_b32 v40, s10, 6 -; GFX10-NEXT: s_mov_b32 s10, 7 -; GFX10-NEXT: v_writelane_b32 v40, s11, 7 -; GFX10-NEXT: s_mov_b32 s11, 8 -; GFX10-NEXT: v_writelane_b32 v40, s30, 8 -; GFX10-NEXT: v_writelane_b32 v40, s31, 9 +; GFX10-NEXT: s_mov_b32 s7, 8 +; GFX10-NEXT: v_writelane_b32 v40, s30, 4 +; GFX10-NEXT: v_writelane_b32 v40, s31, 5 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 9 -; GFX10-NEXT: v_readlane_b32 s30, v40, 8 -; GFX10-NEXT: v_readlane_b32 s11, v40, 7 -; GFX10-NEXT: v_readlane_b32 s10, v40, 6 -; GFX10-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-NEXT: v_readlane_b32 s8, v40, 4 +; GFX10-NEXT: v_readlane_b32 s31, v40, 5 +; GFX10-NEXT: v_readlane_b32 s30, v40, 4 ; GFX10-NEXT: v_readlane_b32 s7, v40, 3 ; GFX10-NEXT: v_readlane_b32 s6, v40, 2 ; GFX10-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 10 +; GFX10-NEXT: v_readlane_b32 s34, v40, 6 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -14392,41 +13794,33 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_imm_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 10 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v8i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v8i32_inreg@abs32@lo -; GFX11-NEXT: s_add_i32 s32, s32, 16 +; GFX11-NEXT: v_writelane_b32 v40, s0, 6 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo +; GFX11-NEXT: s_mov_b32 s0, 1 +; GFX11-NEXT: s_mov_b32 s1, 2 ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 -; GFX11-NEXT: s_mov_b32 s4, 1 +; GFX11-NEXT: s_mov_b32 s2, 3 +; GFX11-NEXT: s_mov_b32 s3, 4 +; GFX11-NEXT: s_mov_b32 s4, 5 +; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s5, 1 -; GFX11-NEXT: s_mov_b32 s5, 2 +; GFX11-NEXT: s_mov_b32 s5, 6 ; GFX11-NEXT: v_writelane_b32 v40, s6, 2 -; GFX11-NEXT: s_mov_b32 s6, 3 +; GFX11-NEXT: s_mov_b32 s6, 7 ; GFX11-NEXT: v_writelane_b32 v40, s7, 3 -; GFX11-NEXT: s_mov_b32 s7, 4 -; GFX11-NEXT: v_writelane_b32 v40, s8, 4 -; GFX11-NEXT: s_mov_b32 s8, 5 -; GFX11-NEXT: v_writelane_b32 v40, s9, 5 -; GFX11-NEXT: s_mov_b32 s9, 6 -; GFX11-NEXT: v_writelane_b32 v40, s10, 6 -; GFX11-NEXT: s_mov_b32 s10, 7 -; GFX11-NEXT: v_writelane_b32 v40, s11, 7 -; GFX11-NEXT: s_mov_b32 s11, 8 -; GFX11-NEXT: v_writelane_b32 v40, s30, 8 -; GFX11-NEXT: v_writelane_b32 v40, s31, 9 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_mov_b32 s7, 8 +; GFX11-NEXT: v_writelane_b32 v40, s30, 4 +; GFX11-NEXT: v_writelane_b32 v40, s31, 5 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 9 -; GFX11-NEXT: v_readlane_b32 s30, v40, 8 -; GFX11-NEXT: v_readlane_b32 s11, v40, 7 -; GFX11-NEXT: v_readlane_b32 s10, v40, 6 -; GFX11-NEXT: v_readlane_b32 s9, v40, 5 -; GFX11-NEXT: v_readlane_b32 s8, v40, 4 +; GFX11-NEXT: v_readlane_b32 s31, v40, 5 +; GFX11-NEXT: v_readlane_b32 s30, v40, 4 ; GFX11-NEXT: v_readlane_b32 s7, v40, 3 ; GFX11-NEXT: v_readlane_b32 s6, v40, 2 ; GFX11-NEXT: v_readlane_b32 s5, v40, 1 ; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 10 +; GFX11-NEXT: v_readlane_b32 s0, v40, 6 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -14444,40 +13838,32 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_imm_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 10 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v8i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v8i32_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 6 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v8i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v8i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 s0, 1 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, 2 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 1 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, 3 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, 4 +; GFX10-SCRATCH-NEXT: s_mov_b32 s4, 5 +; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 -; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s5, 6 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 3 +; GFX10-SCRATCH-NEXT: s_mov_b32 s6, 7 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 -; GFX10-SCRATCH-NEXT: s_mov_b32 s7, 4 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s8, 4 -; GFX10-SCRATCH-NEXT: s_mov_b32 s8, 5 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s9, 5 -; GFX10-SCRATCH-NEXT: s_mov_b32 s9, 6 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s10, 6 -; GFX10-SCRATCH-NEXT: s_mov_b32 s10, 7 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s11, 7 -; GFX10-SCRATCH-NEXT: s_mov_b32 s11, 8 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 8 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 9 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 9 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 8 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s11, v40, 7 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s10, v40, 6 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s9, v40, 5 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s8, v40, 4 +; GFX10-SCRATCH-NEXT: s_mov_b32 s7, 8 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 4 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 5 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 5 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 4 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s7, v40, 3 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 10 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 6 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -14499,38 +13885,47 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 18 +; GFX9-NEXT: v_writelane_b32 v40, s34, 14 ; GFX9-NEXT: v_writelane_b32 v40, s4, 0 ; GFX9-NEXT: v_writelane_b32 v40, s5, 1 ; GFX9-NEXT: v_writelane_b32 v40, s6, 2 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s7, 3 ; GFX9-NEXT: v_writelane_b32 v40, s8, 4 ; GFX9-NEXT: v_writelane_b32 v40, s9, 5 ; GFX9-NEXT: v_writelane_b32 v40, s10, 6 ; GFX9-NEXT: v_writelane_b32 v40, s11, 7 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s12, 8 ; GFX9-NEXT: v_writelane_b32 v40, s13, 9 ; GFX9-NEXT: v_writelane_b32 v40, s14, 10 -; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s15, 11 -; GFX9-NEXT: v_writelane_b32 v40, s16, 12 -; GFX9-NEXT: v_writelane_b32 v40, s17, 13 -; GFX9-NEXT: v_writelane_b32 v40, s18, 14 -; GFX9-NEXT: v_writelane_b32 v40, s19, 15 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_load_dwordx16 s[4:19], s[34:35], 0x0 -; GFX9-NEXT: v_writelane_b32 v40, s30, 16 +; GFX9-NEXT: v_writelane_b32 v40, s30, 12 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v16i32_inreg@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v16i32_inreg@abs32@lo +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_mov_b32 s0, s36 +; GFX9-NEXT: s_mov_b32 s1, s37 +; GFX9-NEXT: s_mov_b32 s2, s38 +; GFX9-NEXT: s_mov_b32 s3, s39 +; GFX9-NEXT: s_mov_b32 s4, s40 +; GFX9-NEXT: s_mov_b32 s5, s41 +; GFX9-NEXT: s_mov_b32 s6, s42 +; GFX9-NEXT: s_mov_b32 s7, s43 +; GFX9-NEXT: s_mov_b32 s8, s44 +; GFX9-NEXT: s_mov_b32 s9, s45 +; GFX9-NEXT: s_mov_b32 s10, s46 +; GFX9-NEXT: s_mov_b32 s11, s47 +; GFX9-NEXT: s_mov_b32 s12, s48 +; GFX9-NEXT: s_mov_b32 s13, s49 +; GFX9-NEXT: s_mov_b32 s14, s50 +; GFX9-NEXT: s_mov_b32 s15, s51 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 17 +; GFX9-NEXT: v_writelane_b32 v40, s31, 13 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 17 -; GFX9-NEXT: v_readlane_b32 s30, v40, 16 -; GFX9-NEXT: v_readlane_b32 s19, v40, 15 -; GFX9-NEXT: v_readlane_b32 s18, v40, 14 -; GFX9-NEXT: v_readlane_b32 s17, v40, 13 -; GFX9-NEXT: v_readlane_b32 s16, v40, 12 +; GFX9-NEXT: v_readlane_b32 s31, v40, 13 +; GFX9-NEXT: v_readlane_b32 s30, v40, 12 ; GFX9-NEXT: v_readlane_b32 s15, v40, 11 ; GFX9-NEXT: v_readlane_b32 s14, v40, 10 ; GFX9-NEXT: v_readlane_b32 s13, v40, 9 @@ -14543,7 +13938,7 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX9-NEXT: v_readlane_b32 s6, v40, 2 ; GFX9-NEXT: v_readlane_b32 s5, v40, 1 ; GFX9-NEXT: v_readlane_b32 s4, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 18 +; GFX9-NEXT: v_readlane_b32 s34, v40, 14 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -14561,38 +13956,47 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 18 +; GFX10-NEXT: v_writelane_b32 v40, s34, 14 ; GFX10-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s4, 0 ; GFX10-NEXT: v_writelane_b32 v40, s5, 1 ; GFX10-NEXT: v_writelane_b32 v40, s6, 2 ; GFX10-NEXT: v_writelane_b32 v40, s7, 3 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x0 +; GFX10-NEXT: s_mov_b32 s35, external_void_func_v16i32_inreg@abs32@hi +; GFX10-NEXT: s_mov_b32 s34, external_void_func_v16i32_inreg@abs32@lo ; GFX10-NEXT: v_writelane_b32 v40, s8, 4 ; GFX10-NEXT: v_writelane_b32 v40, s9, 5 ; GFX10-NEXT: v_writelane_b32 v40, s10, 6 ; GFX10-NEXT: v_writelane_b32 v40, s11, 7 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_mov_b32 s0, s36 +; GFX10-NEXT: s_mov_b32 s1, s37 +; GFX10-NEXT: s_mov_b32 s2, s38 +; GFX10-NEXT: s_mov_b32 s3, s39 ; GFX10-NEXT: v_writelane_b32 v40, s12, 8 +; GFX10-NEXT: s_mov_b32 s4, s40 +; GFX10-NEXT: s_mov_b32 s5, s41 +; GFX10-NEXT: s_mov_b32 s6, s42 +; GFX10-NEXT: s_mov_b32 s7, s43 ; GFX10-NEXT: v_writelane_b32 v40, s13, 9 +; GFX10-NEXT: s_mov_b32 s8, s44 +; GFX10-NEXT: s_mov_b32 s9, s45 +; GFX10-NEXT: s_mov_b32 s10, s46 +; GFX10-NEXT: s_mov_b32 s11, s47 ; GFX10-NEXT: v_writelane_b32 v40, s14, 10 +; GFX10-NEXT: s_mov_b32 s12, s48 +; GFX10-NEXT: s_mov_b32 s13, s49 +; GFX10-NEXT: s_mov_b32 s14, s50 ; GFX10-NEXT: v_writelane_b32 v40, s15, 11 -; GFX10-NEXT: v_writelane_b32 v40, s16, 12 -; GFX10-NEXT: v_writelane_b32 v40, s17, 13 -; GFX10-NEXT: v_writelane_b32 v40, s18, 14 -; GFX10-NEXT: v_writelane_b32 v40, s19, 15 -; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_load_dwordx16 s[4:19], s[34:35], 0x0 -; GFX10-NEXT: s_mov_b32 s35, external_void_func_v16i32_inreg@abs32@hi -; GFX10-NEXT: s_mov_b32 s34, external_void_func_v16i32_inreg@abs32@lo -; GFX10-NEXT: v_writelane_b32 v40, s30, 16 -; GFX10-NEXT: v_writelane_b32 v40, s31, 17 +; GFX10-NEXT: s_mov_b32 s15, s51 +; GFX10-NEXT: v_writelane_b32 v40, s30, 12 +; GFX10-NEXT: v_writelane_b32 v40, s31, 13 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 17 -; GFX10-NEXT: v_readlane_b32 s30, v40, 16 -; GFX10-NEXT: v_readlane_b32 s19, v40, 15 -; GFX10-NEXT: v_readlane_b32 s18, v40, 14 -; GFX10-NEXT: v_readlane_b32 s17, v40, 13 -; GFX10-NEXT: v_readlane_b32 s16, v40, 12 +; GFX10-NEXT: v_readlane_b32 s31, v40, 13 +; GFX10-NEXT: v_readlane_b32 s30, v40, 12 ; GFX10-NEXT: v_readlane_b32 s15, v40, 11 ; GFX10-NEXT: v_readlane_b32 s14, v40, 10 ; GFX10-NEXT: v_readlane_b32 s13, v40, 9 @@ -14605,7 +14009,7 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX10-NEXT: v_readlane_b32 s6, v40, 2 ; GFX10-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 18 +; GFX10-NEXT: v_readlane_b32 s34, v40, 14 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -14623,8 +14027,10 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill ; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 18 +; GFX11-NEXT: v_writelane_b32 v40, s0, 14 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v16i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v16i32_inreg@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 ; GFX11-NEXT: v_writelane_b32 v40, s5, 1 @@ -14638,24 +14044,14 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX11-NEXT: v_writelane_b32 v40, s13, 9 ; GFX11-NEXT: v_writelane_b32 v40, s14, 10 ; GFX11-NEXT: v_writelane_b32 v40, s15, 11 -; GFX11-NEXT: v_writelane_b32 v40, s16, 12 -; GFX11-NEXT: v_writelane_b32 v40, s17, 13 -; GFX11-NEXT: v_writelane_b32 v40, s18, 14 -; GFX11-NEXT: v_writelane_b32 v40, s19, 15 ; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_load_b512 s[4:19], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v16i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v16i32_inreg@abs32@lo -; GFX11-NEXT: v_writelane_b32 v40, s30, 16 -; GFX11-NEXT: v_writelane_b32 v40, s31, 17 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_load_b512 s[0:15], s[0:1], 0x0 +; GFX11-NEXT: v_writelane_b32 v40, s30, 12 +; GFX11-NEXT: v_writelane_b32 v40, s31, 13 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-NEXT: v_readlane_b32 s31, v40, 17 -; GFX11-NEXT: v_readlane_b32 s30, v40, 16 -; GFX11-NEXT: v_readlane_b32 s19, v40, 15 -; GFX11-NEXT: v_readlane_b32 s18, v40, 14 -; GFX11-NEXT: v_readlane_b32 s17, v40, 13 -; GFX11-NEXT: v_readlane_b32 s16, v40, 12 +; GFX11-NEXT: v_readlane_b32 s31, v40, 13 +; GFX11-NEXT: v_readlane_b32 s30, v40, 12 ; GFX11-NEXT: v_readlane_b32 s15, v40, 11 ; GFX11-NEXT: v_readlane_b32 s14, v40, 10 ; GFX11-NEXT: v_readlane_b32 s13, v40, 9 @@ -14668,7 +14064,7 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX11-NEXT: v_readlane_b32 s6, v40, 2 ; GFX11-NEXT: v_readlane_b32 s5, v40, 1 ; GFX11-NEXT: v_readlane_b32 s4, v40, 0 -; GFX11-NEXT: v_readlane_b32 s0, v40, 18 +; GFX11-NEXT: v_readlane_b32 s0, v40, 14 ; GFX11-NEXT: s_or_saveexec_b32 s1, -1 ; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload ; GFX11-NEXT: s_mov_b32 exec_lo, s1 @@ -14686,8 +14082,10 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 18 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 14 ; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v16i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v16i32_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 @@ -14701,23 +14099,13 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s13, 9 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s14, 10 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s15, 11 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s16, 12 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s17, 13 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s18, 14 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s19, 15 ; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[4:19], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v16i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v16i32_inreg@abs32@lo -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 16 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 17 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 17 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 16 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s19, v40, 15 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s18, v40, 14 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s17, v40, 13 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s16, v40, 12 +; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[0:15], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 12 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 13 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 13 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 12 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s15, v40, 11 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s14, v40, 10 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s13, v40, 9 @@ -14730,7 +14118,7 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 { ; GFX10-SCRATCH-NEXT: v_readlane_b32 s6, v40, 2 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s5, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s4, v40, 0 -; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 18 +; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 14 ; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 ; GFX10-SCRATCH-NEXT: scratch_load_dword v40, off, s33 ; 4-byte Folded Reload ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 @@ -14771,49 +14159,47 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 { ; GFX9-NEXT: v_writelane_b32 v40, s17, 13 ; GFX9-NEXT: v_writelane_b32 v40, s18, 14 ; GFX9-NEXT: v_writelane_b32 v40, s19, 15 -; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s20, 16 ; GFX9-NEXT: v_writelane_b32 v40, s21, 17 ; GFX9-NEXT: v_writelane_b32 v40, s22, 18 ; GFX9-NEXT: v_writelane_b32 v40, s23, 19 ; GFX9-NEXT: v_writelane_b32 v40, s24, 20 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x40 -; GFX9-NEXT: s_load_dwordx16 s[4:19], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s25, 21 ; GFX9-NEXT: v_writelane_b32 v40, s26, 22 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s27, 23 -; GFX9-NEXT: s_addk_i32 s32, 0x400 ; GFX9-NEXT: v_writelane_b32 v40, s28, 24 +; GFX9-NEXT: v_writelane_b32 v40, s29, 25 +; GFX9-NEXT: v_writelane_b32 v40, s30, 26 +; GFX9-NEXT: v_writelane_b32 v40, s31, 27 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40 +; GFX9-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x0 +; GFX9-NEXT: s_addk_i32 s32, 0x400 +; GFX9-NEXT: s_mov_b32 s53, external_void_func_v32i32_inreg@abs32@hi +; GFX9-NEXT: s_mov_b32 s52, external_void_func_v32i32_inreg@abs32@lo ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, s46 -; GFX9-NEXT: v_writelane_b32 v40, s29, 25 -; GFX9-NEXT: v_mov_b32_e32 v1, s47 -; GFX9-NEXT: v_mov_b32_e32 v2, s48 -; GFX9-NEXT: v_mov_b32_e32 v3, s49 +; GFX9-NEXT: v_mov_b32_e32 v0, s30 +; GFX9-NEXT: v_mov_b32_e32 v1, s31 ; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 ; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 -; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 -; GFX9-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:12 -; GFX9-NEXT: v_mov_b32_e32 v0, s50 -; GFX9-NEXT: v_writelane_b32 v40, s30, 26 -; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:16 -; GFX9-NEXT: v_mov_b32_e32 v0, s51 -; GFX9-NEXT: s_mov_b32 s35, external_void_func_v32i32_inreg@abs32@hi -; GFX9-NEXT: s_mov_b32 s34, external_void_func_v32i32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s20, s36 -; GFX9-NEXT: s_mov_b32 s21, s37 -; GFX9-NEXT: s_mov_b32 s22, s38 -; GFX9-NEXT: s_mov_b32 s23, s39 -; GFX9-NEXT: s_mov_b32 s24, s40 -; GFX9-NEXT: s_mov_b32 s25, s41 -; GFX9-NEXT: s_mov_b32 s26, s42 -; GFX9-NEXT: s_mov_b32 s27, s43 -; GFX9-NEXT: s_mov_b32 s28, s44 -; GFX9-NEXT: s_mov_b32 s29, s45 -; GFX9-NEXT: v_writelane_b32 v40, s31, 27 -; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:20 -; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX9-NEXT: s_mov_b32 s0, s36 +; GFX9-NEXT: s_mov_b32 s1, s37 +; GFX9-NEXT: s_mov_b32 s2, s38 +; GFX9-NEXT: s_mov_b32 s3, s39 +; GFX9-NEXT: s_mov_b32 s4, s40 +; GFX9-NEXT: s_mov_b32 s5, s41 +; GFX9-NEXT: s_mov_b32 s6, s42 +; GFX9-NEXT: s_mov_b32 s7, s43 +; GFX9-NEXT: s_mov_b32 s8, s44 +; GFX9-NEXT: s_mov_b32 s9, s45 +; GFX9-NEXT: s_mov_b32 s10, s46 +; GFX9-NEXT: s_mov_b32 s11, s47 +; GFX9-NEXT: s_mov_b32 s12, s48 +; GFX9-NEXT: s_mov_b32 s13, s49 +; GFX9-NEXT: s_mov_b32 s14, s50 +; GFX9-NEXT: s_mov_b32 s15, s51 +; GFX9-NEXT: s_swappc_b64 s[30:31], s[52:53] ; GFX9-NEXT: v_readlane_b32 s31, v40, 27 ; GFX9-NEXT: v_readlane_b32 s30, v40, 26 ; GFX9-NEXT: v_readlane_b32 s29, v40, 25 @@ -14879,47 +14265,46 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 { ; GFX10-NEXT: v_writelane_b32 v40, s17, 13 ; GFX10-NEXT: v_writelane_b32 v40, s18, 14 ; GFX10-NEXT: v_writelane_b32 v40, s19, 15 -; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_clause 0x1 -; GFX10-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x40 -; GFX10-NEXT: s_load_dwordx16 s[4:19], s[34:35], 0x0 -; GFX10-NEXT: s_mov_b32 s35, external_void_func_v32i32_inreg@abs32@hi -; GFX10-NEXT: s_mov_b32 s34, external_void_func_v32i32_inreg@abs32@lo ; GFX10-NEXT: v_writelane_b32 v40, s20, 16 ; GFX10-NEXT: v_writelane_b32 v40, s21, 17 ; GFX10-NEXT: v_writelane_b32 v40, s22, 18 -; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v0, s46 ; GFX10-NEXT: v_writelane_b32 v40, s23, 19 -; GFX10-NEXT: v_mov_b32_e32 v1, s47 -; GFX10-NEXT: v_mov_b32_e32 v2, s48 -; GFX10-NEXT: v_mov_b32_e32 v3, s49 -; GFX10-NEXT: s_mov_b32 s20, s36 ; GFX10-NEXT: v_writelane_b32 v40, s24, 20 -; GFX10-NEXT: s_mov_b32 s21, s37 -; GFX10-NEXT: s_mov_b32 s22, s38 -; GFX10-NEXT: s_mov_b32 s23, s39 -; GFX10-NEXT: s_mov_b32 s24, s40 ; GFX10-NEXT: v_writelane_b32 v40, s25, 21 -; GFX10-NEXT: s_mov_b32 s25, s41 -; GFX10-NEXT: v_mov_b32_e32 v4, s50 -; GFX10-NEXT: v_mov_b32_e32 v5, s51 -; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], s32 -; GFX10-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 -; GFX10-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 -; GFX10-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:12 -; GFX10-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16 -; GFX10-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:20 ; GFX10-NEXT: v_writelane_b32 v40, s26, 22 -; GFX10-NEXT: s_mov_b32 s26, s42 ; GFX10-NEXT: v_writelane_b32 v40, s27, 23 -; GFX10-NEXT: s_mov_b32 s27, s43 ; GFX10-NEXT: v_writelane_b32 v40, s28, 24 -; GFX10-NEXT: s_mov_b32 s28, s44 ; GFX10-NEXT: v_writelane_b32 v40, s29, 25 -; GFX10-NEXT: s_mov_b32 s29, s45 ; GFX10-NEXT: v_writelane_b32 v40, s30, 26 ; GFX10-NEXT: v_writelane_b32 v40, s31, 27 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_clause 0x1 +; GFX10-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40 +; GFX10-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x0 +; GFX10-NEXT: s_mov_b32 s35, external_void_func_v32i32_inreg@abs32@hi +; GFX10-NEXT: s_mov_b32 s34, external_void_func_v32i32_inreg@abs32@lo +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: v_mov_b32_e32 v0, s30 +; GFX10-NEXT: v_mov_b32_e32 v1, s31 +; GFX10-NEXT: s_mov_b32 s4, s40 +; GFX10-NEXT: s_mov_b32 s5, s41 +; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], s32 +; GFX10-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 +; GFX10-NEXT: s_mov_b32 s6, s42 +; GFX10-NEXT: s_mov_b32 s7, s43 +; GFX10-NEXT: s_mov_b32 s8, s44 +; GFX10-NEXT: s_mov_b32 s9, s45 +; GFX10-NEXT: s_mov_b32 s10, s46 +; GFX10-NEXT: s_mov_b32 s11, s47 +; GFX10-NEXT: s_mov_b32 s12, s48 +; GFX10-NEXT: s_mov_b32 s13, s49 +; GFX10-NEXT: s_mov_b32 s14, s50 +; GFX10-NEXT: s_mov_b32 s15, s51 +; GFX10-NEXT: s_waitcnt_depctr 0xffe3 +; GFX10-NEXT: s_mov_b32 s0, s36 +; GFX10-NEXT: s_mov_b32 s1, s37 +; GFX10-NEXT: s_mov_b32 s2, s38 +; GFX10-NEXT: s_mov_b32 s3, s39 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX10-NEXT: v_readlane_b32 s31, v40, 27 ; GFX10-NEXT: v_readlane_b32 s30, v40, 26 @@ -14970,8 +14355,8 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 { ; GFX11-NEXT: v_writelane_b32 v40, s0, 28 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-NEXT: s_add_i32 s2, s32, 16 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v32i32_inreg@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v32i32_inreg@abs32@lo ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 ; GFX11-NEXT: v_writelane_b32 v40, s5, 1 ; GFX11-NEXT: v_writelane_b32 v40, s6, 2 @@ -14988,42 +14373,26 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 { ; GFX11-NEXT: v_writelane_b32 v40, s17, 13 ; GFX11-NEXT: v_writelane_b32 v40, s18, 14 ; GFX11-NEXT: v_writelane_b32 v40, s19, 15 -; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_clause 0x1 -; GFX11-NEXT: s_load_b512 s[36:51], s[0:1], 0x40 -; GFX11-NEXT: s_load_b512 s[4:19], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v32i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v32i32_inreg@abs32@lo ; GFX11-NEXT: v_writelane_b32 v40, s20, 16 ; GFX11-NEXT: v_writelane_b32 v40, s21, 17 ; GFX11-NEXT: v_writelane_b32 v40, s22, 18 -; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: v_dual_mov_b32 v4, s50 :: v_dual_mov_b32 v5, s51 ; GFX11-NEXT: v_writelane_b32 v40, s23, 19 -; GFX11-NEXT: v_dual_mov_b32 v0, s46 :: v_dual_mov_b32 v1, s47 -; GFX11-NEXT: v_dual_mov_b32 v2, s48 :: v_dual_mov_b32 v3, s49 ; GFX11-NEXT: v_writelane_b32 v40, s24, 20 -; GFX11-NEXT: s_mov_b32 s20, s36 -; GFX11-NEXT: s_mov_b32 s21, s37 -; GFX11-NEXT: s_mov_b32 s22, s38 -; GFX11-NEXT: s_mov_b32 s23, s39 ; GFX11-NEXT: v_writelane_b32 v40, s25, 21 -; GFX11-NEXT: s_mov_b32 s24, s40 -; GFX11-NEXT: s_mov_b32 s25, s41 -; GFX11-NEXT: scratch_store_b64 off, v[4:5], s2 -; GFX11-NEXT: scratch_store_b128 off, v[0:3], s32 ; GFX11-NEXT: v_writelane_b32 v40, s26, 22 -; GFX11-NEXT: s_mov_b32 s26, s42 ; GFX11-NEXT: v_writelane_b32 v40, s27, 23 -; GFX11-NEXT: s_mov_b32 s27, s43 ; GFX11-NEXT: v_writelane_b32 v40, s28, 24 -; GFX11-NEXT: s_mov_b32 s28, s44 ; GFX11-NEXT: v_writelane_b32 v40, s29, 25 -; GFX11-NEXT: s_mov_b32 s29, s45 ; GFX11-NEXT: v_writelane_b32 v40, s30, 26 ; GFX11-NEXT: v_writelane_b32 v40, s31, 27 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_clause 0x1 +; GFX11-NEXT: s_load_b512 s[16:31], s[0:1], 0x40 +; GFX11-NEXT: s_load_b512 s[0:15], s[0:1], 0x0 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v0, s30 :: v_dual_mov_b32 v1, s31 +; GFX11-NEXT: scratch_store_b64 off, v[0:1], s32 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: v_readlane_b32 s31, v40, 27 ; GFX11-NEXT: v_readlane_b32 s30, v40, 26 ; GFX11-NEXT: v_readlane_b32 s29, v40, 25 @@ -15071,9 +14440,8 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 { ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 28 -; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[34:35], s[0:1], 0x0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: s_add_i32 s2, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 @@ -15090,44 +14458,29 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 { ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s17, 13 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s18, 14 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s19, 15 -; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_clause 0x1 -; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[36:51], s[0:1], 0x40 -; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[4:19], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v32i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v32i32_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s20, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s21, 17 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s22, 18 -; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v4, s50 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s23, 19 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v5, s51 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v0, s46 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v1, s47 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v2, s48 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s24, 20 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v3, s49 -; GFX10-SCRATCH-NEXT: s_mov_b32 s20, s36 -; GFX10-SCRATCH-NEXT: s_mov_b32 s21, s37 -; GFX10-SCRATCH-NEXT: s_mov_b32 s22, s38 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s25, 21 -; GFX10-SCRATCH-NEXT: s_mov_b32 s23, s39 -; GFX10-SCRATCH-NEXT: s_mov_b32 s24, s40 -; GFX10-SCRATCH-NEXT: s_mov_b32 s25, s41 -; GFX10-SCRATCH-NEXT: scratch_store_dwordx2 off, v[4:5], s2 -; GFX10-SCRATCH-NEXT: scratch_store_dwordx4 off, v[0:3], s32 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s26, 22 -; GFX10-SCRATCH-NEXT: s_mov_b32 s26, s42 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s27, 23 -; GFX10-SCRATCH-NEXT: s_mov_b32 s27, s43 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s28, 24 -; GFX10-SCRATCH-NEXT: s_mov_b32 s28, s44 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s29, 25 -; GFX10-SCRATCH-NEXT: s_mov_b32 s29, s45 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 26 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 27 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-SCRATCH-NEXT: s_clause 0x1 +; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40 +; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v32i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v32i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v0, s30 +; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v1, s31 +; GFX10-SCRATCH-NEXT: scratch_store_dwordx2 off, v[0:1], s32 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 27 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 26 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s29, v40, 25 @@ -15196,55 +14549,53 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 { ; GFX9-NEXT: v_writelane_b32 v40, s16, 12 ; GFX9-NEXT: v_writelane_b32 v40, s17, 13 ; GFX9-NEXT: v_writelane_b32 v40, s18, 14 -; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s19, 15 ; GFX9-NEXT: v_writelane_b32 v40, s20, 16 ; GFX9-NEXT: v_writelane_b32 v40, s21, 17 ; GFX9-NEXT: v_writelane_b32 v40, s22, 18 ; GFX9-NEXT: v_writelane_b32 v40, s23, 19 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_load_dword s52, s[34:35], 0x0 -; GFX9-NEXT: ; kill: killed $sgpr34_sgpr35 -; GFX9-NEXT: ; kill: killed $sgpr34_sgpr35 -; GFX9-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x40 -; GFX9-NEXT: s_load_dwordx16 s[4:19], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s24, 20 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s25, 21 -; GFX9-NEXT: s_addk_i32 s32, 0x400 ; GFX9-NEXT: v_writelane_b32 v40, s26, 22 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, s52 ; GFX9-NEXT: v_writelane_b32 v40, s27, 23 -; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24 -; GFX9-NEXT: v_mov_b32_e32 v0, s46 ; GFX9-NEXT: v_writelane_b32 v40, s28, 24 -; GFX9-NEXT: v_mov_b32_e32 v1, s47 -; GFX9-NEXT: v_mov_b32_e32 v2, s48 -; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 -; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 -; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 -; GFX9-NEXT: v_mov_b32_e32 v0, s49 ; GFX9-NEXT: v_writelane_b32 v40, s29, 25 -; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:12 -; GFX9-NEXT: v_mov_b32_e32 v0, s50 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dword s52, s[34:35], 0x0 ; GFX9-NEXT: v_writelane_b32 v40, s30, 26 -; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:16 -; GFX9-NEXT: v_mov_b32_e32 v0, s51 -; GFX9-NEXT: s_mov_b32 s35, external_void_func_v32i32_i32_inreg@abs32@hi -; GFX9-NEXT: s_mov_b32 s34, external_void_func_v32i32_i32_inreg@abs32@lo -; GFX9-NEXT: s_mov_b32 s20, s36 -; GFX9-NEXT: s_mov_b32 s21, s37 -; GFX9-NEXT: s_mov_b32 s22, s38 -; GFX9-NEXT: s_mov_b32 s23, s39 -; GFX9-NEXT: s_mov_b32 s24, s40 -; GFX9-NEXT: s_mov_b32 s25, s41 -; GFX9-NEXT: s_mov_b32 s26, s42 -; GFX9-NEXT: s_mov_b32 s27, s43 -; GFX9-NEXT: s_mov_b32 s28, s44 -; GFX9-NEXT: s_mov_b32 s29, s45 ; GFX9-NEXT: v_writelane_b32 v40, s31, 27 -; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:20 -; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] +; GFX9-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40 +; GFX9-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x0 +; GFX9-NEXT: s_addk_i32 s32, 0x400 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v0, s52 +; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v0, s30 +; GFX9-NEXT: s_mov_b32 s53, external_void_func_v32i32_i32_inreg@abs32@hi +; GFX9-NEXT: v_mov_b32_e32 v1, s31 +; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 +; GFX9-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 +; GFX9-NEXT: s_mov_b32 s52, external_void_func_v32i32_i32_inreg@abs32@lo +; GFX9-NEXT: s_mov_b32 s0, s36 +; GFX9-NEXT: s_mov_b32 s1, s37 +; GFX9-NEXT: s_mov_b32 s2, s38 +; GFX9-NEXT: s_mov_b32 s3, s39 +; GFX9-NEXT: s_mov_b32 s4, s40 +; GFX9-NEXT: s_mov_b32 s5, s41 +; GFX9-NEXT: s_mov_b32 s6, s42 +; GFX9-NEXT: s_mov_b32 s7, s43 +; GFX9-NEXT: s_mov_b32 s8, s44 +; GFX9-NEXT: s_mov_b32 s9, s45 +; GFX9-NEXT: s_mov_b32 s10, s46 +; GFX9-NEXT: s_mov_b32 s11, s47 +; GFX9-NEXT: s_mov_b32 s12, s48 +; GFX9-NEXT: s_mov_b32 s13, s49 +; GFX9-NEXT: s_mov_b32 s14, s50 +; GFX9-NEXT: s_mov_b32 s15, s51 +; GFX9-NEXT: ; kill: killed $sgpr34_sgpr35 +; GFX9-NEXT: ; kill: killed $sgpr34_sgpr35 +; GFX9-NEXT: s_swappc_b64 s[30:31], s[52:53] ; GFX9-NEXT: v_readlane_b32 s31, v40, 27 ; GFX9-NEXT: v_readlane_b32 s30, v40, 26 ; GFX9-NEXT: v_readlane_b32 s29, v40, 25 @@ -15310,52 +14661,51 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 { ; GFX10-NEXT: v_writelane_b32 v40, s17, 13 ; GFX10-NEXT: v_writelane_b32 v40, s18, 14 ; GFX10-NEXT: v_writelane_b32 v40, s19, 15 -; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_clause 0x2 -; GFX10-NEXT: s_load_dword s52, s[34:35], 0x0 -; GFX10-NEXT: ; meta instruction -; GFX10-NEXT: ; meta instruction -; GFX10-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x40 -; GFX10-NEXT: s_load_dwordx16 s[4:19], s[34:35], 0x0 -; GFX10-NEXT: s_mov_b32 s35, external_void_func_v32i32_i32_inreg@abs32@hi -; GFX10-NEXT: s_mov_b32 s34, external_void_func_v32i32_i32_inreg@abs32@lo ; GFX10-NEXT: v_writelane_b32 v40, s20, 16 ; GFX10-NEXT: v_writelane_b32 v40, s21, 17 ; GFX10-NEXT: v_writelane_b32 v40, s22, 18 -; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v0, s52 -; GFX10-NEXT: v_mov_b32_e32 v1, s47 ; GFX10-NEXT: v_writelane_b32 v40, s23, 19 -; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24 -; GFX10-NEXT: v_mov_b32_e32 v0, s46 -; GFX10-NEXT: v_mov_b32_e32 v2, s48 -; GFX10-NEXT: v_mov_b32_e32 v3, s49 ; GFX10-NEXT: v_writelane_b32 v40, s24, 20 -; GFX10-NEXT: s_mov_b32 s20, s36 -; GFX10-NEXT: s_mov_b32 s21, s37 -; GFX10-NEXT: s_mov_b32 s22, s38 -; GFX10-NEXT: s_mov_b32 s23, s39 ; GFX10-NEXT: v_writelane_b32 v40, s25, 21 -; GFX10-NEXT: s_mov_b32 s24, s40 -; GFX10-NEXT: s_mov_b32 s25, s41 -; GFX10-NEXT: v_mov_b32_e32 v4, s50 -; GFX10-NEXT: v_mov_b32_e32 v5, s51 ; GFX10-NEXT: v_writelane_b32 v40, s26, 22 -; GFX10-NEXT: s_mov_b32 s26, s42 -; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], s32 -; GFX10-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:4 -; GFX10-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:8 -; GFX10-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:12 -; GFX10-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16 -; GFX10-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:20 ; GFX10-NEXT: v_writelane_b32 v40, s27, 23 -; GFX10-NEXT: s_mov_b32 s27, s43 ; GFX10-NEXT: v_writelane_b32 v40, s28, 24 -; GFX10-NEXT: s_mov_b32 s28, s44 ; GFX10-NEXT: v_writelane_b32 v40, s29, 25 -; GFX10-NEXT: s_mov_b32 s29, s45 ; GFX10-NEXT: v_writelane_b32 v40, s30, 26 ; GFX10-NEXT: v_writelane_b32 v40, s31, 27 +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: s_clause 0x2 +; GFX10-NEXT: s_load_dword s52, s[34:35], 0x0 +; GFX10-NEXT: ; meta instruction +; GFX10-NEXT: ; meta instruction +; GFX10-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40 +; GFX10-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x0 +; GFX10-NEXT: s_mov_b32 s35, external_void_func_v32i32_i32_inreg@abs32@hi +; GFX10-NEXT: s_mov_b32 s34, external_void_func_v32i32_i32_inreg@abs32@lo +; GFX10-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-NEXT: v_mov_b32_e32 v0, s52 +; GFX10-NEXT: v_mov_b32_e32 v1, s30 +; GFX10-NEXT: v_mov_b32_e32 v2, s31 +; GFX10-NEXT: s_mov_b32 s4, s40 +; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:8 +; GFX10-NEXT: buffer_store_dword v1, off, s[0:3], s32 +; GFX10-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:4 +; GFX10-NEXT: s_mov_b32 s5, s41 +; GFX10-NEXT: s_mov_b32 s6, s42 +; GFX10-NEXT: s_mov_b32 s7, s43 +; GFX10-NEXT: s_mov_b32 s8, s44 +; GFX10-NEXT: s_mov_b32 s9, s45 +; GFX10-NEXT: s_mov_b32 s10, s46 +; GFX10-NEXT: s_mov_b32 s11, s47 +; GFX10-NEXT: s_mov_b32 s12, s48 +; GFX10-NEXT: s_mov_b32 s13, s49 +; GFX10-NEXT: s_mov_b32 s14, s50 +; GFX10-NEXT: s_mov_b32 s15, s51 +; GFX10-NEXT: s_waitcnt_depctr 0xffe3 +; GFX10-NEXT: s_mov_b32 s0, s36 +; GFX10-NEXT: s_mov_b32 s1, s37 +; GFX10-NEXT: s_mov_b32 s2, s38 +; GFX10-NEXT: s_mov_b32 s3, s39 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX10-NEXT: v_readlane_b32 s31, v40, 27 ; GFX10-NEXT: v_readlane_b32 s30, v40, 26 @@ -15406,8 +14756,8 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 { ; GFX11-NEXT: v_writelane_b32 v40, s0, 28 ; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 ; GFX11-NEXT: s_add_i32 s32, s32, 16 -; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) -; GFX11-NEXT: s_add_i32 s3, s32, 16 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v32i32_i32_inreg@abs32@hi +; GFX11-NEXT: s_add_i32 s36, s32, 8 ; GFX11-NEXT: v_writelane_b32 v40, s4, 0 ; GFX11-NEXT: v_writelane_b32 v40, s5, 1 ; GFX11-NEXT: v_writelane_b32 v40, s6, 2 @@ -15424,46 +14774,30 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 { ; GFX11-NEXT: v_writelane_b32 v40, s17, 13 ; GFX11-NEXT: v_writelane_b32 v40, s18, 14 ; GFX11-NEXT: v_writelane_b32 v40, s19, 15 -; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_clause 0x2 -; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x0 -; GFX11-NEXT: s_load_b512 s[36:51], s[0:1], 0x40 -; GFX11-NEXT: s_load_b512 s[4:19], s[0:1], 0x0 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v32i32_i32_inreg@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v32i32_i32_inreg@abs32@lo ; GFX11-NEXT: v_writelane_b32 v40, s20, 16 ; GFX11-NEXT: v_writelane_b32 v40, s21, 17 ; GFX11-NEXT: v_writelane_b32 v40, s22, 18 -; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: v_dual_mov_b32 v6, s2 :: v_dual_mov_b32 v5, s51 ; GFX11-NEXT: v_writelane_b32 v40, s23, 19 -; GFX11-NEXT: v_dual_mov_b32 v4, s50 :: v_dual_mov_b32 v1, s47 -; GFX11-NEXT: v_dual_mov_b32 v0, s46 :: v_dual_mov_b32 v3, s49 ; GFX11-NEXT: v_writelane_b32 v40, s24, 20 -; GFX11-NEXT: v_mov_b32_e32 v2, s48 -; GFX11-NEXT: s_add_i32 s2, s32, 24 -; GFX11-NEXT: s_mov_b32 s20, s36 -; GFX11-NEXT: s_mov_b32 s21, s37 ; GFX11-NEXT: v_writelane_b32 v40, s25, 21 -; GFX11-NEXT: s_mov_b32 s22, s38 -; GFX11-NEXT: s_mov_b32 s23, s39 -; GFX11-NEXT: s_mov_b32 s24, s40 -; GFX11-NEXT: s_mov_b32 s25, s41 ; GFX11-NEXT: v_writelane_b32 v40, s26, 22 -; GFX11-NEXT: s_mov_b32 s26, s42 -; GFX11-NEXT: scratch_store_b32 off, v6, s2 -; GFX11-NEXT: scratch_store_b64 off, v[4:5], s3 -; GFX11-NEXT: scratch_store_b128 off, v[0:3], s32 ; GFX11-NEXT: v_writelane_b32 v40, s27, 23 -; GFX11-NEXT: s_mov_b32 s27, s43 ; GFX11-NEXT: v_writelane_b32 v40, s28, 24 -; GFX11-NEXT: s_mov_b32 s28, s44 ; GFX11-NEXT: v_writelane_b32 v40, s29, 25 -; GFX11-NEXT: s_mov_b32 s29, s45 ; GFX11-NEXT: v_writelane_b32 v40, s30, 26 ; GFX11-NEXT: v_writelane_b32 v40, s31, 27 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_clause 0x2 +; GFX11-NEXT: s_load_b32 s34, s[0:1], 0x0 +; GFX11-NEXT: s_load_b512 s[16:31], s[0:1], 0x40 +; GFX11-NEXT: s_load_b512 s[0:15], s[0:1], 0x0 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: v_dual_mov_b32 v2, s34 :: v_dual_mov_b32 v1, s31 +; GFX11-NEXT: v_mov_b32_e32 v0, s30 +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v32i32_i32_inreg@abs32@lo +; GFX11-NEXT: scratch_store_b32 off, v2, s36 +; GFX11-NEXT: scratch_store_b64 off, v[0:1], s32 +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: v_readlane_b32 s31, v40, 27 ; GFX11-NEXT: v_readlane_b32 s30, v40, 26 ; GFX11-NEXT: v_readlane_b32 s29, v40, 25 @@ -15511,13 +14845,17 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 { ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 28 -; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_clause 0x1 +; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[34:35], s[0:1], 0x0 +; GFX10-SCRATCH-NEXT: s_load_dword s36, s[0:1], 0x0 ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 -; GFX10-SCRATCH-NEXT: s_add_i32 s3, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s4, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s5, 1 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s6, 2 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s7, 3 +; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v2, s36 +; GFX10-SCRATCH-NEXT: s_add_i32 s36, s32, 8 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s8, 4 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s9, 5 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s10, 6 @@ -15530,50 +14868,29 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 { ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s17, 13 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s18, 14 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s19, 15 -; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_clause 0x2 -; GFX10-SCRATCH-NEXT: s_load_dword s2, s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: ; meta instruction -; GFX10-SCRATCH-NEXT: ; meta instruction -; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[36:51], s[0:1], 0x40 -; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[4:19], s[0:1], 0x0 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v32i32_i32_inreg@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v32i32_i32_inreg@abs32@lo ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s20, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s21, 17 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s22, 18 -; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v6, s2 -; GFX10-SCRATCH-NEXT: s_add_i32 s2, s32, 24 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v4, s50 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s23, 19 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v5, s51 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v0, s46 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v1, s47 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v2, s48 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s24, 20 -; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v3, s49 -; GFX10-SCRATCH-NEXT: s_mov_b32 s20, s36 -; GFX10-SCRATCH-NEXT: s_mov_b32 s21, s37 -; GFX10-SCRATCH-NEXT: s_mov_b32 s22, s38 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s25, 21 -; GFX10-SCRATCH-NEXT: s_mov_b32 s23, s39 -; GFX10-SCRATCH-NEXT: s_mov_b32 s24, s40 -; GFX10-SCRATCH-NEXT: s_mov_b32 s25, s41 -; GFX10-SCRATCH-NEXT: scratch_store_dword off, v6, s2 -; GFX10-SCRATCH-NEXT: scratch_store_dwordx2 off, v[4:5], s3 -; GFX10-SCRATCH-NEXT: scratch_store_dwordx4 off, v[0:3], s32 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s26, 22 -; GFX10-SCRATCH-NEXT: s_mov_b32 s26, s42 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s27, 23 -; GFX10-SCRATCH-NEXT: s_mov_b32 s27, s43 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s28, 24 -; GFX10-SCRATCH-NEXT: s_mov_b32 s28, s44 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s29, 25 -; GFX10-SCRATCH-NEXT: s_mov_b32 s29, s45 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 26 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 27 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX10-SCRATCH-NEXT: s_clause 0x1 +; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40 +; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v32i32_i32_inreg@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v32i32_i32_inreg@abs32@lo +; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v0, s30 +; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v1, s31 +; GFX10-SCRATCH-NEXT: scratch_store_dword off, v2, s36 +; GFX10-SCRATCH-NEXT: scratch_store_dwordx2 off, v[0:1], s32 +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 27 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 26 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s29, v40, 25 @@ -17397,6 +16714,7 @@ define amdgpu_gfx void @test_call_external_void_func_bf16_inreg(i16 inreg %arg) ; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_bf16@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_bf16@abs32@lo +; GFX9-NEXT: s_mov_b32 s0, s4 ; GFX9-NEXT: s_addk_i32 s32, 0x400 ; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] @@ -17423,6 +16741,7 @@ define amdgpu_gfx void @test_call_external_void_func_bf16_inreg(i16 inreg %arg) ; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_bf16@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_bf16@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, s4 ; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: v_writelane_b32 v40, s31, 1 @@ -17442,18 +16761,18 @@ define amdgpu_gfx void @test_call_external_void_func_bf16_inreg(i16 inreg %arg) ; GFX11-LABEL: test_call_external_void_func_bf16_inreg: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: s_mov_b32 s0, s33 +; GFX11-NEXT: s_mov_b32 s1, s33 ; GFX11-NEXT: s_mov_b32 s33, s32 -; GFX11-NEXT: s_or_saveexec_b32 s1, -1 +; GFX11-NEXT: s_or_saveexec_b32 s2, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill -; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 2 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_bf16@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_bf16@abs32@lo +; GFX11-NEXT: s_mov_b32 exec_lo, s2 +; GFX11-NEXT: v_writelane_b32 v40, s1, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_bf16@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_bf16@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: v_writelane_b32 v40, s31, 1 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_readlane_b32 s31, v40, 1 ; GFX11-NEXT: v_readlane_b32 s30, v40, 0 @@ -17469,19 +16788,19 @@ define amdgpu_gfx void @test_call_external_void_func_bf16_inreg(i16 inreg %arg) ; GFX10-SCRATCH-LABEL: test_call_external_void_func_bf16_inreg: ; GFX10-SCRATCH: ; %bb.0: ; GFX10-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, s33 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, s33 ; GFX10-SCRATCH-NEXT: s_mov_b32 s33, s32 -; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 +; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s2, -1 ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_bf16@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_bf16@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s2 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s1, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_bf16@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_bf16@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 @@ -17511,6 +16830,7 @@ define amdgpu_gfx void @test_call_external_void_func_v1bf16_inreg(i16 inreg %arg ; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v1bf16@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v1bf16@abs32@lo +; GFX9-NEXT: s_mov_b32 s0, s4 ; GFX9-NEXT: s_addk_i32 s32, 0x400 ; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] @@ -17537,6 +16857,7 @@ define amdgpu_gfx void @test_call_external_void_func_v1bf16_inreg(i16 inreg %arg ; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v1bf16@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v1bf16@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, s4 ; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: v_writelane_b32 v40, s31, 1 @@ -17556,18 +16877,18 @@ define amdgpu_gfx void @test_call_external_void_func_v1bf16_inreg(i16 inreg %arg ; GFX11-LABEL: test_call_external_void_func_v1bf16_inreg: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: s_mov_b32 s0, s33 +; GFX11-NEXT: s_mov_b32 s1, s33 ; GFX11-NEXT: s_mov_b32 s33, s32 -; GFX11-NEXT: s_or_saveexec_b32 s1, -1 +; GFX11-NEXT: s_or_saveexec_b32 s2, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill -; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 2 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v1bf16@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v1bf16@abs32@lo +; GFX11-NEXT: s_mov_b32 exec_lo, s2 +; GFX11-NEXT: v_writelane_b32 v40, s1, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v1bf16@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v1bf16@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: v_writelane_b32 v40, s31, 1 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_readlane_b32 s31, v40, 1 ; GFX11-NEXT: v_readlane_b32 s30, v40, 0 @@ -17583,19 +16904,19 @@ define amdgpu_gfx void @test_call_external_void_func_v1bf16_inreg(i16 inreg %arg ; GFX10-SCRATCH-LABEL: test_call_external_void_func_v1bf16_inreg: ; GFX10-SCRATCH: ; %bb.0: ; GFX10-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, s33 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, s33 ; GFX10-SCRATCH-NEXT: s_mov_b32 s33, s32 -; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 +; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s2, -1 ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v1bf16@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v1bf16@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s2 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s1, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v1bf16@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v1bf16@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 @@ -17625,6 +16946,7 @@ define amdgpu_gfx void @test_call_external_void_func_v2bf16_inreg(i32 inreg %arg ; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v2bf16@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v2bf16@abs32@lo +; GFX9-NEXT: s_mov_b32 s0, s4 ; GFX9-NEXT: s_addk_i32 s32, 0x400 ; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] @@ -17651,6 +16973,7 @@ define amdgpu_gfx void @test_call_external_void_func_v2bf16_inreg(i32 inreg %arg ; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v2bf16@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v2bf16@abs32@lo +; GFX10-NEXT: s_mov_b32 s0, s4 ; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-NEXT: v_writelane_b32 v40, s31, 1 @@ -17670,18 +16993,18 @@ define amdgpu_gfx void @test_call_external_void_func_v2bf16_inreg(i32 inreg %arg ; GFX11-LABEL: test_call_external_void_func_v2bf16_inreg: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: s_mov_b32 s0, s33 +; GFX11-NEXT: s_mov_b32 s1, s33 ; GFX11-NEXT: s_mov_b32 s33, s32 -; GFX11-NEXT: s_or_saveexec_b32 s1, -1 +; GFX11-NEXT: s_or_saveexec_b32 s2, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill -; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 2 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v2bf16@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v2bf16@abs32@lo +; GFX11-NEXT: s_mov_b32 exec_lo, s2 +; GFX11-NEXT: v_writelane_b32 v40, s1, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v2bf16@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v2bf16@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: v_writelane_b32 v40, s31, 1 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_readlane_b32 s31, v40, 1 ; GFX11-NEXT: v_readlane_b32 s30, v40, 0 @@ -17697,19 +17020,19 @@ define amdgpu_gfx void @test_call_external_void_func_v2bf16_inreg(i32 inreg %arg ; GFX10-SCRATCH-LABEL: test_call_external_void_func_v2bf16_inreg: ; GFX10-SCRATCH: ; %bb.0: ; GFX10-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, s33 +; GFX10-SCRATCH-NEXT: s_mov_b32 s1, s33 ; GFX10-SCRATCH-NEXT: s_mov_b32 s33, s32 -; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 +; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s2, -1 ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v2bf16@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v2bf16@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s2 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s1, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v2bf16@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v2bf16@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 @@ -17739,6 +17062,8 @@ define amdgpu_gfx void @test_call_external_void_func_v3bf16_inreg(<3 x i16> inre ; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v3bf16@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v3bf16@abs32@lo +; GFX9-NEXT: s_mov_b32 s1, s5 +; GFX9-NEXT: s_mov_b32 s0, s4 ; GFX9-NEXT: s_addk_i32 s32, 0x400 ; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] @@ -17765,8 +17090,10 @@ define amdgpu_gfx void @test_call_external_void_func_v3bf16_inreg(<3 x i16> inre ; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v3bf16@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v3bf16@abs32@lo -; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: s_mov_b32 s1, s5 +; GFX10-NEXT: s_mov_b32 s0, s4 ; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX10-NEXT: v_readlane_b32 s31, v40, 1 @@ -17784,18 +17111,18 @@ define amdgpu_gfx void @test_call_external_void_func_v3bf16_inreg(<3 x i16> inre ; GFX11-LABEL: test_call_external_void_func_v3bf16_inreg: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: s_mov_b32 s0, s33 +; GFX11-NEXT: s_mov_b32 s2, s33 ; GFX11-NEXT: s_mov_b32 s33, s32 -; GFX11-NEXT: s_or_saveexec_b32 s1, -1 +; GFX11-NEXT: s_or_saveexec_b32 s3, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill -; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 2 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3bf16@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3bf16@abs32@lo +; GFX11-NEXT: s_mov_b32 exec_lo, s3 +; GFX11-NEXT: v_writelane_b32 v40, s2, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v3bf16@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v3bf16@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: v_writelane_b32 v40, s31, 1 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_readlane_b32 s31, v40, 1 ; GFX11-NEXT: v_readlane_b32 s30, v40, 0 @@ -17811,19 +17138,19 @@ define amdgpu_gfx void @test_call_external_void_func_v3bf16_inreg(<3 x i16> inre ; GFX10-SCRATCH-LABEL: test_call_external_void_func_v3bf16_inreg: ; GFX10-SCRATCH: ; %bb.0: ; GFX10-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, s33 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, s33 ; GFX10-SCRATCH-NEXT: s_mov_b32 s33, s32 -; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 +; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s3, -1 ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v3bf16@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v3bf16@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s2, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v3bf16@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v3bf16@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 @@ -17853,6 +17180,8 @@ define amdgpu_gfx void @test_call_external_void_func_v4bf16_inreg(<4 x i16> inre ; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v4bf16@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v4bf16@abs32@lo +; GFX9-NEXT: s_mov_b32 s1, s5 +; GFX9-NEXT: s_mov_b32 s0, s4 ; GFX9-NEXT: s_addk_i32 s32, 0x400 ; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] @@ -17879,8 +17208,10 @@ define amdgpu_gfx void @test_call_external_void_func_v4bf16_inreg(<4 x i16> inre ; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v4bf16@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v4bf16@abs32@lo -; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: s_mov_b32 s1, s5 +; GFX10-NEXT: s_mov_b32 s0, s4 ; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX10-NEXT: v_readlane_b32 s31, v40, 1 @@ -17898,18 +17229,18 @@ define amdgpu_gfx void @test_call_external_void_func_v4bf16_inreg(<4 x i16> inre ; GFX11-LABEL: test_call_external_void_func_v4bf16_inreg: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: s_mov_b32 s0, s33 +; GFX11-NEXT: s_mov_b32 s2, s33 ; GFX11-NEXT: s_mov_b32 s33, s32 -; GFX11-NEXT: s_or_saveexec_b32 s1, -1 +; GFX11-NEXT: s_or_saveexec_b32 s3, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill -; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 2 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v4bf16@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v4bf16@abs32@lo +; GFX11-NEXT: s_mov_b32 exec_lo, s3 +; GFX11-NEXT: v_writelane_b32 v40, s2, 2 +; GFX11-NEXT: s_mov_b32 s3, external_void_func_v4bf16@abs32@hi +; GFX11-NEXT: s_mov_b32 s2, external_void_func_v4bf16@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: v_writelane_b32 v40, s31, 1 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_readlane_b32 s31, v40, 1 ; GFX11-NEXT: v_readlane_b32 s30, v40, 0 @@ -17925,19 +17256,19 @@ define amdgpu_gfx void @test_call_external_void_func_v4bf16_inreg(<4 x i16> inre ; GFX10-SCRATCH-LABEL: test_call_external_void_func_v4bf16_inreg: ; GFX10-SCRATCH: ; %bb.0: ; GFX10-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, s33 +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, s33 ; GFX10-SCRATCH-NEXT: s_mov_b32 s33, s32 -; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 +; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s3, -1 ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v4bf16@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v4bf16@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s3 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s2, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s3, external_void_func_v4bf16@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s2, external_void_func_v4bf16@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[2:3] ; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 @@ -17967,6 +17298,10 @@ define amdgpu_gfx void @test_call_external_void_func_v8bf16_inreg(<8 x i16> inre ; GFX9-NEXT: v_writelane_b32 v40, s30, 0 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v8bf16@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v8bf16@abs32@lo +; GFX9-NEXT: s_mov_b32 s3, s7 +; GFX9-NEXT: s_mov_b32 s2, s6 +; GFX9-NEXT: s_mov_b32 s1, s5 +; GFX9-NEXT: s_mov_b32 s0, s4 ; GFX9-NEXT: s_addk_i32 s32, 0x400 ; GFX9-NEXT: v_writelane_b32 v40, s31, 1 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] @@ -17993,8 +17328,12 @@ define amdgpu_gfx void @test_call_external_void_func_v8bf16_inreg(<8 x i16> inre ; GFX10-NEXT: v_writelane_b32 v40, s34, 2 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v8bf16@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v8bf16@abs32@lo -; GFX10-NEXT: s_addk_i32 s32, 0x200 +; GFX10-NEXT: s_mov_b32 s3, s7 +; GFX10-NEXT: s_mov_b32 s2, s6 ; GFX10-NEXT: v_writelane_b32 v40, s30, 0 +; GFX10-NEXT: s_mov_b32 s1, s5 +; GFX10-NEXT: s_mov_b32 s0, s4 +; GFX10-NEXT: s_addk_i32 s32, 0x200 ; GFX10-NEXT: v_writelane_b32 v40, s31, 1 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX10-NEXT: v_readlane_b32 s31, v40, 1 @@ -18012,18 +17351,18 @@ define amdgpu_gfx void @test_call_external_void_func_v8bf16_inreg(<8 x i16> inre ; GFX11-LABEL: test_call_external_void_func_v8bf16_inreg: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: s_mov_b32 s0, s33 +; GFX11-NEXT: s_mov_b32 s34, s33 ; GFX11-NEXT: s_mov_b32 s33, s32 -; GFX11-NEXT: s_or_saveexec_b32 s1, -1 +; GFX11-NEXT: s_or_saveexec_b32 s35, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill -; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 2 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v8bf16@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v8bf16@abs32@lo +; GFX11-NEXT: s_mov_b32 exec_lo, s35 +; GFX11-NEXT: v_writelane_b32 v40, s34, 2 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v8bf16@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v8bf16@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: v_writelane_b32 v40, s31, 1 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_readlane_b32 s31, v40, 1 ; GFX11-NEXT: v_readlane_b32 s30, v40, 0 @@ -18039,19 +17378,19 @@ define amdgpu_gfx void @test_call_external_void_func_v8bf16_inreg(<8 x i16> inre ; GFX10-SCRATCH-LABEL: test_call_external_void_func_v8bf16_inreg: ; GFX10-SCRATCH: ; %bb.0: ; GFX10-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, s33 +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, s33 ; GFX10-SCRATCH-NEXT: s_mov_b32 s33, s32 -; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 +; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v8bf16@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v8bf16@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s35 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s34, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v8bf16@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v8bf16@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 @@ -18077,16 +17416,32 @@ define amdgpu_gfx void @test_call_external_void_func_v16bf16_inreg(<16 x i16> in ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX9-NEXT: s_mov_b64 exec, s[36:37] -; GFX9-NEXT: v_writelane_b32 v40, s34, 2 -; GFX9-NEXT: v_writelane_b32 v40, s30, 0 +; GFX9-NEXT: v_writelane_b32 v40, s34, 6 +; GFX9-NEXT: v_writelane_b32 v40, s4, 0 +; GFX9-NEXT: v_writelane_b32 v40, s5, 1 +; GFX9-NEXT: v_writelane_b32 v40, s6, 2 +; GFX9-NEXT: v_writelane_b32 v40, s7, 3 +; GFX9-NEXT: v_writelane_b32 v40, s30, 4 ; GFX9-NEXT: s_mov_b32 s35, external_void_func_v16bf16@abs32@hi ; GFX9-NEXT: s_mov_b32 s34, external_void_func_v16bf16@abs32@lo +; GFX9-NEXT: s_mov_b32 s3, s7 +; GFX9-NEXT: s_mov_b32 s2, s6 +; GFX9-NEXT: s_mov_b32 s1, s5 +; GFX9-NEXT: s_mov_b32 s0, s4 +; GFX9-NEXT: s_mov_b32 s4, s8 +; GFX9-NEXT: s_mov_b32 s5, s9 +; GFX9-NEXT: s_mov_b32 s6, s10 +; GFX9-NEXT: s_mov_b32 s7, s11 ; GFX9-NEXT: s_addk_i32 s32, 0x400 -; GFX9-NEXT: v_writelane_b32 v40, s31, 1 +; GFX9-NEXT: v_writelane_b32 v40, s31, 5 ; GFX9-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX9-NEXT: v_readlane_b32 s31, v40, 1 -; GFX9-NEXT: v_readlane_b32 s30, v40, 0 -; GFX9-NEXT: v_readlane_b32 s34, v40, 2 +; GFX9-NEXT: v_readlane_b32 s31, v40, 5 +; GFX9-NEXT: v_readlane_b32 s30, v40, 4 +; GFX9-NEXT: v_readlane_b32 s7, v40, 3 +; GFX9-NEXT: v_readlane_b32 s6, v40, 2 +; GFX9-NEXT: v_readlane_b32 s5, v40, 1 +; GFX9-NEXT: v_readlane_b32 s4, v40, 0 +; GFX9-NEXT: v_readlane_b32 s34, v40, 6 ; GFX9-NEXT: s_or_saveexec_b64 s[36:37], -1 ; GFX9-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX9-NEXT: s_mov_b64 exec, s[36:37] @@ -18104,16 +17459,32 @@ define amdgpu_gfx void @test_call_external_void_func_v16bf16_inreg(<16 x i16> in ; GFX10-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 ; GFX10-NEXT: s_mov_b32 exec_lo, s35 -; GFX10-NEXT: v_writelane_b32 v40, s34, 2 +; GFX10-NEXT: v_writelane_b32 v40, s34, 6 ; GFX10-NEXT: s_mov_b32 s35, external_void_func_v16bf16@abs32@hi ; GFX10-NEXT: s_mov_b32 s34, external_void_func_v16bf16@abs32@lo +; GFX10-NEXT: s_mov_b32 s3, s7 +; GFX10-NEXT: s_mov_b32 s2, s6 +; GFX10-NEXT: v_writelane_b32 v40, s4, 0 +; GFX10-NEXT: s_mov_b32 s1, s5 +; GFX10-NEXT: s_mov_b32 s0, s4 +; GFX10-NEXT: s_mov_b32 s4, s8 ; GFX10-NEXT: s_addk_i32 s32, 0x200 -; GFX10-NEXT: v_writelane_b32 v40, s30, 0 -; GFX10-NEXT: v_writelane_b32 v40, s31, 1 +; GFX10-NEXT: v_writelane_b32 v40, s5, 1 +; GFX10-NEXT: s_mov_b32 s5, s9 +; GFX10-NEXT: v_writelane_b32 v40, s6, 2 +; GFX10-NEXT: s_mov_b32 s6, s10 +; GFX10-NEXT: v_writelane_b32 v40, s7, 3 +; GFX10-NEXT: s_mov_b32 s7, s11 +; GFX10-NEXT: v_writelane_b32 v40, s30, 4 +; GFX10-NEXT: v_writelane_b32 v40, s31, 5 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[34:35] -; GFX10-NEXT: v_readlane_b32 s31, v40, 1 -; GFX10-NEXT: v_readlane_b32 s30, v40, 0 -; GFX10-NEXT: v_readlane_b32 s34, v40, 2 +; GFX10-NEXT: v_readlane_b32 s31, v40, 5 +; GFX10-NEXT: v_readlane_b32 s30, v40, 4 +; GFX10-NEXT: v_readlane_b32 s7, v40, 3 +; GFX10-NEXT: v_readlane_b32 s6, v40, 2 +; GFX10-NEXT: v_readlane_b32 s5, v40, 1 +; GFX10-NEXT: v_readlane_b32 s4, v40, 0 +; GFX10-NEXT: v_readlane_b32 s34, v40, 6 ; GFX10-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload ; GFX10-NEXT: s_waitcnt_depctr 0xffe3 @@ -18126,18 +17497,18 @@ define amdgpu_gfx void @test_call_external_void_func_v16bf16_inreg(<16 x i16> in ; GFX11-LABEL: test_call_external_void_func_v16bf16_inreg: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: s_mov_b32 s0, s33 +; GFX11-NEXT: s_mov_b32 s34, s33 ; GFX11-NEXT: s_mov_b32 s33, s32 -; GFX11-NEXT: s_or_saveexec_b32 s1, -1 +; GFX11-NEXT: s_or_saveexec_b32 s35, -1 ; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill -; GFX11-NEXT: s_mov_b32 exec_lo, s1 -; GFX11-NEXT: v_writelane_b32 v40, s0, 2 -; GFX11-NEXT: s_mov_b32 s1, external_void_func_v16bf16@abs32@hi -; GFX11-NEXT: s_mov_b32 s0, external_void_func_v16bf16@abs32@lo +; GFX11-NEXT: s_mov_b32 exec_lo, s35 +; GFX11-NEXT: v_writelane_b32 v40, s34, 2 +; GFX11-NEXT: s_mov_b32 s35, external_void_func_v16bf16@abs32@hi +; GFX11-NEXT: s_mov_b32 s34, external_void_func_v16bf16@abs32@lo ; GFX11-NEXT: s_add_i32 s32, s32, 16 ; GFX11-NEXT: v_writelane_b32 v40, s30, 0 ; GFX11-NEXT: v_writelane_b32 v40, s31, 1 -; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX11-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_readlane_b32 s31, v40, 1 ; GFX11-NEXT: v_readlane_b32 s30, v40, 0 @@ -18153,19 +17524,19 @@ define amdgpu_gfx void @test_call_external_void_func_v16bf16_inreg(<16 x i16> in ; GFX10-SCRATCH-LABEL: test_call_external_void_func_v16bf16_inreg: ; GFX10-SCRATCH: ; %bb.0: ; GFX10-SCRATCH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, s33 +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, s33 ; GFX10-SCRATCH-NEXT: s_mov_b32 s33, s32 -; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s1, -1 +; GFX10-SCRATCH-NEXT: s_or_saveexec_b32 s35, -1 ; GFX10-SCRATCH-NEXT: scratch_store_dword off, v40, s33 ; 4-byte Folded Spill ; GFX10-SCRATCH-NEXT: s_waitcnt_depctr 0xffe3 -; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s1 -; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s0, 2 -; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v16bf16@abs32@hi -; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v16bf16@abs32@lo +; GFX10-SCRATCH-NEXT: s_mov_b32 exec_lo, s35 +; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s34, 2 +; GFX10-SCRATCH-NEXT: s_mov_b32 s35, external_void_func_v16bf16@abs32@hi +; GFX10-SCRATCH-NEXT: s_mov_b32 s34, external_void_func_v16bf16@abs32@lo ; GFX10-SCRATCH-NEXT: s_add_i32 s32, s32, 16 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 0 ; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 1 -; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[34:35] ; GFX10-SCRATCH-NEXT: v_readlane_b32 s31, v40, 1 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 0 ; GFX10-SCRATCH-NEXT: v_readlane_b32 s0, v40, 2 diff --git a/llvm/test/CodeGen/AMDGPU/indirect-call.ll b/llvm/test/CodeGen/AMDGPU/indirect-call.ll index 7799b9509ceb..25c684004446 100644 --- a/llvm/test/CodeGen/AMDGPU/indirect-call.ll +++ b/llvm/test/CodeGen/AMDGPU/indirect-call.ll @@ -847,11 +847,11 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) { ; GCN-LABEL: test_indirect_call_vgpr_ptr_inreg_arg: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_mov_b32 s5, s33 +; GCN-NEXT: s_mov_b32 s10, s33 ; GCN-NEXT: s_mov_b32 s33, s32 -; GCN-NEXT: s_or_saveexec_b64 s[6:7], -1 +; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1 ; GCN-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill -; GCN-NEXT: s_mov_b64 exec, s[6:7] +; GCN-NEXT: s_mov_b64 exec, s[4:5] ; GCN-NEXT: s_addk_i32 s32, 0x400 ; GCN-NEXT: v_writelane_b32 v40, s30, 0 ; GCN-NEXT: v_writelane_b32 v40, s31, 1 @@ -885,19 +885,19 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) { ; GCN-NEXT: v_writelane_b32 v40, s61, 29 ; GCN-NEXT: v_writelane_b32 v40, s62, 30 ; GCN-NEXT: v_writelane_b32 v40, s63, 31 -; GCN-NEXT: s_mov_b64 s[6:7], exec -; GCN-NEXT: s_movk_i32 s4, 0x7b +; GCN-NEXT: s_mov_b64 s[4:5], exec ; GCN-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1 -; GCN-NEXT: v_readfirstlane_b32 s8, v0 -; GCN-NEXT: v_readfirstlane_b32 s9, v1 -; GCN-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1] -; GCN-NEXT: s_and_saveexec_b64 s[10:11], vcc -; GCN-NEXT: s_swappc_b64 s[30:31], s[8:9] +; GCN-NEXT: v_readfirstlane_b32 s6, v0 +; GCN-NEXT: v_readfirstlane_b32 s7, v1 +; GCN-NEXT: v_cmp_eq_u64_e32 vcc, s[6:7], v[0:1] +; GCN-NEXT: s_and_saveexec_b64 s[8:9], vcc +; GCN-NEXT: s_movk_i32 s0, 0x7b +; GCN-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1 -; GCN-NEXT: s_xor_b64 exec, exec, s[10:11] +; GCN-NEXT: s_xor_b64 exec, exec, s[8:9] ; GCN-NEXT: s_cbranch_execnz .LBB6_1 ; GCN-NEXT: ; %bb.2: -; GCN-NEXT: s_mov_b64 exec, s[6:7] +; GCN-NEXT: s_mov_b64 exec, s[4:5] ; GCN-NEXT: v_readlane_b32 s63, v40, 31 ; GCN-NEXT: v_readlane_b32 s62, v40, 30 ; GCN-NEXT: v_readlane_b32 s61, v40, 29 @@ -930,22 +930,22 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) { ; GCN-NEXT: v_readlane_b32 s34, v40, 2 ; GCN-NEXT: v_readlane_b32 s31, v40, 1 ; GCN-NEXT: v_readlane_b32 s30, v40, 0 -; GCN-NEXT: s_or_saveexec_b64 s[6:7], -1 +; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1 ; GCN-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload -; GCN-NEXT: s_mov_b64 exec, s[6:7] +; GCN-NEXT: s_mov_b64 exec, s[4:5] ; GCN-NEXT: s_addk_i32 s32, 0xfc00 -; GCN-NEXT: s_mov_b32 s33, s5 +; GCN-NEXT: s_mov_b32 s33, s10 ; GCN-NEXT: s_waitcnt vmcnt(0) ; GCN-NEXT: s_setpc_b64 s[30:31] ; ; GISEL-LABEL: test_indirect_call_vgpr_ptr_inreg_arg: ; GISEL: ; %bb.0: ; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GISEL-NEXT: s_mov_b32 s5, s33 +; GISEL-NEXT: s_mov_b32 s10, s33 ; GISEL-NEXT: s_mov_b32 s33, s32 -; GISEL-NEXT: s_or_saveexec_b64 s[6:7], -1 +; GISEL-NEXT: s_or_saveexec_b64 s[4:5], -1 ; GISEL-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill -; GISEL-NEXT: s_mov_b64 exec, s[6:7] +; GISEL-NEXT: s_mov_b64 exec, s[4:5] ; GISEL-NEXT: s_addk_i32 s32, 0x400 ; GISEL-NEXT: v_writelane_b32 v40, s30, 0 ; GISEL-NEXT: v_writelane_b32 v40, s31, 1 @@ -979,19 +979,19 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) { ; GISEL-NEXT: v_writelane_b32 v40, s61, 29 ; GISEL-NEXT: v_writelane_b32 v40, s62, 30 ; GISEL-NEXT: v_writelane_b32 v40, s63, 31 -; GISEL-NEXT: s_mov_b64 s[6:7], exec -; GISEL-NEXT: s_movk_i32 s4, 0x7b +; GISEL-NEXT: s_mov_b64 s[4:5], exec ; GISEL-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1 -; GISEL-NEXT: v_readfirstlane_b32 s8, v0 -; GISEL-NEXT: v_readfirstlane_b32 s9, v1 -; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1] -; GISEL-NEXT: s_and_saveexec_b64 s[10:11], vcc -; GISEL-NEXT: s_swappc_b64 s[30:31], s[8:9] +; GISEL-NEXT: v_readfirstlane_b32 s6, v0 +; GISEL-NEXT: v_readfirstlane_b32 s7, v1 +; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, s[6:7], v[0:1] +; GISEL-NEXT: s_and_saveexec_b64 s[8:9], vcc +; GISEL-NEXT: s_movk_i32 s0, 0x7b +; GISEL-NEXT: s_swappc_b64 s[30:31], s[6:7] ; GISEL-NEXT: ; implicit-def: $vgpr0 -; GISEL-NEXT: s_xor_b64 exec, exec, s[10:11] +; GISEL-NEXT: s_xor_b64 exec, exec, s[8:9] ; GISEL-NEXT: s_cbranch_execnz .LBB6_1 ; GISEL-NEXT: ; %bb.2: -; GISEL-NEXT: s_mov_b64 exec, s[6:7] +; GISEL-NEXT: s_mov_b64 exec, s[4:5] ; GISEL-NEXT: v_readlane_b32 s63, v40, 31 ; GISEL-NEXT: v_readlane_b32 s62, v40, 30 ; GISEL-NEXT: v_readlane_b32 s61, v40, 29 @@ -1024,11 +1024,11 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) { ; GISEL-NEXT: v_readlane_b32 s34, v40, 2 ; GISEL-NEXT: v_readlane_b32 s31, v40, 1 ; GISEL-NEXT: v_readlane_b32 s30, v40, 0 -; GISEL-NEXT: s_or_saveexec_b64 s[6:7], -1 +; GISEL-NEXT: s_or_saveexec_b64 s[4:5], -1 ; GISEL-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload -; GISEL-NEXT: s_mov_b64 exec, s[6:7] +; GISEL-NEXT: s_mov_b64 exec, s[4:5] ; GISEL-NEXT: s_addk_i32 s32, 0xfc00 -; GISEL-NEXT: s_mov_b32 s33, s5 +; GISEL-NEXT: s_mov_b32 s33, s10 ; GISEL-NEXT: s_waitcnt vmcnt(0) ; GISEL-NEXT: s_setpc_b64 s[30:31] call amdgpu_gfx void %fptr(i32 inreg 123) diff --git a/llvm/test/CodeGen/AMDGPU/schedule-addrspaces.ll b/llvm/test/CodeGen/AMDGPU/schedule-addrspaces.ll index 0139c52db1d5..27e9d0fce942 100644 --- a/llvm/test/CodeGen/AMDGPU/schedule-addrspaces.ll +++ b/llvm/test/CodeGen/AMDGPU/schedule-addrspaces.ll @@ -10,7 +10,7 @@ define amdgpu_gfx void @example(<4 x i32> inreg %rsrc, ptr addrspace(5) %src, i3 ; CHECK-NEXT: scratch_load_b32 v2, v0, off ; CHECK-NEXT: scratch_load_b32 v3, v3, off ; CHECK-NEXT: s_waitcnt vmcnt(0) -; CHECK-NEXT: buffer_store_b64 v[2:3], v1, s[4:7], 0 offen +; CHECK-NEXT: buffer_store_b64 v[2:3], v1, s[0:3], 0 offen ; CHECK-NEXT: s_setpc_b64 s[30:31] %x0 = load i32, ptr addrspace(5) %src diff --git a/llvm/test/CodeGen/AMDGPU/scratch-pointer-sink.ll b/llvm/test/CodeGen/AMDGPU/scratch-pointer-sink.ll index cdaac14833e0..ec1de0222d5d 100644 --- a/llvm/test/CodeGen/AMDGPU/scratch-pointer-sink.ll +++ b/llvm/test/CodeGen/AMDGPU/scratch-pointer-sink.ll @@ -6,7 +6,7 @@ define amdgpu_gfx i32 @sink_scratch_pointer(ptr addrspace(5) %stack, i32 inreg % ; GCN-LABEL: sink_scratch_pointer: ; GCN: ; %bb.0: ; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GCN-NEXT: s_cmp_lg_u32 s4, 0 +; GCN-NEXT: s_cmp_lg_u32 s0, 0 ; GCN-NEXT: s_cbranch_scc0 .LBB0_2 ; GCN-NEXT: ; %bb.1: ; %bb2 ; GCN-NEXT: scratch_load_b32 v0, v0, off offset:-4 @@ -21,7 +21,7 @@ define amdgpu_gfx i32 @sink_scratch_pointer(ptr addrspace(5) %stack, i32 inreg % ; GISEL-LABEL: sink_scratch_pointer: ; GISEL: ; %bb.0: ; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GISEL-NEXT: s_cmp_lg_u32 s4, 0 +; GISEL-NEXT: s_cmp_lg_u32 s0, 0 ; GISEL-NEXT: s_cbranch_scc0 .LBB0_2 ; GISEL-NEXT: ; %bb.1: ; %bb2 ; GISEL-NEXT: scratch_load_b32 v0, v0, off offset:-4 -- GitLab From 34f0a8aaba11bf703ddd2de92eee8ecbb77be5c8 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Thu, 21 Mar 2024 04:13:23 -0700 Subject: [PATCH 129/296] [SLP]Fix comparison in bitwidth check. Projected bitwidth should be less than the original, not greater. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 2 +- .../orig-btiwidth-les-projected.ll | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 llvm/test/Transforms/SLPVectorizer/orig-btiwidth-les-projected.ll diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index a52064e5417b..4853c2006fea 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -13995,7 +13995,7 @@ bool BoUpSLP::collectValuesToDemote( if (MultiNodeScalars.contains(V)) return false; uint32_t OrigBitWidth = DL->getTypeSizeInBits(V->getType()); - if (OrigBitWidth < BitWidth) { + if (OrigBitWidth > BitWidth) { APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth); if (MaskedValueIsZero(V, Mask, SimplifyQuery(*DL))) return true; diff --git a/llvm/test/Transforms/SLPVectorizer/orig-btiwidth-les-projected.ll b/llvm/test/Transforms/SLPVectorizer/orig-btiwidth-les-projected.ll new file mode 100644 index 000000000000..531e96405348 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/orig-btiwidth-les-projected.ll @@ -0,0 +1,22 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S --passes=slp-vectorizer < %s | FileCheck %s + +define i32 @test(i4 %0) { +; CHECK-LABEL: define i32 @test( +; CHECK-SAME: i4 [[TMP0:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP1:%.*]] = trunc i8 0 to i4 +; CHECK-NEXT: [[TMP2:%.*]] = trunc i8 0 to i4 +; CHECK-NEXT: [[ADD_R:%.*]] = or i4 [[TMP1]], [[TMP0]] +; CHECK-NEXT: [[ADD_R14:%.*]] = or i4 0, [[TMP2]] +; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i4 [[ADD_R]], [[ADD_R14]] +; CHECK-NEXT: ret i32 0 +; +entry: + %1 = trunc i8 0 to i4 + %2 = trunc i8 0 to i4 + %add.r = or i4 %1, %0 + %add.r14 = or i4 0, %2 + %cmp.not = icmp eq i4 %add.r, %add.r14 + ret i32 0 +} -- GitLab From df6a1d44094e187d7d5ea3ee5b54b9bccc8a4798 Mon Sep 17 00:00:00 2001 From: Christian Sigg Date: Thu, 21 Mar 2024 12:32:12 +0100 Subject: [PATCH 130/296] [mlir][tensor] NFC: fully qualify verifyEncoding arguments. --- mlir/include/mlir/IR/TensorEncoding.td | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlir/include/mlir/IR/TensorEncoding.td b/mlir/include/mlir/IR/TensorEncoding.td index 3991520d72a5..4907dcbb5de9 100644 --- a/mlir/include/mlir/IR/TensorEncoding.td +++ b/mlir/include/mlir/IR/TensorEncoding.td @@ -34,8 +34,8 @@ def VerifiableTensorEncoding : AttrInterface<"VerifiableTensorEncoding"> { /*retTy=*/"::mlir::LogicalResult", /*methodName=*/"verifyEncoding", /*args=*/(ins - "ArrayRef":$shape, - "Type":$elementType, + "::mlir::ArrayRef":$shape, + "::mlir::Type":$elementType, "::llvm::function_ref<::mlir::InFlightDiagnostic()>":$emitError) >, ]; -- GitLab From 2699072b4bc8d8d5e84eb66af38face73ceeb4d3 Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Thu, 21 Mar 2024 12:57:24 +0100 Subject: [PATCH 131/296] [clang] Accept lambdas in C++03 as an extensions (#73376) Implements https://discourse.llvm.org/t/rfc-allow-c-11-lambdas-in-c-03-as-an-extension/75262 --- clang/docs/LanguageExtensions.rst | 73 ++++++----- clang/docs/ReleaseNotes.rst | 2 + .../clang/Basic/DiagnosticParseKinds.td | 1 + clang/include/clang/Basic/Features.def | 1 + clang/lib/Parse/ParseExpr.cpp | 2 +- clang/lib/Parse/ParseExprCXX.cpp | 9 +- clang/lib/Parse/ParseInit.cpp | 2 +- clang/lib/Sema/SemaDecl.cpp | 2 +- clang/lib/Sema/SemaDeclCXX.cpp | 3 +- clang/test/Lexer/has_extension_cxx.cpp | 5 + .../OpenMP/declare_reduction_messages.cpp | 14 +-- clang/test/OpenMP/openmp_check.cpp | 2 +- clang/test/Parser/cxx03-lambda-extension.cpp | 5 + .../test/Parser/cxx0x-lambda-expressions.cpp | 116 +++++++----------- clang/test/Parser/cxx2b-lambdas.cpp | 45 +++++-- .../Parser/objcxx-lambda-expressions-neg.mm | 9 +- clang/test/ParserHLSL/group_shared.hlsl | 4 +- clang/test/SemaCXX/cxx2a-template-lambdas.cpp | 26 ++-- clang/test/SemaCXX/lambda-expressions.cpp | 64 ++++++---- .../SemaCXX/lambda-implicit-this-capture.cpp | 1 + clang/test/SemaCXX/lambda-invalid-capture.cpp | 1 + clang/test/SemaCXX/new-delete.cpp | 7 +- 22 files changed, 213 insertions(+), 181 deletions(-) create mode 100644 clang/test/Parser/cxx03-lambda-extension.cpp diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst index 13d7261d83d7..201a4c27f7dd 100644 --- a/clang/docs/LanguageExtensions.rst +++ b/clang/docs/LanguageExtensions.rst @@ -1459,40 +1459,45 @@ More information could be found `here Language Extensions Back-ported to Previous Standards ===================================================== -====================================== ================================ ============= ============= -Feature Feature Test Macro Introduced In Backported To -====================================== ================================ ============= ============= -variadic templates __cpp_variadic_templates C++11 C++03 -Alias templates __cpp_alias_templates C++11 C++03 -Non-static data member initializers __cpp_nsdmi C++11 C++03 -Range-based ``for`` loop __cpp_range_based_for C++11 C++03 -RValue references __cpp_rvalue_references C++11 C++03 -Attributes __cpp_attributes C++11 C++03 -variable templates __cpp_variable_templates C++14 C++03 -Binary literals __cpp_binary_literals C++14 C++03 -Relaxed constexpr __cpp_constexpr C++14 C++11 -``if constexpr`` __cpp_if_constexpr C++17 C++11 -fold expressions __cpp_fold_expressions C++17 C++03 -Lambda capture of \*this by value __cpp_capture_star_this C++17 C++11 -Attributes on enums __cpp_enumerator_attributes C++17 C++03 -Guaranteed copy elision __cpp_guaranteed_copy_elision C++17 C++03 -Hexadecimal floating literals __cpp_hex_float C++17 C++03 -``inline`` variables __cpp_inline_variables C++17 C++03 -Attributes on namespaces __cpp_namespace_attributes C++17 C++11 -Structured bindings __cpp_structured_bindings C++17 C++03 -template template arguments __cpp_template_template_args C++17 C++03 -``static operator[]`` __cpp_multidimensional_subscript C++20 C++03 -Designated initializers __cpp_designated_initializers C++20 C++03 -Conditional ``explicit`` __cpp_conditional_explicit C++20 C++03 -``using enum`` __cpp_using_enum C++20 C++03 -``if consteval`` __cpp_if_consteval C++23 C++20 -``static operator()`` __cpp_static_call_operator C++23 C++03 -Attributes on Lambda-Expressions C++23 C++11 --------------------------------------- -------------------------------- ------------- ------------- -Designated initializers (N494) C99 C89 -Array & element qualification (N2607) C23 C89 -Attributes (N2335) C23 C89 -====================================== ================================ ============= ============= +============================================ ================================ ============= ============= +Feature Feature Test Macro Introduced In Backported To +============================================ ================================ ============= ============= +variadic templates __cpp_variadic_templates C++11 C++03 +Alias templates __cpp_alias_templates C++11 C++03 +Non-static data member initializers __cpp_nsdmi C++11 C++03 +Range-based ``for`` loop __cpp_range_based_for C++11 C++03 +RValue references __cpp_rvalue_references C++11 C++03 +Attributes __cpp_attributes C++11 C++03 +Lambdas __cpp_lambdas C++11 C++03 +Generalized lambda captures __cpp_init_captures C++14 C++03 +Generic lambda expressions __cpp_generic_lambdas C++14 C++03 +variable templates __cpp_variable_templates C++14 C++03 +Binary literals __cpp_binary_literals C++14 C++03 +Relaxed constexpr __cpp_constexpr C++14 C++11 +Pack expansion in generalized lambda-capture __cpp_init_captures C++17 C++03 +``if constexpr`` __cpp_if_constexpr C++17 C++11 +fold expressions __cpp_fold_expressions C++17 C++03 +Lambda capture of \*this by value __cpp_capture_star_this C++17 C++03 +Attributes on enums __cpp_enumerator_attributes C++17 C++03 +Guaranteed copy elision __cpp_guaranteed_copy_elision C++17 C++03 +Hexadecimal floating literals __cpp_hex_float C++17 C++03 +``inline`` variables __cpp_inline_variables C++17 C++03 +Attributes on namespaces __cpp_namespace_attributes C++17 C++11 +Structured bindings __cpp_structured_bindings C++17 C++03 +template template arguments __cpp_template_template_args C++17 C++03 +Familiar template syntax for generic lambdas __cpp_generic_lambdas C++20 C++03 +``static operator[]`` __cpp_multidimensional_subscript C++20 C++03 +Designated initializers __cpp_designated_initializers C++20 C++03 +Conditional ``explicit`` __cpp_conditional_explicit C++20 C++03 +``using enum`` __cpp_using_enum C++20 C++03 +``if consteval`` __cpp_if_consteval C++23 C++20 +``static operator()`` __cpp_static_call_operator C++23 C++03 +Attributes on Lambda-Expressions C++23 C++11 +-------------------------------------------- -------------------------------- ------------- ------------- +Designated initializers (N494) C99 C89 +Array & element qualification (N2607) C23 C89 +Attributes (N2335) C23 C89 +============================================ ================================ ============= ============= Type Trait Primitives ===================== diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index a9c55ef662a0..50990140a53a 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -180,6 +180,8 @@ Non-comprehensive list of changes in this release the previous builtins, this new builtin is constexpr and may be used in constant expressions. +- Lambda expressions are now accepted in C++03 mode as an extension. + New Compiler Flags ------------------ diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td index 816c3ff5f8b2..48de5e2ef5f4 100644 --- a/clang/include/clang/Basic/DiagnosticParseKinds.td +++ b/clang/include/clang/Basic/DiagnosticParseKinds.td @@ -1029,6 +1029,7 @@ def err_expected_lambda_body : Error<"expected body of lambda expression">; def warn_cxx98_compat_lambda : Warning< "lambda expressions are incompatible with C++98">, InGroup, DefaultIgnore; +def ext_lambda : ExtWarn<"lambdas are a C++11 extension">, InGroup; def err_lambda_decl_specifier_repeated : Error< "%select{'mutable'|'static'|'constexpr'|'consteval'}0 cannot " "appear multiple times in a lambda declarator">; diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def index 726ead4b5ab5..b41aadc73f20 100644 --- a/clang/include/clang/Basic/Features.def +++ b/clang/include/clang/Basic/Features.def @@ -261,6 +261,7 @@ EXTENSION(cxx_defaulted_functions, LangOpts.CPlusPlus) EXTENSION(cxx_deleted_functions, LangOpts.CPlusPlus) EXTENSION(cxx_explicit_conversions, LangOpts.CPlusPlus) EXTENSION(cxx_inline_namespaces, LangOpts.CPlusPlus) +EXTENSION(cxx_lambdas, LangOpts.CPlusPlus) EXTENSION(cxx_local_type_template_args, LangOpts.CPlusPlus) EXTENSION(cxx_nonstatic_member_init, LangOpts.CPlusPlus) EXTENSION(cxx_override_control, LangOpts.CPlusPlus) diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp index 88c3a1469e8e..ae23cb432c43 100644 --- a/clang/lib/Parse/ParseExpr.cpp +++ b/clang/lib/Parse/ParseExpr.cpp @@ -1823,7 +1823,7 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind, } goto ExpectedExpression; case tok::l_square: - if (getLangOpts().CPlusPlus11) { + if (getLangOpts().CPlusPlus) { if (getLangOpts().ObjC) { // C++11 lambda expressions and Objective-C message sends both start with a // square bracket. There are three possibilities here: diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp index 9471f6f725ef..73c85c585baa 100644 --- a/clang/lib/Parse/ParseExprCXX.cpp +++ b/clang/lib/Parse/ParseExprCXX.cpp @@ -806,9 +806,8 @@ ExprResult Parser::ParseLambdaExpression() { /// /// If we are not looking at a lambda expression, returns ExprError(). ExprResult Parser::TryParseLambdaExpression() { - assert(getLangOpts().CPlusPlus11 - && Tok.is(tok::l_square) - && "Not at the start of a possible lambda expression."); + assert(getLangOpts().CPlusPlus && Tok.is(tok::l_square) && + "Not at the start of a possible lambda expression."); const Token Next = NextToken(); if (Next.is(tok::eof)) // Nothing else to lookup here... @@ -1326,7 +1325,9 @@ static void DiagnoseStaticSpecifierRestrictions(Parser &P, ExprResult Parser::ParseLambdaExpressionAfterIntroducer( LambdaIntroducer &Intro) { SourceLocation LambdaBeginLoc = Intro.Range.getBegin(); - Diag(LambdaBeginLoc, diag::warn_cxx98_compat_lambda); + Diag(LambdaBeginLoc, getLangOpts().CPlusPlus11 + ? diag::warn_cxx98_compat_lambda + : diag::ext_lambda); PrettyStackTraceLoc CrashInfo(PP.getSourceManager(), LambdaBeginLoc, "lambda expression parsing"); diff --git a/clang/lib/Parse/ParseInit.cpp b/clang/lib/Parse/ParseInit.cpp index 637f21176792..423497bfcb66 100644 --- a/clang/lib/Parse/ParseInit.cpp +++ b/clang/lib/Parse/ParseInit.cpp @@ -35,7 +35,7 @@ bool Parser::MayBeDesignationStart() { return true; case tok::l_square: { // designator: array-designator - if (!PP.getLangOpts().CPlusPlus11) + if (!PP.getLangOpts().CPlusPlus) return true; // C++11 lambda expressions and C99 designators can be ambiguous all the diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 8ceb79555fb5..aa754d47a0c4 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -16109,7 +16109,7 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body, FD->setInvalidDecl(); } } - } else if (getLangOpts().CPlusPlus11 && isLambdaCallOperator(FD)) { + } else if (getLangOpts().CPlusPlus && isLambdaCallOperator(FD)) { // In C++11, we don't use 'auto' deduction rules for lambda call // operators because we don't support return type deduction. auto *LSI = getCurLambda(); diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp index e258a4f7c894..ee732679417e 100644 --- a/clang/lib/Sema/SemaDeclCXX.cpp +++ b/clang/lib/Sema/SemaDeclCXX.cpp @@ -9738,7 +9738,8 @@ bool Sema::ShouldDeleteSpecialMember(CXXMethodDecl *MD, CXXSpecialMember CSM, return false; CXXRecordDecl *RD = MD->getParent(); assert(!RD->isDependentType() && "do deletion after instantiation"); - if (!LangOpts.CPlusPlus11 || RD->isInvalidDecl()) + if (!LangOpts.CPlusPlus || (!LangOpts.CPlusPlus11 && !RD->isLambda()) || + RD->isInvalidDecl()) return false; // C++11 [expr.lambda.prim]p19: diff --git a/clang/test/Lexer/has_extension_cxx.cpp b/clang/test/Lexer/has_extension_cxx.cpp index 7941997428ac..7366029d3727 100644 --- a/clang/test/Lexer/has_extension_cxx.cpp +++ b/clang/test/Lexer/has_extension_cxx.cpp @@ -33,6 +33,11 @@ int has_deleted_functions(); int has_inline_namespaces(); #endif +// CHECK: has_lambdas +#if __has_extension(cxx_lambdas) +int has_lambdas(); +#endif + // CHECK: has_override_control #if __has_extension(cxx_override_control) int has_override_control(); diff --git a/clang/test/OpenMP/declare_reduction_messages.cpp b/clang/test/OpenMP/declare_reduction_messages.cpp index 38a5d766eead..752cc4fb05a1 100644 --- a/clang/test/OpenMP/declare_reduction_messages.cpp +++ b/clang/test/OpenMP/declare_reduction_messages.cpp @@ -58,16 +58,10 @@ class Class2 : public Class1 { #pragma omp declare reduction(fun1 : long : omp_out += omp_in) initializer // expected-error {{expected '(' after 'initializer'}} #pragma omp declare reduction(fun2 : long : omp_out += omp_in) initializer { // expected-error {{expected '(' after 'initializer'}} expected-error {{expected expression}} expected-warning {{extra tokens at the end of '#pragma omp declare reduction' are ignored}} #pragma omp declare reduction(fun3 : long : omp_out += omp_in) initializer[ -#if __cplusplus <= 199711L -// expected-error@-2 {{expected '(' after 'initializer'}} -// expected-error@-3 {{expected expression}} -// expected-warning@-4 {{extra tokens at the end of '#pragma omp declare reduction' are ignored}} -#else -// expected-error@-6 {{expected '(' after 'initializer'}} -// expected-error@-7 {{expected variable name or 'this' in lambda capture list}} -// expected-error@-8 {{expected ')'}} -// expected-note@-9 {{to match this '('}} -#endif +// expected-error@-1 {{expected '(' after 'initializer'}} +// expected-error@-2 {{expected variable name or 'this' in lambda capture list}} +// expected-error@-3 {{expected ')'}} +// expected-note@-4 {{to match this '('}} #pragma omp declare reduction(fun4 : long : omp_out += omp_in) initializer() // expected-error {{expected expression}} #pragma omp declare reduction(fun5 : long : omp_out += omp_in) initializer(temp) // expected-error {{only 'omp_priv' or 'omp_orig' variables are allowed in initializer expression}} #pragma omp declare reduction(fun6 : long : omp_out += omp_in) initializer(omp_orig // expected-error {{expected ')'}} expected-note {{to match this '('}} diff --git a/clang/test/OpenMP/openmp_check.cpp b/clang/test/OpenMP/openmp_check.cpp index 6a8dd17fc836..b52ce0c06692 100644 --- a/clang/test/OpenMP/openmp_check.cpp +++ b/clang/test/OpenMP/openmp_check.cpp @@ -18,7 +18,7 @@ int nested(int a) { auto F = [&]() { #if __cplusplus <= 199711L // expected-warning@-2 {{'auto' type specifier is a C++11 extension}} - // expected-error@-3 {{expected expression}} + // expected-warning@-3 {{lambdas are a C++11 extension}} #endif #pragma omp parallel diff --git a/clang/test/Parser/cxx03-lambda-extension.cpp b/clang/test/Parser/cxx03-lambda-extension.cpp new file mode 100644 index 000000000000..82ae7da30530 --- /dev/null +++ b/clang/test/Parser/cxx03-lambda-extension.cpp @@ -0,0 +1,5 @@ +// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++03 %s + +void func() { + []() {}; // expected-warning {{lambdas are a C++11 extension}} +} diff --git a/clang/test/Parser/cxx0x-lambda-expressions.cpp b/clang/test/Parser/cxx0x-lambda-expressions.cpp index 72b315a497c0..a786a964163e 100644 --- a/clang/test/Parser/cxx0x-lambda-expressions.cpp +++ b/clang/test/Parser/cxx0x-lambda-expressions.cpp @@ -1,10 +1,15 @@ -// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++11 -Wno-c99-designator %s -// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++20 -Wno-c99-designator %s -// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++23 -Wno-c99-designator %s +// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify=expected,cxx14ext,cxx17ext,cxx20ext,cxx23ext -std=c++03 -Wno-c99-designator %s -Wno-c++11-extensions +// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify=expected,cxx14ext,cxx17ext,cxx20ext,cxx23ext -std=c++11 -Wno-c99-designator %s +// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify=expected,cxx17ext,cxx20ext,cxx23ext -std=c++14 -Wno-c99-designator %s +// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify=expected,cxx20ext,cxx23ext -std=c++17 -Wno-c99-designator %s +// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify=expected,cxx23ext -std=c++20 -Wno-c99-designator %s +// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify=expected -std=c++23 -Wno-c99-designator %s enum E { e }; +#if __cplusplus >= 201103L constexpr int id(int n) { return n; } +#endif class C { @@ -19,28 +24,25 @@ class C { [&,] {}; // expected-error {{expected variable name or 'this' in lambda capture list}} [=,] {}; // expected-error {{expected variable name or 'this' in lambda capture list}} [] {}; - [=] (int i) {}; - [&] (int) mutable -> void {}; - [foo,bar] () { return 3; }; - [=,&foo] () {}; - [&,foo] () {}; - [this] () {}; + [=] (int i) {}; + [&] (int) mutable -> void {}; + [foo,bar] () { return 3; }; + [=,&foo] () {}; + [&,foo] () {}; + [this] () {}; [] () -> class C { return C(); }; [] () -> enum E { return e; }; - [] -> int { return 0; }; - [] mutable -> int { return 0; }; -#if __cplusplus <= 202002L - // expected-warning@-3 {{lambda without a parameter clause is a C++23 extension}} - // expected-warning@-3 {{is a C++23 extension}} -#endif + [] -> int { return 0; }; // cxx23ext-warning {{lambda without a parameter clause is a C++23 extension}} + [] mutable -> int { return 0; }; // cxx23ext-warning {{is a C++23 extension}} + [](int) -> {}; // PR13652 expected-error {{expected a type}} return 1; } void designator_or_lambda() { - typedef int T; - const int b = 0; + typedef int T; + const int b = 0; const int c = 1; int d; int a1[1] = {[b] (T()) {}}; // expected-error{{no viable conversion from '(lambda}} @@ -49,19 +51,18 @@ class C { int a4[1] = {[&b] = 1 }; // expected-error{{integral constant expression must have integral or unscoped enumeration type, not 'const int *'}} int a5[3] = { []{return 0;}() }; int a6[1] = {[this] = 1 }; // expected-error{{integral constant expression must have integral or unscoped enumeration type, not 'C *'}} - int a7[1] = {[d(0)] { return d; } ()}; - int a8[1] = {[d = 0] { return d; } ()}; - int a10[1] = {[id(0)] { return id; } ()}; -#if __cplusplus <= 201103L - // expected-warning@-4{{extension}} - // expected-warning@-4{{extension}} - // expected-warning@-4{{extension}} + int a7[1] = {[d(0)] { return d; } ()}; // cxx14ext-warning {{initialized lambda captures are a C++14 extension}} + int a8[1] = {[d = 0] { return d; } ()}; // cxx14ext-warning {{initialized lambda captures are a C++14 extension}} +#if __cplusplus >= 201103L + int a10[1] = {[id(0)] { return id; } ()}; // cxx14ext-warning {{initialized lambda captures are a C++14 extension}} #endif int a9[1] = {[d = 0] = 1}; // expected-error{{is not an integral constant expression}} #if __cplusplus >= 201402L // expected-note@-2{{constant expression cannot modify an object that is visible outside that expression}} #endif +#if __cplusplus >= 201103L int a11[1] = {[id(0)] = 1}; +#endif } void delete_lambda(int *p) { @@ -80,43 +81,33 @@ class C { // We support init-captures in C++11 as an extension. int z; void init_capture() { - [n(0)] () mutable -> int { return ++n; }; - [n{0}] { return; }; - [a([&b = z]{})](){}; - [n = 0] { return ++n; }; // expected-error {{captured by copy in a non-mutable}} - [n = {0}] { return; }; // expected-error {{}} -#if __cplusplus <= 201103L - // expected-warning@-6{{extension}} - // expected-warning@-6{{extension}} - // expected-warning@-6{{extension}} - // expected-warning@-7{{extension}} - // expected-warning@-7{{extension}} - // expected-warning@-7{{extension}} -#endif + [n(0)] () mutable -> int { return ++n; }; // cxx14ext-warning {{initialized lambda captures are a C++14 extension}} + [n{0}] { return; }; // cxx14ext-warning {{initialized lambda captures are a C++14 extension}} + [a([&b = z]{})](){}; // cxx14ext-warning 2 {{initialized lambda captures are a C++14 extension}} + [n = 0] { return ++n; }; // expected-error {{captured by copy in a non-mutable}} + // cxx14ext-warning@-1 {{initialized lambda captures are a C++14 extension}} + [n = {0}] { return; }; // expected-error {{}} + // cxx14ext-warning@-1 {{initialized lambda captures are a C++14 extension}} int x = 4; - auto y = [&r = x, x = x + 1]() -> int { -#if __cplusplus <= 201103L - // expected-warning@-2{{extension}} - // expected-warning@-3{{extension}} -#endif + auto y = [&r = x, x = x + 1]() -> int { // cxx14ext-warning 2 {{initialized lambda captures are a C++14 extension}} r += 2; return x + 2; } (); } void attributes() { - [] __attribute__((noreturn)){}; -#if __cplusplus <= 202002L - // expected-warning@-2 {{is a C++23 extension}} -#endif + [] __attribute__((noreturn)){}; // cxx23ext-warning {{lambda without a parameter clause is a C++23 extension}} + []() [[]] mutable {}; // expected-error {{expected body of lambda expression}} []() [[]] {}; []() [[]] -> void {}; []() mutable [[]] -> void {}; +#if __cplusplus >= 201103L []() mutable noexcept [[]] -> void {}; +#endif // Testing GNU-style attributes on lambdas -- the attribute is specified // before the mutable specifier instead of after (unlike C++11). @@ -126,28 +117,18 @@ class C { // Testing support for P2173 on adding attributes to the declaration // rather than the type. - [][[]](){}; -#if __cplusplus <= 202002L - // expected-warning@-2 {{an attribute specifier sequence in this position is a C++23 extension}} -#endif -#if __cplusplus > 201703L - [][[]](){}; -#if __cplusplus <= 202002L - // expected-warning@-2 {{an attribute specifier sequence in this position is a C++23 extension}} -#endif -#endif - [][[]]{}; -#if __cplusplus <= 202002L - // expected-warning@-2 {{an attribute specifier sequence in this position is a C++23 extension}} -#endif + [][[]](){}; // cxx23ext-warning {{an attribute specifier sequence in this position is a C++23 extension}} + + [][[]](){}; // cxx20ext-warning {{explicit template parameter list for lambdas is a C++20 extension}} + // cxx23ext-warning@-1 {{an attribute specifier sequence in this position is a C++23 extension}} + + [][[]]{}; // cxx23ext-warning {{an attribute specifier sequence in this position is a C++23 extension}} } void missing_parens() { - [] mutable {}; - [] noexcept {}; -#if __cplusplus <= 202002L - // expected-warning@-3 {{is a C++23 extension}} - // expected-warning@-3 {{is a C++23 extension}} + [] mutable {}; // cxx23ext-warning {{is a C++23 extension}} +#if __cplusplus >= 201103L + [] noexcept {}; // cxx23ext-warning {{is a C++23 extension}} #endif } }; @@ -165,10 +146,7 @@ struct A { }; struct S { - void mf() { A{[*this]{}}; } -#if __cplusplus < 201703L - // expected-warning@-2 {{C++17 extension}} -#endif + void mf() { A(([*this]{})); } // cxx17ext-warning {{'*this' by copy is a C++17 extension}} }; } diff --git a/clang/test/Parser/cxx2b-lambdas.cpp b/clang/test/Parser/cxx2b-lambdas.cpp index ad975a17b6e4..758ec9a42f56 100644 --- a/clang/test/Parser/cxx2b-lambdas.cpp +++ b/clang/test/Parser/cxx2b-lambdas.cpp @@ -1,30 +1,48 @@ +// RUN: %clang_cc1 -std=c++03 %s -verify -Wno-c++23-extensions -Wno-c++20-extensions -Wno-c++17-extensions -Wno-c++14-extensions -Wno-c++11-extensions +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx11 -Wno-c++23-extensions -Wno-c++20-extensions -Wno-c++17-extensions -Wno-c++14-extensions +// RUN: %clang_cc1 -std=c++14 %s -verify -Wno-c++23-extensions -Wno-c++20-extensions -Wno-c++17-extensions +// RUN: %clang_cc1 -std=c++17 %s -verify -Wno-c++23-extensions -Wno-c++20-extensions +// RUN: %clang_cc1 -std=c++20 %s -verify -Wno-c++23-extensions // RUN: %clang_cc1 -std=c++23 %s -verify auto LL0 = [] {}; auto LL1 = []() {}; auto LL2 = []() mutable {}; -auto LL3 = []() constexpr {}; +#if __cplusplus >= 201103L +auto LL3 = []() constexpr {}; // cxx11-error {{return type 'void' is not a literal type}} +#endif -auto L0 = [] constexpr {}; +#if __cplusplus >= 201103L +auto L0 = [] constexpr {}; // cxx11-error {{return type 'void' is not a literal type}} +#endif auto L1 = [] mutable {}; +#if __cplusplus >= 201103L auto L2 = [] noexcept {}; -auto L3 = [] constexpr mutable {}; -auto L4 = [] mutable constexpr {}; -auto L5 = [] constexpr mutable noexcept {}; +auto L3 = [] constexpr mutable {}; // cxx11-error {{return type 'void' is not a literal type}} +auto L4 = [] mutable constexpr {}; // cxx11-error {{return type 'void' is not a literal type}} +auto L5 = [] constexpr mutable noexcept {}; // cxx11-error {{return type 'void' is not a literal type}} +#endif auto L6 = [s = 1] mutable {}; -auto L7 = [s = 1] constexpr mutable noexcept {}; +#if __cplusplus >= 201103L +auto L7 = [s = 1] constexpr mutable noexcept {}; // cxx11-error {{return type 'void' is not a literal type}} +#endif auto L8 = [] -> bool { return true; }; auto L9 = [] { return true; }; +#if __cplusplus >= 201103L auto L10 = [] noexcept { return true; }; +#endif auto L11 = [] -> bool { return true; }; +#if __cplusplus >= 202002L auto L12 = [] consteval {}; auto L13 = []() requires true {}; // expected-error{{non-templated function cannot have a requires clause}} auto L14 = [] requires true() requires true {}; auto L15 = [] requires true noexcept {}; +#endif auto L16 = [] [[maybe_unused]]{}; -auto XL0 = [] mutable constexpr mutable {}; // expected-error{{cannot appear multiple times}} -auto XL1 = [] constexpr mutable constexpr {}; // expected-error{{cannot appear multiple times}} +#if __cplusplus >= 201103L +auto XL0 = [] mutable constexpr mutable {}; // expected-error{{cannot appear multiple times}} cxx11-error {{return type 'void' is not a literal type}} +auto XL1 = [] constexpr mutable constexpr {}; // expected-error{{cannot appear multiple times}} cxx11-error {{return type 'void' is not a literal type}} auto XL2 = []) constexpr mutable constexpr {}; // expected-error{{expected body of lambda expression}} auto XL3 = []( constexpr mutable constexpr {}; // expected-error{{invalid storage class specifier}} \ // expected-error{{function parameter cannot be constexpr}} \ @@ -33,16 +51,23 @@ auto XL3 = []( constexpr mutable constexpr {}; // expected-error{{invalid storag // expected-note{{to match this '('}} \ // expected-error{{expected body}} \ // expected-warning{{duplicate 'constexpr'}} +#endif // http://llvm.org/PR49736 auto XL4 = [] requires true {}; // expected-error{{expected body}} +#if __cplusplus >= 201703L auto XL5 = [] requires true requires true {}; // expected-error{{expected body}} auto XL6 = [] requires true noexcept requires true {}; // expected-error{{expected body}} +#endif auto XL7 = []() static static {}; // expected-error {{cannot appear multiple times}} auto XL8 = []() static mutable {}; // expected-error {{cannot be both mutable and static}} +#if __cplusplus >= 202002L auto XL9 = []() static consteval {}; -auto XL10 = []() static constexpr {}; +#endif +#if __cplusplus >= 201103L +auto XL10 = []() static constexpr {}; // cxx11-error {{return type 'void' is not a literal type}} +#endif auto XL11 = [] static {}; auto XL12 = []() static {}; @@ -67,6 +92,7 @@ void static_captures() { }; } +#if __cplusplus >= 201703L constexpr auto static_capture_constexpr() { char n = 'n'; return [n] static { return n; }(); // expected-error {{a static lambda cannot have any captures}} @@ -78,3 +104,4 @@ constexpr auto capture_constexpr() { return [n] { return n; }(); } static_assert(capture_constexpr()); +#endif diff --git a/clang/test/Parser/objcxx-lambda-expressions-neg.mm b/clang/test/Parser/objcxx-lambda-expressions-neg.mm index b2fe39dfbf70..795157816dcf 100644 --- a/clang/test/Parser/objcxx-lambda-expressions-neg.mm +++ b/clang/test/Parser/objcxx-lambda-expressions-neg.mm @@ -1,13 +1,8 @@ // RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify %s -// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++98 %s +// RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify=cxx03 -std=c++98 %s // RUN: %clang_cc1 -fsyntax-only -Wno-unused-value -verify -std=c++11 %s int main() { - []{}; -#if __cplusplus <= 199711L - // expected-error@-2 {{expected expression}} -#else + []{}; // cxx03-warning {{lambdas are a C++11 extension}} // expected-no-diagnostics -#endif - } diff --git a/clang/test/ParserHLSL/group_shared.hlsl b/clang/test/ParserHLSL/group_shared.hlsl index 0b9f28395ee4..44f3a2e5b450 100644 --- a/clang/test/ParserHLSL/group_shared.hlsl +++ b/clang/test/ParserHLSL/group_shared.hlsl @@ -3,8 +3,8 @@ extern groupshared float f; extern float groupshared f; // Ok, redeclaration? -// NOTE:lambda is not enabled except for hlsl202x. -// expected-error@+2 {{expected expression}} +// expected-warning@+3 {{lambdas are a C++11 extension}} +// expected-error@+2 {{expected body of lambda expression}} // expected-warning@+1 {{'auto' type specifier is a C++11 extension}} auto l = []() groupshared {}; diff --git a/clang/test/SemaCXX/cxx2a-template-lambdas.cpp b/clang/test/SemaCXX/cxx2a-template-lambdas.cpp index 7ac481369891..fff524e77d3b 100644 --- a/clang/test/SemaCXX/cxx2a-template-lambdas.cpp +++ b/clang/test/SemaCXX/cxx2a-template-lambdas.cpp @@ -1,10 +1,14 @@ -// RUN: %clang_cc1 -std=c++2a -verify %s +// RUN: %clang_cc1 -std=c++03 -verify -Dstatic_assert=_Static_assert -Wno-c++11-extensions -Wno-c++14-extensions -Wno-c++17-extensions -Wno-c++20-extensions %s +// RUN: %clang_cc1 -std=c++11 -verify=expected,cxx11,cxx11-cxx14 -Wno-c++20-extensions -Wno-c++17-extensions -Wno-c++14-extensions %s +// RUN: %clang_cc1 -std=c++14 -verify=expected,cxx11-cxx14,cxx14 -Wno-c++20-extensions -Wno-c++17-extensions %s +// RUN: %clang_cc1 -std=c++17 -verify -Wno-c++20-extensions %s +// RUN: %clang_cc1 -std=c++20 -verify %s template -constexpr bool is_same = false; +inline const bool is_same = false; template -constexpr bool is_same = true; +inline const bool is_same = true; template struct DummyTemplate { }; @@ -23,7 +27,7 @@ void func() { L1.operator()<6>(); // expected-note {{in instantiation}} auto L2 = [] class T, class U>(T &&arg) { - static_assert(is_same, DummyTemplate>); // // expected-error {{static assertion failed}} + static_assert(is_same, DummyTemplate >); // // expected-error {{static assertion failed}} }; L2(DummyTemplate()); L2(DummyTemplate()); // expected-note {{in instantiation}} @@ -36,15 +40,20 @@ struct ShadowMe { } }; +#if __cplusplus >= 201102L template constexpr T outer() { - return []() { return x; }.template operator()<123>(); // expected-error {{no matching member function}} \ - expected-note {{candidate template ignored}} + // FIXME: The C++11 error seems wrong + return []() { return x; }.template operator()<123>(); // expected-error {{no matching member function}} \ + expected-note {{candidate template ignored}} \ + cxx11-note {{non-literal type '' cannot be used in a constant expression}} \ + cxx14-note {{non-literal type}} } -static_assert(outer() == 123); +static_assert(outer() == 123); // cxx11-cxx14-error {{not an integral constant expression}} cxx11-cxx14-note {{in call}} template int *outer(); // expected-note {{in instantiation}} +#endif - +#if __cplusplus >= 202002L namespace GH62611 { template struct C { @@ -87,3 +96,4 @@ void foo() { } } +#endif diff --git a/clang/test/SemaCXX/lambda-expressions.cpp b/clang/test/SemaCXX/lambda-expressions.cpp index 389002ab0e34..151d74f21d64 100644 --- a/clang/test/SemaCXX/lambda-expressions.cpp +++ b/clang/test/SemaCXX/lambda-expressions.cpp @@ -1,6 +1,7 @@ -// RUN: %clang_cc1 -std=c++11 -Wno-unused-value -fsyntax-only -verify=expected,expected-cxx14,cxx11 -fblocks %s -// RUN: %clang_cc1 -std=c++14 -Wno-unused-value -fsyntax-only -verify -verify=expected-cxx14 -fblocks %s -// RUN: %clang_cc1 -std=c++17 -Wno-unused-value -verify -ast-dump -fblocks %s | FileCheck %s +// RUN: %clang_cc1 -std=c++11 -Wno-unused-value -fsyntax-only -verify=expected,not-cxx03,cxx03-cxx11,cxx11,expected-cxx14 -fblocks %s +// RUN: %clang_cc1 -std=c++03 -Wno-unused-value -fsyntax-only -verify=expected,cxx03,cxx03-cxx11,expected-cxx14 -fblocks %s -Ddecltype=__decltype -Dstatic_assert=_Static_assert -Wno-c++11-extensions +// RUN: %clang_cc1 -std=c++14 -Wno-unused-value -fsyntax-only -verify=expected,not-cxx03,expected-cxx14 -fblocks %s +// RUN: %clang_cc1 -std=c++17 -Wno-unused-value -verify=expected,not-cxx03 -ast-dump -fblocks %s | FileCheck %s namespace std { class type_info; }; @@ -93,14 +94,14 @@ namespace ImplicitCapture { [] { return ref_i; }; // expected-error {{variable 'ref_i' cannot be implicitly captured in a lambda with no capture-default specified}} expected-note {{lambda expression begins here}} expected-note 2 {{capture 'ref_i' by}} expected-note 2 {{default capture by}} static int j; - int &ref_j = j; - [] { return ref_j; }; // ok + int &ref_j = j; // cxx03-note {{declared here}} + [] { return ref_j; }; // cxx03-error {{variable 'ref_j' cannot be implicitly captured in a lambda with no capture-default specified}} cxx03-note 4 {{capture}} cxx03-note {{lambda expression begins here}} } } namespace SpecialMembers { void f() { - auto a = []{}; // expected-note 2{{here}} expected-note 2{{candidate}} + auto a = []{}; // expected-note 2{{here}} expected-note {{candidate}} not-cxx03-note {{candidate}} decltype(a) b; // expected-error {{no matching constructor}} decltype(a) c = a; decltype(a) d = static_cast(a); @@ -213,7 +214,7 @@ namespace VariadicPackExpansion { }; template void local_class() { - sink { + sink s( [] (Ts t) { struct S : Ts { void f(Ts t) { @@ -226,7 +227,7 @@ namespace VariadicPackExpansion { s.f(t); return s; } (Ts()).g() ... - }; + ); }; struct X {}; struct Y {}; template void local_class(); @@ -296,7 +297,7 @@ namespace PR16708 { namespace TypeDeduction { struct S {}; void f() { - const S s {}; + const S s = S(); S &&t = [&] { return s; } (); #if __cplusplus > 201103L S &&u = [&] () -> auto { return s; } (); @@ -308,7 +309,7 @@ namespace TypeDeduction { namespace lambdas_in_NSDMIs { template struct L { - T t{}; + T t = T(); T t2 = ([](int a) { return [](int b) { return b; };})(t)(t); }; L l; @@ -345,6 +346,7 @@ namespace CaptureIncomplete { } } +#if __cplusplus >= 201103L namespace CaptureAbstract { struct S { virtual void f() = 0; // expected-note {{unimplemented}} @@ -362,6 +364,7 @@ namespace CaptureAbstract { [=] { return s.n; }; // expected-error {{abstract}} } } +#endif namespace PR18128 { auto l = [=]{}; // expected-error {{non-local lambda expression cannot have a capture-default}} @@ -372,6 +375,8 @@ namespace PR18128 { // expected-error@-1 {{non-local lambda expression cannot have a capture-default}} // expected-error@-2 {{invalid use of non-static data member 'n'}} // expected-cxx14-error@-3 {{a lambda expression may not appear inside of a constant expression}} + // cxx03-error@-4 {{function declaration cannot have variably modified type}} + // cxx03-warning@-5 {{variable length arrays in C++ are a Clang extension}} int g(int k = ([=]{ return n; }(), 0)); // expected-error@-1 {{non-local lambda expression cannot have a capture-default}} // expected-error@-2 {{invalid use of non-static data member 'n'}} @@ -434,13 +439,13 @@ struct A { template void g(F f) { - auto a = A{}; + auto a = A(); // expected-note@-1 {{in instantiation of template class 'PR20731::A' requested here}} auto xf = [a, f]() {}; int x = sizeof(xf); }; void f() { - g([] {}); + g([] {}); // cxx03-warning {{template argument uses local type}} // expected-note-re@-1 {{in instantiation of function template specialization 'PR20731::g<(lambda at {{.*}}>' requested here}} } @@ -491,8 +496,8 @@ namespace PR21857 { fun() = default; using Fn::operator(); }; - template fun wrap(Fn fn); - auto x = wrap([](){}); + template fun wrap(Fn fn); // cxx03-warning {{template argument uses unnamed type}} + auto x = wrap([](){}); // cxx03-warning {{template argument uses unnamed type}} cxx03-note 2 {{unnamed type used in template argument was declared here}} } namespace PR13987 { @@ -559,8 +564,8 @@ struct B { int x; A a = [&] { int y = x; }; A b = [&] { [&] { [&] { int y = x; }; }; }; - A d = [&](auto param) { int y = x; }; // cxx11-error {{'auto' not allowed in lambda parameter}} - A e = [&](auto param) { [&] { [&](auto param2) { int y = x; }; }; }; // cxx11-error 2 {{'auto' not allowed in lambda parameter}} + A d = [&](auto param) { int y = x; }; // cxx03-cxx11-error {{'auto' not allowed in lambda parameter}} + A e = [&](auto param) { [&] { [&](auto param2) { int y = x; }; }; }; // cxx03-cxx11-error 2 {{'auto' not allowed in lambda parameter}} }; B b; @@ -588,9 +593,9 @@ struct S1 { }; void foo1() { - auto s0 = S1{[name=]() {}}; // expected-error 2 {{expected expression}} - auto s1 = S1{[name=name]() {}}; // expected-error {{use of undeclared identifier 'name'; did you mean 'name1'?}} - // cxx11-warning@-1 {{initialized lambda captures are a C++14 extension}} + auto s0 = S1([name=]() {}); // expected-error {{expected expression}} + auto s1 = S1([name=name]() {}); // expected-error {{use of undeclared identifier 'name'; did you mean 'name1'?}} + // cxx03-cxx11-warning@-1 {{initialized lambda captures are a C++14 extension}} } } @@ -606,7 +611,7 @@ namespace PR25627_dont_odr_use_local_consts { namespace ConversionOperatorDoesNotHaveDeducedReturnType { auto x = [](int){}; - auto y = [](auto &v) -> void { v.n = 0; }; // cxx11-error {{'auto' not allowed in lambda parameter}} cxx11-note {{candidate function not viable}} cxx11-note {{conversion candidate}} + auto y = [](auto &v) -> void { v.n = 0; }; // cxx03-cxx11-error {{'auto' not allowed in lambda parameter}} cxx03-cxx11-note {{candidate function not viable}} cxx03-cxx11-note {{conversion candidate}} using T = decltype(x); using U = decltype(y); using ExpectedTypeT = void (*)(int); @@ -626,14 +631,15 @@ namespace ConversionOperatorDoesNotHaveDeducedReturnType { template friend constexpr U::operator ExpectedTypeU() const noexcept; #else - friend auto T::operator()(int) const; // cxx11-error {{'auto' return without trailing return type; deduced return types are a C++14 extension}} + friend auto T::operator()(int) const; // cxx11-error {{'auto' return without trailing return type; deduced return types are a C++14 extension}} \ + cxx03-error {{'auto' not allowed in function return type}} friend T::operator ExpectedTypeT() const; template - friend void U::operator()(T&) const; // cxx11-error {{friend declaration of 'operator()' does not match any declaration}} + friend void U::operator()(T&) const; // cxx03-cxx11-error {{friend declaration of 'operator()' does not match any declaration}} // FIXME: This should not match, as above. template - friend U::operator ExpectedTypeU() const; // cxx11-error {{friend declaration of 'operator void (*)(type-parameter-0-0 &)' does not match any declaration}} + friend U::operator ExpectedTypeU() const; // cxx03-cxx11-error {{friend declaration of 'operator void (*)(type-parameter-0-0 &)' does not match any declaration}} #endif private: @@ -641,7 +647,7 @@ namespace ConversionOperatorDoesNotHaveDeducedReturnType { }; // Should be OK in C++14 and later: lambda's call operator is a friend. - void use(X &x) { y(x); } // cxx11-error {{no matching function for call to object}} + void use(X &x) { y(x); } // cxx03-cxx11-error {{no matching function for call to object}} // This used to crash in return type deduction for the conversion opreator. struct A { int n; void f() { +[](decltype(n)) {}; } }; @@ -682,8 +688,8 @@ namespace GH60518 { // function parameters that are used in enable_if struct StringLiteral { template -StringLiteral(const char (&array)[N]) - __attribute__((enable_if(__builtin_strlen(array) == 2, +StringLiteral(const char (&array)[N]) // cxx03-note {{declared here}} + __attribute__((enable_if(__builtin_strlen(array) == 2, // cxx03-error {{'enable_if' attribute expression never produces a constant expression}} cxx03-note {{read of variable}} "invalid string literal"))); }; @@ -695,7 +701,7 @@ StringLiteral(const char (&array)[N]) [[clang::annotate_type("test", array)]]; } void Func1() { - [[maybe_unused]] auto y = [&](decltype(StringLiteral("xx"))) {}; + [[maybe_unused]] auto y = [&](decltype(StringLiteral("xx"))) {}; // cxx03-note {{in instantiation of function template specialization}} [[maybe_unused]] auto z = [&](decltype(cpp_attribute::StringLiteral("xx"))) {}; } @@ -718,6 +724,7 @@ static_assert([]() constexpr { // Call operator attributes refering to a variable should // be properly handled after D124351 +#if __cplusplus >= 201103L constexpr int i = 2; void foo() { (void)[=][[gnu::aligned(i)]] () {}; // expected-warning{{C++23 extension}} @@ -725,15 +732,18 @@ void foo() { // CHECK-NEXT: ConstantExpr // CHECK-NEXT: value: Int 2 } +#endif void GH48527() { auto a = []()__attribute__((b(({ return 0; })))){}; // expected-warning {{unknown attribute 'b' ignored}} } +#if __cplusplus >= 201103L void GH67492() { constexpr auto test = 42; auto lambda = (test, []() noexcept(true) {}); } +#endif // FIXME: This currently causes clang to crash in C++11 mode. #if __cplusplus >= 201402L diff --git a/clang/test/SemaCXX/lambda-implicit-this-capture.cpp b/clang/test/SemaCXX/lambda-implicit-this-capture.cpp index 7e0e347a8fee..eb1f9e880aec 100644 --- a/clang/test/SemaCXX/lambda-implicit-this-capture.cpp +++ b/clang/test/SemaCXX/lambda-implicit-this-capture.cpp @@ -1,3 +1,4 @@ +// RUN: %clang_cc1 -std=c++03 -verify=cxx11 %s -Wno-c++11-extensions // RUN: %clang_cc1 -std=c++11 -verify=cxx11 %s // RUN: %clang_cc1 -std=c++2a -verify=cxx2a %s // RUN: %clang_cc1 -std=c++2a -verify=cxx2a-no-deprecated %s -Wno-deprecated diff --git a/clang/test/SemaCXX/lambda-invalid-capture.cpp b/clang/test/SemaCXX/lambda-invalid-capture.cpp index 236753871d70..5be8c8c5078f 100644 --- a/clang/test/SemaCXX/lambda-invalid-capture.cpp +++ b/clang/test/SemaCXX/lambda-invalid-capture.cpp @@ -1,3 +1,4 @@ +// RUN: %clang_cc1 -fsyntax-only -verify -std=c++03 -Wno-c++11-extensions %s // RUN: %clang_cc1 -fsyntax-only -verify %s // Don't crash. diff --git a/clang/test/SemaCXX/new-delete.cpp b/clang/test/SemaCXX/new-delete.cpp index 4f78b7c71a91..1a99c6aac604 100644 --- a/clang/test/SemaCXX/new-delete.cpp +++ b/clang/test/SemaCXX/new-delete.cpp @@ -171,12 +171,7 @@ void good_deletes() void bad_deletes() { delete 0; // expected-error {{cannot delete expression of type 'int'}} - delete [0] (int*)0; -#if __cplusplus <= 199711L - // expected-error@-2 {{expected expression}} -#else - // expected-error@-4 {{expected variable name or 'this' in lambda capture list}} -#endif + delete [0] (int*)0; // expected-error {{expected variable name or 'this' in lambda capture list}} delete (void*)0; // expected-warning {{cannot delete expression with pointer-to-'void' type 'void *'}} delete (T*)0; // expected-warning {{deleting pointer to incomplete type}} ::S::delete (int*)0; // expected-error {{expected unqualified-id}} -- GitLab From 4946cc37f4865b89fbebcfa0120183a11ae8d4ab Mon Sep 17 00:00:00 2001 From: Ilia Kuklin Date: Thu, 21 Mar 2024 17:05:35 +0500 Subject: [PATCH 132/296] [llvm-objcopy] Add --skip-symbol and --skip-symbols options (#80873) Add --skip-symbol and --skip-symbols options that allow to skip symbols when executing other options that can change the symbol's name, binding or visibility, similar to an existing option --keep-symbol that keeps a symbol from being removed by other options. --- llvm/docs/CommandGuide/llvm-objcopy.rst | 13 +++ llvm/docs/ReleaseNotes.rst | 4 + llvm/include/llvm/ObjCopy/CommonConfig.h | 1 + llvm/lib/ObjCopy/ConfigManager.cpp | 8 +- llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp | 3 + .../tools/llvm-objcopy/ELF/skip-symbol.test | 100 ++++++++++++++++++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp | 9 ++ llvm/tools/llvm-objcopy/ObjcopyOpts.td | 14 +++ 8 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 llvm/test/tools/llvm-objcopy/ELF/skip-symbol.test diff --git a/llvm/docs/CommandGuide/llvm-objcopy.rst b/llvm/docs/CommandGuide/llvm-objcopy.rst index 9d0cb7ad1195..985d16eb11cf 100644 --- a/llvm/docs/CommandGuide/llvm-objcopy.rst +++ b/llvm/docs/CommandGuide/llvm-objcopy.rst @@ -464,6 +464,19 @@ them. Read a list of symbols from and change their visibility to the specified value. Visibility values: default, internal, hidden, protected. +.. option:: --skip-symbol + + Do not change the parameters of symbol ```` when executing other + options that can change the symbol's name, binding or visibility. + +.. option:: --skip-symbols + + Do not change the parameters of symbols named in the file ```` when + executing other options that can change the symbol's name, binding or + visibility. In the file, each line represents a single symbol, with leading + and trailing whitespace ignored, as is anything following a '#'. + Can be specified multiple times to read names from multiple files. + .. option:: --split-dwo Equivalent to running :program:`llvm-objcopy` with :option:`--extract-dwo` and diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst index 03691efe836f..01ecbdba5060 100644 --- a/llvm/docs/ReleaseNotes.rst +++ b/llvm/docs/ReleaseNotes.rst @@ -175,6 +175,10 @@ Changes to the LLVM tools ``--set-symbols-visibility`` options for ELF input to change the visibility of symbols. +* llvm-objcopy now supports ``--skip-symbol`` and ``--skip-symbols`` options + for ELF input to skip the specified symbols when executing other options + that can change a symbol's name, binding or visibility. + Changes to LLDB --------------------------------- diff --git a/llvm/include/llvm/ObjCopy/CommonConfig.h b/llvm/include/llvm/ObjCopy/CommonConfig.h index 8f69c9fbeaf5..9d6d5fb23b18 100644 --- a/llvm/include/llvm/ObjCopy/CommonConfig.h +++ b/llvm/include/llvm/ObjCopy/CommonConfig.h @@ -233,6 +233,7 @@ struct CommonConfig { NameMatcher UnneededSymbolsToRemove; NameMatcher SymbolsToWeaken; NameMatcher SymbolsToKeepGlobal; + NameMatcher SymbolsToSkip; // Map options StringMap SectionsToRename; diff --git a/llvm/lib/ObjCopy/ConfigManager.cpp b/llvm/lib/ObjCopy/ConfigManager.cpp index e46b595a56dc..6442f1b958fb 100644 --- a/llvm/lib/ObjCopy/ConfigManager.cpp +++ b/llvm/lib/ObjCopy/ConfigManager.cpp @@ -15,7 +15,7 @@ namespace objcopy { Expected ConfigManager::getCOFFConfig() const { if (!Common.SplitDWO.empty() || !Common.SymbolsPrefix.empty() || - !Common.SymbolsPrefixRemove.empty() || + !Common.SymbolsPrefixRemove.empty() || !Common.SymbolsToSkip.empty() || !Common.AllocSectionsPrefix.empty() || !Common.KeepSection.empty() || !Common.SymbolsToGlobalize.empty() || !Common.SymbolsToKeep.empty() || !Common.SymbolsToLocalize.empty() || !Common.SymbolsToWeaken.empty() || @@ -34,7 +34,7 @@ Expected ConfigManager::getCOFFConfig() const { Expected ConfigManager::getMachOConfig() const { if (!Common.SplitDWO.empty() || !Common.SymbolsPrefix.empty() || - !Common.SymbolsPrefixRemove.empty() || + !Common.SymbolsPrefixRemove.empty() || !Common.SymbolsToSkip.empty() || !Common.AllocSectionsPrefix.empty() || !Common.KeepSection.empty() || !Common.SymbolsToGlobalize.empty() || !Common.SymbolsToKeep.empty() || !Common.SymbolsToLocalize.empty() || @@ -56,7 +56,7 @@ Expected ConfigManager::getMachOConfig() const { Expected ConfigManager::getWasmConfig() const { if (!Common.AddGnuDebugLink.empty() || Common.ExtractPartition || !Common.SplitDWO.empty() || !Common.SymbolsPrefix.empty() || - !Common.SymbolsPrefixRemove.empty() || + !Common.SymbolsPrefixRemove.empty() || !Common.SymbolsToSkip.empty() || !Common.AllocSectionsPrefix.empty() || Common.DiscardMode != DiscardType::None || !Common.SymbolsToAdd.empty() || !Common.SymbolsToGlobalize.empty() || !Common.SymbolsToLocalize.empty() || @@ -77,7 +77,7 @@ Expected ConfigManager::getWasmConfig() const { Expected ConfigManager::getXCOFFConfig() const { if (!Common.AddGnuDebugLink.empty() || Common.ExtractPartition || !Common.SplitDWO.empty() || !Common.SymbolsPrefix.empty() || - !Common.SymbolsPrefixRemove.empty() || + !Common.SymbolsPrefixRemove.empty() || !Common.SymbolsToSkip.empty() || !Common.AllocSectionsPrefix.empty() || Common.DiscardMode != DiscardType::None || !Common.AddSection.empty() || !Common.DumpSection.empty() || !Common.SymbolsToAdd.empty() || diff --git a/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp b/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp index e4d6e02f3aa6..205bc1ef5b1a 100644 --- a/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp +++ b/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp @@ -291,6 +291,9 @@ static Error updateAndRemoveSymbols(const CommonConfig &Config, return Error::success(); Obj.SymbolTable->updateSymbols([&](Symbol &Sym) { + if (Config.SymbolsToSkip.matches(Sym.Name)) + return; + // Common and undefined symbols don't make sense as local symbols, and can // even cause crashes if we localize those, so skip them. if (!Sym.isCommon() && Sym.getShndx() != SHN_UNDEF && diff --git a/llvm/test/tools/llvm-objcopy/ELF/skip-symbol.test b/llvm/test/tools/llvm-objcopy/ELF/skip-symbol.test new file mode 100644 index 000000000000..0f3ab808482b --- /dev/null +++ b/llvm/test/tools/llvm-objcopy/ELF/skip-symbol.test @@ -0,0 +1,100 @@ +## This test checks the functionality of options --skip-symbol and --skip-symbols. +# RUN: yaml2obj %s -o %t.o +# RUN: echo 'foo[2-3]' > %t.skip.regex + +## Check --skip-symbol functionality when changing symbol bindings. +# RUN: llvm-objcopy %t.o %t2.o --localize-hidden --skip-symbol=foo3 +# RUN: llvm-readelf -s %t2.o | FileCheck %s --check-prefix=LH-SYM +# LH-SYM-DAG: LOCAL HIDDEN 1 foo1 +# LH-SYM-DAG: LOCAL HIDDEN 1 foo2 +# LH-SYM-DAG: GLOBAL HIDDEN 1 foo3 +# LH-SYM-DAG: LOCAL HIDDEN 1 foo4 +# LH-SYM-DAG: LOCAL HIDDEN 1 foo5 + +## Check --skip-symbols functionality when changing symbol bindings. +# RUN: llvm-objcopy %t.o %t1.o --localize-hidden --skip-symbols=%t.skip.regex --regex +# RUN: llvm-readelf -s %t1.o | FileCheck %s --check-prefix=LH-SYMS +# LH-SYMS-DAG: LOCAL HIDDEN 1 foo1 +# LH-SYMS-DAG: GLOBAL HIDDEN 1 foo2 +# LH-SYMS-DAG: GLOBAL HIDDEN 1 foo3 +# LH-SYMS-DAG: LOCAL HIDDEN 1 foo4 +# LH-SYMS-DAG: LOCAL HIDDEN 1 foo5 + +## Check --skip-symbol functionality when changing symbol names. +# RUN: echo -e "foo1 bar1\nfoo2 bar2" > %t.renames.list +# RUN: llvm-objcopy %t.o %t4.o --redefine-syms=%t.renames.list \ +# RUN: --skip-symbol='fo*' --wildcard +# RUN: llvm-readelf -s %t4.o | FileCheck %s --check-prefix=RS-SYM +# RS-SYM-DAG: foo1 +# RS-SYM-DAG: foo2 +# RS-SYM-DAG: foo3 +# RS-SYM-DAG: foo4 +# RS-SYM-DAG: foo5 + +## Check --skip-symbols functionality when changing symbol names. +# RUN: llvm-objcopy %t.o %t3.o --redefine-syms=%t.renames.list \ +# RUN: --skip-symbols=%t.skip.regex --regex +# RUN: llvm-readelf -s %t3.o | FileCheck %s --check-prefix=RS-SYMS +# RS-SYMS-DAG: bar1 +# RS-SYMS-DAG: foo2 +# RS-SYMS-DAG: foo3 +# RS-SYMS-DAG: foo4 +# RS-SYMS-DAG: foo5 + +## Check the functionality when using skip options multiple times. +# RUN: echo "foo3" > %t.symbol0.list +# RUN: echo "foo4" > %t.symbol1.list +# RUN: llvm-objcopy %t.o %t5.o --set-symbol-visibility='foo*'=internal --wildcard \ +# RUN: --skip-symbol=foo1 --skip-symbol=foo2 \ +# RUN: --skip-symbols=%t.symbol0.list --skip-symbols=%t.symbol1.list +# RUN: llvm-readelf -s %t5.o | FileCheck %s --check-prefix=BOTH +# BOTH-DAG: GLOBAL HIDDEN 1 foo1 +# BOTH-DAG: GLOBAL HIDDEN 1 foo2 +# BOTH-DAG: GLOBAL HIDDEN 1 foo3 +# BOTH-DAG: GLOBAL HIDDEN 1 foo4 +## Only foo5 is not skipped. +# BOTH-DAG: GLOBAL INTERNAL 1 foo5 + +## Check that using an invalid symbol name regex generates an error. +# RUN: echo '*.' > %t.symbols.regex +# RUN: not llvm-objcopy %t.o --skip-symbols=%t.symbols.regex --regex 2>&1 | \ +# RUN: FileCheck %s --check-prefix=SYMBOL +# RUN: not llvm-objcopy %t.o --skip-symbol='*.' --regex 2>&1 | \ +# RUN: FileCheck %s --check-prefix=SYMBOL +# SYMBOL: error: cannot compile regular expression '*.': repetition-operator operand invalid + +## Check passing an invalid filename generates an error. +# RUN: not llvm-objcopy %t.o --skip-symbols=no_file 2>&1 | \ +# RUN: FileCheck %s --check-prefix=FILE -DMSG=%errc_ENOENT +# FILE: error: 'no_file': [[MSG]] + +!ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_REL + Machine: EM_X86_64 +Sections: + - Name: .text + Type: SHT_PROGBITS +Symbols: + - Name: foo1 + Section: .text + Binding: STB_GLOBAL + Other: [ STV_HIDDEN ] + - Name: foo2 + Section: .text + Binding: STB_GLOBAL + Other: [ STV_HIDDEN ] + - Name: foo3 + Section: .text + Binding: STB_GLOBAL + Other: [ STV_HIDDEN ] + - Name: foo4 + Section: .text + Binding: STB_GLOBAL + Other: [ STV_HIDDEN ] + - Name: foo5 + Section: .text + Binding: STB_GLOBAL + Other: [ STV_HIDDEN ] diff --git a/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp b/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp index a0c6415bf0e6..7269c51a08d6 100644 --- a/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp +++ b/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp @@ -978,6 +978,15 @@ objcopy::parseObjcopyOptions(ArrayRef RawArgsArr, addSymbolsFromFile(Config.SymbolsToKeep, DC.Alloc, Arg->getValue(), SymbolMatchStyle, ErrorCallback)) return std::move(E); + for (auto *Arg : InputArgs.filtered(OBJCOPY_skip_symbol)) + if (Error E = Config.SymbolsToSkip.addMatcher(NameOrPattern::create( + Arg->getValue(), SymbolMatchStyle, ErrorCallback))) + return std::move(E); + for (auto *Arg : InputArgs.filtered(OBJCOPY_skip_symbols)) + if (Error E = + addSymbolsFromFile(Config.SymbolsToSkip, DC.Alloc, Arg->getValue(), + SymbolMatchStyle, ErrorCallback)) + return std::move(E); for (auto *Arg : InputArgs.filtered(OBJCOPY_add_symbol)) { Expected SymInfo = parseNewSymbolInfo(Arg->getValue()); if (!SymInfo) diff --git a/llvm/tools/llvm-objcopy/ObjcopyOpts.td b/llvm/tools/llvm-objcopy/ObjcopyOpts.td index 3c0e5cd475a3..be02616e8c68 100644 --- a/llvm/tools/llvm-objcopy/ObjcopyOpts.td +++ b/llvm/tools/llvm-objcopy/ObjcopyOpts.td @@ -206,6 +206,20 @@ defm keep_symbols "be repeated to read symbols from many files">, MetaVarName<"filename">; +defm skip_symbol : Eq<"skip-symbol", "Do not change parameters of symbol " + "when executing other options that can change the symbol's " + "name, binding or visibility">, + MetaVarName<"symbol">; + +defm skip_symbols + : Eq<"skip-symbols", + "Read a list of symbols from and run as if " + "--skip-symbol= is set for each one. " + "contains one symbol per line and may contain comments beginning with " + "'#'. Leading and trailing whitespace is stripped from each line. May " + "be repeated to read symbols from many files">, + MetaVarName<"filename">; + defm dump_section : Eq<"dump-section", "Dump contents of section named
into file ">, -- GitLab From 02cb89b36a7ae9be4ab657306b69dc9d2830d0d5 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Thu, 21 Mar 2024 20:09:49 +0800 Subject: [PATCH 133/296] [NewPM] Handle error in TargetPassRegistry.inc (#86112) Mistakenly believing that checking Expected is sufficient. --- llvm/include/llvm/Passes/TargetPassRegistry.inc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llvm/include/llvm/Passes/TargetPassRegistry.inc b/llvm/include/llvm/Passes/TargetPassRegistry.inc index 50766a99f6a7..b618331c6998 100644 --- a/llvm/include/llvm/Passes/TargetPassRegistry.inc +++ b/llvm/include/llvm/Passes/TargetPassRegistry.inc @@ -80,8 +80,10 @@ if (PopulateClassToPassNames) { #define ADD_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \ if (PassBuilder::checkParametrizedPassName(Name, NAME)) { \ auto Params = PassBuilder::parsePassParameters(PARSER, Name, NAME); \ - if (!Params) \ + if (!Params) { \ + errs() << NAME ": " << toString(Params.takeError()) << '\n'; \ return false; \ + } \ PM.addPass(CREATE_PASS(Params.get())); \ return true; \ } -- GitLab From 734026347cca85cf0e242ef5f04896f55e0ac113 Mon Sep 17 00:00:00 2001 From: Sergio Afonso Date: Thu, 21 Mar 2024 12:25:48 +0000 Subject: [PATCH 134/296] Reapply "[Flang][OpenMP][Lower] NFC: Move clause processing helpers into the ClauseProcessor (#85258)" (#85807) This patch contains slight modifications to the reverted PR #85258 to avoid issues with constructs containing multiple reduction clauses, uncovered by a test on the gfortran testsuite. This reverts commit 9f80444c2e669237a5c92013f1a42b91b5609012. --- flang/lib/Lower/OpenMP/ClauseProcessor.cpp | 59 ++++++++++++-- flang/lib/Lower/OpenMP/ClauseProcessor.h | 15 ++-- flang/lib/Lower/OpenMP/OpenMP.cpp | 74 +++-------------- flang/lib/Lower/OpenMP/Utils.cpp | 19 +++++ flang/lib/Lower/OpenMP/Utils.h | 3 + .../Lower/OpenMP/wsloop-reduction-multi.f90 | 81 +++++++++++++++++++ 6 files changed, 171 insertions(+), 80 deletions(-) create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-multi.f90 diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp index 95faa0767e36..52c3479b1ea9 100644 --- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp +++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp @@ -208,6 +208,25 @@ addUseDeviceClause(Fortran::lower::AbstractConverter &converter, useDeviceSymbols.push_back(object.id()); } +static void convertLoopBounds(Fortran::lower::AbstractConverter &converter, + mlir::Location loc, + llvm::SmallVectorImpl &lowerBound, + llvm::SmallVectorImpl &upperBound, + llvm::SmallVectorImpl &step, + std::size_t loopVarTypeSize) { + fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); + // The types of lower bound, upper bound, and step are converted into the + // type of the loop variable if necessary. + mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize); + for (unsigned it = 0; it < (unsigned)lowerBound.size(); it++) { + lowerBound[it] = + firOpBuilder.createConvert(loc, loopVarType, lowerBound[it]); + upperBound[it] = + firOpBuilder.createConvert(loc, loopVarType, upperBound[it]); + step[it] = firOpBuilder.createConvert(loc, loopVarType, step[it]); + } +} + //===----------------------------------------------------------------------===// // ClauseProcessor unique clauses //===----------------------------------------------------------------------===// @@ -217,8 +236,7 @@ bool ClauseProcessor::processCollapse( llvm::SmallVectorImpl &lowerBound, llvm::SmallVectorImpl &upperBound, llvm::SmallVectorImpl &step, - llvm::SmallVectorImpl &iv, - std::size_t &loopVarTypeSize) const { + llvm::SmallVectorImpl &iv) const { bool found = false; fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); @@ -236,7 +254,7 @@ bool ClauseProcessor::processCollapse( found = true; } - loopVarTypeSize = 0; + std::size_t loopVarTypeSize = 0; do { Fortran::lower::pft::Evaluation *doLoop = &doConstructEval->getFirstNestedEvaluation(); @@ -267,6 +285,9 @@ bool ClauseProcessor::processCollapse( &*std::next(doConstructEval->getNestedEvaluations().begin()); } while (collapseValue > 0); + convertLoopBounds(converter, currentLocation, lowerBound, upperBound, step, + loopVarTypeSize); + return found; } @@ -902,17 +923,39 @@ bool ClauseProcessor::processMap( bool ClauseProcessor::processReduction( mlir::Location currentLocation, - llvm::SmallVectorImpl &reductionVars, - llvm::SmallVectorImpl &reductionDeclSymbols, - llvm::SmallVectorImpl *reductionSymbols) - const { + llvm::SmallVectorImpl &outReductionVars, + llvm::SmallVectorImpl &outReductionTypes, + llvm::SmallVectorImpl &outReductionDeclSymbols, + llvm::SmallVectorImpl + *outReductionSymbols) const { return findRepeatableClause( [&](const omp::clause::Reduction &clause, const Fortran::parser::CharBlock &) { + // Use local lists of reductions to prevent variables from other + // already-processed reduction clauses from impacting this reduction. + // For example, the whole `reductionVars` array is queried to decide + // whether to do the reduction byref. + llvm::SmallVector reductionVars; + llvm::SmallVector reductionDeclSymbols; + llvm::SmallVector reductionSymbols; ReductionProcessor rp; rp.addDeclareReduction(currentLocation, converter, clause, reductionVars, reductionDeclSymbols, - reductionSymbols); + outReductionSymbols ? &reductionSymbols + : nullptr); + + // Copy local lists into the output. + llvm::copy(reductionVars, std::back_inserter(outReductionVars)); + llvm::copy(reductionDeclSymbols, + std::back_inserter(outReductionDeclSymbols)); + if (outReductionSymbols) + llvm::copy(reductionSymbols, + std::back_inserter(*outReductionSymbols)); + + outReductionTypes.reserve(outReductionTypes.size() + + reductionVars.size()); + llvm::transform(reductionVars, std::back_inserter(outReductionTypes), + [](mlir::Value v) { return v.getType(); }); }); } diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h index ffa8a5e05593..1b76eb97e823 100644 --- a/flang/lib/Lower/OpenMP/ClauseProcessor.h +++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h @@ -56,14 +56,12 @@ public: clauses(makeList(clauses, semaCtx)) {} // 'Unique' clauses: They can appear at most once in the clause list. - bool - processCollapse(mlir::Location currentLocation, - Fortran::lower::pft::Evaluation &eval, - llvm::SmallVectorImpl &lowerBound, - llvm::SmallVectorImpl &upperBound, - llvm::SmallVectorImpl &step, - llvm::SmallVectorImpl &iv, - std::size_t &loopVarTypeSize) const; + bool processCollapse( + mlir::Location currentLocation, Fortran::lower::pft::Evaluation &eval, + llvm::SmallVectorImpl &lowerBound, + llvm::SmallVectorImpl &upperBound, + llvm::SmallVectorImpl &step, + llvm::SmallVectorImpl &iv) const; bool processDefault() const; bool processDevice(Fortran::lower::StatementContext &stmtCtx, mlir::Value &result) const; @@ -126,6 +124,7 @@ public: bool processReduction(mlir::Location currentLocation, llvm::SmallVectorImpl &reductionVars, + llvm::SmallVectorImpl &reductionTypes, llvm::SmallVectorImpl &reductionDeclSymbols, llvm::SmallVectorImpl *reductionSymbols = nullptr) const; diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp index d335129565b4..160ada379a08 100644 --- a/flang/lib/Lower/OpenMP/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP/OpenMP.cpp @@ -214,24 +214,6 @@ static void threadPrivatizeVars(Fortran::lower::AbstractConverter &converter, firOpBuilder.restoreInsertionPoint(insPt); } -static mlir::Type getLoopVarType(Fortran::lower::AbstractConverter &converter, - std::size_t loopVarTypeSize) { - // OpenMP runtime requires 32-bit or 64-bit loop variables. - loopVarTypeSize = loopVarTypeSize * 8; - if (loopVarTypeSize < 32) { - loopVarTypeSize = 32; - } else if (loopVarTypeSize > 64) { - loopVarTypeSize = 64; - mlir::emitWarning(converter.getCurrentLocation(), - "OpenMP loop iteration variable cannot have more than 64 " - "bits size and will be narrowed into 64 bits."); - } - assert((loopVarTypeSize == 32 || loopVarTypeSize == 64) && - "OpenMP loop iteration variable size must be transformed into 32-bit " - "or 64-bit"); - return converter.getFirOpBuilder().getIntegerType(loopVarTypeSize); -} - static mlir::Operation * createAndSetPrivatizedLoopVar(Fortran::lower::AbstractConverter &converter, mlir::Location loc, mlir::Value indexVal, @@ -568,6 +550,7 @@ genParallelOp(Fortran::lower::AbstractConverter &converter, mlir::omp::ClauseProcBindKindAttr procBindKindAttr; llvm::SmallVector allocateOperands, allocatorOperands, reductionVars; + llvm::SmallVector reductionTypes; llvm::SmallVector reductionDeclSymbols; llvm::SmallVector reductionSymbols; @@ -578,13 +561,8 @@ genParallelOp(Fortran::lower::AbstractConverter &converter, cp.processDefault(); cp.processAllocate(allocatorOperands, allocateOperands); if (!outerCombined) - cp.processReduction(currentLocation, reductionVars, reductionDeclSymbols, - &reductionSymbols); - - llvm::SmallVector reductionTypes; - reductionTypes.reserve(reductionVars.size()); - llvm::transform(reductionVars, std::back_inserter(reductionTypes), - [](mlir::Value v) { return v.getType(); }); + cp.processReduction(currentLocation, reductionVars, reductionTypes, + reductionDeclSymbols, &reductionSymbols); auto reductionCallback = [&](mlir::Operation *op) { llvm::SmallVector locs(reductionVars.size(), @@ -1465,25 +1443,6 @@ genOMP(Fortran::lower::AbstractConverter &converter, standaloneConstruct.u); } -static void convertLoopBounds(Fortran::lower::AbstractConverter &converter, - mlir::Location loc, - llvm::SmallVectorImpl &lowerBound, - llvm::SmallVectorImpl &upperBound, - llvm::SmallVectorImpl &step, - std::size_t loopVarTypeSize) { - fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); - // The types of lower bound, upper bound, and step are converted into the - // type of the loop variable if necessary. - mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize); - for (unsigned it = 0; it < (unsigned)lowerBound.size(); it++) { - lowerBound[it] = - firOpBuilder.createConvert(loc, loopVarType, lowerBound[it]); - upperBound[it] = - firOpBuilder.createConvert(loc, loopVarType, upperBound[it]); - step[it] = firOpBuilder.createConvert(loc, loopVarType, step[it]); - } -} - static llvm::SmallVector genLoopVars(mlir::Operation *op, Fortran::lower::AbstractConverter &converter, mlir::Location &loc, @@ -1517,7 +1476,7 @@ genLoopAndReductionVars( mlir::Location &loc, llvm::ArrayRef loopArgs, llvm::ArrayRef reductionArgs, - llvm::SmallVectorImpl &reductionTypes) { + llvm::ArrayRef reductionTypes) { fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); llvm::SmallVector blockArgTypes; @@ -1579,16 +1538,15 @@ createSimdLoop(Fortran::lower::AbstractConverter &converter, llvm::SmallVector lowerBound, upperBound, step, reductionVars; llvm::SmallVector alignedVars, nontemporalVars; llvm::SmallVector iv; + llvm::SmallVector reductionTypes; llvm::SmallVector reductionDeclSymbols; mlir::omp::ClauseOrderKindAttr orderClauseOperand; mlir::IntegerAttr simdlenClauseOperand, safelenClauseOperand; - std::size_t loopVarTypeSize; ClauseProcessor cp(converter, semaCtx, loopOpClauseList); - cp.processCollapse(loc, eval, lowerBound, upperBound, step, iv, - loopVarTypeSize); + cp.processCollapse(loc, eval, lowerBound, upperBound, step, iv); cp.processScheduleChunk(stmtCtx, scheduleChunkClauseOperand); - cp.processReduction(loc, reductionVars, reductionDeclSymbols); + cp.processReduction(loc, reductionVars, reductionTypes, reductionDeclSymbols); cp.processIf(clause::If::DirectiveNameModifier::Simd, ifClauseOperand); cp.processSimdlen(simdlenClauseOperand); cp.processSafelen(safelenClauseOperand); @@ -1598,9 +1556,6 @@ createSimdLoop(Fortran::lower::AbstractConverter &converter, Fortran::parser::OmpClause::Nontemporal, Fortran::parser::OmpClause::Order>(loc, ompDirective); - convertLoopBounds(converter, loc, lowerBound, upperBound, step, - loopVarTypeSize); - mlir::TypeRange resultType; auto simdLoopOp = firOpBuilder.create( loc, resultType, lowerBound, upperBound, step, alignedVars, @@ -1638,6 +1593,7 @@ static void createWsloop(Fortran::lower::AbstractConverter &converter, llvm::SmallVector lowerBound, upperBound, step, reductionVars; llvm::SmallVector linearVars, linearStepVars; llvm::SmallVector iv; + llvm::SmallVector reductionTypes; llvm::SmallVector reductionDeclSymbols; llvm::SmallVector reductionSymbols; mlir::omp::ClauseOrderKindAttr orderClauseOperand; @@ -1645,20 +1601,15 @@ static void createWsloop(Fortran::lower::AbstractConverter &converter, mlir::UnitAttr nowaitClauseOperand, byrefOperand, scheduleSimdClauseOperand; mlir::IntegerAttr orderedClauseOperand; mlir::omp::ScheduleModifierAttr scheduleModClauseOperand; - std::size_t loopVarTypeSize; ClauseProcessor cp(converter, semaCtx, beginClauseList); - cp.processCollapse(loc, eval, lowerBound, upperBound, step, iv, - loopVarTypeSize); + cp.processCollapse(loc, eval, lowerBound, upperBound, step, iv); cp.processScheduleChunk(stmtCtx, scheduleChunkClauseOperand); - cp.processReduction(loc, reductionVars, reductionDeclSymbols, + cp.processReduction(loc, reductionVars, reductionTypes, reductionDeclSymbols, &reductionSymbols); cp.processTODO(loc, ompDirective); - convertLoopBounds(converter, loc, lowerBound, upperBound, step, - loopVarTypeSize); - if (ReductionProcessor::doReductionByRef(reductionVars)) byrefOperand = firOpBuilder.getUnitAttr(); @@ -1699,11 +1650,6 @@ static void createWsloop(Fortran::lower::AbstractConverter &converter, auto *nestedEval = getCollapsedLoopEval( eval, Fortran::lower::getCollapseValue(beginClauseList)); - llvm::SmallVector reductionTypes; - reductionTypes.reserve(reductionVars.size()); - llvm::transform(reductionVars, std::back_inserter(reductionTypes), - [](mlir::Value v) { return v.getType(); }); - auto ivCallback = [&](mlir::Operation *op) { return genLoopAndReductionVars(op, converter, loc, iv, reductionSymbols, reductionTypes); diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp index fa4a51e33848..b9c0660aa4da 100644 --- a/flang/lib/Lower/OpenMP/Utils.cpp +++ b/flang/lib/Lower/OpenMP/Utils.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -70,6 +71,24 @@ void genObjectList2(const Fortran::parser::OmpObjectList &objectList, } } +mlir::Type getLoopVarType(Fortran::lower::AbstractConverter &converter, + std::size_t loopVarTypeSize) { + // OpenMP runtime requires 32-bit or 64-bit loop variables. + loopVarTypeSize = loopVarTypeSize * 8; + if (loopVarTypeSize < 32) { + loopVarTypeSize = 32; + } else if (loopVarTypeSize > 64) { + loopVarTypeSize = 64; + mlir::emitWarning(converter.getCurrentLocation(), + "OpenMP loop iteration variable cannot have more than 64 " + "bits size and will be narrowed into 64 bits."); + } + assert((loopVarTypeSize == 32 || loopVarTypeSize == 64) && + "OpenMP loop iteration variable size must be transformed into 32-bit " + "or 64-bit"); + return converter.getFirOpBuilder().getIntegerType(loopVarTypeSize); +} + void gatherFuncAndVarSyms( const ObjectList &objects, mlir::omp::DeclareTargetCaptureClause clause, llvm::SmallVectorImpl &symbolAndClause) { diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h index 3ab0823a4621..4074bf73987d 100644 --- a/flang/lib/Lower/OpenMP/Utils.h +++ b/flang/lib/Lower/OpenMP/Utils.h @@ -51,6 +51,9 @@ createMapInfoOp(fir::FirOpBuilder &builder, mlir::Location loc, mlir::omp::VariableCaptureKind mapCaptureType, mlir::Type retTy, bool isVal = false); +mlir::Type getLoopVarType(Fortran::lower::AbstractConverter &converter, + std::size_t loopVarTypeSize); + void gatherFuncAndVarSyms( const ObjectList &objects, mlir::omp::DeclareTargetCaptureClause clause, llvm::SmallVectorImpl &symbolAndClause); diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-multi.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-multi.f90 new file mode 100644 index 000000000000..9e9951c399c9 --- /dev/null +++ b/flang/test/Lower/OpenMP/wsloop-reduction-multi.f90 @@ -0,0 +1,81 @@ +! RUN: bbc -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s +! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s + +!CHECK-LABEL: omp.declare_reduction +!CHECK-SAME: @[[MIN_RED_I32_NAME:.*]] : i32 init { +!CHECK: ^bb0(%{{.*}}: i32): +!CHECK: %[[C0_1:.*]] = arith.constant 2147483647 : i32 +!CHECK: omp.yield(%[[C0_1]] : i32) +!CHECK: } combiner { +!CHECK: ^bb0(%[[ARG0:.*]]: i32, %[[ARG1:.*]]: i32): +!CHECK: %[[RES:.*]] = arith.minsi %[[ARG0]], %[[ARG1]] : i32 +!CHECK: omp.yield(%[[RES]] : i32) +!CHECK: } + +!CHECK-LABEL: omp.declare_reduction +!CHECK-SAME: @[[ADD_RED_F32_NAME:.*]] : f32 init { +!CHECK: ^bb0(%{{.*}}: f32): +!CHECK: %[[C0_1:.*]] = arith.constant 0.000000e+00 : f32 +!CHECK: omp.yield(%[[C0_1]] : f32) +!CHECK: } combiner { +!CHECK: ^bb0(%[[ARG0:.*]]: f32, %[[ARG1:.*]]: f32): +!CHECK: %[[RES:.*]] = arith.addf %[[ARG0]], %[[ARG1]] {{.*}} : f32 +!CHECK: omp.yield(%[[RES]] : f32) +!CHECK: } + +!CHECK-LABEL: omp.declare_reduction +!CHECK-SAME: @[[ADD_RED_I32_NAME:.*]] : i32 init { +!CHECK: ^bb0(%{{.*}}: i32): +!CHECK: %[[C0_1:.*]] = arith.constant 0 : i32 +!CHECK: omp.yield(%[[C0_1]] : i32) +!CHECK: } combiner { +!CHECK: ^bb0(%[[ARG0:.*]]: i32, %[[ARG1:.*]]: i32): +!CHECK: %[[RES:.*]] = arith.addi %[[ARG0]], %[[ARG1]] : i32 +!CHECK: omp.yield(%[[RES]] : i32) +!CHECK: } + +!CHECK-LABEL: func.func @_QPmultiple_reduction +!CHECK: %[[X_REF:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFmultiple_reductionEx"} +!CHECK: %[[X_DECL:.*]]:2 = hlfir.declare %[[X_REF]] {uniq_name = "_QFmultiple_reductionEx"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[Y_REF:.*]] = fir.alloca f32 {bindc_name = "y", uniq_name = "_QFmultiple_reductionEy"} +!CHECK: %[[Y_DECL:.*]]:2 = hlfir.declare %[[Y_REF]] {uniq_name = "_QFmultiple_reductionEy"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[Z_REF:.*]] = fir.alloca i32 {bindc_name = "z", uniq_name = "_QFmultiple_reductionEz"} +!CHECK: %[[Z_DECL:.*]]:2 = hlfir.declare %[[Z_REF]] {uniq_name = "_QFmultiple_reductionEz"} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: omp.wsloop reduction( +!CHECK-SAME: @[[ADD_RED_I32_NAME]] %[[X_DECL]]#0 -> %[[PRV_X:.+]] : !fir.ref, +!CHECK-SAME: @[[ADD_RED_F32_NAME]] %[[Y_DECL]]#0 -> %[[PRV_Y:.+]] : !fir.ref, +!CHECK-SAME: @[[MIN_RED_I32_NAME]] %[[Z_DECL]]#0 -> %[[PRV_Z:.+]] : !fir.ref) {{.*}}{ +!CHECK: %[[PRV_X_DECL:.+]]:2 = hlfir.declare %[[PRV_X]] {{.*}} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[PRV_Y_DECL:.+]]:2 = hlfir.declare %[[PRV_Y]] {{.*}} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[PRV_Z_DECL:.+]]:2 = hlfir.declare %[[PRV_Z]] {{.*}} : (!fir.ref) -> (!fir.ref, !fir.ref) +!CHECK: %[[LPRV_X:.+]] = fir.load %[[PRV_X_DECL]]#0 : !fir.ref +!CHECK: %[[RES_X:.+]] = arith.addi %[[LPRV_X]], %{{.+}} : i32 +!CHECK: hlfir.assign %[[RES_X]] to %[[PRV_X_DECL]]#0 : i32, !fir.ref +!CHECK: %[[LPRV_Y:.+]] = fir.load %[[PRV_Y_DECL]]#0 : !fir.ref +!CHECK: %[[RES_Y:.+]] = arith.addf %[[LPRV_Y]], %{{.+}} : f32 +!CHECK: hlfir.assign %[[RES_Y]] to %[[PRV_Y_DECL]]#0 : f32, !fir.ref +!CHECK: %[[LPRV_Z:.+]] = fir.load %[[PRV_Z_DECL]]#0 : !fir.ref +!CHECK: %[[RES_Z:.+]] = arith.select %{{.+}}, %[[LPRV_Z]], %{{.+}} : i32 +!CHECK: hlfir.assign %[[RES_Z]] to %[[PRV_Z_DECL]]#0 : i32, !fir.ref +!CHECK: omp.yield +!CHECK: } +!CHECK: return +subroutine multiple_reduction(v) + implicit none + integer, intent(in) :: v(:) + integer :: i + integer :: x + real :: y + integer:: z + x = 0 + y = 0.0 + z = 10 + + !$omp do reduction(+:x,y) reduction(min:z) + do i=1, 100 + x = x + v(i) + y = y + 1.5 * v(i) + z = min(z, v(i)) + end do + !$omp end do +end subroutine -- GitLab From fa6e4338369c787710f1fe682cf6bd62348b9104 Mon Sep 17 00:00:00 2001 From: Spenser Bauman Date: Thu, 21 Mar 2024 09:02:21 -0400 Subject: [PATCH 135/296] [mlir][tosa] Fix assertion failure in tosa-layerwise-constant-fold (#85670) The existing implementation of tosa-layerwise-constant-fold only works for constant values backed by DenseElementsAttr. For constants which hold DenseResourceAttrs, the folder will end up asserting at runtime, as it assumes that the backing data can always be accessed through ElementsAttr::getValues. This change reworks the logic so that types types used to perform folding are based on whether the ElementsAttr can be converted to a range of that particular type. --------- Co-authored-by: Spenser Bauman Co-authored-by: Tina Jung --- .../Dialect/Tosa/Transforms/TosaFolders.cpp | 73 ++++++++++--------- mlir/test/Dialect/Tosa/constant-op-fold.mlir | 17 +++++ 2 files changed, 56 insertions(+), 34 deletions(-) diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaFolders.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaFolders.cpp index 050f8ca3f32a..6575b39fd45a 100644 --- a/mlir/lib/Dialect/Tosa/Transforms/TosaFolders.cpp +++ b/mlir/lib/Dialect/Tosa/Transforms/TosaFolders.cpp @@ -132,14 +132,17 @@ bool constantUnaryOpShouldBeFolded(TosaOp unaryOp, DenseElementsAttr values) { return inputOp.hasOneUse(); } -template -DenseElementsAttr transposeType(ElementsAttr attr, ShapedType inputType, +template +DenseElementsAttr transposeType(const RangeType &data, ShapedType inputType, ShapedType outputType, llvm::ArrayRef permValues) { + using ElementType = std::decay_t; + + assert(inputType.getElementType() == outputType.getElementType()); + if (inputType.getNumElements() == 0) - return DenseElementsAttr::get(outputType, llvm::ArrayRef{}); + return DenseElementsAttr::get(outputType, llvm::ArrayRef{}); - auto attrValues = attr.getValues(); auto inputShape = inputType.getShape(); // The inverted permutation map and strides of the output are used to compute @@ -148,10 +151,11 @@ DenseElementsAttr transposeType(ElementsAttr attr, ShapedType inputType, auto outputStrides = computeStrides(outputType.getShape()); auto invertedPermValues = invertPermutationVector(permValues); - auto initialValue = *std::begin(attrValues); - SmallVector outputValues(inputType.getNumElements(), initialValue); + auto initialValue = *std::begin(data); + SmallVector outputValues(inputType.getNumElements(), + initialValue); - for (const auto &it : llvm::enumerate(attrValues)) { + for (const auto &it : llvm::enumerate(data)) { auto srcLinearIndex = it.index(); uint64_t dstLinearIndex = 0; @@ -170,7 +174,7 @@ DenseElementsAttr transposeType(ElementsAttr attr, ShapedType inputType, } return DenseElementsAttr::get(outputType, - llvm::ArrayRef(outputValues)); + llvm::ArrayRef(outputValues)); } // A type specialized transposition of an ElementsAttr. @@ -180,32 +184,28 @@ DenseElementsAttr transposeType(ElementsAttr attr, ShapedType inputType, DenseElementsAttr transpose(ElementsAttr attr, ShapedType inputType, ShapedType outputType, llvm::ArrayRef permValues) { - auto baseType = inputType.getElementType(); - - // Handle possible integer types - if (auto intType = dyn_cast(baseType)) { - switch (intType.getWidth()) { - case 1: - return transposeType(attr, inputType, outputType, permValues); - case 8: - return transposeType(attr, inputType, outputType, permValues); - case 16: - return transposeType(attr, inputType, outputType, permValues); - case 32: - return transposeType(attr, inputType, outputType, permValues); - case 64: - return transposeType(attr, inputType, outputType, permValues); - default: - return transposeType(attr, inputType, outputType, permValues); - } - } + if (auto data = attr.tryGetValues()) + return transposeType(*data, inputType, outputType, permValues); - // Handle possible float types - if (baseType.isF32()) { - return transposeType(attr, inputType, outputType, permValues); - } + if (auto data = attr.tryGetValues()) + return transposeType(*data, inputType, outputType, permValues); + + if (auto data = attr.tryGetValues()) + return transposeType(*data, inputType, outputType, permValues); + + if (auto data = attr.tryGetValues()) + return transposeType(*data, inputType, outputType, permValues); - return transposeType(attr, inputType, outputType, permValues); + if (auto data = attr.tryGetValues()) + return transposeType(*data, inputType, outputType, permValues); + + if (auto data = attr.tryGetValues()) + return transposeType(*data, inputType, outputType, permValues); + + if (auto data = attr.tryGetValues()) + return transposeType(*data, inputType, outputType, permValues); + + return nullptr; } struct TosaFoldConstantTranspose : public OpRewritePattern { @@ -228,14 +228,19 @@ struct TosaFoldConstantTranspose : public OpRewritePattern { DenseIntElementsAttr permAttr; if (!matchPattern(op.getPerms(), m_Constant(&permAttr))) return failure(); - auto permValues = llvm::to_vector<6>(llvm::map_range( + auto permValues = llvm::map_to_vector( // TOSA allows both 32- and 64-bit integer tensors here. permAttr.getValues(), - [](const APInt &val) { return val.getSExtValue(); })); + [](const APInt &val) { return val.getSExtValue(); }); auto inputType = cast(op.getInput1().getType()); auto resultAttr = transpose(inputValues, inputType, outputType, permValues); + if (!resultAttr) { + return rewriter.notifyMatchFailure( + op, "unsupported attribute or element type"); + } + rewriter.replaceOpWithNewOp(op, outputType, resultAttr); return success(); } diff --git a/mlir/test/Dialect/Tosa/constant-op-fold.mlir b/mlir/test/Dialect/Tosa/constant-op-fold.mlir index 27ca3ae3c21b..de752f31fcba 100644 --- a/mlir/test/Dialect/Tosa/constant-op-fold.mlir +++ b/mlir/test/Dialect/Tosa/constant-op-fold.mlir @@ -112,6 +112,23 @@ func.func @transpose_nofold_quantized_types() -> tensor<1x1x2x2x!quant.uniform:f32:3, {1.000000e-01,1.000000e-01}>> } +// CHECK-LABEL: @transpose_nofold_dense_resource +func.func @transpose_nofold_dense_resource() -> tensor<2x2xf32> { + %0 = "tosa.const"() <{value = dense_resource : tensor<2x2xf32>}> : () -> tensor<2x2xf32> + %1 = "tosa.const"() <{value = dense<[1, 0]> : tensor<2xi32>}> : () -> tensor<2xi32> + + // CHECK: tosa.transpose + %2 = tosa.transpose %0, %1 : (tensor<2x2xf32>, tensor<2xi32>) -> tensor<2x2xf32> + return %2 : tensor<2x2xf32> +} +{-# + dialect_resources: { + builtin: { + resource: "0x08000000010000000000000002000000000000000300000000000000" + } + } +#-} + // ----- // CHECK-LABEL: @fold_add_zero_rhs_f32 -- GitLab From 49b520856967c2354339d3c2a05fcf1d2d637f30 Mon Sep 17 00:00:00 2001 From: Paul T Robinson Date: Thu, 21 Mar 2024 06:09:34 -0700 Subject: [PATCH 136/296] [X86][Headers] Specify result of NaN comparisons (#85862) Make sure all float/double comparison intrinsics specify what happens with a NaN input. Update some existing descriptions of comparison results to make them all consistent. Also replace "yields" with "returns" throughout. --- clang/lib/Headers/avxintrin.h | 42 +++++---- clang/lib/Headers/emmintrin.h | 166 ++++++++++++++++++++-------------- clang/lib/Headers/xmmintrin.h | 98 ++++++++++++++------ 3 files changed, 193 insertions(+), 113 deletions(-) diff --git a/clang/lib/Headers/avxintrin.h b/clang/lib/Headers/avxintrin.h index a8882e82e171..be7a0b247e03 100644 --- a/clang/lib/Headers/avxintrin.h +++ b/clang/lib/Headers/avxintrin.h @@ -207,6 +207,8 @@ _mm256_div_ps(__m256 __a, __m256 __b) /// Compares two 256-bit vectors of [4 x double] and returns the greater /// of each pair of values. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMAXPD instruction. @@ -226,6 +228,8 @@ _mm256_max_pd(__m256d __a, __m256d __b) /// Compares two 256-bit vectors of [8 x float] and returns the greater /// of each pair of values. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMAXPS instruction. @@ -245,6 +249,8 @@ _mm256_max_ps(__m256 __a, __m256 __b) /// Compares two 256-bit vectors of [4 x double] and returns the lesser /// of each pair of values. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMINPD instruction. @@ -264,6 +270,8 @@ _mm256_min_pd(__m256d __a, __m256d __b) /// Compares two 256-bit vectors of [8 x float] and returns the lesser /// of each pair of values. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMINPS instruction. @@ -1604,9 +1612,9 @@ _mm256_blendv_ps(__m256 __a, __m256 __b, __m256 __c) /// 128-bit vectors of [2 x double], using the operation specified by the /// immediate integer operand. /// -/// Returns a [2 x double] vector consisting of two doubles corresponding to -/// the two comparison results: zero if the comparison is false, and all 1's -/// if the comparison is true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// @@ -1663,9 +1671,9 @@ _mm256_blendv_ps(__m256 __a, __m256 __b, __m256 __c) /// [4 x float], using the operation specified by the immediate integer /// operand. /// -/// Returns a [4 x float] vector consisting of four floats corresponding to -/// the four comparison results: zero if the comparison is false, and all 1's -/// if the comparison is true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// @@ -1721,9 +1729,9 @@ _mm256_blendv_ps(__m256 __a, __m256 __b, __m256 __c) /// 256-bit vectors of [4 x double], using the operation specified by the /// immediate integer operand. /// -/// Returns a [4 x double] vector consisting of four doubles corresponding to -/// the four comparison results: zero if the comparison is false, and all 1's -/// if the comparison is true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// @@ -1781,9 +1789,9 @@ _mm256_blendv_ps(__m256 __a, __m256 __b, __m256 __c) /// [8 x float], using the operation specified by the immediate integer /// operand. /// -/// Returns a [8 x float] vector consisting of eight floats corresponding to -/// the eight comparison results: zero if the comparison is false, and all -/// 1's if the comparison is true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// @@ -1842,8 +1850,9 @@ _mm256_blendv_ps(__m256 __a, __m256 __b, __m256 __c) /// two 128-bit vectors of [2 x double], using the operation specified by the /// immediate integer operand. /// -/// If the result is true, all 64 bits of the destination vector are set; -/// otherwise they are cleared. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// @@ -1900,8 +1909,9 @@ _mm256_blendv_ps(__m256 __a, __m256 __b, __m256 __c) /// vectors of [4 x float], using the operation specified by the immediate /// integer operand. /// -/// If the result is true, all 32 bits of the destination vector are set; -/// otherwise they are cleared. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// diff --git a/clang/lib/Headers/emmintrin.h b/clang/lib/Headers/emmintrin.h index f0c2db752195..e85bfc47aa5c 100644 --- a/clang/lib/Headers/emmintrin.h +++ b/clang/lib/Headers/emmintrin.h @@ -259,6 +259,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_sqrt_pd(__m128d __a) { /// result. The upper 64 bits of the result are copied from the upper /// double-precision value of the first operand. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMINSD / MINSD instruction. @@ -278,9 +280,11 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_min_sd(__m128d __a, } /// Performs element-by-element comparison of the two 128-bit vectors of -/// [2 x double] and returns the vector containing the lesser of each pair of +/// [2 x double] and returns a vector containing the lesser of each pair of /// values. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMINPD / MINPD instruction. @@ -301,6 +305,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_min_pd(__m128d __a, /// result. The upper 64 bits of the result are copied from the upper /// double-precision value of the first operand. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMAXSD / MAXSD instruction. @@ -320,9 +326,11 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_max_sd(__m128d __a, } /// Performs element-by-element comparison of the two 128-bit vectors of -/// [2 x double] and returns the vector containing the greater of each pair +/// [2 x double] and returns a vector containing the greater of each pair /// of values. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMAXPD / MAXPD instruction. @@ -412,7 +420,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_xor_pd(__m128d __a, /// Compares each of the corresponding double-precision values of the /// 128-bit vectors of [2 x double] for equality. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -432,7 +441,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpeq_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are less than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -452,7 +462,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmplt_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are less than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -472,7 +483,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmple_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -492,7 +504,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpgt_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are greater than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -512,8 +525,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpge_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are ordered with respect to those in the second operand. /// -/// A pair of double-precision values are "ordered" with respect to each -/// other if neither value is a NaN. Each comparison yields 0x0 for false, +/// A pair of double-precision values are ordered with respect to each +/// other if neither value is a NaN. Each comparison returns 0x0 for false, /// 0xFFFFFFFFFFFFFFFF for true. /// /// \headerfile @@ -534,8 +547,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpord_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are unordered with respect to those in the second operand. /// -/// A pair of double-precision values are "unordered" with respect to each -/// other if one or both values are NaN. Each comparison yields 0x0 for +/// A pair of double-precision values are unordered with respect to each +/// other if one or both values are NaN. Each comparison returns 0x0 for /// false, 0xFFFFFFFFFFFFFFFF for true. /// /// \headerfile @@ -557,7 +570,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpunord_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are unequal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -577,7 +591,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpneq_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are not less than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -597,7 +612,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpnlt_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are not less than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -617,7 +633,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpnle_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are not greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -637,7 +654,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpngt_pd(__m128d __a, /// 128-bit vectors of [2 x double] to determine if the values in the first /// operand are not greater than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -656,7 +674,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpnge_pd(__m128d __a, /// Compares the lower double-precision floating-point values in each of /// the two 128-bit floating-point vectors of [2 x double] for equality. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -680,7 +699,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpeq_sd(__m128d __a, /// the value in the first parameter is less than the corresponding value in /// the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -704,7 +724,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmplt_sd(__m128d __a, /// the value in the first parameter is less than or equal to the /// corresponding value in the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -728,7 +749,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmple_sd(__m128d __a, /// the value in the first parameter is greater than the corresponding value /// in the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -753,7 +775,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpgt_sd(__m128d __a, /// the value in the first parameter is greater than or equal to the /// corresponding value in the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -775,11 +798,11 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpge_sd(__m128d __a, /// Compares the lower double-precision floating-point values in each of /// the two 128-bit floating-point vectors of [2 x double] to determine if -/// the value in the first parameter is "ordered" with respect to the +/// the value in the first parameter is ordered with respect to the /// corresponding value in the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. A pair -/// of double-precision values are "ordered" with respect to each other if +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. A pair +/// of double-precision values are ordered with respect to each other if /// neither value is a NaN. /// /// \headerfile @@ -801,11 +824,11 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpord_sd(__m128d __a, /// Compares the lower double-precision floating-point values in each of /// the two 128-bit floating-point vectors of [2 x double] to determine if -/// the value in the first parameter is "unordered" with respect to the +/// the value in the first parameter is unordered with respect to the /// corresponding value in the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. A pair -/// of double-precision values are "unordered" with respect to each other if +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. A pair +/// of double-precision values are unordered with respect to each other if /// one or both values are NaN. /// /// \headerfile @@ -831,7 +854,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpunord_sd(__m128d __a, /// the value in the first parameter is unequal to the corresponding value in /// the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -855,7 +879,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpneq_sd(__m128d __a, /// the value in the first parameter is not less than the corresponding /// value in the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -879,7 +904,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpnlt_sd(__m128d __a, /// the value in the first parameter is not less than or equal to the /// corresponding value in the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -903,7 +929,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpnle_sd(__m128d __a, /// the value in the first parameter is not greater than the corresponding /// value in the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -928,7 +955,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpngt_sd(__m128d __a, /// the value in the first parameter is not greater than or equal to the /// corresponding value in the second parameter. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -951,8 +979,8 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_cmpnge_sd(__m128d __a, /// Compares the lower double-precision floating-point values in each of /// the two 128-bit floating-point vectors of [2 x double] for equality. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -975,8 +1003,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_comieq_sd(__m128d __a, /// the value in the first parameter is less than the corresponding value in /// the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -999,8 +1027,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_comilt_sd(__m128d __a, /// the value in the first parameter is less than or equal to the /// corresponding value in the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1023,8 +1051,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_comile_sd(__m128d __a, /// the value in the first parameter is greater than the corresponding value /// in the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1047,8 +1075,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_comigt_sd(__m128d __a, /// the value in the first parameter is greater than or equal to the /// corresponding value in the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1071,8 +1099,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_comige_sd(__m128d __a, /// the value in the first parameter is unequal to the corresponding value in /// the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, 1 is returned. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 1. /// /// \headerfile /// @@ -1093,8 +1121,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_comineq_sd(__m128d __a, /// Compares the lower double-precision floating-point values in each of /// the two 128-bit floating-point vectors of [2 x double] for equality. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1117,8 +1145,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_ucomieq_sd(__m128d __a, /// the value in the first parameter is less than the corresponding value in /// the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1141,8 +1169,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_ucomilt_sd(__m128d __a, /// the value in the first parameter is less than or equal to the /// corresponding value in the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1165,8 +1193,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_ucomile_sd(__m128d __a, /// the value in the first parameter is greater than the corresponding value /// in the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1189,8 +1217,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_ucomigt_sd(__m128d __a, /// the value in the first parameter is greater than or equal to the /// corresponding value in the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1213,8 +1241,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_ucomige_sd(__m128d __a, /// the value in the first parameter is unequal to the corresponding value in /// the second parameter. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower double-precision values is NaN, 1 is returned. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 1. /// /// \headerfile /// @@ -3033,7 +3061,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_srl_epi64(__m128i __a, /// Compares each of the corresponding 8-bit values of the 128-bit /// integer vectors for equality. /// -/// Each comparison yields 0x0 for false, 0xFF for true. +/// Each comparison returns 0x0 for false, 0xFF for true. /// /// \headerfile /// @@ -3052,7 +3080,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpeq_epi8(__m128i __a, /// Compares each of the corresponding 16-bit values of the 128-bit /// integer vectors for equality. /// -/// Each comparison yields 0x0 for false, 0xFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFF for true. /// /// \headerfile /// @@ -3071,7 +3099,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpeq_epi16(__m128i __a, /// Compares each of the corresponding 32-bit values of the 128-bit /// integer vectors for equality. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// /// \headerfile /// @@ -3091,7 +3119,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpeq_epi32(__m128i __a, /// integer vectors to determine if the values in the first operand are /// greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFF for true. +/// Each comparison returns 0x0 for false, 0xFF for true. /// /// \headerfile /// @@ -3113,7 +3141,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpgt_epi8(__m128i __a, /// 128-bit integer vectors to determine if the values in the first operand /// are greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFF for true. /// /// \headerfile /// @@ -3133,7 +3161,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpgt_epi16(__m128i __a, /// 128-bit integer vectors to determine if the values in the first operand /// are greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// /// \headerfile /// @@ -3153,7 +3181,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpgt_epi32(__m128i __a, /// integer vectors to determine if the values in the first operand are less /// than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFF for true. +/// Each comparison returns 0x0 for false, 0xFF for true. /// /// \headerfile /// @@ -3173,7 +3201,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmplt_epi8(__m128i __a, /// 128-bit integer vectors to determine if the values in the first operand /// are less than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFF for true. /// /// \headerfile /// @@ -3193,7 +3221,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmplt_epi16(__m128i __a, /// 128-bit integer vectors to determine if the values in the first operand /// are less than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// /// \headerfile /// @@ -4777,7 +4805,9 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_castsi128_pd(__m128i __a) { /// 128-bit vectors of [2 x double], using the operation specified by the /// immediate integer operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// @@ -4811,7 +4841,9 @@ static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_castsi128_pd(__m128i __a) { /// two 128-bit vectors of [2 x double], using the operation specified by the /// immediate integer operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// diff --git a/clang/lib/Headers/xmmintrin.h b/clang/lib/Headers/xmmintrin.h index b2c68c3b7be9..040194786a27 100644 --- a/clang/lib/Headers/xmmintrin.h +++ b/clang/lib/Headers/xmmintrin.h @@ -316,6 +316,8 @@ _mm_rsqrt_ps(__m128 __a) /// operands and returns the lesser value in the low-order bits of the /// vector of [4 x float]. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMINSS / MINSS instructions. @@ -338,6 +340,8 @@ _mm_min_ss(__m128 __a, __m128 __b) /// Compares two 128-bit vectors of [4 x float] and returns the lesser /// of each pair of values. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMINPS / MINPS instructions. @@ -358,6 +362,8 @@ _mm_min_ps(__m128 __a, __m128 __b) /// operands and returns the greater value in the low-order bits of a 128-bit /// vector of [4 x float]. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMAXSS / MAXSS instructions. @@ -380,6 +386,8 @@ _mm_max_ss(__m128 __a, __m128 __b) /// Compares two 128-bit vectors of [4 x float] and returns the greater /// of each pair of values. /// +/// If either value in a comparison is NaN, returns the value from \a __b. +/// /// \headerfile /// /// This intrinsic corresponds to the VMAXPS / MAXPS instructions. @@ -478,6 +486,7 @@ _mm_xor_ps(__m128 __a, __m128 __b) /// /// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector [4 x float]. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -501,6 +510,7 @@ _mm_cmpeq_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] for equality. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -523,6 +533,7 @@ _mm_cmpeq_ps(__m128 __a, __m128 __b) /// /// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -547,6 +558,7 @@ _mm_cmplt_ss(__m128 __a, __m128 __b) /// operand are less than those in the second operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -569,6 +581,7 @@ _mm_cmplt_ps(__m128 __a, __m128 __b) /// /// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true, in /// the low-order bits of a vector of [4 x float]. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -593,6 +606,7 @@ _mm_cmple_ss(__m128 __a, __m128 __b) /// operand are less than or equal to those in the second operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -615,6 +629,7 @@ _mm_cmple_ps(__m128 __a, __m128 __b) /// /// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -641,6 +656,7 @@ _mm_cmpgt_ss(__m128 __a, __m128 __b) /// operand are greater than those in the second operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -663,6 +679,7 @@ _mm_cmpgt_ps(__m128 __a, __m128 __b) /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -689,6 +706,7 @@ _mm_cmpge_ss(__m128 __a, __m128 __b) /// operand are greater than or equal to those in the second operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// If either value in a comparison is NaN, returns false. /// /// \headerfile /// @@ -710,6 +728,7 @@ _mm_cmpge_ps(__m128 __a, __m128 __b) /// /// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -734,6 +753,7 @@ _mm_cmpneq_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] for inequality. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -757,6 +777,7 @@ _mm_cmpneq_ps(__m128 __a, __m128 __b) /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -782,6 +803,7 @@ _mm_cmpnlt_ss(__m128 __a, __m128 __b) /// operand are not less than those in the second operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -805,6 +827,7 @@ _mm_cmpnlt_ps(__m128 __a, __m128 __b) /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -830,6 +853,7 @@ _mm_cmpnle_ss(__m128 __a, __m128 __b) /// operand are not less than or equal to those in the second operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -853,6 +877,7 @@ _mm_cmpnle_ps(__m128 __a, __m128 __b) /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -880,6 +905,7 @@ _mm_cmpngt_ss(__m128 __a, __m128 __b) /// operand are not greater than those in the second operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -903,6 +929,7 @@ _mm_cmpngt_ps(__m128 __a, __m128 __b) /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -930,6 +957,7 @@ _mm_cmpnge_ss(__m128 __a, __m128 __b) /// operand are not greater than or equal to those in the second operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, returns true. /// /// \headerfile /// @@ -951,8 +979,9 @@ _mm_cmpnge_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is ordered with /// respect to the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the -/// low-order bits of a vector of [4 x float]. +/// A pair of floating-point values are ordered with respect to each +/// other if neither value is a NaN. Each comparison returns 0x0 for false, +/// 0xFFFFFFFF for true. /// /// \headerfile /// @@ -977,7 +1006,9 @@ _mm_cmpord_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are ordered with respect to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// A pair of floating-point values are ordered with respect to each +/// other if neither value is a NaN. Each comparison returns 0x0 for false, +/// 0xFFFFFFFF for true. /// /// \headerfile /// @@ -999,8 +1030,9 @@ _mm_cmpord_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is unordered /// with respect to the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the -/// low-order bits of a vector of [4 x float]. +/// A pair of double-precision values are unordered with respect to each +/// other if one or both values are NaN. Each comparison returns 0x0 for +/// false, 0xFFFFFFFF for true. /// /// \headerfile /// @@ -1025,7 +1057,9 @@ _mm_cmpunord_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are unordered with respect to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// A pair of double-precision values are unordered with respect to each +/// other if one or both values are NaN. Each comparison returns 0x0 for +/// false, 0xFFFFFFFFFFFFFFFF for true. /// /// \headerfile /// @@ -1046,8 +1080,8 @@ _mm_cmpunord_ps(__m128 __a, __m128 __b) /// Compares two 32-bit float values in the low-order bits of both /// operands for equality. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1071,8 +1105,8 @@ _mm_comieq_ss(__m128 __a, __m128 __b) /// operands to determine if the first operand is less than the second /// operand. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1096,8 +1130,8 @@ _mm_comilt_ss(__m128 __a, __m128 __b) /// operands to determine if the first operand is less than or equal to the /// second operand. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1120,8 +1154,8 @@ _mm_comile_ss(__m128 __a, __m128 __b) /// operands to determine if the first operand is greater than the second /// operand. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1144,8 +1178,8 @@ _mm_comigt_ss(__m128 __a, __m128 __b) /// operands to determine if the first operand is greater than or equal to /// the second operand. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1168,8 +1202,8 @@ _mm_comige_ss(__m128 __a, __m128 __b) /// operands to determine if the first operand is not equal to the second /// operand. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 1. /// /// \headerfile /// @@ -1191,8 +1225,8 @@ _mm_comineq_ss(__m128 __a, __m128 __b) /// Performs an unordered comparison of two 32-bit float values using /// the low-order bits of both operands to determine equality. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1215,8 +1249,8 @@ _mm_ucomieq_ss(__m128 __a, __m128 __b) /// the low-order bits of both operands to determine if the first operand is /// less than the second operand. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1239,8 +1273,8 @@ _mm_ucomilt_ss(__m128 __a, __m128 __b) /// the low-order bits of both operands to determine if the first operand is /// less than or equal to the second operand. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1263,8 +1297,8 @@ _mm_ucomile_ss(__m128 __a, __m128 __b) /// the low-order bits of both operands to determine if the first operand is /// greater than the second operand. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1287,8 +1321,8 @@ _mm_ucomigt_ss(__m128 __a, __m128 __b) /// the low-order bits of both operands to determine if the first operand is /// greater than or equal to the second operand. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -1310,8 +1344,8 @@ _mm_ucomige_ss(__m128 __a, __m128 __b) /// Performs an unordered comparison of two 32-bit float values using /// the low-order bits of both operands to determine inequality. /// -/// The comparison returns 0 for false, 1 for true. If either of the two -/// lower floating-point values is NaN, returns 0. +/// The comparison returns 0 for false, 1 for true. If either value in a +/// comparison is NaN, returns 0. /// /// \headerfile /// @@ -3028,6 +3062,8 @@ _mm_movemask_ps(__m128 __a) /// operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// @@ -3061,6 +3097,8 @@ _mm_movemask_ps(__m128 __a) /// integer operand. /// /// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// If either value in a comparison is NaN, comparisons that are ordered +/// return false, and comparisons that are unordered return true. /// /// \headerfile /// -- GitLab From aa4cbaba1dd3882932ab8772392325242f3c7bee Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Thu, 21 Mar 2024 06:08:40 -0700 Subject: [PATCH 137/296] [SLP][NFC]Add a test with @llvm.abs nodes, which can be analyzed for better bitwidth. --- .../X86/store-abs-minbitwidth.ll | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/store-abs-minbitwidth.ll diff --git a/llvm/test/Transforms/SLPVectorizer/X86/store-abs-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/store-abs-minbitwidth.ll new file mode 100644 index 000000000000..e8b854b7cea6 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/X86/store-abs-minbitwidth.ll @@ -0,0 +1,70 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -S -mtriple=x86_64-unknown -mattr=+avx512vl -passes=slp-vectorizer -slp-threshold=-3 | FileCheck %s + + +define i32 @test(ptr noalias %in, ptr noalias %inn, ptr %out) { +; CHECK-LABEL: @test( +; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i8>, ptr [[IN:%.*]], align 1 +; CHECK-NEXT: [[GEP_2:%.*]] = getelementptr inbounds i8, ptr [[IN]], i64 2 +; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i8>, ptr [[GEP_2]], align 1 +; CHECK-NEXT: [[TMP3:%.*]] = load <2 x i8>, ptr [[INN:%.*]], align 1 +; CHECK-NEXT: [[GEP_5:%.*]] = getelementptr inbounds i8, ptr [[INN]], i64 2 +; CHECK-NEXT: [[TMP4:%.*]] = load <2 x i8>, ptr [[GEP_5]], align 1 +; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i8> [[TMP3]], <2 x i8> poison, <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i8> [[TMP2]], <2 x i8> poison, <4 x i32> +; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i8> [[TMP5]], <4 x i8> [[TMP6]], <4 x i32> +; CHECK-NEXT: [[TMP8:%.*]] = sext <4 x i8> [[TMP7]] to <4 x i32> +; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <4 x i32> +; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x i8> [[TMP4]], <2 x i8> poison, <4 x i32> +; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i8> [[TMP9]], <4 x i8> [[TMP10]], <4 x i32> +; CHECK-NEXT: [[TMP12:%.*]] = sext <4 x i8> [[TMP11]] to <4 x i32> +; CHECK-NEXT: [[TMP13:%.*]] = sub <4 x i32> [[TMP12]], [[TMP8]] +; CHECK-NEXT: [[TMP14:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> [[TMP13]], i1 true) +; CHECK-NEXT: [[TMP15:%.*]] = trunc <4 x i32> [[TMP14]] to <4 x i16> +; CHECK-NEXT: store <4 x i16> [[TMP15]], ptr [[OUT:%.*]], align 2 +; CHECK-NEXT: ret i32 undef +; + %load.1 = load i8, ptr %in, align 1 + %gep.1 = getelementptr inbounds i8, ptr %in, i64 1 + %load.2 = load i8, ptr %gep.1, align 1 + %gep.2 = getelementptr inbounds i8, ptr %in, i64 2 + %load.3 = load i8, ptr %gep.2, align 1 + %gep.3 = getelementptr inbounds i8, ptr %in, i64 3 + %load.4 = load i8, ptr %gep.3, align 1 + %load.5 = load i8, ptr %inn, align 1 + %gep.4 = getelementptr inbounds i8, ptr %inn, i64 1 + %load.6 = load i8, ptr %gep.4, align 1 + %gep.5 = getelementptr inbounds i8, ptr %inn, i64 2 + %load.7 = load i8, ptr %gep.5, align 1 + %gep.6 = getelementptr inbounds i8, ptr %inn, i64 3 + %load.8 = load i8, ptr %gep.6, align 1 + %sext1 = sext i8 %load.1 to i32 + %sext2 = sext i8 %load.2 to i32 + %sext3 = sext i8 %load.3 to i32 + %sext4 = sext i8 %load.4 to i32 + %sext5 = sext i8 %load.5 to i32 + %sext6 = sext i8 %load.6 to i32 + %sext7 = sext i8 %load.7 to i32 + %sext8 = sext i8 %load.8 to i32 + %sub1 = sub i32 %sext1, %sext5 + %sub2 = sub i32 %sext2, %sext6 + %sub3 = sub i32 %sext7, %sext3 + %sub4 = sub i32 %sext8, %sext4 + %call1 = call i32 @llvm.abs(i32 %sub1, i1 true) + %call2 = call i32 @llvm.abs(i32 %sub2, i1 true) + %call3 = call i32 @llvm.abs(i32 %sub3, i1 true) + %call4 = call i32 @llvm.abs(i32 %sub4, i1 true) + %t1 = trunc i32 %call1 to i16 + %t2 = trunc i32 %call2 to i16 + %t3 = trunc i32 %call3 to i16 + %t4 = trunc i32 %call4 to i16 + %gep.8 = getelementptr inbounds i16, ptr %out, i64 1 + %gep.9 = getelementptr inbounds i16, ptr %out, i64 2 + %gep.10 = getelementptr inbounds i16, ptr %out, i64 3 + store i16 %t1, ptr %out, align 2 + store i16 %t2, ptr %gep.8, align 2 + store i16 %t3, ptr %gep.9, align 2 + store i16 %t4, ptr %gep.10, align 2 + + ret i32 undef +} -- GitLab From 538257bf00960f6134a51a17c8477b298ff87c30 Mon Sep 17 00:00:00 2001 From: Christian Sigg Date: Thu, 21 Mar 2024 14:24:55 +0100 Subject: [PATCH 138/296] [mlir][bazel] Update BUILD after 61b24c61a90802e06e40a7ab0aa5e2138486bd73 --- utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index e20aebe95063..201c7f653398 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -3765,6 +3765,7 @@ cc_library( ":DialectUtils", ":IR", ":ShapedOpInterfaces", + ":SideEffectInterfaces", ":ViewLikeInterface", ":XeGPUIncGen", "//llvm:Core", -- GitLab From 0aa6d57e575dd920db81bef7ff509c4d3a9c6891 Mon Sep 17 00:00:00 2001 From: Matthias Gehre Date: Thu, 21 Mar 2024 14:27:37 +0100 Subject: [PATCH 139/296] [MLIR] Add initial convert-memref-to-emitc pass (#85389) This converts `memref.alloca`, `memref.load` & `memref.store` to `emitc.variable`, `emitc.subscript` and `emitc.assign`. --- .../Conversion/MemRefToEmitC/MemRefToEmitC.h | 21 ++++ .../MemRefToEmitC/MemRefToEmitCPass.h | 20 +++ mlir/include/mlir/Conversion/Passes.h | 1 + mlir/include/mlir/Conversion/Passes.td | 9 ++ mlir/lib/Conversion/CMakeLists.txt | 1 + .../Conversion/MemRefToEmitC/CMakeLists.txt | 18 +++ .../MemRefToEmitC/MemRefToEmitC.cpp | 114 ++++++++++++++++++ .../MemRefToEmitC/MemRefToEmitCPass.cpp | 55 +++++++++ .../MemRefToEmitC/memref-to-emitc-failed.mlir | 40 ++++++ .../MemRefToEmitC/memref-to-emitc.mlir | 28 +++++ .../llvm-project-overlay/mlir/BUILD.bazel | 27 +++++ 11 files changed, 334 insertions(+) create mode 100644 mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitC.h create mode 100644 mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitCPass.h create mode 100644 mlir/lib/Conversion/MemRefToEmitC/CMakeLists.txt create mode 100644 mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp create mode 100644 mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitCPass.cpp create mode 100644 mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-failed.mlir create mode 100644 mlir/test/Conversion/MemRefToEmitC/memref-to-emitc.mlir diff --git a/mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitC.h b/mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitC.h new file mode 100644 index 000000000000..734ffdba520c --- /dev/null +++ b/mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitC.h @@ -0,0 +1,21 @@ +//===- MemRefToEmitC.h - Convert MemRef to EmitC --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef MLIR_CONVERSION_MEMREFTOEMITC_MEMREFTOEMITC_H +#define MLIR_CONVERSION_MEMREFTOEMITC_MEMREFTOEMITC_H + +namespace mlir { +class RewritePatternSet; +class TypeConverter; + +void populateMemRefToEmitCTypeConversion(TypeConverter &typeConverter); + +void populateMemRefToEmitCConversionPatterns(RewritePatternSet &patterns, + TypeConverter &converter); +} // namespace mlir + +#endif // MLIR_CONVERSION_MEMREFTOEMITC_MEMREFTOEMITC_H diff --git a/mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitCPass.h b/mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitCPass.h new file mode 100644 index 000000000000..4a63014c19ad --- /dev/null +++ b/mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitCPass.h @@ -0,0 +1,20 @@ +//===- MemRefToEmitCPass.h - A Pass to convert MemRef to EmitC ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef MLIR_CONVERSION_MEMREFTOEMITC_MEMREFTOEMITCPASS_H +#define MLIR_CONVERSION_MEMREFTOEMITC_MEMREFTOEMITCPASS_H + +#include + +namespace mlir { +class Pass; + +#define GEN_PASS_DECL_CONVERTMEMREFTOEMITC +#include "mlir/Conversion/Passes.h.inc" +} // namespace mlir + +#endif // MLIR_CONVERSION_MEMREFTOEMITC_MEMREFTOEMITCPASS_H diff --git a/mlir/include/mlir/Conversion/Passes.h b/mlir/include/mlir/Conversion/Passes.h index f2aa4fb53540..2179ae18ac07 100644 --- a/mlir/include/mlir/Conversion/Passes.h +++ b/mlir/include/mlir/Conversion/Passes.h @@ -45,6 +45,7 @@ #include "mlir/Conversion/MathToLLVM/MathToLLVM.h" #include "mlir/Conversion/MathToLibm/MathToLibm.h" #include "mlir/Conversion/MathToSPIRV/MathToSPIRVPass.h" +#include "mlir/Conversion/MemRefToEmitC/MemRefToEmitCPass.h" #include "mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h" #include "mlir/Conversion/MemRefToSPIRV/MemRefToSPIRVPass.h" #include "mlir/Conversion/NVGPUToNVVM/NVGPUToNVVM.h" diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td index bd81cc6d5323..7e7ee3a2f780 100644 --- a/mlir/include/mlir/Conversion/Passes.td +++ b/mlir/include/mlir/Conversion/Passes.td @@ -753,6 +753,15 @@ def ConvertMathToFuncs : Pass<"convert-math-to-funcs", "ModuleOp"> { ]; } +//===----------------------------------------------------------------------===// +// MemRefToEmitC +//===----------------------------------------------------------------------===// + +def ConvertMemRefToEmitC : Pass<"convert-memref-to-emitc"> { + let summary = "Convert MemRef dialect to EmitC dialect"; + let dependentDialects = ["emitc::EmitCDialect"]; +} + //===----------------------------------------------------------------------===// // MemRefToLLVM //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Conversion/CMakeLists.txt b/mlir/lib/Conversion/CMakeLists.txt index 8219cf98575f..41ab7046b91c 100644 --- a/mlir/lib/Conversion/CMakeLists.txt +++ b/mlir/lib/Conversion/CMakeLists.txt @@ -35,6 +35,7 @@ add_subdirectory(MathToFuncs) add_subdirectory(MathToLibm) add_subdirectory(MathToLLVM) add_subdirectory(MathToSPIRV) +add_subdirectory(MemRefToEmitC) add_subdirectory(MemRefToLLVM) add_subdirectory(MemRefToSPIRV) add_subdirectory(NVGPUToNVVM) diff --git a/mlir/lib/Conversion/MemRefToEmitC/CMakeLists.txt b/mlir/lib/Conversion/MemRefToEmitC/CMakeLists.txt new file mode 100644 index 000000000000..8a72e747d024 --- /dev/null +++ b/mlir/lib/Conversion/MemRefToEmitC/CMakeLists.txt @@ -0,0 +1,18 @@ +add_mlir_conversion_library(MLIRMemRefToEmitC + MemRefToEmitC.cpp + MemRefToEmitCPass.cpp + + ADDITIONAL_HEADER_DIRS + ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/MemRefToEmitC + + DEPENDS + MLIRConversionPassIncGen + + LINK_COMPONENTS + Core + + LINK_LIBS PUBLIC + MLIREmitCDialect + MLIRMemRefDialect + MLIRTransforms + ) diff --git a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp new file mode 100644 index 000000000000..0e3b64692126 --- /dev/null +++ b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp @@ -0,0 +1,114 @@ +//===- MemRefToEmitC.cpp - MemRef to EmitC conversion ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements patterns to convert memref ops into emitc ops. +// +//===----------------------------------------------------------------------===// + +#include "mlir/Conversion/MemRefToEmitC/MemRefToEmitC.h" + +#include "mlir/Dialect/EmitC/IR/EmitC.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/PatternMatch.h" +#include "mlir/Transforms/DialectConversion.h" + +using namespace mlir; + +namespace { +struct ConvertAlloca final : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + + LogicalResult + matchAndRewrite(memref::AllocaOp op, OpAdaptor operands, + ConversionPatternRewriter &rewriter) const override { + + if (!op.getType().hasStaticShape()) { + return rewriter.notifyMatchFailure( + op.getLoc(), "cannot transform alloca with dynamic shape"); + } + + if (op.getAlignment().value_or(1) > 1) { + // TODO: Allow alignment if it is not more than the natural alignment + // of the C array. + return rewriter.notifyMatchFailure( + op.getLoc(), "cannot transform alloca with alignment requirement"); + } + + auto resultTy = getTypeConverter()->convertType(op.getType()); + if (!resultTy) { + return rewriter.notifyMatchFailure(op.getLoc(), "cannot convert type"); + } + auto noInit = emitc::OpaqueAttr::get(getContext(), ""); + rewriter.replaceOpWithNewOp(op, resultTy, noInit); + return success(); + } +}; + +struct ConvertLoad final : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + + LogicalResult + matchAndRewrite(memref::LoadOp op, OpAdaptor operands, + ConversionPatternRewriter &rewriter) const override { + + auto resultTy = getTypeConverter()->convertType(op.getType()); + if (!resultTy) { + return rewriter.notifyMatchFailure(op.getLoc(), "cannot convert type"); + } + + auto subscript = rewriter.create( + op.getLoc(), operands.getMemref(), operands.getIndices()); + + auto noInit = emitc::OpaqueAttr::get(getContext(), ""); + auto var = + rewriter.create(op.getLoc(), resultTy, noInit); + + rewriter.create(op.getLoc(), var, subscript); + rewriter.replaceOp(op, var); + return success(); + } +}; + +struct ConvertStore final : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + + LogicalResult + matchAndRewrite(memref::StoreOp op, OpAdaptor operands, + ConversionPatternRewriter &rewriter) const override { + + auto subscript = rewriter.create( + op.getLoc(), operands.getMemref(), operands.getIndices()); + rewriter.replaceOpWithNewOp(op, subscript, + operands.getValue()); + return success(); + } +}; +} // namespace + +void mlir::populateMemRefToEmitCTypeConversion(TypeConverter &typeConverter) { + typeConverter.addConversion( + [&](MemRefType memRefType) -> std::optional { + if (!memRefType.hasStaticShape() || + !memRefType.getLayout().isIdentity() || memRefType.getRank() == 0) { + return {}; + } + Type convertedElementType = + typeConverter.convertType(memRefType.getElementType()); + if (!convertedElementType) + return {}; + return emitc::ArrayType::get(memRefType.getShape(), + convertedElementType); + }); +} + +void mlir::populateMemRefToEmitCConversionPatterns(RewritePatternSet &patterns, + TypeConverter &converter) { + patterns.add(converter, + patterns.getContext()); +} diff --git a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitCPass.cpp b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitCPass.cpp new file mode 100644 index 000000000000..4e5d1912d157 --- /dev/null +++ b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitCPass.cpp @@ -0,0 +1,55 @@ +//===- MemRefToEmitC.cpp - MemRef to EmitC conversion ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements a pass to convert memref ops into emitc ops. +// +//===----------------------------------------------------------------------===// + +#include "mlir/Conversion/MemRefToEmitC/MemRefToEmitCPass.h" + +#include "mlir/Conversion/MemRefToEmitC/MemRefToEmitC.h" +#include "mlir/Dialect/EmitC/IR/EmitC.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Pass/Pass.h" +#include "mlir/Transforms/DialectConversion.h" + +namespace mlir { +#define GEN_PASS_DEF_CONVERTMEMREFTOEMITC +#include "mlir/Conversion/Passes.h.inc" +} // namespace mlir + +using namespace mlir; + +namespace { +struct ConvertMemRefToEmitCPass + : public impl::ConvertMemRefToEmitCBase { + void runOnOperation() override { + TypeConverter converter; + + // Fallback for other types. + converter.addConversion([](Type type) -> std::optional { + if (isa(type)) + return {}; + return type; + }); + + populateMemRefToEmitCTypeConversion(converter); + + RewritePatternSet patterns(&getContext()); + populateMemRefToEmitCConversionPatterns(patterns, converter); + + ConversionTarget target(getContext()); + target.addIllegalDialect(); + target.addLegalDialect(); + + if (failed(applyPartialConversion(getOperation(), target, + std::move(patterns)))) + return signalPassFailure(); + } +}; +} // namespace diff --git a/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-failed.mlir b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-failed.mlir new file mode 100644 index 000000000000..390190d341e5 --- /dev/null +++ b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-failed.mlir @@ -0,0 +1,40 @@ +// RUN: mlir-opt -convert-memref-to-emitc %s -split-input-file -verify-diagnostics + +func.func @memref_op(%arg0 : memref<2x4xf32>) { + // expected-error@+1 {{failed to legalize operation 'memref.copy'}} + memref.copy %arg0, %arg0 : memref<2x4xf32> to memref<2x4xf32> + return +} + +// ----- + +func.func @alloca_with_dynamic_shape() { + %0 = index.constant 1 + // expected-error@+1 {{failed to legalize operation 'memref.alloca'}} + %1 = memref.alloca(%0) : memref<4x?xf32> + return +} + +// ----- + +func.func @alloca_with_alignment() { + // expected-error@+1 {{failed to legalize operation 'memref.alloca'}} + %0 = memref.alloca() {alignment = 64 : i64}: memref<4xf32> + return +} + +// ----- + +func.func @non_identity_layout() { + // expected-error@+1 {{failed to legalize operation 'memref.alloca'}} + %0 = memref.alloca() : memref<4x3xf32, affine_map<(d0, d1) -> (d1, d0)>> + return +} + +// ----- + +func.func @zero_rank() { + // expected-error@+1 {{failed to legalize operation 'memref.alloca'}} + %0 = memref.alloca() : memref + return +} diff --git a/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc.mlir b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc.mlir new file mode 100644 index 000000000000..9793b2d6d783 --- /dev/null +++ b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc.mlir @@ -0,0 +1,28 @@ +// RUN: mlir-opt -convert-memref-to-emitc %s -split-input-file | FileCheck %s + +// CHECK-LABEL: memref_store +// CHECK-SAME: %[[v:.*]]: f32, %[[i:.*]]: index, %[[j:.*]]: index +func.func @memref_store(%v : f32, %i: index, %j: index) { + // CHECK: %[[ALLOCA:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> !emitc.array<4x8xf32> + %0 = memref.alloca() : memref<4x8xf32> + + // CHECK: %[[SUBSCRIPT:.*]] = emitc.subscript %[[ALLOCA]][%[[i]], %[[j]]] : <4x8xf32> + // CHECK: emitc.assign %[[v]] : f32 to %[[SUBSCRIPT:.*]] : f32 + memref.store %v, %0[%i, %j] : memref<4x8xf32> + return +} +// ----- + +// CHECK-LABEL: memref_load +// CHECK-SAME: %[[i:.*]]: index, %[[j:.*]]: index +func.func @memref_load(%i: index, %j: index) -> f32 { + // CHECK: %[[ALLOCA:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> !emitc.array<4x8xf32> + %0 = memref.alloca() : memref<4x8xf32> + + // CHECK: %[[LOAD:.*]] = emitc.subscript %[[ALLOCA]][%[[i]], %[[j]]] : <4x8xf32> + // CHECK: %[[VAR:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f32 + // CHECK: emitc.assign %[[LOAD]] : f32 to %[[VAR]] : f32 + %1 = memref.load %0[%i, %j] : memref<4x8xf32> + // CHECK: return %[[VAR]] : f32 + return %1 : f32 +} diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 201c7f653398..3b575d4a413c 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -4186,6 +4186,7 @@ cc_library( ":MathToLLVM", ":MathToLibm", ":MathToSPIRV", + ":MemRefToEmitC", ":MemRefToLLVM", ":MemRefToSPIRV", ":NVGPUToNVVM", @@ -8256,6 +8257,32 @@ cc_library( ], ) +cc_library( + name = "MemRefToEmitC", + srcs = glob([ + "lib/Conversion/MemRefToEmitC/*.cpp", + "lib/Conversion/MemRefToEmitC/*.h", + ]), + hdrs = glob([ + "include/mlir/Conversion/MemRefToEmitC/*.h", + ]), + includes = [ + "include", + "lib/Conversion/MemRefToEmitC", + ], + deps = [ + ":ConversionPassIncGen", + ":EmitCDialect", + ":MemRefDialect", + ":IR", + ":Pass", + ":Support", + ":TransformUtils", + ":Transforms", + "//llvm:Support", + ], +) + cc_library( name = "MemRefToLLVM", srcs = glob(["lib/Conversion/MemRefToLLVM/*.cpp"]), -- GitLab From 276283d8641f13b6ecde736ab1a8720f742d9d02 Mon Sep 17 00:00:00 2001 From: chrulski-intel Date: Thu, 21 Mar 2024 06:34:01 -0700 Subject: [PATCH 140/296] [LLD] [MinGW] Implement the -lto-sample-profile option (#85841) This has been a supported option for ELF and is added to the COFF Linker in #85701 --- lld/MinGW/Driver.cpp | 2 ++ lld/MinGW/Options.td | 2 ++ lld/test/MinGW/driver.test | 3 +++ 3 files changed, 7 insertions(+) diff --git a/lld/MinGW/Driver.cpp b/lld/MinGW/Driver.cpp index efd643f9a322..bb08c77b2e11 100644 --- a/lld/MinGW/Driver.cpp +++ b/lld/MinGW/Driver.cpp @@ -455,6 +455,8 @@ bool link(ArrayRef argsArr, llvm::raw_ostream &stdoutOS, add("-lldemit:llvm"); if (args.hasArg(OPT_lto_emit_asm)) add("-lldemit:asm"); + if (auto *arg = args.getLastArg(OPT_lto_sample_profile)) + add("-lto-sample-profile:" + StringRef(arg->getValue())); if (auto *a = args.getLastArg(OPT_thinlto_cache_dir)) add("-lldltocache:" + StringRef(a->getValue())); diff --git a/lld/MinGW/Options.td b/lld/MinGW/Options.td index 9a0a96aac7f1..56f67e3dd96c 100644 --- a/lld/MinGW/Options.td +++ b/lld/MinGW/Options.td @@ -160,6 +160,8 @@ def lto_cs_profile_file: JJ<"lto-cs-profile-file=">, HelpText<"Context sensitive profile file path">; def lto_emit_asm: FF<"lto-emit-asm">, HelpText<"Emit assembly code">; +def lto_sample_profile: JJ<"lto-sample-profile=">, + HelpText<"Sample profile file path">; def thinlto_cache_dir: JJ<"thinlto-cache-dir=">, HelpText<"Path to ThinLTO cached object file directory">; diff --git a/lld/test/MinGW/driver.test b/lld/test/MinGW/driver.test index a4e9e5e1b19b..619fee8dee7c 100644 --- a/lld/test/MinGW/driver.test +++ b/lld/test/MinGW/driver.test @@ -422,6 +422,9 @@ LTO_EMIT_ASM: -lldemit:asm RUN: ld.lld -### foo.o -m i386pe -plugin-opt=emit-llvm 2>&1 | FileCheck -check-prefix=LTO_EMIT_LLVM %s LTO_EMIT_LLVM: -lldemit:llvm +RUN: ld.lld -### foo.o -m i386pep --lto-sample-profile=foo 2>&1 | FileCheck -check-prefix=LTO_SAMPLE_PROFILE %s +LTO_SAMPLE_PROFILE: -lto-sample-profile:foo + Test GCC specific LTO options that GCC passes unconditionally, that we ignore. RUN: ld.lld -### foo.o -m i386pep -plugin /usr/lib/gcc/x86_64-w64-mingw32/10-posix/liblto_plugin.so -plugin-opt=/usr/lib/gcc/x86_64-w64-mingw32/10-posix/lto-wrapper -plugin-opt=-fresolution=/tmp/ccM9d4fP.res -plugin-opt=-pass-through=-lmingw32 2> /dev/null -- GitLab From 5344a370fe85d9119729d9036540bbd91956da38 Mon Sep 17 00:00:00 2001 From: Kirill Chibisov Date: Thu, 21 Mar 2024 17:35:00 +0400 Subject: [PATCH 141/296] [mlir][emitc] Fix form-expressions inside expression (#86081) Make form-expressions not create `emitc.expression`s for operations inside the `emitc.expression`s, since they are invalid. --- .../EmitC/Transforms/FormExpressions.cpp | 3 ++- mlir/test/Dialect/EmitC/transforms.mlir | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/mlir/lib/Dialect/EmitC/Transforms/FormExpressions.cpp b/mlir/lib/Dialect/EmitC/Transforms/FormExpressions.cpp index 5b03f81b305f..e7c431f39e3f 100644 --- a/mlir/lib/Dialect/EmitC/Transforms/FormExpressions.cpp +++ b/mlir/lib/Dialect/EmitC/Transforms/FormExpressions.cpp @@ -36,7 +36,8 @@ struct FormExpressionsPass // Wrap each C operator op with an expression op. OpBuilder builder(context); auto matchFun = [&](Operation *op) { - if (op->hasTrait()) + if (op->hasTrait() && + !op->getParentOfType()) createExpression(op, builder); }; rootOp->walk(matchFun); diff --git a/mlir/test/Dialect/EmitC/transforms.mlir b/mlir/test/Dialect/EmitC/transforms.mlir index ad167fa455a1..8ac606a2c8c0 100644 --- a/mlir/test/Dialect/EmitC/transforms.mlir +++ b/mlir/test/Dialect/EmitC/transforms.mlir @@ -107,3 +107,20 @@ func.func @expression_with_address_taken(%arg0: i32, %arg1: i32, %arg2: !emitc.p %d = emitc.cmp lt, %c, %arg2 :(!emitc.ptr, !emitc.ptr) -> i1 return %d : i1 } + +// CHECK-LABEL: func.func @no_nested_expression( +// CHECK-SAME: %[[VAL_0:.*]]: i32, %[[VAL_1:.*]]: i32) -> i1 { +// CHECK: %[[VAL_2:.*]] = emitc.expression : i1 { +// CHECK: %[[VAL_3:.*]] = emitc.cmp lt, %[[VAL_0]], %[[VAL_1]] : (i32, i32) -> i1 +// CHECK: emitc.yield %[[VAL_3]] : i1 +// CHECK: } +// CHECK: return %[[VAL_2]] : i1 +// CHECK: } + +func.func @no_nested_expression(%arg0: i32, %arg1: i32) -> i1 { + %a = emitc.expression : i1 { + %b = emitc.cmp lt, %arg0, %arg1 :(i32, i32) -> i1 + emitc.yield %b : i1 + } + return %a : i1 +} -- GitLab From 15eba9c12a1486ee600e35ecb83b1f2c8459416e Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 21 Mar 2024 12:48:19 +0000 Subject: [PATCH 142/296] [VectorCombine] Add DataLayout to VectorCombine class instead of repeated calls to getDataLayout(). NFC. --- .../Transforms/Vectorize/VectorCombine.cpp | 37 +++++++++---------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp index 23494314f132..7e86137f23f3 100644 --- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp +++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp @@ -66,8 +66,8 @@ class VectorCombine { public: VectorCombine(Function &F, const TargetTransformInfo &TTI, const DominatorTree &DT, AAResults &AA, AssumptionCache &AC, - bool TryEarlyFoldsOnly) - : F(F), Builder(F.getContext()), TTI(TTI), DT(DT), AA(AA), AC(AC), + const DataLayout *DL, bool TryEarlyFoldsOnly) + : F(F), Builder(F.getContext()), TTI(TTI), DT(DT), AA(AA), AC(AC), DL(DL), TryEarlyFoldsOnly(TryEarlyFoldsOnly) {} bool run(); @@ -79,6 +79,7 @@ private: const DominatorTree &DT; AAResults &AA; AssumptionCache &AC; + const DataLayout *DL; /// If true, only perform beneficial early IR transforms. Do not introduce new /// vector operations. @@ -181,7 +182,6 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) { // We use minimal alignment (maximum flexibility) because we only care about // the dereferenceable region. When calculating cost and creating a new op, // we may use a larger value based on alignment attributes. - const DataLayout &DL = I.getModule()->getDataLayout(); Value *SrcPtr = Load->getPointerOperand()->stripPointerCasts(); assert(isa(SrcPtr->getType()) && "Expected a pointer type"); @@ -189,15 +189,15 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) { auto *MinVecTy = VectorType::get(ScalarTy, MinVecNumElts, false); unsigned OffsetEltIndex = 0; Align Alignment = Load->getAlign(); - if (!isSafeToLoadUnconditionally(SrcPtr, MinVecTy, Align(1), DL, Load, &AC, + if (!isSafeToLoadUnconditionally(SrcPtr, MinVecTy, Align(1), *DL, Load, &AC, &DT)) { // It is not safe to load directly from the pointer, but we can still peek // through gep offsets and check if it safe to load from a base address with // updated alignment. If it is, we can shuffle the element(s) into place // after loading. - unsigned OffsetBitWidth = DL.getIndexTypeSizeInBits(SrcPtr->getType()); + unsigned OffsetBitWidth = DL->getIndexTypeSizeInBits(SrcPtr->getType()); APInt Offset(OffsetBitWidth, 0); - SrcPtr = SrcPtr->stripAndAccumulateInBoundsConstantOffsets(DL, Offset); + SrcPtr = SrcPtr->stripAndAccumulateInBoundsConstantOffsets(*DL, Offset); // We want to shuffle the result down from a high element of a vector, so // the offset must be positive. @@ -215,7 +215,7 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) { if (OffsetEltIndex >= MinVecNumElts) return false; - if (!isSafeToLoadUnconditionally(SrcPtr, MinVecTy, Align(1), DL, Load, &AC, + if (!isSafeToLoadUnconditionally(SrcPtr, MinVecTy, Align(1), *DL, Load, &AC, &DT)) return false; @@ -227,7 +227,7 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) { // Original pattern: insertelt undef, load [free casts of] PtrOp, 0 // Use the greater of the alignment on the load or its source pointer. - Alignment = std::max(SrcPtr->getPointerAlignment(DL), Alignment); + Alignment = std::max(SrcPtr->getPointerAlignment(*DL), Alignment); Type *LoadTy = Load->getType(); unsigned AS = Load->getPointerAddressSpace(); InstructionCost OldCost = @@ -298,14 +298,13 @@ bool VectorCombine::widenSubvectorLoad(Instruction &I) { // the dereferenceable region. When calculating cost and creating a new op, // we may use a larger value based on alignment attributes. auto *Ty = cast(I.getType()); - const DataLayout &DL = I.getModule()->getDataLayout(); Value *SrcPtr = Load->getPointerOperand()->stripPointerCasts(); assert(isa(SrcPtr->getType()) && "Expected a pointer type"); Align Alignment = Load->getAlign(); - if (!isSafeToLoadUnconditionally(SrcPtr, Ty, Align(1), DL, Load, &AC, &DT)) + if (!isSafeToLoadUnconditionally(SrcPtr, Ty, Align(1), *DL, Load, &AC, &DT)) return false; - Alignment = std::max(SrcPtr->getPointerAlignment(DL), Alignment); + Alignment = std::max(SrcPtr->getPointerAlignment(*DL), Alignment); Type *LoadTy = Load->getType(); unsigned AS = Load->getPointerAddressSpace(); @@ -854,7 +853,6 @@ bool VectorCombine::scalarizeVPIntrinsic(Instruction &I) { // Scalarize the intrinsic ElementCount EC = cast(Op0->getType())->getElementCount(); Value *EVL = VPI.getArgOperand(3); - const DataLayout &DL = VPI.getModule()->getDataLayout(); // If the VP op might introduce UB or poison, we can scalarize it provided // that we know the EVL > 0: If the EVL is zero, then the original VP op @@ -867,7 +865,7 @@ bool VectorCombine::scalarizeVPIntrinsic(Instruction &I) { else SafeToSpeculate = isSafeToSpeculativelyExecuteWithOpcode( *FunctionalOpcode, &VPI, nullptr, &AC, &DT); - if (!SafeToSpeculate && !isKnownNonZero(EVL, DL, 0, &AC, &VPI, &DT)) + if (!SafeToSpeculate && !isKnownNonZero(EVL, *DL, 0, &AC, &VPI, &DT)) return false; Value *ScalarVal = @@ -1246,12 +1244,11 @@ bool VectorCombine::foldSingleElementStore(Instruction &I) { if (auto *Load = dyn_cast(Source)) { auto VecTy = cast(SI->getValueOperand()->getType()); - const DataLayout &DL = I.getModule()->getDataLayout(); Value *SrcAddr = Load->getPointerOperand()->stripPointerCasts(); // Don't optimize for atomic/volatile load or store. Ensure memory is not // modified between, vector type matches store size, and index is inbounds. if (!Load->isSimple() || Load->getParent() != SI->getParent() || - !DL.typeSizeEqualsStoreSize(Load->getType()->getScalarType()) || + !DL->typeSizeEqualsStoreSize(Load->getType()->getScalarType()) || SrcAddr != SI->getPointerOperand()->stripPointerCasts()) return false; @@ -1270,7 +1267,7 @@ bool VectorCombine::foldSingleElementStore(Instruction &I) { NSI->copyMetadata(*SI); Align ScalarOpAlignment = computeAlignmentAfterScalarization( std::max(SI->getAlign(), Load->getAlign()), NewElement->getType(), Idx, - DL); + *DL); NSI->setAlignment(ScalarOpAlignment); replaceValue(I, *NSI); eraseInstruction(I); @@ -1288,8 +1285,7 @@ bool VectorCombine::scalarizeLoadExtract(Instruction &I) { auto *VecTy = cast(I.getType()); auto *LI = cast(&I); - const DataLayout &DL = I.getModule()->getDataLayout(); - if (LI->isVolatile() || !DL.typeSizeEqualsStoreSize(VecTy->getScalarType())) + if (LI->isVolatile() || !DL->typeSizeEqualsStoreSize(VecTy->getScalarType())) return false; InstructionCost OriginalCost = @@ -1367,7 +1363,7 @@ bool VectorCombine::scalarizeLoadExtract(Instruction &I) { VecTy->getElementType(), GEP, EI->getName() + ".scalar")); Align ScalarOpAlignment = computeAlignmentAfterScalarization( - LI->getAlign(), VecTy->getElementType(), Idx, DL); + LI->getAlign(), VecTy->getElementType(), Idx, *DL); NewLoad->setAlignment(ScalarOpAlignment); replaceValue(*EI, *NewLoad); @@ -2042,7 +2038,8 @@ PreservedAnalyses VectorCombinePass::run(Function &F, TargetTransformInfo &TTI = FAM.getResult(F); DominatorTree &DT = FAM.getResult(F); AAResults &AA = FAM.getResult(F); - VectorCombine Combiner(F, TTI, DT, AA, AC, TryEarlyFoldsOnly); + const DataLayout *DL = &F.getParent()->getDataLayout(); + VectorCombine Combiner(F, TTI, DT, AA, AC, DL, TryEarlyFoldsOnly); if (!Combiner.run()) return PreservedAnalyses::all(); PreservedAnalyses PA; -- GitLab From 686f4599cfa444aa62db4e22bf752f3d9614c30d Mon Sep 17 00:00:00 2001 From: David Green Date: Thu, 21 Mar 2024 13:45:44 +0000 Subject: [PATCH 143/296] [ARM] Regenerate some check lines. NFC --- llvm/test/CodeGen/ARM/arm-and-tst-peephole.ll | 15 +- llvm/test/CodeGen/ARM/select.ll | 399 ++++++++++++++++-- 2 files changed, 361 insertions(+), 53 deletions(-) diff --git a/llvm/test/CodeGen/ARM/arm-and-tst-peephole.ll b/llvm/test/CodeGen/ARM/arm-and-tst-peephole.ll index 365727c9dd27..0795525fba1b 100644 --- a/llvm/test/CodeGen/ARM/arm-and-tst-peephole.ll +++ b/llvm/test/CodeGen/ARM/arm-and-tst-peephole.ll @@ -8,10 +8,8 @@ %struct.Foo = type { ptr } -; ARM-LABEL: foo: -; THUMB-LABEL: foo: -; T2-LABEL: foo: define ptr @foo(ptr %this, i32 %acc) nounwind readonly align 2 { +; ARM-LABEL: foo: ; ARM: @ %bb.0: @ %entry ; ARM-NEXT: add r2, r0, #4 ; ARM-NEXT: mov r12, #1 @@ -44,6 +42,7 @@ define ptr @foo(ptr %this, i32 %acc) nounwind readonly align 2 { ; ARM-NEXT: add r0, r0, r1, lsl #2 ; ARM-NEXT: mov pc, lr ; +; THUMB-LABEL: foo: ; THUMB: @ %bb.0: @ %entry ; THUMB-NEXT: .save {r4, r5, r7, lr} ; THUMB-NEXT: push {r4, r5, r7, lr} @@ -91,6 +90,7 @@ define ptr @foo(ptr %this, i32 %acc) nounwind readonly align 2 { ; THUMB-NEXT: pop {r0} ; THUMB-NEXT: bx r0 ; +; T2-LABEL: foo: ; T2: @ %bb.0: @ %entry ; T2-NEXT: adds r2, r0, #4 ; T2-NEXT: mov.w r12, #1 @@ -125,6 +125,7 @@ define ptr @foo(ptr %this, i32 %acc) nounwind readonly align 2 { ; T2-NEXT: add.w r0, r0, r1, lsl #2 ; T2-NEXT: bx lr ; +; V8-LABEL: foo: ; V8: @ %bb.0: @ %entry ; V8-NEXT: adds r2, r0, #4 ; V8-NEXT: mov.w r12, #1 @@ -210,11 +211,8 @@ sw.epilog: ; preds = %tailrecurse.switch %struct.S = type { ptr, [1 x i8] } -; ARM-LABEL: bar: -; THUMB-LABEL: bar: -; T2-LABEL: bar: -; V8-LABEL: bar: define internal zeroext i8 @bar(ptr %x, ptr nocapture %y) nounwind readonly { +; ARM-LABEL: bar: ; ARM: @ %bb.0: @ %entry ; ARM-NEXT: ldrb r2, [r0, #4] ; ARM-NEXT: ands r2, r2, #112 @@ -230,6 +228,7 @@ define internal zeroext i8 @bar(ptr %x, ptr nocapture %y) nounwind readonly { ; ARM-NEXT: mov r0, #1 ; ARM-NEXT: mov pc, lr ; +; THUMB-LABEL: bar: ; THUMB: @ %bb.0: @ %entry ; THUMB-NEXT: ldrb r2, [r0, #4] ; THUMB-NEXT: movs r3, #112 @@ -253,6 +252,7 @@ define internal zeroext i8 @bar(ptr %x, ptr nocapture %y) nounwind readonly { ; THUMB-NEXT: ands r0, r1 ; THUMB-NEXT: bx lr ; +; T2-LABEL: bar: ; T2: @ %bb.0: @ %entry ; T2-NEXT: ldrb r2, [r0, #4] ; T2-NEXT: ands r2, r2, #112 @@ -270,6 +270,7 @@ define internal zeroext i8 @bar(ptr %x, ptr nocapture %y) nounwind readonly { ; T2-NEXT: movs r0, #1 ; T2-NEXT: bx lr ; +; V8-LABEL: bar: ; V8: @ %bb.0: @ %entry ; V8-NEXT: ldrb r2, [r0, #4] ; V8-NEXT: ands r2, r2, #112 diff --git a/llvm/test/CodeGen/ARM/select.ll b/llvm/test/CodeGen/ARM/select.ll index 4bb79651f040..24ca9aeac7f2 100644 --- a/llvm/test/CodeGen/ARM/select.ll +++ b/llvm/test/CodeGen/ARM/select.ll @@ -1,14 +1,25 @@ -; RUN: llc -mtriple=arm-apple-darwin %s -o - | FileCheck %s - -; RUN: llc -mtriple=arm-eabi -mattr=+vfp2 %s -o - \ -; RUN: | FileCheck %s --check-prefix=CHECK-VFP - -; RUN: llc -mtriple=thumbv7-apple-darwin -mattr=+neon,+thumb2 %s -o - \ -; RUN: | FileCheck %s --check-prefix=CHECK-NEON +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=armv7-eabi -mattr=-fpregs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-ARM +; RUN: llc -mtriple=armv7-eabi -mattr=+vfp2 %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-VFP +; RUN: llc -mtriple=thumbv7-apple-darwin -mattr=+neon,+thumb2 %s -o - | FileCheck %s --check-prefix=CHECK-NEON define i32 @f1(i32 %a.s) { -;CHECK-LABEL: f1: -;CHECK: moveq +; CHECK-LABEL: f1: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: mov r1, #3 +; CHECK-NEXT: cmp r0, #4 +; CHECK-NEXT: movweq r1, #2 +; CHECK-NEXT: mov r0, r1 +; CHECK-NEXT: bx lr +; +; CHECK-NEON-LABEL: f1: +; CHECK-NEON: @ %bb.0: @ %entry +; CHECK-NEON-NEXT: movs r1, #3 +; CHECK-NEON-NEXT: cmp r0, #4 +; CHECK-NEON-NEXT: it eq +; CHECK-NEON-NEXT: moveq r1, #2 +; CHECK-NEON-NEXT: mov r0, r1 +; CHECK-NEON-NEXT: bx lr entry: %tmp = icmp eq i32 %a.s, 4 %tmp1.s = select i1 %tmp, i32 2, i32 3 @@ -16,8 +27,22 @@ entry: } define i32 @f2(i32 %a.s) { -;CHECK-LABEL: f2: -;CHECK: movgt +; CHECK-LABEL: f2: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: mov r1, #3 +; CHECK-NEXT: cmp r0, #4 +; CHECK-NEXT: movwgt r1, #2 +; CHECK-NEXT: mov r0, r1 +; CHECK-NEXT: bx lr +; +; CHECK-NEON-LABEL: f2: +; CHECK-NEON: @ %bb.0: @ %entry +; CHECK-NEON-NEXT: movs r1, #3 +; CHECK-NEON-NEXT: cmp r0, #4 +; CHECK-NEON-NEXT: it gt +; CHECK-NEON-NEXT: movgt r1, #2 +; CHECK-NEON-NEXT: mov r0, r1 +; CHECK-NEON-NEXT: bx lr entry: %tmp = icmp sgt i32 %a.s, 4 %tmp1.s = select i1 %tmp, i32 2, i32 3 @@ -25,8 +50,22 @@ entry: } define i32 @f3(i32 %a.s, i32 %b.s) { -;CHECK-LABEL: f3: -;CHECK: movlt +; CHECK-LABEL: f3: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: mov r2, #3 +; CHECK-NEXT: cmp r0, r1 +; CHECK-NEXT: movwlt r2, #2 +; CHECK-NEXT: mov r0, r2 +; CHECK-NEXT: bx lr +; +; CHECK-NEON-LABEL: f3: +; CHECK-NEON: @ %bb.0: @ %entry +; CHECK-NEON-NEXT: movs r2, #3 +; CHECK-NEON-NEXT: cmp r0, r1 +; CHECK-NEON-NEXT: it lt +; CHECK-NEON-NEXT: movlt r2, #2 +; CHECK-NEON-NEXT: mov r0, r2 +; CHECK-NEON-NEXT: bx lr entry: %tmp = icmp slt i32 %a.s, %b.s %tmp1.s = select i1 %tmp, i32 2, i32 3 @@ -34,8 +73,22 @@ entry: } define i32 @f4(i32 %a.s, i32 %b.s) { -;CHECK-LABEL: f4: -;CHECK: movle +; CHECK-LABEL: f4: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: mov r2, #3 +; CHECK-NEXT: cmp r0, r1 +; CHECK-NEXT: movwle r2, #2 +; CHECK-NEXT: mov r0, r2 +; CHECK-NEXT: bx lr +; +; CHECK-NEON-LABEL: f4: +; CHECK-NEON: @ %bb.0: @ %entry +; CHECK-NEON-NEXT: movs r2, #3 +; CHECK-NEON-NEXT: cmp r0, r1 +; CHECK-NEON-NEXT: it le +; CHECK-NEON-NEXT: movle r2, #2 +; CHECK-NEON-NEXT: mov r0, r2 +; CHECK-NEON-NEXT: bx lr entry: %tmp = icmp sle i32 %a.s, %b.s %tmp1.s = select i1 %tmp, i32 2, i32 3 @@ -43,8 +96,22 @@ entry: } define i32 @f5(i32 %a.u, i32 %b.u) { -;CHECK-LABEL: f5: -;CHECK: movls +; CHECK-LABEL: f5: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: mov r2, #3 +; CHECK-NEXT: cmp r0, r1 +; CHECK-NEXT: movwls r2, #2 +; CHECK-NEXT: mov r0, r2 +; CHECK-NEXT: bx lr +; +; CHECK-NEON-LABEL: f5: +; CHECK-NEON: @ %bb.0: @ %entry +; CHECK-NEON-NEXT: movs r2, #3 +; CHECK-NEON-NEXT: cmp r0, r1 +; CHECK-NEON-NEXT: it ls +; CHECK-NEON-NEXT: movls r2, #2 +; CHECK-NEON-NEXT: mov r0, r2 +; CHECK-NEON-NEXT: bx lr entry: %tmp = icmp ule i32 %a.u, %b.u %tmp1.s = select i1 %tmp, i32 2, i32 3 @@ -52,8 +119,22 @@ entry: } define i32 @f6(i32 %a.u, i32 %b.u) { -;CHECK-LABEL: f6: -;CHECK: movhi +; CHECK-LABEL: f6: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: mov r2, #3 +; CHECK-NEXT: cmp r0, r1 +; CHECK-NEXT: movwhi r2, #2 +; CHECK-NEXT: mov r0, r2 +; CHECK-NEXT: bx lr +; +; CHECK-NEON-LABEL: f6: +; CHECK-NEON: @ %bb.0: @ %entry +; CHECK-NEON-NEXT: movs r2, #3 +; CHECK-NEON-NEXT: cmp r0, r1 +; CHECK-NEON-NEXT: it hi +; CHECK-NEON-NEXT: movhi r2, #2 +; CHECK-NEON-NEXT: mov r0, r2 +; CHECK-NEON-NEXT: bx lr entry: %tmp = icmp ugt i32 %a.u, %b.u %tmp1.s = select i1 %tmp, i32 2, i32 3 @@ -61,11 +142,61 @@ entry: } define double @f7(double %a, double %b) { -;CHECK-LABEL: f7: -;CHECK: movmi -;CHECK: movpl -;CHECK-VFP-LABEL: f7: -;CHECK-VFP: vmovmi +; CHECK-ARM-LABEL: f7: +; CHECK-ARM: @ %bb.0: +; CHECK-ARM-NEXT: .save {r4, r5, r11, lr} +; CHECK-ARM-NEXT: push {r4, r5, r11, lr} +; CHECK-ARM-NEXT: mov r4, r3 +; CHECK-ARM-NEXT: movw r3, #48758 +; CHECK-ARM-NEXT: mov r5, r2 +; CHECK-ARM-NEXT: movw r2, #14680 +; CHECK-ARM-NEXT: movt r2, #51380 +; CHECK-ARM-NEXT: movt r3, #16371 +; CHECK-ARM-NEXT: bl __aeabi_dcmplt +; CHECK-ARM-NEXT: cmp r0, #0 +; CHECK-ARM-NEXT: movwne r4, #0 +; CHECK-ARM-NEXT: movwne r5, #0 +; CHECK-ARM-NEXT: movtne r4, #49136 +; CHECK-ARM-NEXT: mov r0, r5 +; CHECK-ARM-NEXT: mov r1, r4 +; CHECK-ARM-NEXT: pop {r4, r5, r11, pc} +; +; CHECK-VFP-LABEL: f7: +; CHECK-VFP: @ %bb.0: +; CHECK-VFP-NEXT: vldr d17, .LCPI6_0 +; CHECK-VFP-NEXT: vmov d19, r0, r1 +; CHECK-VFP-NEXT: vmov.f64 d16, #-1.000000e+00 +; CHECK-VFP-NEXT: vcmp.f64 d19, d17 +; CHECK-VFP-NEXT: vmrs APSR_nzcv, fpscr +; CHECK-VFP-NEXT: vmov d18, r2, r3 +; CHECK-VFP-NEXT: vmovmi.f64 d18, d16 +; CHECK-VFP-NEXT: vmov r0, r1, d18 +; CHECK-VFP-NEXT: bx lr +; CHECK-VFP-NEXT: .p2align 3 +; CHECK-VFP-NEXT: @ %bb.1: +; CHECK-VFP-NEXT: .LCPI6_0: +; CHECK-VFP-NEXT: .long 3367254360 @ double 1.234 +; CHECK-VFP-NEXT: .long 1072938614 +; +; CHECK-NEON-LABEL: f7: +; CHECK-NEON: @ %bb.0: +; CHECK-NEON-NEXT: vldr d17, LCPI6_0 +; CHECK-NEON-NEXT: vmov d19, r0, r1 +; CHECK-NEON-NEXT: vmov d18, r2, r3 +; CHECK-NEON-NEXT: vcmp.f64 d19, d17 +; CHECK-NEON-NEXT: vmov.f64 d16, #-1.000000e+00 +; CHECK-NEON-NEXT: vmrs APSR_nzcv, fpscr +; CHECK-NEON-NEXT: it mi +; CHECK-NEON-NEXT: vmovmi.f64 d18, d16 +; CHECK-NEON-NEXT: vmov r0, r1, d18 +; CHECK-NEON-NEXT: bx lr +; CHECK-NEON-NEXT: .p2align 3 +; CHECK-NEON-NEXT: @ %bb.1: +; CHECK-NEON-NEXT: .data_region +; CHECK-NEON-NEXT: LCPI6_0: +; CHECK-NEON-NEXT: .long 3367254360 @ double 1.234 +; CHECK-NEON-NEXT: .long 1072938614 +; CHECK-NEON-NEXT: .end_data_region %tmp = fcmp olt double %a, 1.234e+00 %tmp1 = select i1 %tmp, double -1.000e+00, double %b ret double %tmp1 @@ -77,18 +208,49 @@ define double @f7(double %a, double %b) { ; a lack of a custom lowering routine for an ISD::SELECT. This would result in ; two "it" blocks in the code: one for the "icmp" and another to move the index ; into the constant pool based on the value of the "icmp". If we have one "it" -; block generated, odds are good that we have close to the ideal code for this: +; block generated, odds are good that we have close to the ideal code for this. +define arm_apcscc float @f8(i32 %a) nounwind { +; CHECK-ARM-LABEL: f8: +; CHECK-ARM: @ %bb.0: +; CHECK-ARM-NEXT: movw r1, #29905 +; CHECK-ARM-NEXT: movw r2, #1123 +; CHECK-ARM-NEXT: movt r1, #16408 +; CHECK-ARM-NEXT: cmp r0, r2 +; CHECK-ARM-NEXT: movweq r1, #62390 +; CHECK-ARM-NEXT: movteq r1, #16285 +; CHECK-ARM-NEXT: mov r0, r1 +; CHECK-ARM-NEXT: bx lr +; +; CHECK-VFP-LABEL: f8: +; CHECK-VFP: @ %bb.0: +; CHECK-VFP-NEXT: movw r2, #1123 +; CHECK-VFP-NEXT: adr r1, .LCPI7_0 +; CHECK-VFP-NEXT: cmp r0, r2 +; CHECK-VFP-NEXT: addeq r1, r1, #4 +; CHECK-VFP-NEXT: ldr r0, [r1] +; CHECK-VFP-NEXT: bx lr +; CHECK-VFP-NEXT: .p2align 2 +; CHECK-VFP-NEXT: @ %bb.1: +; CHECK-VFP-NEXT: .LCPI7_0: +; CHECK-VFP-NEXT: .long 0x401874d1 @ float 2.38212991 +; CHECK-VFP-NEXT: .long 0x3f9df3b6 @ float 1.23399997 ; ; CHECK-NEON-LABEL: f8: -; CHECK-NEON: adr [[R2:r[0-9]+]], LCPI7_0 -; CHECK-NEON: movw [[R3:r[0-9]+]], #1123 -; CHECK-NEON-NEXT: cmp r0, [[R3]] -; CHECK-NEON-NEXT: it eq -; CHECK-NEON-NEXT: addeq{{.*}} [[R2]], #4 -; CHECK-NEON-NEXT: ldr -; CHECK-NEON: bx - -define arm_apcscc float @f8(i32 %a) nounwind { +; CHECK-NEON: @ %bb.0: +; CHECK-NEON-NEXT: adr r1, LCPI7_0 +; CHECK-NEON-NEXT: movw r2, #1123 +; CHECK-NEON-NEXT: cmp r0, r2 +; CHECK-NEON-NEXT: it eq +; CHECK-NEON-NEXT: addeq r1, #4 +; CHECK-NEON-NEXT: ldr r0, [r1] +; CHECK-NEON-NEXT: bx lr +; CHECK-NEON-NEXT: .p2align 2 +; CHECK-NEON-NEXT: @ %bb.1: +; CHECK-NEON-NEXT: .data_region +; CHECK-NEON-NEXT: LCPI7_0: +; CHECK-NEON-NEXT: .long 0x401874d1 @ float 2.38212991 +; CHECK-NEON-NEXT: .long 0x3f9df3b6 @ float 1.23399997 +; CHECK-NEON-NEXT: .end_data_region %tmp = icmp eq i32 %a, 1123 %tmp1 = select i1 %tmp, float 0x3FF3BE76C0000000, float 0x40030E9A20000000 ret float %tmp1 @@ -98,10 +260,40 @@ define arm_apcscc float @f8(i32 %a) nounwind { ; Glue values can only have a single use, but the following test exposed a ; case where a SELECT was lowered with 2 uses of a comparison, causing the ; scheduler to assert. -; CHECK-VFP-LABEL: f9: - declare ptr @objc_msgSend(ptr, ptr, ...) define void @f9() optsize { +; CHECK-LABEL: f9: +; CHECK: @ %bb.0: @ %entry +; CHECK-NEXT: .save {r11, lr} +; CHECK-NEXT: push {r11, lr} +; CHECK-NEXT: .pad #8 +; CHECK-NEXT: sub sp, sp, #8 +; CHECK-NEXT: movw r2, #0 +; CHECK-NEXT: movw r3, #0 +; CHECK-NEXT: mov r1, #1065353216 +; CHECK-NEXT: mov r0, #0 +; CHECK-NEXT: movt r2, #16672 +; CHECK-NEXT: movt r3, #32704 +; CHECK-NEXT: strd r0, r1, [sp] +; CHECK-NEXT: bl objc_msgSend +; CHECK-NEXT: add sp, sp, #8 +; CHECK-NEXT: pop {r11, pc} +; +; CHECK-NEON-LABEL: f9: +; CHECK-NEON: @ %bb.0: @ %entry +; CHECK-NEON-NEXT: str lr, [sp, #-4]! +; CHECK-NEON-NEXT: sub sp, #8 +; CHECK-NEON-NEXT: movs r2, #0 +; CHECK-NEON-NEXT: movs r3, #0 +; CHECK-NEON-NEXT: mov.w r0, #1065353216 +; CHECK-NEON-NEXT: movs r1, #0 +; CHECK-NEON-NEXT: movt r2, #16672 +; CHECK-NEON-NEXT: movt r3, #32704 +; CHECK-NEON-NEXT: strd r1, r0, [sp] +; CHECK-NEON-NEXT: bl _objc_msgSend +; CHECK-NEON-NEXT: add sp, #8 +; CHECK-NEON-NEXT: ldr lr, [sp], #4 +; CHECK-NEON-NEXT: bx lr entry: %cmp = icmp eq ptr undef, inttoptr (i32 4 to ptr) %conv191 = select i1 %cmp, float -3.000000e+00, float 0.000000e+00 @@ -117,36 +309,151 @@ entry: ret void } -; CHECK-LABEL: f10: define float @f10(i32 %a, i32 %b) nounwind uwtable readnone ssp { -; CHECK-NOT: floatsisf +; CHECK-ARM-LABEL: f10: +; CHECK-ARM: @ %bb.0: +; CHECK-ARM-NEXT: mov r2, #0 +; CHECK-ARM-NEXT: cmp r0, r1 +; CHECK-ARM-NEXT: moveq r2, #1065353216 +; CHECK-ARM-NEXT: mov r0, r2 +; CHECK-ARM-NEXT: bx lr +; +; CHECK-VFP-LABEL: f10: +; CHECK-VFP: @ %bb.0: +; CHECK-VFP-NEXT: vmov.f32 s2, #1.000000e+00 +; CHECK-VFP-NEXT: vldr s0, .LCPI9_0 +; CHECK-VFP-NEXT: cmp r0, r1 +; CHECK-VFP-NEXT: vmoveq.f32 s0, s2 +; CHECK-VFP-NEXT: vmov r0, s0 +; CHECK-VFP-NEXT: bx lr +; CHECK-VFP-NEXT: .p2align 2 +; CHECK-VFP-NEXT: @ %bb.1: +; CHECK-VFP-NEXT: .LCPI9_0: +; CHECK-VFP-NEXT: .long 0x00000000 @ float 0 +; +; CHECK-NEON-LABEL: f10: +; CHECK-NEON: @ %bb.0: +; CHECK-NEON-NEXT: vldr s0, LCPI9_0 +; CHECK-NEON-NEXT: vmov.f32 s2, #1.000000e+00 +; CHECK-NEON-NEXT: cmp r0, r1 +; CHECK-NEON-NEXT: it eq +; CHECK-NEON-NEXT: vmoveq.f32 s0, s2 +; CHECK-NEON-NEXT: vmov r0, s0 +; CHECK-NEON-NEXT: bx lr +; CHECK-NEON-NEXT: .p2align 2 +; CHECK-NEON-NEXT: @ %bb.1: +; CHECK-NEON-NEXT: .data_region +; CHECK-NEON-NEXT: LCPI9_0: +; CHECK-NEON-NEXT: .long 0x00000000 @ float 0 +; CHECK-NEON-NEXT: .end_data_region %1 = icmp eq i32 %a, %b %2 = zext i1 %1 to i32 %3 = sitofp i32 %2 to float ret float %3 } -; CHECK-LABEL: f11: define float @f11(i32 %a, i32 %b) nounwind uwtable readnone ssp { -; CHECK-NOT: floatsisf +; CHECK-ARM-LABEL: f11: +; CHECK-ARM: @ %bb.0: +; CHECK-ARM-NEXT: mov r2, #0 +; CHECK-ARM-NEXT: cmp r0, r1 +; CHECK-ARM-NEXT: movweq r2, #0 +; CHECK-ARM-NEXT: movteq r2, #49024 +; CHECK-ARM-NEXT: mov r0, r2 +; CHECK-ARM-NEXT: bx lr +; +; CHECK-VFP-LABEL: f11: +; CHECK-VFP: @ %bb.0: +; CHECK-VFP-NEXT: vmov.f32 s2, #-1.000000e+00 +; CHECK-VFP-NEXT: vldr s0, .LCPI10_0 +; CHECK-VFP-NEXT: cmp r0, r1 +; CHECK-VFP-NEXT: vmoveq.f32 s0, s2 +; CHECK-VFP-NEXT: vmov r0, s0 +; CHECK-VFP-NEXT: bx lr +; CHECK-VFP-NEXT: .p2align 2 +; CHECK-VFP-NEXT: @ %bb.1: +; CHECK-VFP-NEXT: .LCPI10_0: +; CHECK-VFP-NEXT: .long 0x00000000 @ float 0 +; +; CHECK-NEON-LABEL: f11: +; CHECK-NEON: @ %bb.0: +; CHECK-NEON-NEXT: vldr s0, LCPI10_0 +; CHECK-NEON-NEXT: vmov.f32 s2, #-1.000000e+00 +; CHECK-NEON-NEXT: cmp r0, r1 +; CHECK-NEON-NEXT: it eq +; CHECK-NEON-NEXT: vmoveq.f32 s0, s2 +; CHECK-NEON-NEXT: vmov r0, s0 +; CHECK-NEON-NEXT: bx lr +; CHECK-NEON-NEXT: .p2align 2 +; CHECK-NEON-NEXT: @ %bb.1: +; CHECK-NEON-NEXT: .data_region +; CHECK-NEON-NEXT: LCPI10_0: +; CHECK-NEON-NEXT: .long 0x00000000 @ float 0 +; CHECK-NEON-NEXT: .end_data_region %1 = icmp eq i32 %a, %b %2 = sitofp i1 %1 to float ret float %2 } -; CHECK-LABEL: f12: define float @f12(i32 %a, i32 %b) nounwind uwtable readnone ssp { -; CHECK-NOT: floatunsisf +; CHECK-ARM-LABEL: f12: +; CHECK-ARM: @ %bb.0: +; CHECK-ARM-NEXT: mov r2, #0 +; CHECK-ARM-NEXT: cmp r0, r1 +; CHECK-ARM-NEXT: moveq r2, #1065353216 +; CHECK-ARM-NEXT: mov r0, r2 +; CHECK-ARM-NEXT: bx lr +; +; CHECK-VFP-LABEL: f12: +; CHECK-VFP: @ %bb.0: +; CHECK-VFP-NEXT: vmov.f32 s2, #1.000000e+00 +; CHECK-VFP-NEXT: vldr s0, .LCPI11_0 +; CHECK-VFP-NEXT: cmp r0, r1 +; CHECK-VFP-NEXT: vmoveq.f32 s0, s2 +; CHECK-VFP-NEXT: vmov r0, s0 +; CHECK-VFP-NEXT: bx lr +; CHECK-VFP-NEXT: .p2align 2 +; CHECK-VFP-NEXT: @ %bb.1: +; CHECK-VFP-NEXT: .LCPI11_0: +; CHECK-VFP-NEXT: .long 0x00000000 @ float 0 +; +; CHECK-NEON-LABEL: f12: +; CHECK-NEON: @ %bb.0: +; CHECK-NEON-NEXT: vldr s0, LCPI11_0 +; CHECK-NEON-NEXT: vmov.f32 s2, #1.000000e+00 +; CHECK-NEON-NEXT: cmp r0, r1 +; CHECK-NEON-NEXT: it eq +; CHECK-NEON-NEXT: vmoveq.f32 s0, s2 +; CHECK-NEON-NEXT: vmov r0, s0 +; CHECK-NEON-NEXT: bx lr +; CHECK-NEON-NEXT: .p2align 2 +; CHECK-NEON-NEXT: @ %bb.1: +; CHECK-NEON-NEXT: .data_region +; CHECK-NEON-NEXT: LCPI11_0: +; CHECK-NEON-NEXT: .long 0x00000000 @ float 0 +; CHECK-NEON-NEXT: .end_data_region %1 = icmp eq i32 %a, %b %2 = uitofp i1 %1 to float ret float %2 } -; CHECK-LABEL: test_overflow_recombine: define i1 @test_overflow_recombine(i32 %in1, i32 %in2) { -; CHECK: smull [[LO:r[0-9]+]], [[HI:r[0-9]+]] -; CHECK: subs [[ZERO:r[0-9]+]], [[HI]], [[LO]], asr #31 -; CHECK: movne [[ZERO]], #1 +; CHECK-LABEL: test_overflow_recombine: +; CHECK: @ %bb.0: +; CHECK-NEXT: mul r2, r0, r1 +; CHECK-NEXT: smmul r0, r0, r1 +; CHECK-NEXT: subs r0, r0, r2, asr #31 +; CHECK-NEXT: movwne r0, #1 +; CHECK-NEXT: bx lr +; +; CHECK-NEON-LABEL: test_overflow_recombine: +; CHECK-NEON: @ %bb.0: +; CHECK-NEON-NEXT: mul r2, r0, r1 +; CHECK-NEON-NEXT: smmul r0, r0, r1 +; CHECK-NEON-NEXT: subs.w r0, r0, r2, asr #31 +; CHECK-NEON-NEXT: it ne +; CHECK-NEON-NEXT: movne r0, #1 +; CHECK-NEON-NEXT: bx lr %prod = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 %in1, i32 %in2) %overflow = extractvalue { i32, i1 } %prod, 1 ret i1 %overflow -- GitLab From 2bfa7d0e1691dcff095c602a0387a6d13213dc91 Mon Sep 17 00:00:00 2001 From: Yingwei Zheng Date: Thu, 21 Mar 2024 21:48:10 +0800 Subject: [PATCH 144/296] [InstCombine] Fold `fmul X, -0.0` into `copysign(0.0, -X)` (#85772) `fneg + copysign` is better than fmul for analysis/codegen. godbolt: https://godbolt.org/z/eEs6dGd1G Alive2: https://alive2.llvm.org/ce/z/K3M5BA --- .../InstCombine/InstCombineMulDivRem.cpp | 15 ++- .../Transforms/InstCombine/binop-itofp.ll | 12 ++- llvm/test/Transforms/InstCombine/fmul.ll | 99 ++++++++++++++++++- llvm/test/Transforms/InstCombine/fpcast.ll | 5 +- 4 files changed, 116 insertions(+), 15 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp index 9d4c271f990d..6e05fd8fb4d6 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp @@ -814,8 +814,19 @@ Instruction *InstCombinerImpl::visitFMul(BinaryOperator &I) { if (match(Op1, m_SpecificFP(-1.0))) return UnaryOperator::CreateFNegFMF(Op0, &I); - // With no-nans: X * 0.0 --> copysign(0.0, X) - if (I.hasNoNaNs() && match(Op1, m_PosZeroFP())) { + // With no-nans/no-infs: + // X * 0.0 --> copysign(0.0, X) + // X * -0.0 --> copysign(0.0, -X) + const APFloat *FPC; + if (match(Op1, m_APFloatAllowUndef(FPC)) && FPC->isZero() && + ((I.hasNoInfs() && + isKnownNeverNaN(Op0, /*Depth=*/0, SQ.getWithInstruction(&I))) || + isKnownNeverNaN(&I, /*Depth=*/0, SQ.getWithInstruction(&I)))) { + if (FPC->isNegative()) + Op0 = Builder.CreateFNegFMF(Op0, &I); + Op1 = Constant::replaceUndefsWith( + cast(Op1), + ConstantFP::get(Op1->getType()->getScalarType(), *FPC)); CallInst *CopySign = Builder.CreateIntrinsic(Intrinsic::copysign, {I.getType()}, {Op1, Op0}, &I); return replaceInstUsesWith(I, CopySign); diff --git a/llvm/test/Transforms/InstCombine/binop-itofp.ll b/llvm/test/Transforms/InstCombine/binop-itofp.ll index 82cdb3ce6bee..cd9ec1e59203 100644 --- a/llvm/test/Transforms/InstCombine/binop-itofp.ll +++ b/llvm/test/Transforms/InstCombine/binop-itofp.ll @@ -1012,7 +1012,7 @@ define float @missed_nonzero_check_on_constant_for_si_fmul(i1 %c, i1 %.b, ptr %g ; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C:%.*]], i32 65529, i32 53264 ; CHECK-NEXT: [[CONV_I:%.*]] = trunc i32 [[SEL]] to i16 ; CHECK-NEXT: [[CONV1_I:%.*]] = sitofp i16 [[CONV_I]] to float -; CHECK-NEXT: [[MUL3_I_I:%.*]] = fmul float [[CONV1_I]], 0.000000e+00 +; CHECK-NEXT: [[MUL3_I_I:%.*]] = call float @llvm.copysign.f32(float 0.000000e+00, float [[CONV1_I]]) ; CHECK-NEXT: store i32 [[SEL]], ptr [[G_2345:%.*]], align 4 ; CHECK-NEXT: ret float [[MUL3_I_I]] ; @@ -1031,7 +1031,7 @@ define <2 x float> @missed_nonzero_check_on_constant_for_si_fmul_vec(i1 %c, i1 % ; CHECK-NEXT: [[CONV_I_V:%.*]] = insertelement <2 x i16> poison, i16 [[CONV_I_S]], i64 0 ; CHECK-NEXT: [[CONV_I:%.*]] = shufflevector <2 x i16> [[CONV_I_V]], <2 x i16> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[CONV1_I:%.*]] = sitofp <2 x i16> [[CONV_I]] to <2 x float> -; CHECK-NEXT: [[MUL3_I_I:%.*]] = fmul <2 x float> [[CONV1_I]], zeroinitializer +; CHECK-NEXT: [[MUL3_I_I:%.*]] = call <2 x float> @llvm.copysign.v2f32(<2 x float> zeroinitializer, <2 x float> [[CONV1_I]]) ; CHECK-NEXT: store i32 [[SEL]], ptr [[G_2345:%.*]], align 4 ; CHECK-NEXT: ret <2 x float> [[MUL3_I_I]] ; @@ -1050,7 +1050,8 @@ define float @negzero_check_on_constant_for_si_fmul(i1 %c, i1 %.b, ptr %g_2345) ; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C:%.*]], i32 65529, i32 53264 ; CHECK-NEXT: [[CONV_I:%.*]] = trunc i32 [[SEL]] to i16 ; CHECK-NEXT: [[CONV1_I:%.*]] = sitofp i16 [[CONV_I]] to float -; CHECK-NEXT: [[MUL3_I_I:%.*]] = fmul float [[CONV1_I]], -0.000000e+00 +; CHECK-NEXT: [[TMP1:%.*]] = fneg float [[CONV1_I]] +; CHECK-NEXT: [[MUL3_I_I:%.*]] = call float @llvm.copysign.f32(float 0.000000e+00, float [[TMP1]]) ; CHECK-NEXT: store i32 [[SEL]], ptr [[G_2345:%.*]], align 4 ; CHECK-NEXT: ret float [[MUL3_I_I]] ; @@ -1069,7 +1070,7 @@ define <2 x float> @nonzero_check_on_constant_for_si_fmul_vec_w_undef(i1 %c, i1 ; CHECK-NEXT: [[CONV_I_V:%.*]] = insertelement <2 x i16> poison, i16 [[CONV_I_S]], i64 0 ; CHECK-NEXT: [[CONV_I:%.*]] = shufflevector <2 x i16> [[CONV_I_V]], <2 x i16> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[CONV1_I:%.*]] = sitofp <2 x i16> [[CONV_I]] to <2 x float> -; CHECK-NEXT: [[MUL3_I_I:%.*]] = fmul <2 x float> [[CONV1_I]], +; CHECK-NEXT: [[MUL3_I_I:%.*]] = call <2 x float> @llvm.copysign.v2f32(<2 x float> zeroinitializer, <2 x float> [[CONV1_I]]) ; CHECK-NEXT: store i32 [[SEL]], ptr [[G_2345:%.*]], align 4 ; CHECK-NEXT: ret <2 x float> [[MUL3_I_I]] ; @@ -1111,7 +1112,8 @@ define <2 x float> @nonzero_check_on_constant_for_si_fmul_negz_vec_w_undef(i1 %c ; CHECK-NEXT: [[CONV_I_V:%.*]] = insertelement <2 x i16> poison, i16 [[CONV_I_S]], i64 0 ; CHECK-NEXT: [[CONV_I:%.*]] = shufflevector <2 x i16> [[CONV_I_V]], <2 x i16> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: [[CONV1_I:%.*]] = sitofp <2 x i16> [[CONV_I]] to <2 x float> -; CHECK-NEXT: [[MUL3_I_I:%.*]] = fmul <2 x float> [[CONV1_I]], +; CHECK-NEXT: [[TMP1:%.*]] = fneg <2 x float> [[CONV1_I]] +; CHECK-NEXT: [[MUL3_I_I:%.*]] = call <2 x float> @llvm.copysign.v2f32(<2 x float> zeroinitializer, <2 x float> [[TMP1]]) ; CHECK-NEXT: store i32 [[SEL]], ptr [[G_2345:%.*]], align 4 ; CHECK-NEXT: ret <2 x float> [[MUL3_I_I]] ; diff --git a/llvm/test/Transforms/InstCombine/fmul.ll b/llvm/test/Transforms/InstCombine/fmul.ll index 96e57939d285..f6435f003289 100644 --- a/llvm/test/Transforms/InstCombine/fmul.ll +++ b/llvm/test/Transforms/InstCombine/fmul.ll @@ -1250,7 +1250,7 @@ define half @mul_zero_nnan(half %x) { define <2 x float> @mul_zero_nnan_vec_poison(<2 x float> %x) { ; CHECK-LABEL: @mul_zero_nnan_vec_poison( -; CHECK-NEXT: [[R:%.*]] = call nnan <2 x float> @llvm.copysign.v2f32(<2 x float> , <2 x float> [[X:%.*]]) +; CHECK-NEXT: [[R:%.*]] = call nnan <2 x float> @llvm.copysign.v2f32(<2 x float> zeroinitializer, <2 x float> [[X:%.*]]) ; CHECK-NEXT: ret <2 x float> [[R]] ; %r = fmul nnan <2 x float> %x, @@ -1268,13 +1268,104 @@ define half @mul_zero(half %x) { ret half %r } -; TODO: This could be fneg+copysign. - define half @mul_negzero_nnan(half %x) { ; CHECK-LABEL: @mul_negzero_nnan( -; CHECK-NEXT: [[R:%.*]] = fmul nnan half [[X:%.*]], 0xH8000 +; CHECK-NEXT: [[TMP1:%.*]] = fneg nnan half [[X:%.*]] +; CHECK-NEXT: [[R:%.*]] = call nnan half @llvm.copysign.f16(half 0xH0000, half [[TMP1]]) ; CHECK-NEXT: ret half [[R]] ; %r = fmul nnan half %x, -0.0 ret half %r } + +define float @mul_pos_zero_nnan_ninf(float nofpclass(inf nan) %a) { +; CHECK-LABEL: @mul_pos_zero_nnan_ninf( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[RET:%.*]] = call float @llvm.copysign.f32(float 0.000000e+00, float [[A:%.*]]) +; CHECK-NEXT: ret float [[RET]] +; +entry: + %ret = fmul float %a, 0.000000e+00 + ret float %ret +} + +define float @mul_pos_zero_nnan(float nofpclass(nan) %a) { +; CHECK-LABEL: @mul_pos_zero_nnan( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[RET:%.*]] = fmul float [[A:%.*]], 0.000000e+00 +; CHECK-NEXT: ret float [[RET]] +; +entry: + %ret = fmul float %a, 0.000000e+00 + ret float %ret +} + +define float @mul_pos_zero_nnan_ninf_fmf(float nofpclass(nan) %a) { +; CHECK-LABEL: @mul_pos_zero_nnan_ninf_fmf( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[RET:%.*]] = call ninf float @llvm.copysign.f32(float 0.000000e+00, float [[A:%.*]]) +; CHECK-NEXT: ret float [[RET]] +; +entry: + %ret = fmul ninf float %a, 0.000000e+00 + ret float %ret +} + +define float @mul_neg_zero_nnan_ninf(float nofpclass(inf nan) %a) { +; CHECK-LABEL: @mul_neg_zero_nnan_ninf( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = fneg float [[A:%.*]] +; CHECK-NEXT: [[RET:%.*]] = call float @llvm.copysign.f32(float 0.000000e+00, float [[TMP0]]) +; CHECK-NEXT: ret float [[RET]] +; +entry: + %ret = fmul float %a, -0.000000e+00 + ret float %ret +} + +define float @mul_neg_zero_nnan_fmf(float %a) { +; CHECK-LABEL: @mul_neg_zero_nnan_fmf( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = fneg nnan float [[A:%.*]] +; CHECK-NEXT: [[RET:%.*]] = call nnan float @llvm.copysign.f32(float 0.000000e+00, float [[TMP0]]) +; CHECK-NEXT: ret float [[RET]] +; +entry: + %ret = fmul nnan float %a, -0.000000e+00 + ret float %ret +} + +define float @mul_neg_zero_nnan_ninf_fmf(float nofpclass(inf nan) %a) { +; CHECK-LABEL: @mul_neg_zero_nnan_ninf_fmf( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = fneg nnan ninf float [[A:%.*]] +; CHECK-NEXT: [[RET:%.*]] = call nnan ninf float @llvm.copysign.f32(float 0.000000e+00, float [[TMP0]]) +; CHECK-NEXT: ret float [[RET]] +; +entry: + %ret = fmul nnan ninf float %a, -0.000000e+00 + ret float %ret +} + +define <3 x float> @mul_neg_zero_nnan_ninf_vec(<3 x float> nofpclass(inf nan) %a) { +; CHECK-LABEL: @mul_neg_zero_nnan_ninf_vec( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = fneg <3 x float> [[A:%.*]] +; CHECK-NEXT: [[RET:%.*]] = call <3 x float> @llvm.copysign.v3f32(<3 x float> zeroinitializer, <3 x float> [[TMP0]]) +; CHECK-NEXT: ret <3 x float> [[RET]] +; +entry: + %ret = fmul <3 x float> %a, + ret <3 x float> %ret +} + +define <3 x float> @mul_mixed_zero_nnan_ninf_vec(<3 x float> nofpclass(inf nan) %a) { +; CHECK-LABEL: @mul_mixed_zero_nnan_ninf_vec( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[RET:%.*]] = fmul <3 x float> [[A:%.*]], +; CHECK-NEXT: ret <3 x float> [[RET]] +; +entry: + %ret = fmul <3 x float> %a, + ret <3 x float> %ret +} diff --git a/llvm/test/Transforms/InstCombine/fpcast.ll b/llvm/test/Transforms/InstCombine/fpcast.ll index 32bfdb52bb5f..ac4b88fcddd7 100644 --- a/llvm/test/Transforms/InstCombine/fpcast.ll +++ b/llvm/test/Transforms/InstCombine/fpcast.ll @@ -424,10 +424,7 @@ define i32 @fptosi_select(i1 %cond) { define i32 @mul_pos_zero_convert(i32 %a) { ; CHECK-LABEL: @mul_pos_zero_convert( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[FP:%.*]] = sitofp i32 [[A:%.*]] to float -; CHECK-NEXT: [[RET:%.*]] = fmul float [[FP]], 0.000000e+00 -; CHECK-NEXT: [[CONV:%.*]] = fptosi float [[RET]] to i32 -; CHECK-NEXT: ret i32 [[CONV]] +; CHECK-NEXT: ret i32 0 ; entry: %fp = sitofp i32 %a to float -- GitLab From 857161c367a1cdca926dbe0d2601e3afc52f03f9 Mon Sep 17 00:00:00 2001 From: Janek van Oirschot <5994977+JanekvO@users.noreply.github.com> Date: Thu, 21 Mar 2024 13:57:10 +0000 Subject: [PATCH 145/296] [AMDGPU] MCExpr-ify MC layer kernel descriptor (#80855) Kernel descriptor attributes, with their respective emit and asm parse functionality, converted to MCExpr. --- llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp | 40 +- llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h | 11 +- .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 193 ++++++--- .../MCTargetDesc/AMDGPUMCKernelDescriptor.cpp | 32 ++ .../MCTargetDesc/AMDGPUMCKernelDescriptor.h | 51 +++ .../MCTargetDesc/AMDGPUTargetStreamer.cpp | 404 +++++++++++------- .../MCTargetDesc/AMDGPUTargetStreamer.h | 33 +- .../Target/AMDGPU/MCTargetDesc/CMakeLists.txt | 1 + .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 88 ++-- llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 10 +- llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s | 27 ++ llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s | 281 ++++++++++++ llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s | 190 ++++++++ llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s | 186 ++++++++ llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s | 184 ++++++++ llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s | 168 ++++++++ llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s | 171 ++++++++ llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s | 148 +++++++ llvm/test/MC/AMDGPU/hsa-tg-split.s | 74 ++++ 19 files changed, 1988 insertions(+), 304 deletions(-) create mode 100644 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp create mode 100644 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h create mode 100644 llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s create mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s create mode 100644 llvm/test/MC/AMDGPU/hsa-tg-split.s diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp index 72e8b59e0a40..052b231d62a3 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp @@ -22,6 +22,7 @@ #include "AMDKernelCodeT.h" #include "GCNSubtarget.h" #include "MCTargetDesc/AMDGPUInstPrinter.h" +#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h" #include "MCTargetDesc/AMDGPUTargetStreamer.h" #include "R600AsmPrinter.h" #include "SIMachineFunctionInfo.h" @@ -428,38 +429,43 @@ uint16_t AMDGPUAsmPrinter::getAmdhsaKernelCodeProperties( return KernelCodeProperties; } -amdhsa::kernel_descriptor_t AMDGPUAsmPrinter::getAmdhsaKernelDescriptor( - const MachineFunction &MF, - const SIProgramInfo &PI) const { +MCKernelDescriptor +AMDGPUAsmPrinter::getAmdhsaKernelDescriptor(const MachineFunction &MF, + const SIProgramInfo &PI) const { const GCNSubtarget &STM = MF.getSubtarget(); const Function &F = MF.getFunction(); const SIMachineFunctionInfo *Info = MF.getInfo(); + MCContext &Ctx = MF.getContext(); - amdhsa::kernel_descriptor_t KernelDescriptor; - memset(&KernelDescriptor, 0x0, sizeof(KernelDescriptor)); + MCKernelDescriptor KernelDescriptor; assert(isUInt<32>(PI.ScratchSize)); assert(isUInt<32>(PI.getComputePGMRSrc1(STM))); assert(isUInt<32>(PI.getComputePGMRSrc2())); - KernelDescriptor.group_segment_fixed_size = PI.LDSSize; - KernelDescriptor.private_segment_fixed_size = PI.ScratchSize; + KernelDescriptor.group_segment_fixed_size = + MCConstantExpr::create(PI.LDSSize, Ctx); + KernelDescriptor.private_segment_fixed_size = + MCConstantExpr::create(PI.ScratchSize, Ctx); Align MaxKernArgAlign; - KernelDescriptor.kernarg_size = STM.getKernArgSegmentSize(F, MaxKernArgAlign); + KernelDescriptor.kernarg_size = MCConstantExpr::create( + STM.getKernArgSegmentSize(F, MaxKernArgAlign), Ctx); - KernelDescriptor.compute_pgm_rsrc1 = PI.getComputePGMRSrc1(STM); - KernelDescriptor.compute_pgm_rsrc2 = PI.getComputePGMRSrc2(); - KernelDescriptor.kernel_code_properties = getAmdhsaKernelCodeProperties(MF); + KernelDescriptor.compute_pgm_rsrc1 = + MCConstantExpr::create(PI.getComputePGMRSrc1(STM), Ctx); + KernelDescriptor.compute_pgm_rsrc2 = + MCConstantExpr::create(PI.getComputePGMRSrc2(), Ctx); + KernelDescriptor.kernel_code_properties = + MCConstantExpr::create(getAmdhsaKernelCodeProperties(MF), Ctx); assert(STM.hasGFX90AInsts() || CurrentProgramInfo.ComputePGMRSrc3GFX90A == 0); - if (STM.hasGFX90AInsts()) - KernelDescriptor.compute_pgm_rsrc3 = - CurrentProgramInfo.ComputePGMRSrc3GFX90A; + KernelDescriptor.compute_pgm_rsrc3 = MCConstantExpr::create( + STM.hasGFX90AInsts() ? CurrentProgramInfo.ComputePGMRSrc3GFX90A : 0, Ctx); - if (AMDGPU::hasKernargPreload(STM)) - KernelDescriptor.kernarg_preload = - static_cast(Info->getNumKernargPreloadedSGPRs()); + KernelDescriptor.kernarg_preload = MCConstantExpr::create( + AMDGPU::hasKernargPreload(STM) ? Info->getNumKernargPreloadedSGPRs() : 0, + Ctx); return KernelDescriptor; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h index 79326cd3d328..b8b2718d293e 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h @@ -28,15 +28,12 @@ class MCCodeEmitter; class MCOperand; namespace AMDGPU { +struct MCKernelDescriptor; namespace HSAMD { class MetadataStreamer; } } // namespace AMDGPU -namespace amdhsa { -struct kernel_descriptor_t; -} - class AMDGPUAsmPrinter final : public AsmPrinter { private: unsigned CodeObjectVersion; @@ -75,9 +72,9 @@ private: uint16_t getAmdhsaKernelCodeProperties( const MachineFunction &MF) const; - amdhsa::kernel_descriptor_t getAmdhsaKernelDescriptor( - const MachineFunction &MF, - const SIProgramInfo &PI) const; + AMDGPU::MCKernelDescriptor + getAmdhsaKernelDescriptor(const MachineFunction &MF, + const SIProgramInfo &PI) const; void initTargetStreamer(Module &M); diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp index 529705479646..38850f5acadd 100644 --- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -8,6 +8,7 @@ #include "AMDKernelCodeT.h" #include "MCTargetDesc/AMDGPUMCExpr.h" +#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "MCTargetDesc/AMDGPUTargetStreamer.h" #include "SIDefines.h" @@ -5417,7 +5418,8 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { if (getParser().parseIdentifier(KernelName)) return true; - kernel_descriptor_t KD = getDefaultAmdhsaKernelDescriptor(&getSTI()); + AMDGPU::MCKernelDescriptor KD = + getDefaultAmdhsaKernelDescriptor(&getSTI(), getContext()); StringSet<> Seen; @@ -5457,89 +5459,111 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { return TokError(".amdhsa_ directives cannot be repeated"); SMLoc ValStart = getLoc(); - int64_t IVal; - if (getParser().parseAbsoluteExpression(IVal)) + const MCExpr *ExprVal; + if (getParser().parseExpression(ExprVal)) return true; SMLoc ValEnd = getLoc(); SMRange ValRange = SMRange(ValStart, ValEnd); - if (IVal < 0) - return OutOfRangeError(ValRange); - + int64_t IVal = 0; uint64_t Val = IVal; + bool EvaluatableExpr; + if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(IVal))) { + if (IVal < 0) + return OutOfRangeError(ValRange); + Val = IVal; + } #define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \ - if (!isUInt(VALUE)) \ + if (!isUInt(Val)) \ return OutOfRangeError(RANGE); \ - AMDHSA_BITS_SET(FIELD, ENTRY, VALUE); + AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY, \ + getContext()); + +// Some fields use the parsed value immediately which requires the expression to +// be solvable. +#define EXPR_RESOLVE_OR_ERROR(RESOLVED) \ + if (!(RESOLVED)) \ + return Error(IDRange.Start, "directive should have resolvable expression", \ + IDRange); if (ID == ".amdhsa_group_segment_fixed_size") { - if (!isUInt(Val)) + if (!isUInt(Val)) return OutOfRangeError(ValRange); - KD.group_segment_fixed_size = Val; + KD.group_segment_fixed_size = ExprVal; } else if (ID == ".amdhsa_private_segment_fixed_size") { - if (!isUInt(Val)) + if (!isUInt(Val)) return OutOfRangeError(ValRange); - KD.private_segment_fixed_size = Val; + KD.private_segment_fixed_size = ExprVal; } else if (ID == ".amdhsa_kernarg_size") { - if (!isUInt(Val)) + if (!isUInt(Val)) return OutOfRangeError(ValRange); - KD.kernarg_size = Val; + KD.kernarg_size = ExprVal; } else if (ID == ".amdhsa_user_sgpr_count") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); ExplicitUserSGPRCount = Val; } else if (ID == ".amdhsa_user_sgpr_private_segment_buffer") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (hasArchitectedFlatScratch()) return Error(IDRange.Start, "directive is not supported with architected flat scratch", IDRange); PARSE_BITS_ENTRY(KD.kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER, - Val, ValRange); + ExprVal, ValRange); if (Val) ImpliedUserSGPRCount += 4; } else if (ID == ".amdhsa_user_sgpr_kernarg_preload_length") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (!hasKernargPreload()) return Error(IDRange.Start, "directive requires gfx90a+", IDRange); if (Val > getMaxNumUserSGPRs()) return OutOfRangeError(ValRange); - PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_LENGTH, Val, + PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_LENGTH, ExprVal, ValRange); if (Val) { ImpliedUserSGPRCount += Val; PreloadLength = Val; } } else if (ID == ".amdhsa_user_sgpr_kernarg_preload_offset") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (!hasKernargPreload()) return Error(IDRange.Start, "directive requires gfx90a+", IDRange); if (Val >= 1024) return OutOfRangeError(ValRange); - PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_OFFSET, Val, + PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_OFFSET, ExprVal, ValRange); if (Val) PreloadOffset = Val; } else if (ID == ".amdhsa_user_sgpr_dispatch_ptr") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, Val, + KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal, ValRange); if (Val) ImpliedUserSGPRCount += 2; } else if (ID == ".amdhsa_user_sgpr_queue_ptr") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, Val, + KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal, ValRange); if (Val) ImpliedUserSGPRCount += 2; } else if (ID == ".amdhsa_user_sgpr_kernarg_segment_ptr") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR, - Val, ValRange); + ExprVal, ValRange); if (Val) ImpliedUserSGPRCount += 2; } else if (ID == ".amdhsa_user_sgpr_dispatch_id") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, Val, + KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal, ValRange); if (Val) ImpliedUserSGPRCount += 2; @@ -5548,34 +5572,39 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { return Error(IDRange.Start, "directive is not supported with architected flat scratch", IDRange); + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT, Val, - ValRange); + KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT, + ExprVal, ValRange); if (Val) ImpliedUserSGPRCount += 2; } else if (ID == ".amdhsa_user_sgpr_private_segment_size") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE, - Val, ValRange); + ExprVal, ValRange); if (Val) ImpliedUserSGPRCount += 1; } else if (ID == ".amdhsa_wavefront_size32") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (IVersion.Major < 10) return Error(IDRange.Start, "directive requires gfx10+", IDRange); EnableWavefrontSize32 = Val; PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, - Val, ValRange); + KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal, + ValRange); } else if (ID == ".amdhsa_uses_dynamic_stack") { PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, Val, ValRange); + KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal, + ValRange); } else if (ID == ".amdhsa_system_sgpr_private_segment_wavefront_offset") { if (hasArchitectedFlatScratch()) return Error(IDRange.Start, "directive is not supported with architected flat scratch", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, Val, ValRange); + COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal, + ValRange); } else if (ID == ".amdhsa_enable_private_segment") { if (!hasArchitectedFlatScratch()) return Error( @@ -5583,42 +5612,48 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { "directive is not supported without architected flat scratch", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, Val, ValRange); + COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal, + ValRange); } else if (ID == ".amdhsa_system_sgpr_workgroup_id_x") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, Val, + COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal, ValRange); } else if (ID == ".amdhsa_system_sgpr_workgroup_id_y") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, Val, + COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal, ValRange); } else if (ID == ".amdhsa_system_sgpr_workgroup_id_z") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, Val, + COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal, ValRange); } else if (ID == ".amdhsa_system_sgpr_workgroup_info") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, Val, + COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal, ValRange); } else if (ID == ".amdhsa_system_vgpr_workitem_id") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, Val, + COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal, ValRange); } else if (ID == ".amdhsa_next_free_vgpr") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); VGPRRange = ValRange; NextFreeVGPR = Val; } else if (ID == ".amdhsa_next_free_sgpr") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); SGPRRange = ValRange; NextFreeSGPR = Val; } else if (ID == ".amdhsa_accum_offset") { if (!isGFX90A()) return Error(IDRange.Start, "directive requires gfx90a+", IDRange); + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); AccumOffset = Val; } else if (ID == ".amdhsa_reserve_vcc") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (!isUInt<1>(Val)) return OutOfRangeError(ValRange); ReserveVCC = Val; } else if (ID == ".amdhsa_reserve_flat_scratch") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (IVersion.Major < 7) return Error(IDRange.Start, "directive requires gfx7+", IDRange); if (hasArchitectedFlatScratch()) @@ -5638,97 +5673,105 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { IDRange); } else if (ID == ".amdhsa_float_round_mode_32") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, Val, ValRange); + COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal, + ValRange); } else if (ID == ".amdhsa_float_round_mode_16_64") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, Val, ValRange); + COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal, + ValRange); } else if (ID == ".amdhsa_float_denorm_mode_32") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, Val, ValRange); + COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal, + ValRange); } else if (ID == ".amdhsa_float_denorm_mode_16_64") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, Val, + COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal, ValRange); } else if (ID == ".amdhsa_dx10_clamp") { if (IVersion.Major >= 12) return Error(IDRange.Start, "directive unsupported on gfx12+", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, Val, + COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal, ValRange); } else if (ID == ".amdhsa_ieee_mode") { if (IVersion.Major >= 12) return Error(IDRange.Start, "directive unsupported on gfx12+", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, Val, + COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal, ValRange); } else if (ID == ".amdhsa_fp16_overflow") { if (IVersion.Major < 9) return Error(IDRange.Start, "directive requires gfx9+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, Val, + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, + COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal, ValRange); } else if (ID == ".amdhsa_tg_split") { if (!isGFX90A()) return Error(IDRange.Start, "directive requires gfx90a+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Val, - ValRange); + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, + ExprVal, ValRange); } else if (ID == ".amdhsa_workgroup_processor_mode") { if (IVersion.Major < 10) return Error(IDRange.Start, "directive requires gfx10+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, Val, + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, + COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal, ValRange); } else if (ID == ".amdhsa_memory_ordered") { if (IVersion.Major < 10) return Error(IDRange.Start, "directive requires gfx10+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, Val, + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, + COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal, ValRange); } else if (ID == ".amdhsa_forward_progress") { if (IVersion.Major < 10) return Error(IDRange.Start, "directive requires gfx10+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, Val, + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, + COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal, ValRange); } else if (ID == ".amdhsa_shared_vgpr_count") { + EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (IVersion.Major < 10 || IVersion.Major >= 12) return Error(IDRange.Start, "directive requires gfx10 or gfx11", IDRange); SharedVGPRCount = Val; PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, - COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, Val, + COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal, ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_invalid_op") { PARSE_BITS_ENTRY( KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, Val, - ValRange); + COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, + ExprVal, ValRange); } else if (ID == ".amdhsa_exception_fp_denorm_src") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE, - Val, ValRange); + ExprVal, ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_div_zero") { PARSE_BITS_ENTRY( KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, Val, - ValRange); + COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, + ExprVal, ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_overflow") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW, - Val, ValRange); + ExprVal, ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_underflow") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW, - Val, ValRange); + ExprVal, ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_inexact") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT, - Val, ValRange); + ExprVal, ValRange); } else if (ID == ".amdhsa_exception_int_div_zero") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO, - Val, ValRange); + ExprVal, ValRange); } else if (ID == ".amdhsa_round_robin_scheduling") { if (IVersion.Major < 12) return Error(IDRange.Start, "directive requires gfx12+", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, Val, + COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal, ValRange); } else { return Error(IDRange.Start, "unknown .amdhsa_kernel directive", IDRange); @@ -5755,15 +5798,18 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { if (!isUInt( VGPRBlocks)) return OutOfRangeError(VGPRRange); - AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, VGPRBlocks); + AMDGPU::MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc1, MCConstantExpr::create(VGPRBlocks, getContext()), + COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT, + COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, getContext()); if (!isUInt( SGPRBlocks)) return OutOfRangeError(SGPRRange); - AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, - SGPRBlocks); + AMDGPU::MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc1, MCConstantExpr::create(SGPRBlocks, getContext()), + COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT, + COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, getContext()); if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount) return TokError("amdgpu_user_sgpr_count smaller than than implied by " @@ -5774,11 +5820,17 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { if (!isUInt(UserSGPRCount)) return TokError("too many user SGPRs enabled"); - AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, - UserSGPRCount); - - if (PreloadLength && KD.kernarg_size && - (PreloadLength * 4 + PreloadOffset * 4 > KD.kernarg_size)) + AMDGPU::MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc2, MCConstantExpr::create(UserSGPRCount, getContext()), + COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_SHIFT, + COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, getContext()); + + int64_t IVal = 0; + if (!KD.kernarg_size->evaluateAsAbsolute(IVal)) + return TokError("Kernarg size should be resolvable"); + uint64_t kernarg_size = IVal; + if (PreloadLength && kernarg_size && + (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size)) return TokError("Kernarg preload length + offset is larger than the " "kernarg segment size"); @@ -5790,8 +5842,11 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { "increments of 4"); if (AccumOffset > alignTo(std::max((uint64_t)1, NextFreeVGPR), 4)) return TokError("accum_offset exceeds total VGPR allocation"); - AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, - (AccumOffset / 4 - 1)); + MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc3, + MCConstantExpr::create(AccumOffset / 4 - 1, getContext()), + COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT, + COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, getContext()); } if (IVersion.Major >= 10 && IVersion.Major < 12) { diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp new file mode 100644 index 000000000000..0179d575464d --- /dev/null +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp @@ -0,0 +1,32 @@ +//===--- AMDHSAKernelDescriptor.h -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "AMDGPUMCKernelDescriptor.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCExpr.h" + +using namespace llvm; +using namespace llvm::AMDGPU; + +void MCKernelDescriptor::bits_set(const MCExpr *&Dst, const MCExpr *Value, + uint32_t Shift, uint32_t Mask, + MCContext &Ctx) { + auto Sft = MCConstantExpr::create(Shift, Ctx); + auto Msk = MCConstantExpr::create(Mask, Ctx); + Dst = MCBinaryExpr::createAnd(Dst, MCUnaryExpr::createNot(Msk, Ctx), Ctx); + Dst = MCBinaryExpr::createOr(Dst, MCBinaryExpr::createShl(Value, Sft, Ctx), + Ctx); +} + +const MCExpr *MCKernelDescriptor::bits_get(const MCExpr *Src, uint32_t Shift, + uint32_t Mask, MCContext &Ctx) { + auto Sft = MCConstantExpr::create(Shift, Ctx); + auto Msk = MCConstantExpr::create(Mask, Ctx); + return MCBinaryExpr::createLShr(MCBinaryExpr::createAnd(Src, Msk, Ctx), Sft, + Ctx); +} diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h new file mode 100644 index 000000000000..71659e642dd7 --- /dev/null +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h @@ -0,0 +1,51 @@ +//===--- AMDGPUMCKernelDescriptor.h ---------------------------*- C++ -*---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file +/// AMDHSA kernel descriptor MCExpr struct for use in MC layer. Uses +/// AMDHSAKernelDescriptor.h for sizes and constants. +/// +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCKERNELDESCRIPTOR_H +#define LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCKERNELDESCRIPTOR_H + +#include "llvm/Support/AMDHSAKernelDescriptor.h" + +namespace llvm { +class MCExpr; +class MCContext; +namespace AMDGPU { + +struct MCKernelDescriptor { + const MCExpr *group_segment_fixed_size = nullptr; + const MCExpr *private_segment_fixed_size = nullptr; + const MCExpr *kernarg_size = nullptr; + const MCExpr *compute_pgm_rsrc3 = nullptr; + const MCExpr *compute_pgm_rsrc1 = nullptr; + const MCExpr *compute_pgm_rsrc2 = nullptr; + const MCExpr *kernel_code_properties = nullptr; + const MCExpr *kernarg_preload = nullptr; + + // MCExpr for: + // Dst = Dst & ~Mask + // Dst = Dst | (Value << Shift) + static void bits_set(const MCExpr *&Dst, const MCExpr *Value, uint32_t Shift, + uint32_t Mask, MCContext &Ctx); + + // MCExpr for: + // return (Src & Mask) >> Shift + static const MCExpr *bits_get(const MCExpr *Src, uint32_t Shift, + uint32_t Mask, MCContext &Ctx); +}; + +} // end namespace AMDGPU +} // end namespace llvm + +#endif // LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCKERNELDESCRIPTOR_H diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp index 4742b0b3e52e..3006fcdb9282 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "AMDGPUTargetStreamer.h" +#include "AMDGPUMCKernelDescriptor.h" #include "AMDGPUPTNote.h" #include "AMDKernelCodeT.h" #include "Utils/AMDGPUBaseInfo.h" @@ -307,94 +308,142 @@ bool AMDGPUTargetAsmStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) { void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor( const MCSubtargetInfo &STI, StringRef KernelName, - const amdhsa::kernel_descriptor_t &KD, uint64_t NextVGPR, uint64_t NextSGPR, + const MCKernelDescriptor &KD, uint64_t NextVGPR, uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) { IsaVersion IVersion = getIsaVersion(STI.getCPU()); + const MCAsmInfo *MAI = getContext().getAsmInfo(); OS << "\t.amdhsa_kernel " << KernelName << '\n'; -#define PRINT_FIELD(STREAM, DIRECTIVE, KERNEL_DESC, MEMBER_NAME, FIELD_NAME) \ - STREAM << "\t\t" << DIRECTIVE << " " \ - << AMDHSA_BITS_GET(KERNEL_DESC.MEMBER_NAME, FIELD_NAME) << '\n'; - - OS << "\t\t.amdhsa_group_segment_fixed_size " << KD.group_segment_fixed_size - << '\n'; - OS << "\t\t.amdhsa_private_segment_fixed_size " - << KD.private_segment_fixed_size << '\n'; - OS << "\t\t.amdhsa_kernarg_size " << KD.kernarg_size << '\n'; - - PRINT_FIELD(OS, ".amdhsa_user_sgpr_count", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT); + auto PrintField = [&](const MCExpr *Expr, uint32_t Shift, uint32_t Mask, + StringRef Directive) { + int64_t IVal; + OS << "\t\t" << Directive << ' '; + const MCExpr *pgm_rsrc1_bits = + MCKernelDescriptor::bits_get(Expr, Shift, Mask, getContext()); + if (pgm_rsrc1_bits->evaluateAsAbsolute(IVal)) + OS << static_cast(IVal); + else + pgm_rsrc1_bits->print(OS, MAI); + OS << '\n'; + }; + + OS << "\t\t.amdhsa_group_segment_fixed_size "; + KD.group_segment_fixed_size->print(OS, MAI); + OS << '\n'; + + OS << "\t\t.amdhsa_private_segment_fixed_size "; + KD.private_segment_fixed_size->print(OS, MAI); + OS << '\n'; + + OS << "\t\t.amdhsa_kernarg_size "; + KD.kernarg_size->print(OS, MAI); + OS << '\n'; + + PrintField( + KD.compute_pgm_rsrc2, amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, ".amdhsa_user_sgpr_count"); if (!hasArchitectedFlatScratch(STI)) - PRINT_FIELD( - OS, ".amdhsa_user_sgpr_private_segment_buffer", KD, - kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER); - PRINT_FIELD(OS, ".amdhsa_user_sgpr_dispatch_ptr", KD, - kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR); - PRINT_FIELD(OS, ".amdhsa_user_sgpr_queue_ptr", KD, - kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR); - PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_segment_ptr", KD, - kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR); - PRINT_FIELD(OS, ".amdhsa_user_sgpr_dispatch_id", KD, - kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID); + PrintField( + KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER, + ".amdhsa_user_sgpr_private_segment_buffer"); + PrintField(KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, + ".amdhsa_user_sgpr_dispatch_ptr"); + PrintField(KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, + ".amdhsa_user_sgpr_queue_ptr"); + PrintField(KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR, + ".amdhsa_user_sgpr_kernarg_segment_ptr"); + PrintField(KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, + ".amdhsa_user_sgpr_dispatch_id"); if (!hasArchitectedFlatScratch(STI)) - PRINT_FIELD(OS, ".amdhsa_user_sgpr_flat_scratch_init", KD, - kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT); + PrintField(KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT, + ".amdhsa_user_sgpr_flat_scratch_init"); if (hasKernargPreload(STI)) { - PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_preload_length ", KD, - kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_LENGTH); - PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_preload_offset ", KD, - kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_OFFSET); + PrintField(KD.kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_LENGTH_SHIFT, + amdhsa::KERNARG_PRELOAD_SPEC_LENGTH, + ".amdhsa_user_sgpr_kernarg_preload_length"); + PrintField(KD.kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_OFFSET_SHIFT, + amdhsa::KERNARG_PRELOAD_SPEC_OFFSET, + ".amdhsa_user_sgpr_kernarg_preload_offset"); } - PRINT_FIELD(OS, ".amdhsa_user_sgpr_private_segment_size", KD, - kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE); + PrintField( + KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE, + ".amdhsa_user_sgpr_private_segment_size"); if (IVersion.Major >= 10) - PRINT_FIELD(OS, ".amdhsa_wavefront_size32", KD, - kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32); + PrintField(KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, + ".amdhsa_wavefront_size32"); if (CodeObjectVersion >= AMDGPU::AMDHSA_COV5) - PRINT_FIELD(OS, ".amdhsa_uses_dynamic_stack", KD, kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK); - PRINT_FIELD(OS, - (hasArchitectedFlatScratch(STI) - ? ".amdhsa_enable_private_segment" - : ".amdhsa_system_sgpr_private_segment_wavefront_offset"), - KD, compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT); - PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_x", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X); - PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_y", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y); - PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_z", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z); - PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_info", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO); - PRINT_FIELD(OS, ".amdhsa_system_vgpr_workitem_id", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID); + PrintField(KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, + ".amdhsa_uses_dynamic_stack"); + PrintField(KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, + (hasArchitectedFlatScratch(STI) + ? ".amdhsa_enable_private_segment" + : ".amdhsa_system_sgpr_private_segment_wavefront_offset")); + PrintField(KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, + ".amdhsa_system_sgpr_workgroup_id_x"); + PrintField(KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, + ".amdhsa_system_sgpr_workgroup_id_y"); + PrintField(KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, + ".amdhsa_system_sgpr_workgroup_id_z"); + PrintField(KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, + ".amdhsa_system_sgpr_workgroup_info"); + PrintField(KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, + ".amdhsa_system_vgpr_workitem_id"); // These directives are required. OS << "\t\t.amdhsa_next_free_vgpr " << NextVGPR << '\n'; OS << "\t\t.amdhsa_next_free_sgpr " << NextSGPR << '\n'; - if (AMDGPU::isGFX90A(STI)) - OS << "\t\t.amdhsa_accum_offset " << - (AMDHSA_BITS_GET(KD.compute_pgm_rsrc3, - amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET) + 1) * 4 - << '\n'; + if (AMDGPU::isGFX90A(STI)) { + // MCExpr equivalent of taking the (accum_offset + 1) * 4. + const MCExpr *accum_bits = MCKernelDescriptor::bits_get( + KD.compute_pgm_rsrc3, + amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT, + amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, getContext()); + accum_bits = MCBinaryExpr::createAdd( + accum_bits, MCConstantExpr::create(1, getContext()), getContext()); + accum_bits = MCBinaryExpr::createMul( + accum_bits, MCConstantExpr::create(4, getContext()), getContext()); + OS << "\t\t.amdhsa_accum_offset "; + int64_t IVal; + if (accum_bits->evaluateAsAbsolute(IVal)) { + OS << static_cast(IVal); + } else { + accum_bits->print(OS, MAI); + } + OS << '\n'; + } if (!ReserveVCC) OS << "\t\t.amdhsa_reserve_vcc " << ReserveVCC << '\n'; @@ -411,74 +460,105 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor( break; } - PRINT_FIELD(OS, ".amdhsa_float_round_mode_32", KD, - compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32); - PRINT_FIELD(OS, ".amdhsa_float_round_mode_16_64", KD, - compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64); - PRINT_FIELD(OS, ".amdhsa_float_denorm_mode_32", KD, - compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32); - PRINT_FIELD(OS, ".amdhsa_float_denorm_mode_16_64", KD, - compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64); + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, + ".amdhsa_float_round_mode_32"); + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, + ".amdhsa_float_round_mode_16_64"); + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, + ".amdhsa_float_denorm_mode_32"); + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, + ".amdhsa_float_denorm_mode_16_64"); if (IVersion.Major < 12) { - PRINT_FIELD(OS, ".amdhsa_dx10_clamp", KD, compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP); - PRINT_FIELD(OS, ".amdhsa_ieee_mode", KD, compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE); + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, + ".amdhsa_dx10_clamp"); + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, + ".amdhsa_ieee_mode"); + } + if (IVersion.Major >= 9) { + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, + ".amdhsa_fp16_overflow"); } - if (IVersion.Major >= 9) - PRINT_FIELD(OS, ".amdhsa_fp16_overflow", KD, - compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL); if (AMDGPU::isGFX90A(STI)) - PRINT_FIELD(OS, ".amdhsa_tg_split", KD, - compute_pgm_rsrc3, - amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT); + PrintField(KD.compute_pgm_rsrc3, + amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT, + amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, ".amdhsa_tg_split"); if (IVersion.Major >= 10) { - PRINT_FIELD(OS, ".amdhsa_workgroup_processor_mode", KD, - compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE); - PRINT_FIELD(OS, ".amdhsa_memory_ordered", KD, - compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED); - PRINT_FIELD(OS, ".amdhsa_forward_progress", KD, - compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS); + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, + ".amdhsa_workgroup_processor_mode"); + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, + ".amdhsa_memory_ordered"); + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, + ".amdhsa_forward_progress"); } if (IVersion.Major >= 10 && IVersion.Major < 12) { - PRINT_FIELD(OS, ".amdhsa_shared_vgpr_count", KD, compute_pgm_rsrc3, - amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT); + PrintField(KD.compute_pgm_rsrc3, + amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT_SHIFT, + amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, + ".amdhsa_shared_vgpr_count"); } - if (IVersion.Major >= 12) - PRINT_FIELD(OS, ".amdhsa_round_robin_scheduling", KD, compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN); - PRINT_FIELD( - OS, ".amdhsa_exception_fp_ieee_invalid_op", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION); - PRINT_FIELD(OS, ".amdhsa_exception_fp_denorm_src", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE); - PRINT_FIELD( - OS, ".amdhsa_exception_fp_ieee_div_zero", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO); - PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_overflow", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW); - PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_underflow", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW); - PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_inexact", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT); - PRINT_FIELD(OS, ".amdhsa_exception_int_div_zero", KD, - compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO); -#undef PRINT_FIELD + if (IVersion.Major >= 12) { + PrintField(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, + ".amdhsa_round_robin_scheduling"); + } + PrintField( + KD.compute_pgm_rsrc2, + amdhsa:: + COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, + ".amdhsa_exception_fp_ieee_invalid_op"); + PrintField( + KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE, + ".amdhsa_exception_fp_denorm_src"); + PrintField( + KD.compute_pgm_rsrc2, + amdhsa:: + COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, + ".amdhsa_exception_fp_ieee_div_zero"); + PrintField( + KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW, + ".amdhsa_exception_fp_ieee_overflow"); + PrintField( + KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW, + ".amdhsa_exception_fp_ieee_underflow"); + PrintField( + KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT, + ".amdhsa_exception_fp_ieee_inexact"); + PrintField( + KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO, + ".amdhsa_exception_int_div_zero"); OS << "\t.end_amdhsa_kernel\n"; } @@ -835,7 +915,7 @@ bool AMDGPUTargetELFStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) { void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor( const MCSubtargetInfo &STI, StringRef KernelName, - const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, + const MCKernelDescriptor &KernelDescriptor, uint64_t NextVGPR, uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) { auto &Streamer = getStreamer(); auto &Context = Streamer.getContext(); @@ -853,7 +933,7 @@ void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor( // Kernel descriptor symbol's type and size are fixed. KernelDescriptorSymbol->setType(ELF::STT_OBJECT); KernelDescriptorSymbol->setSize( - MCConstantExpr::create(sizeof(KernelDescriptor), Context)); + MCConstantExpr::create(sizeof(amdhsa::kernel_descriptor_t), Context)); // The visibility of the kernel code symbol must be protected or less to allow // static relocations from the kernel descriptor to be used. @@ -861,31 +941,43 @@ void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor( KernelCodeSymbol->setVisibility(ELF::STV_PROTECTED); Streamer.emitLabel(KernelDescriptorSymbol); - Streamer.emitInt32(KernelDescriptor.group_segment_fixed_size); - Streamer.emitInt32(KernelDescriptor.private_segment_fixed_size); - Streamer.emitInt32(KernelDescriptor.kernarg_size); - - for (uint8_t Res : KernelDescriptor.reserved0) - Streamer.emitInt8(Res); + Streamer.emitValue( + KernelDescriptor.group_segment_fixed_size, + sizeof(amdhsa::kernel_descriptor_t::group_segment_fixed_size)); + Streamer.emitValue( + KernelDescriptor.private_segment_fixed_size, + sizeof(amdhsa::kernel_descriptor_t::private_segment_fixed_size)); + Streamer.emitValue(KernelDescriptor.kernarg_size, + sizeof(amdhsa::kernel_descriptor_t::kernarg_size)); + + for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved0); ++i) + Streamer.emitInt8(0u); // FIXME: Remove the use of VK_AMDGPU_REL64 in the expression below. The // expression being created is: // (start of kernel code) - (start of kernel descriptor) // It implies R_AMDGPU_REL64, but ends up being R_AMDGPU_ABS64. - Streamer.emitValue(MCBinaryExpr::createSub( - MCSymbolRefExpr::create( - KernelCodeSymbol, MCSymbolRefExpr::VK_AMDGPU_REL64, Context), - MCSymbolRefExpr::create( - KernelDescriptorSymbol, MCSymbolRefExpr::VK_None, Context), - Context), - sizeof(KernelDescriptor.kernel_code_entry_byte_offset)); - for (uint8_t Res : KernelDescriptor.reserved1) - Streamer.emitInt8(Res); - Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc3); - Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc1); - Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc2); - Streamer.emitInt16(KernelDescriptor.kernel_code_properties); - Streamer.emitInt16(KernelDescriptor.kernarg_preload); - for (uint8_t Res : KernelDescriptor.reserved3) - Streamer.emitInt8(Res); + Streamer.emitValue( + MCBinaryExpr::createSub( + MCSymbolRefExpr::create(KernelCodeSymbol, + MCSymbolRefExpr::VK_AMDGPU_REL64, Context), + MCSymbolRefExpr::create(KernelDescriptorSymbol, + MCSymbolRefExpr::VK_None, Context), + Context), + sizeof(amdhsa::kernel_descriptor_t::kernel_code_entry_byte_offset)); + for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved1); ++i) + Streamer.emitInt8(0u); + Streamer.emitValue(KernelDescriptor.compute_pgm_rsrc3, + sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc3)); + Streamer.emitValue(KernelDescriptor.compute_pgm_rsrc1, + sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc1)); + Streamer.emitValue(KernelDescriptor.compute_pgm_rsrc2, + sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc2)); + Streamer.emitValue( + KernelDescriptor.kernel_code_properties, + sizeof(amdhsa::kernel_descriptor_t::kernel_code_properties)); + Streamer.emitValue(KernelDescriptor.kernarg_preload, + sizeof(amdhsa::kernel_descriptor_t::kernarg_preload)); + for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved3); ++i) + Streamer.emitInt8(0u); } diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h index 5aa80ff578c6..706897a5dc1f 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h @@ -22,15 +22,13 @@ class MCSymbol; class formatted_raw_ostream; namespace AMDGPU { + +struct MCKernelDescriptor; namespace HSAMD { struct Metadata; } } // namespace AMDGPU -namespace amdhsa { -struct kernel_descriptor_t; -} - class AMDGPUTargetStreamer : public MCTargetStreamer { AMDGPUPALMetadata PALMetadata; @@ -94,10 +92,11 @@ public: return true; } - virtual void EmitAmdhsaKernelDescriptor( - const MCSubtargetInfo &STI, StringRef KernelName, - const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, - uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) {} + virtual void + EmitAmdhsaKernelDescriptor(const MCSubtargetInfo &STI, StringRef KernelName, + const AMDGPU::MCKernelDescriptor &KernelDescriptor, + uint64_t NextVGPR, uint64_t NextSGPR, + bool ReserveVCC, bool ReserveFlatScr) {} static StringRef getArchNameFromElfMach(unsigned ElfMach); static unsigned getElfMach(StringRef GPU); @@ -150,10 +149,11 @@ public: bool EmitKernargPreloadHeader(const MCSubtargetInfo &STI, bool TrapEnabled) override; - void EmitAmdhsaKernelDescriptor( - const MCSubtargetInfo &STI, StringRef KernelName, - const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, - uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) override; + void + EmitAmdhsaKernelDescriptor(const MCSubtargetInfo &STI, StringRef KernelName, + const AMDGPU::MCKernelDescriptor &KernelDescriptor, + uint64_t NextVGPR, uint64_t NextSGPR, + bool ReserveVCC, bool ReserveFlatScr) override; }; class AMDGPUTargetELFStreamer final : public AMDGPUTargetStreamer { @@ -205,10 +205,11 @@ public: bool EmitKernargPreloadHeader(const MCSubtargetInfo &STI, bool TrapEnabled) override; - void EmitAmdhsaKernelDescriptor( - const MCSubtargetInfo &STI, StringRef KernelName, - const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, - uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) override; + void + EmitAmdhsaKernelDescriptor(const MCSubtargetInfo &STI, StringRef KernelName, + const AMDGPU::MCKernelDescriptor &KernelDescriptor, + uint64_t NextVGPR, uint64_t NextSGPR, + bool ReserveVCC, bool ReserveFlatScr) override; }; } #endif diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt index 0842a58f794b..14a02b6d8e36 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt @@ -8,6 +8,7 @@ add_llvm_component_library(LLVMAMDGPUDesc AMDGPUMCExpr.cpp AMDGPUMCTargetDesc.cpp AMDGPUTargetStreamer.cpp + AMDGPUMCKernelDescriptor.cpp R600InstPrinter.cpp R600MCCodeEmitter.cpp R600MCTargetDesc.cpp diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index 6d53f68ace70..4970055c4bdb 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -10,6 +10,7 @@ #include "AMDGPU.h" #include "AMDGPUAsmUtils.h" #include "AMDKernelCodeT.h" +#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "llvm/ADT/StringExtras.h" #include "llvm/BinaryFormat/ELF.h" @@ -20,6 +21,7 @@ #include "llvm/IR/IntrinsicsAMDGPU.h" #include "llvm/IR/IntrinsicsR600.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSubtargetInfo.h" @@ -1215,44 +1217,64 @@ void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, } } -amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor( - const MCSubtargetInfo *STI) { +MCKernelDescriptor getDefaultAmdhsaKernelDescriptor(const MCSubtargetInfo *STI, + MCContext &Ctx) { IsaVersion Version = getIsaVersion(STI->getCPU()); - amdhsa::kernel_descriptor_t KD; - memset(&KD, 0, sizeof(KD)); - - AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, - amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE); - if (Version.Major >= 12) { - AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, 0); - AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_DISABLE_PERF, 0); - } else { - AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, 1); - AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, 1); + MCKernelDescriptor KD; + const MCExpr *ZeroMCExpr = MCConstantExpr::create(0, Ctx); + const MCExpr *OneMCExpr = MCConstantExpr::create(1, Ctx); + + KD.group_segment_fixed_size = ZeroMCExpr; + KD.private_segment_fixed_size = ZeroMCExpr; + KD.compute_pgm_rsrc1 = ZeroMCExpr; + KD.compute_pgm_rsrc2 = ZeroMCExpr; + KD.compute_pgm_rsrc3 = ZeroMCExpr; + KD.kernarg_size = ZeroMCExpr; + KD.kernel_code_properties = ZeroMCExpr; + KD.kernarg_preload = ZeroMCExpr; + + MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc1, + MCConstantExpr::create(amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE, Ctx), + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, Ctx); + if (Version.Major < 12) { + MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc1, OneMCExpr, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, Ctx); + MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc1, OneMCExpr, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, Ctx); } - AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1); + MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc2, OneMCExpr, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X_SHIFT, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, Ctx); if (Version.Major >= 10) { - AMDHSA_BITS_SET(KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, - STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0); - AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, - STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1); - AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, 1); - } - if (AMDGPU::isGFX90A(*STI)) { - AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, - amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, - STI->getFeatureBits().test(FeatureTgSplit) ? 1 : 0); + if (STI->getFeatureBits().test(FeatureWavefrontSize32)) + MCKernelDescriptor::bits_set( + KD.kernel_code_properties, OneMCExpr, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32_SHIFT, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, Ctx); + if (!STI->getFeatureBits().test(FeatureCuMode)) + MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc1, OneMCExpr, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, Ctx); + + MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc1, OneMCExpr, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED_SHIFT, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, Ctx); } + if (AMDGPU::isGFX90A(*STI) && STI->getFeatureBits().test(FeatureTgSplit)) + MCKernelDescriptor::bits_set( + KD.compute_pgm_rsrc3, OneMCExpr, + amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT, + amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Ctx); return KD; } diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index 29ac402d9535..32b73f1d868d 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -26,6 +26,7 @@ struct Align; class Argument; class Function; class GlobalValue; +class MCContext; class MCInstrInfo; class MCRegisterClass; class MCRegisterInfo; @@ -34,12 +35,9 @@ class StringRef; class Triple; class raw_ostream; -namespace amdhsa { -struct kernel_descriptor_t; -} - namespace AMDGPU { +struct MCKernelDescriptor; struct IsaVersion; /// Generic target versions emitted by this version of LLVM. @@ -852,8 +850,8 @@ unsigned mapWMMA3AddrTo2AddrOpcode(unsigned Opc); void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, const MCSubtargetInfo *STI); -amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor( - const MCSubtargetInfo *STI); +MCKernelDescriptor getDefaultAmdhsaKernelDescriptor(const MCSubtargetInfo *STI, + MCContext &Ctx); bool isGroupSegment(const GlobalValue *GV); bool isGlobalSegment(const GlobalValue *GV); diff --git a/llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s b/llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s new file mode 100644 index 000000000000..4623500987be --- /dev/null +++ b/llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s @@ -0,0 +1,27 @@ +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck --check-prefix=ASM %s +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -filetype=obj < %s > %t +// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s + +// OBJDUMP: 0000 00000000 0f000000 00000000 00000000 + +.text + +.p2align 8 +.type caller,@function +caller: + s_endpgm + +.rodata + +.p2align 6 +.amdhsa_kernel caller + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_private_segment_fixed_size max(7, callee1.private_seg_size, callee2.private_seg_size) +.end_amdhsa_kernel + +.set callee1.private_seg_size, 4 +.set callee2.private_seg_size, 15 + +// ASM: .amdhsa_private_segment_fixed_size max(7, callee1.private_seg_size, callee2.private_seg_size) diff --git a/llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s b/llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s new file mode 100644 index 000000000000..fab3e893352b --- /dev/null +++ b/llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s @@ -0,0 +1,281 @@ +// RUN: not llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a %s 2>&1 | FileCheck --check-prefix=ASM %s + +// Some expression currently require (immediately) solvable expressions, i.e., +// they don't depend on yet-unknown symbolic values. + +.text +// ASM: .text + +.amdhsa_code_object_version 4 +// ASM: .amdhsa_code_object_version 4 + +.p2align 8 +.type user_sgpr_count,@function +user_sgpr_count: + s_endpgm + +.p2align 6 +.amdhsa_kernel user_sgpr_count + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_count defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_count + +.p2align 8 +.type user_sgpr_private_segment_buffer,@function +user_sgpr_private_segment_buffer: + s_endpgm + +.amdhsa_kernel user_sgpr_private_segment_buffer + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_private_segment_buffer defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer + +.p2align 8 +.type user_sgpr_kernarg_preload_length,@function +user_sgpr_kernarg_preload_length: + s_endpgm + +.amdhsa_kernel user_sgpr_kernarg_preload_length + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_kernarg_preload_length defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length defined_boolean + +.p2align 8 +.type user_sgpr_kernarg_preload_offset,@function +user_sgpr_kernarg_preload_offset: + s_endpgm + +.amdhsa_kernel user_sgpr_kernarg_preload_offset + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_kernarg_preload_offset defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset defined_boolean + +.p2align 8 +.type user_sgpr_dispatch_ptr,@function +user_sgpr_dispatch_ptr: + s_endpgm + +.p2align 6 +.amdhsa_kernel user_sgpr_dispatch_ptr + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_dispatch_ptr defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr + +.p2align 8 +.type user_sgpr_queue_ptr,@function +user_sgpr_queue_ptr: + s_endpgm + +.p2align 6 +.amdhsa_kernel user_sgpr_queue_ptr + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_queue_ptr defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr + +.p2align 8 +.type user_sgpr_kernarg_segment_ptr,@function +user_sgpr_kernarg_segment_ptr: + s_endpgm + +.p2align 6 +.amdhsa_kernel user_sgpr_kernarg_segment_ptr + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_kernarg_segment_ptr defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr + +.p2align 8 +.type user_sgpr_dispatch_id,@function +user_sgpr_dispatch_id: + s_endpgm + +.p2align 6 +.amdhsa_kernel user_sgpr_dispatch_id + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_dispatch_id defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id + +.p2align 8 +.type user_sgpr_flat_scratch_init,@function +user_sgpr_flat_scratch_init: + s_endpgm + +.p2align 6 +.amdhsa_kernel user_sgpr_flat_scratch_init + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_flat_scratch_init defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init + +.p2align 8 +.type user_sgpr_private_segment_size,@function +user_sgpr_private_segment_size: + s_endpgm + +.p2align 6 +.amdhsa_kernel user_sgpr_private_segment_size + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_user_sgpr_private_segment_size defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size + +.p2align 8 +.type wavefront_size32,@function +wavefront_size32: + s_endpgm + +.p2align 6 +.amdhsa_kernel wavefront_size32 + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_wavefront_size32 defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_wavefront_size32 + +.p2align 8 +.type next_free_vgpr,@function +next_free_vgpr: + s_endpgm + +.p2align 6 +.amdhsa_kernel next_free_vgpr + .amdhsa_next_free_vgpr defined_boolean + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_next_free_vgpr + +.p2align 8 +.type next_free_sgpr,@function +next_free_sgpr: + s_endpgm + +.p2align 6 +.amdhsa_kernel next_free_sgpr + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr defined_boolean + .amdhsa_accum_offset 4 +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_next_free_sgpr + +.p2align 8 +.type accum_offset,@function +accum_offset: + s_endpgm + +.p2align 6 +.amdhsa_kernel accum_offset + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_accum_offset + +.p2align 8 +.type reserve_vcc,@function +reserve_vcc: + s_endpgm + +.p2align 6 +.amdhsa_kernel reserve_vcc + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_reserve_vcc defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_reserve_vcc + +.p2align 8 +.type reserve_flat_scratch,@function +reserve_flat_scratch: + s_endpgm + +.p2align 6 +.amdhsa_kernel reserve_flat_scratch + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_reserve_flat_scratch defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_reserve_flat_scratch + +.p2align 8 +.type shared_vgpr_count,@function +shared_vgpr_count: + s_endpgm + +.p2align 6 +.amdhsa_kernel shared_vgpr_count + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 + .amdhsa_shared_vgpr_count defined_boolean +.end_amdhsa_kernel + +// ASM: error: directive should have resolvable expression +// ASM-NEXT: .amdhsa_shared_vgpr_count + +.set defined_boolean, 1 + +// ASM: .set defined_boolean, 1 +// ASM-NEXT: .no_dead_strip defined_boolean diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s new file mode 100644 index 000000000000..95af59c413ae --- /dev/null +++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s @@ -0,0 +1,190 @@ +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck --check-prefix=ASM %s +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 -filetype=obj < %s > %t +// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s + +// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). +// When going from asm -> obj, the expressions should get resolved (through fixups), + +// OBJDUMP: Contents of section .rodata +// expr_defined_later +// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 +// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0030 00f0afe4 801f007f 000c0000 00000000 +// expr_defined +// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 +// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0070 00f0afe4 801f007f 000c0000 00000000 + +.text +// ASM: .text + +.amdhsa_code_object_version 4 +// ASM: .amdhsa_code_object_version 4 + +.p2align 8 +.type expr_defined_later,@function +expr_defined_later: + s_endpgm + +.p2align 8 +.type expr_defined,@function +expr_defined: + s_endpgm + +.rodata +// ASM: .rodata + +.p2align 6 +.amdhsa_kernel expr_defined_later + .amdhsa_group_segment_fixed_size defined_value+2 + .amdhsa_private_segment_fixed_size defined_value+3 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_fp16_overflow defined_boolean + .amdhsa_workgroup_processor_mode defined_boolean + .amdhsa_memory_ordered defined_boolean + .amdhsa_forward_progress defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +.set defined_value, 41 +.set defined_2_bits, 3 +.set defined_boolean, 1 + +.p2align 6 +.amdhsa_kernel expr_defined + .amdhsa_group_segment_fixed_size defined_value+1 + .amdhsa_private_segment_fixed_size defined_value+2 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_fp16_overflow defined_boolean + .amdhsa_workgroup_processor_mode defined_boolean + .amdhsa_memory_ordered defined_boolean + .amdhsa_forward_progress defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +// ASM: .amdhsa_kernel expr_defined_later +// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 +// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1)>>0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4 +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&32)>>5 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6 +// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10 +// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 +// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 +// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 +// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 +// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26 +// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&536870912)>>29 +// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&1073741824)>>30 +// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2147483648)>>31 +// ASM-NEXT: .amdhsa_shared_vgpr_count 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29 +// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30 +// ASM-NEXT: .end_amdhsa_kernel + +// ASM: .set defined_value, 41 +// ASM-NEXT: .no_dead_strip defined_value +// ASM-NEXT: .set defined_2_bits, 3 +// ASM-NEXT: .no_dead_strip defined_2_bits +// ASM-NEXT: .set defined_boolean, 1 +// ASM-NEXT: .no_dead_strip defined_boolean + +// ASM: .amdhsa_kernel expr_defined +// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 +// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 +// ASM-NEXT: .amdhsa_wavefront_size32 1 +// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 +// ASM-NEXT: .amdhsa_float_round_mode_32 3 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 +// ASM-NEXT: .amdhsa_dx10_clamp 1 +// ASM-NEXT: .amdhsa_ieee_mode 1 +// ASM-NEXT: .amdhsa_fp16_overflow 1 +// ASM-NEXT: .amdhsa_workgroup_processor_mode 1 +// ASM-NEXT: .amdhsa_memory_ordered 1 +// ASM-NEXT: .amdhsa_forward_progress 1 +// ASM-NEXT: .amdhsa_shared_vgpr_count 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 +// ASM-NEXT: .amdhsa_exception_int_div_zero 1 +// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s new file mode 100644 index 000000000000..e1107fb69ba4 --- /dev/null +++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s @@ -0,0 +1,186 @@ +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefix=ASM %s +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1100 -filetype=obj < %s > %t +// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s + +// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). +// When going from asm -> obj, the expressions should get resolved (through fixups), + +// OBJDUMP: Contents of section .rodata +// expr_defined_later +// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 +// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0030 00f0afe4 811f007f 000c0000 00000000 +// expr_defined +// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 +// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0070 00f0afe4 811f007f 000c0000 00000000 + +.text +// ASM: .text + +.amdhsa_code_object_version 4 +// ASM: .amdhsa_code_object_version 4 + +.p2align 8 +.type expr_defined_later,@function +expr_defined_later: + s_endpgm + +.p2align 8 +.type expr_defined,@function +expr_defined: + s_endpgm + +.rodata +// ASM: .rodata + +.p2align 6 +.amdhsa_kernel expr_defined_later + .amdhsa_group_segment_fixed_size defined_value+2 + .amdhsa_private_segment_fixed_size defined_value+3 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_fp16_overflow defined_boolean + .amdhsa_workgroup_processor_mode defined_boolean + .amdhsa_memory_ordered defined_boolean + .amdhsa_forward_progress defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_enable_private_segment defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +.set defined_value, 41 +.set defined_2_bits, 3 +.set defined_boolean, 1 + +.p2align 6 +.amdhsa_kernel expr_defined + .amdhsa_group_segment_fixed_size defined_value+1 + .amdhsa_private_segment_fixed_size defined_value+2 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_fp16_overflow defined_boolean + .amdhsa_workgroup_processor_mode defined_boolean + .amdhsa_memory_ordered defined_boolean + .amdhsa_forward_progress defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_enable_private_segment defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +// ASM: .amdhsa_kernel expr_defined_later +// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 +// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6 +// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10 +// ASM-NEXT: .amdhsa_enable_private_segment (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 +// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 +// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 +// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26 +// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&536870912)>>29 +// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&1073741824)>>30 +// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2147483648)>>31 +// ASM-NEXT: .amdhsa_shared_vgpr_count 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29 +// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30 +// ASM-NEXT: .end_amdhsa_kernel + +// ASM: .set defined_value, 41 +// ASM-NEXT: .no_dead_strip defined_value +// ASM-NEXT: .set defined_2_bits, 3 +// ASM-NEXT: .no_dead_strip defined_2_bits +// ASM-NEXT: .set defined_boolean, 1 +// ASM-NEXT: .no_dead_strip defined_boolean + +// ASM: .amdhsa_kernel expr_defined +// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 +// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 +// ASM-NEXT: .amdhsa_wavefront_size32 1 +// ASM-NEXT: .amdhsa_enable_private_segment 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_float_round_mode_32 3 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 +// ASM-NEXT: .amdhsa_dx10_clamp 1 +// ASM-NEXT: .amdhsa_ieee_mode 1 +// ASM-NEXT: .amdhsa_fp16_overflow 1 +// ASM-NEXT: .amdhsa_workgroup_processor_mode 1 +// ASM-NEXT: .amdhsa_memory_ordered 1 +// ASM-NEXT: .amdhsa_forward_progress 1 +// ASM-NEXT: .amdhsa_shared_vgpr_count 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 +// ASM-NEXT: .amdhsa_exception_int_div_zero 1 +// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s new file mode 100644 index 000000000000..449616d35186 --- /dev/null +++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s @@ -0,0 +1,184 @@ +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefix=ASM %s +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1200 -filetype=obj < %s > %t +// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s + +// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). +// When going from asm -> obj, the expressions should get resolved (through fixups), + +// OBJDUMP: Contents of section .rodata +// expr_defined_later +// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 +// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0030 00f02fe4 811f007f 000c0000 00000000 +// expr_defined +// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 +// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0070 00f02fe4 811f007f 000c0000 00000000 + +.text +// ASM: .text + +.amdhsa_code_object_version 4 +// ASM: .amdhsa_code_object_version 4 + +.p2align 8 +.type expr_defined_later,@function +expr_defined_later: + s_endpgm + +.p2align 8 +.type expr_defined,@function +expr_defined: + s_endpgm + +.rodata +// ASM: .rodata + +.p2align 6 +.amdhsa_kernel expr_defined_later + .amdhsa_group_segment_fixed_size defined_value+2 + .amdhsa_private_segment_fixed_size defined_value+3 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_fp16_overflow defined_boolean + .amdhsa_workgroup_processor_mode defined_boolean + .amdhsa_memory_ordered defined_boolean + .amdhsa_forward_progress defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_round_robin_scheduling defined_boolean + .amdhsa_enable_private_segment defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +.set defined_value, 41 +.set defined_2_bits, 3 +.set defined_boolean, 1 + +.p2align 6 +.amdhsa_kernel expr_defined + .amdhsa_group_segment_fixed_size defined_value+1 + .amdhsa_private_segment_fixed_size defined_value+2 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_fp16_overflow defined_boolean + .amdhsa_workgroup_processor_mode defined_boolean + .amdhsa_memory_ordered defined_boolean + .amdhsa_forward_progress defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_round_robin_scheduling defined_boolean + .amdhsa_enable_private_segment defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +// ASM: .amdhsa_kernel expr_defined_later +// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 +// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6 +// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10 +// ASM-NEXT: .amdhsa_enable_private_segment (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 +// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26 +// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&536870912)>>29 +// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&1073741824)>>30 +// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&2147483648)>>31 +// ASM-NEXT: .amdhsa_round_robin_scheduling (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29 +// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30 +// ASM-NEXT: .end_amdhsa_kernel + +// ASM: .set defined_value, 41 +// ASM-NEXT: .no_dead_strip defined_value +// ASM-NEXT: .set defined_2_bits, 3 +// ASM-NEXT: .no_dead_strip defined_2_bits +// ASM-NEXT: .set defined_boolean, 1 +// ASM-NEXT: .no_dead_strip defined_boolean + +// ASM: .amdhsa_kernel expr_defined +// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 +// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 +// ASM-NEXT: .amdhsa_wavefront_size32 1 +// ASM-NEXT: .amdhsa_enable_private_segment 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_float_round_mode_32 3 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 +// ASM-NEXT: .amdhsa_fp16_overflow 1 +// ASM-NEXT: .amdhsa_workgroup_processor_mode 1 +// ASM-NEXT: .amdhsa_memory_ordered 1 +// ASM-NEXT: .amdhsa_forward_progress 1 +// ASM-NEXT: .amdhsa_round_robin_scheduling 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 +// ASM-NEXT: .amdhsa_exception_int_div_zero 1 +// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s new file mode 100644 index 000000000000..c7e05441b45f --- /dev/null +++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s @@ -0,0 +1,168 @@ +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck --check-prefix=ASM %s +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj < %s > %t +// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s + +// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). +// When going from asm -> obj, the expressions should get resolved (through fixups), + +// OBJDUMP: Contents of section .rodata +// expr_defined_later +// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 +// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0030 00f0af00 801f007f 00080000 00000000 +// expr_defined +// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 +// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0070 00f0af00 801f007f 00080000 00000000 + +.text +// ASM: .text + +.amdhsa_code_object_version 4 +// ASM: .amdhsa_code_object_version 4 + +.p2align 8 +.type expr_defined_later,@function +expr_defined_later: + s_endpgm + +.p2align 8 +.type expr_defined,@function +expr_defined: + s_endpgm + +.rodata +// ASM: .rodata + +.p2align 6 +.amdhsa_kernel expr_defined_later + .amdhsa_group_segment_fixed_size defined_value+2 + .amdhsa_private_segment_fixed_size defined_value+3 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +.set defined_value, 41 +.set defined_2_bits, 3 +.set defined_boolean, 1 + +.p2align 6 +.amdhsa_kernel expr_defined + .amdhsa_group_segment_fixed_size defined_value+1 + .amdhsa_private_segment_fixed_size defined_value+2 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +// ASM: .amdhsa_kernel expr_defined_later +// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 +// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((0&(~2048))|(defined_boolean<<11))&1)>>0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((0&(~2048))|(defined_boolean<<11))&2)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((0&(~2048))|(defined_boolean<<11))&4)>>2 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((0&(~2048))|(defined_boolean<<11))&8)>>3 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((0&(~2048))|(defined_boolean<<11))&16)>>4 +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((0&(~2048))|(defined_boolean<<11))&32)>>5 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((0&(~2048))|(defined_boolean<<11))&64)>>6 +// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 +// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 +// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29 +// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30 +// ASM-NEXT: .end_amdhsa_kernel + +// ASM: .set defined_value, 41 +// ASM-NEXT: .no_dead_strip defined_value +// ASM-NEXT: .set defined_2_bits, 3 +// ASM-NEXT: .no_dead_strip defined_2_bits +// ASM-NEXT: .set defined_boolean, 1 +// ASM-NEXT: .no_dead_strip defined_boolean + +// ASM: .amdhsa_kernel expr_defined +// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 +// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 +// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_float_round_mode_32 3 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 +// ASM-NEXT: .amdhsa_dx10_clamp 1 +// ASM-NEXT: .amdhsa_ieee_mode 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 +// ASM-NEXT: .amdhsa_exception_int_div_zero 1 +// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s new file mode 100644 index 000000000000..49a5015987a6 --- /dev/null +++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s @@ -0,0 +1,171 @@ +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx801 < %s | FileCheck --check-prefix=ASM %s + +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx801 -filetype=obj < %s > %t +// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s + +// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). +// When going from asm -> obj, the expressions should get resolved (through fixups), + +// OBJDUMP: Contents of section .rodata +// expr_defined_later +// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 +// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0030 00f0af00 801f007f 00080000 00000000 +// expr_defined +// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 +// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0070 00f0af00 801f007f 00080000 00000000 + +.text +// ASM: .text + +.amdhsa_code_object_version 4 +// ASM: .amdhsa_code_object_version 4 + +.p2align 8 +.type expr_defined_later,@function +expr_defined_later: + s_endpgm + +.p2align 8 +.type expr_defined,@function +expr_defined: + s_endpgm + +.rodata +// ASM: .rodata + +.p2align 6 +.amdhsa_kernel expr_defined_later + .amdhsa_group_segment_fixed_size defined_value+2 + .amdhsa_private_segment_fixed_size defined_value+3 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +.set defined_value, 41 +.set defined_2_bits, 3 +.set defined_boolean, 1 + +.p2align 6 +.amdhsa_kernel expr_defined + .amdhsa_group_segment_fixed_size defined_value+1 + .amdhsa_private_segment_fixed_size defined_value+2 + .amdhsa_system_vgpr_workitem_id defined_2_bits + .amdhsa_float_round_mode_32 defined_2_bits + .amdhsa_float_round_mode_16_64 defined_2_bits + .amdhsa_float_denorm_mode_32 defined_2_bits + .amdhsa_float_denorm_mode_16_64 defined_2_bits + .amdhsa_system_sgpr_workgroup_id_x defined_boolean + .amdhsa_system_sgpr_workgroup_id_y defined_boolean + .amdhsa_system_sgpr_workgroup_id_z defined_boolean + .amdhsa_system_sgpr_workgroup_info defined_boolean + .amdhsa_exception_fp_ieee_invalid_op defined_boolean + .amdhsa_exception_fp_denorm_src defined_boolean + .amdhsa_exception_fp_ieee_div_zero defined_boolean + .amdhsa_exception_fp_ieee_overflow defined_boolean + .amdhsa_exception_fp_ieee_underflow defined_boolean + .amdhsa_exception_fp_ieee_inexact defined_boolean + .amdhsa_exception_int_div_zero defined_boolean + .amdhsa_uses_dynamic_stack defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 +.end_amdhsa_kernel + +// ASM: .amdhsa_kernel expr_defined_later +// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 +// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((0&(~2048))|(defined_boolean<<11))&1)>>0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((0&(~2048))|(defined_boolean<<11))&2)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((0&(~2048))|(defined_boolean<<11))&4)>>2 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((0&(~2048))|(defined_boolean<<11))&8)>>3 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((0&(~2048))|(defined_boolean<<11))&16)>>4 +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((0&(~2048))|(defined_boolean<<11))&32)>>5 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((0&(~2048))|(defined_boolean<<11))&64)>>6 +// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 +// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 +// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 +// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29 +// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30 +// ASM-NEXT: .end_amdhsa_kernel + +// ASM: .set defined_value, 41 +// ASM-NEXT: .no_dead_strip defined_value +// ASM-NEXT: .set defined_2_bits, 3 +// ASM-NEXT: .no_dead_strip defined_2_bits +// ASM-NEXT: .set defined_boolean, 1 +// ASM-NEXT: .no_dead_strip defined_boolean + +// ASM: .amdhsa_kernel expr_defined +// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 +// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 +// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 +// ASM-NEXT: .amdhsa_float_round_mode_32 3 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 +// ASM-NEXT: .amdhsa_dx10_clamp 1 +// ASM-NEXT: .amdhsa_ieee_mode 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 +// ASM-NEXT: .amdhsa_exception_int_div_zero 1 +// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s new file mode 100644 index 000000000000..b7f89239160f --- /dev/null +++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s @@ -0,0 +1,148 @@ +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck --check-prefix=ASM %s +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -filetype=obj < %s > %t +// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s + +// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). +// When going from asm -> obj, the expressions should get resolved (through fixups), + +// OBJDUMP: Contents of section .rodata +// expr_defined_later +// OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000100 +// OBJDUMP-NEXT: 0030 0000ac04 81000000 00000000 00000000 +// expr_defined +// OBJDUMP-NEXT: 0040 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000100 +// OBJDUMP-NEXT: 0070 0000ac04 81000000 00000000 00000000 + +.text +// ASM: .text + +.amdhsa_code_object_version 4 +// ASM: .amdhsa_code_object_version 4 + +.p2align 8 +.type expr_defined_later,@function +expr_defined_later: + s_endpgm + +.p2align 8 +.type expr_defined,@function +expr_defined: + s_endpgm + +.rodata +// ASM: .rodata + +.p2align 6 +.amdhsa_kernel expr_defined_later + .amdhsa_system_sgpr_private_segment_wavefront_offset defined_boolean + .amdhsa_dx10_clamp defined_boolean + .amdhsa_ieee_mode defined_boolean + .amdhsa_fp16_overflow defined_boolean + .amdhsa_tg_split defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 +.end_amdhsa_kernel + +.set defined_boolean, 1 + +.p2align 6 +.amdhsa_kernel expr_defined + .amdhsa_system_sgpr_private_segment_wavefront_offset defined_boolean + .amdhsa_dx10_clamp defined_boolean + .amdhsa_ieee_mode defined_boolean + .amdhsa_fp16_overflow defined_boolean + .amdhsa_tg_split defined_boolean + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 +.end_amdhsa_kernel + +// ASM: .amdhsa_kernel expr_defined_later +// ASM-NEXT: .amdhsa_group_segment_fixed_size 0 +// ASM-NEXT: .amdhsa_private_segment_fixed_size 0 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 +// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_accum_offset (((((((0&(~65536))|(defined_boolean<<16))&(~63))|(0<<0))&63)>>0)+1)*4 +// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 +// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 +// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 +// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 +// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26 +// ASM-NEXT: .amdhsa_tg_split (((((0&(~65536))|(defined_boolean<<16))&(~63))|(0<<0))&65536)>>16 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29 +// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30 +// ASM-NEXT: .end_amdhsa_kernel + +// ASM: .set defined_boolean, 1 +// ASM-NEXT: .no_dead_strip defined_boolean + +// ASM: .amdhsa_kernel expr_defined +// ASM-NEXT: .amdhsa_group_segment_fixed_size 0 +// ASM-NEXT: .amdhsa_private_segment_fixed_size 0 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 +// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 0 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 0 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_accum_offset 4 +// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 +// ASM-NEXT: .amdhsa_float_round_mode_32 0 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 0 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 0 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 +// ASM-NEXT: .amdhsa_dx10_clamp 1 +// ASM-NEXT: .amdhsa_ieee_mode 1 +// ASM-NEXT: .amdhsa_fp16_overflow 1 +// ASM-NEXT: .amdhsa_tg_split 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 0 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 0 +// ASM-NEXT: .amdhsa_exception_int_div_zero 0 +// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-tg-split.s b/llvm/test/MC/AMDGPU/hsa-tg-split.s new file mode 100644 index 000000000000..5a4d3e2c279c --- /dev/null +++ b/llvm/test/MC/AMDGPU/hsa-tg-split.s @@ -0,0 +1,74 @@ +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+xnack,+tgsplit < %s | FileCheck --check-prefix=ASM %s +// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+xnack,+tgsplit -filetype=obj < %s > %t +// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s + +// OBJDUMP: Contents of section .rodata +// OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 +// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000100 +// OBJDUMP-NEXT: 0030 0000ac00 80000000 00000000 00000000 + +.text +// ASM: .text + +.amdgcn_target "amdgcn-amd-amdhsa--gfx90a:xnack+" +// ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx90a:xnack+" + +.amdhsa_code_object_version 4 +// ASM: .amdhsa_code_object_version 4 + +.p2align 8 +.type minimal,@function +minimal: + s_endpgm + +.rodata +// ASM: .rodata + +.p2align 6 +.amdhsa_kernel minimal + .amdhsa_next_free_vgpr 0 + .amdhsa_next_free_sgpr 0 + .amdhsa_accum_offset 4 +.end_amdhsa_kernel + +// ASM: .amdhsa_kernel minimal +// ASM-NEXT: .amdhsa_group_segment_fixed_size 0 +// ASM-NEXT: .amdhsa_private_segment_fixed_size 0 +// ASM-NEXT: .amdhsa_kernarg_size 0 +// ASM-NEXT: .amdhsa_user_sgpr_count 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 +// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 +// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length 0 +// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset 0 +// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 +// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0 +// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 0 +// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 0 +// ASM-NEXT: .amdhsa_next_free_vgpr 0 +// ASM-NEXT: .amdhsa_next_free_sgpr 0 +// ASM-NEXT: .amdhsa_accum_offset 4 +// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 +// ASM-NEXT: .amdhsa_float_round_mode_32 0 +// ASM-NEXT: .amdhsa_float_round_mode_16_64 0 +// ASM-NEXT: .amdhsa_float_denorm_mode_32 0 +// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 +// ASM-NEXT: .amdhsa_dx10_clamp 1 +// ASM-NEXT: .amdhsa_ieee_mode 1 +// ASM-NEXT: .amdhsa_fp16_overflow 0 +// ASM-NEXT: .amdhsa_tg_split 1 +// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 0 +// ASM-NEXT: .amdhsa_exception_fp_denorm_src 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 0 +// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 0 +// ASM-NEXT: .amdhsa_exception_int_div_zero 0 +// ASM-NEXT: .end_amdhsa_kernel -- GitLab From 26c3d018b1f57b33779c53d4fed5ea895810c314 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Thu, 21 Mar 2024 13:57:14 +0000 Subject: [PATCH 146/296] [gn build] Port 857161c367a1 --- .../gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn index 12d875cf40c9..5ba91fcec83a 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn @@ -104,6 +104,7 @@ static_library("MCTargetDesc") { "AMDGPUMCAsmInfo.cpp", "AMDGPUMCCodeEmitter.cpp", "AMDGPUMCExpr.cpp", + "AMDGPUMCKernelDescriptor.cpp", "AMDGPUMCTargetDesc.cpp", "AMDGPUTargetStreamer.cpp", "R600InstPrinter.cpp", -- GitLab From a11d9b463966d31ecedb373115abdcca54f704c3 Mon Sep 17 00:00:00 2001 From: Akira Hatanaka Date: Thu, 21 Mar 2024 07:10:42 -0700 Subject: [PATCH 147/296] Disable driver tests on macosx that are currently disabled on darwin (#85990) macosx and darwin in triples are equivalent. rdar://124246653 --- clang/test/Driver/clang-offload-bundler-asserts-on.c | 2 +- clang/test/Driver/clang-offload-bundler-standardize.c | 2 +- clang/test/Driver/clang-offload-bundler.c | 2 +- clang/test/Driver/fat-archive-unbundle-ext.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/clang/test/Driver/clang-offload-bundler-asserts-on.c b/clang/test/Driver/clang-offload-bundler-asserts-on.c index 521c8641ff54..eb11d5fbbee4 100644 --- a/clang/test/Driver/clang-offload-bundler-asserts-on.c +++ b/clang/test/Driver/clang-offload-bundler-asserts-on.c @@ -1,6 +1,6 @@ // REQUIRES: x86-registered-target // REQUIRES: asserts -// UNSUPPORTED: target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}} +// UNSUPPORTED: target={{.*}}-macosx{{.*}}, target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}} // Generate the file we can bundle. // RUN: %clang -O0 -target %itanium_abi_triple %s -c -o %t.o diff --git a/clang/test/Driver/clang-offload-bundler-standardize.c b/clang/test/Driver/clang-offload-bundler-standardize.c index 6a24968c30ef..91dc8947aabb 100644 --- a/clang/test/Driver/clang-offload-bundler-standardize.c +++ b/clang/test/Driver/clang-offload-bundler-standardize.c @@ -1,6 +1,6 @@ // REQUIRES: x86-registered-target // REQUIRES: asserts -// UNSUPPORTED: target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}} +// UNSUPPORTED: target={{.*}}-macosx{{.*}}, target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}} // REQUIRES: asserts // Generate the file we can bundle. diff --git a/clang/test/Driver/clang-offload-bundler.c b/clang/test/Driver/clang-offload-bundler.c index f3cd2493e052..a56a5424abf8 100644 --- a/clang/test/Driver/clang-offload-bundler.c +++ b/clang/test/Driver/clang-offload-bundler.c @@ -1,5 +1,5 @@ // REQUIRES: x86-registered-target -// UNSUPPORTED: target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}} +// UNSUPPORTED: target={{.*}}-macosx{{.*}}, target={{.*}}-darwin{{.*}}, target={{.*}}-aix{{.*}} // // Generate all the types of files we can bundle. diff --git a/clang/test/Driver/fat-archive-unbundle-ext.c b/clang/test/Driver/fat-archive-unbundle-ext.c index b409aa6313b1..e98b872f0c0c 100644 --- a/clang/test/Driver/fat-archive-unbundle-ext.c +++ b/clang/test/Driver/fat-archive-unbundle-ext.c @@ -1,5 +1,5 @@ // REQUIRES: x86-registered-target -// UNSUPPORTED: target={{.*-windows.*}}, target={{.*-darwin.*}}, target={{.*}}-aix{{.*}} +// UNSUPPORTED: target={{.*-windows.*}}, target={{.*}}-macosx{{.*}}, target={{.*-darwin.*}}, target={{.*}}-aix{{.*}} // Generate dummy fat object // RUN: %clang -O0 -target %itanium_abi_triple %s -c -o %t.host.o -- GitLab From 2152094a45af98c9ccfef6d5913f38c66ab8b165 Mon Sep 17 00:00:00 2001 From: Mark de Wever Date: Thu, 21 Mar 2024 15:11:45 +0100 Subject: [PATCH 148/296] [clang] Improves -print-library-module-manifest-path. (#85943) This adds a libc++ to modules.json as is currently used by libc++. When libc++.so is not found the function will search for libc++.a as fallback. --- clang/lib/Driver/Driver.cpp | 49 +++++++++++-------- ...les-print-library-module-manifest-path.cpp | 19 ++++++- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index 767c1cd47e8c..7a53764364ce 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -6203,28 +6203,35 @@ std::string Driver::GetStdModuleManifestPath(const Compilation &C, switch (TC.GetCXXStdlibType(C.getArgs())) { case ToolChain::CST_Libcxx: { - std::string lib = GetFilePath("libc++.so", TC); - - // Note when there are multiple flavours of libc++ the module json needs to - // look at the command-line arguments for the proper json. - // These flavours do not exist at the moment, but there are plans to - // provide a variant that is built with sanitizer instrumentation enabled. - - // For example - // StringRef modules = [&] { - // const SanitizerArgs &Sanitize = TC.getSanitizerArgs(C.getArgs()); - // if (Sanitize.needsAsanRt()) - // return "modules-asan.json"; - // return "modules.json"; - // }(); - - SmallString<128> path(lib.begin(), lib.end()); - llvm::sys::path::remove_filename(path); - llvm::sys::path::append(path, "modules.json"); - if (TC.getVFS().exists(path)) - return static_cast(path); + auto evaluate = [&](const char *library) -> std::optional { + std::string lib = GetFilePath(library, TC); + + // Note when there are multiple flavours of libc++ the module json needs + // to look at the command-line arguments for the proper json. These + // flavours do not exist at the moment, but there are plans to provide a + // variant that is built with sanitizer instrumentation enabled. + + // For example + // StringRef modules = [&] { + // const SanitizerArgs &Sanitize = TC.getSanitizerArgs(C.getArgs()); + // if (Sanitize.needsAsanRt()) + // return "libc++.modules-asan.json"; + // return "libc++.modules.json"; + // }(); + + SmallString<128> path(lib.begin(), lib.end()); + llvm::sys::path::remove_filename(path); + llvm::sys::path::append(path, "libc++.modules.json"); + if (TC.getVFS().exists(path)) + return static_cast(path); + + return {}; + }; - return error; + if (std::optional result = evaluate("libc++.so"); result) + return *result; + + return evaluate("libc++.a").value_or(error); } case ToolChain::CST_Libstdcxx: diff --git a/clang/test/Driver/modules-print-library-module-manifest-path.cpp b/clang/test/Driver/modules-print-library-module-manifest-path.cpp index 24797002b80f..3ba2709ad95c 100644 --- a/clang/test/Driver/modules-print-library-module-manifest-path.cpp +++ b/clang/test/Driver/modules-print-library-module-manifest-path.cpp @@ -3,6 +3,7 @@ // RUN: rm -rf %t && split-file %s %t && cd %t // RUN: mkdir -p %t/Inputs/usr/lib/x86_64-linux-gnu // RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.so +// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.a // RUN: %clang -print-library-module-manifest-path \ // RUN: -stdlib=libc++ \ @@ -10,13 +11,21 @@ // RUN: --target=x86_64-linux-gnu 2>&1 \ // RUN: | FileCheck libcxx-no-module-json.cpp -// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/modules.json +// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.modules.json // RUN: %clang -print-library-module-manifest-path \ // RUN: -stdlib=libc++ \ // RUN: -resource-dir=%t/Inputs/usr/lib/x86_64-linux-gnu \ // RUN: --target=x86_64-linux-gnu 2>&1 \ // RUN: | FileCheck libcxx.cpp +// RUN: rm %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.so +// RUN: touch %t/Inputs/usr/lib/x86_64-linux-gnu/libc++.a +// RUN: %clang -print-library-module-manifest-path \ +// RUN: -stdlib=libc++ \ +// RUN: -resource-dir=%t/Inputs/usr/lib/x86_64-linux-gnu \ +// RUN: --target=x86_64-linux-gnu 2>&1 \ +// RUN: | FileCheck libcxx-no-shared-lib.cpp + // RUN: %clang -print-library-module-manifest-path \ // RUN: -stdlib=libstdc++ \ // RUN: -resource-dir=%t/Inputs/usr/lib/x86_64-linux-gnu \ @@ -29,7 +38,13 @@ //--- libcxx.cpp -// CHECK: {{.*}}/Inputs/usr/lib/x86_64-linux-gnu{{/|\\}}modules.json +// CHECK: {{.*}}/Inputs/usr/lib/x86_64-linux-gnu{{/|\\}}libc++.modules.json + +//--- libcxx-no-shared-lib.cpp + +// Note this might find a different path depending whether search path +// contains a different libc++.so. +// CHECK: {{.*}}libc++.modules.json //--- libstdcxx.cpp -- GitLab From 2861856baf16e43a5e465e87022c6c2c2d238969 Mon Sep 17 00:00:00 2001 From: Benjamin Maxwell Date: Thu, 21 Mar 2024 14:18:56 +0000 Subject: [PATCH 149/296] [mlir][Vector] Add utility for computing scalable value bounds (#83876) This adds a new API built with the `ValueBoundsConstraintSet` to compute the bounds of possibly scalable quantities. It uses knowledge of the range of vscale (which is defined by the target architecture), to solve for the bound as either a constant or an expression in terms of vscale. The result is an `AffineMap` that will always take at most one parameter, vscale, and returns a single result, which is the bound of `value`. The API is defined as follows: ```c++ FailureOr vector::ScalableValueBoundsConstraintSet::computeScalableBound( Value value, std::optional dim, unsigned vscaleMin, unsigned vscaleMax, presburger::BoundType boundType, bool closedUB = true, StopConditionFn stopCondition = nullptr); ``` Note: `ConstantOrScalableBound` is a thin wrapper over the `AffineMap` with a utility for converting the bound to a single quantity (i.e. a size and scalable flag). We believe this API could prove useful downstream in IREE (which uses a similar analysis to hoist allocas, which currently fails for scalable vectors). --- .../IR/ScalableValueBoundsConstraintSet.h | 104 +++++++++++ .../Vector/IR/ValueBoundsOpInterfaceImpl.h | 20 +++ mlir/include/mlir/InitAllDialects.h | 2 + .../mlir/Interfaces/ValueBoundsOpInterface.h | 16 +- mlir/lib/Dialect/Vector/IR/CMakeLists.txt | 2 + .../IR/ScalableValueBoundsConstraintSet.cpp | 103 +++++++++++ .../Vector/IR/ValueBoundsOpInterfaceImpl.cpp | 51 ++++++ .../lib/Interfaces/ValueBoundsOpInterface.cpp | 86 +++++++--- .../Dialect/Vector/test-scalable-bounds.mlir | 161 ++++++++++++++++++ .../Dialect/Affine/TestReifyValueBounds.cpp | 38 ++++- 10 files changed, 555 insertions(+), 28 deletions(-) create mode 100644 mlir/include/mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h create mode 100644 mlir/include/mlir/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.h create mode 100644 mlir/lib/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.cpp create mode 100644 mlir/lib/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.cpp create mode 100644 mlir/test/Dialect/Vector/test-scalable-bounds.mlir diff --git a/mlir/include/mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h b/mlir/include/mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h new file mode 100644 index 000000000000..31e19ff1ad39 --- /dev/null +++ b/mlir/include/mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h @@ -0,0 +1,104 @@ +//===- ScalableValueBoundsConstraintSet.h - Scalable Value Bounds ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_VECTOR_IR_SCALABLEVALUEBOUNDSCONSTRAINTSET_H +#define MLIR_DIALECT_VECTOR_IR_SCALABLEVALUEBOUNDSCONSTRAINTSET_H + +#include "mlir/Analysis/Presburger/IntegerRelation.h" +#include "mlir/Dialect/Vector/IR/VectorOps.h" +#include "mlir/Interfaces/ValueBoundsOpInterface.h" + +namespace mlir::vector { + +namespace detail { + +/// Parent class for the value bounds RTTIExtends. Uses protected inheritance to +/// hide all ValueBoundsConstraintSet methods by default (as some do not use the +/// ScalableValueBoundsConstraintSet, so may produce unexpected results). +struct ValueBoundsConstraintSet : protected ::mlir::ValueBoundsConstraintSet { + using ::mlir::ValueBoundsConstraintSet::ValueBoundsConstraintSet; +}; +} // namespace detail + +/// A version of `ValueBoundsConstraintSet` that can solve for scalable bounds. +struct ScalableValueBoundsConstraintSet + : public llvm::RTTIExtends { + ScalableValueBoundsConstraintSet(MLIRContext *context, unsigned vscaleMin, + unsigned vscaleMax) + : RTTIExtends(context), vscaleMin(vscaleMin), vscaleMax(vscaleMax){}; + + using RTTIExtends::bound; + using RTTIExtends::StopConditionFn; + + /// A thin wrapper over an `AffineMap` which can represent a constant bound, + /// or a scalable bound (in terms of vscale). The `AffineMap` will always + /// take at most one parameter, vscale, and returns a single result, which is + /// the bound of value. + struct ConstantOrScalableBound { + AffineMap map; + + struct BoundSize { + int64_t baseSize{0}; + bool scalable{false}; + }; + + /// Get the (possibly) scalable size of the bound, returns failure if + /// the bound cannot be represented as a single quantity. + FailureOr getSize() const; + }; + + /// Computes a (possibly) scalable bound for a given value. This is + /// similar to `ValueBoundsConstraintSet::computeConstantBound()`, but + /// uses knowledge of the range of vscale to compute either a constant + /// bound, an expression in terms of vscale, or failure if no bound can + /// be computed. + /// + /// The resulting `AffineMap` will always take at most one parameter, + /// vscale, and return a single result, which is the bound of `value`. + /// + /// Note: `vscaleMin` must be `<=` to `vscaleMax`. If `vscaleMin` == + /// `vscaleMax`, the resulting bound (if found), will be constant. + static FailureOr + computeScalableBound(Value value, std::optional dim, + unsigned vscaleMin, unsigned vscaleMax, + presburger::BoundType boundType, bool closedUB = true, + StopConditionFn stopCondition = nullptr); + + /// Get the value of vscale. Returns `nullptr` vscale as not been encountered. + Value getVscaleValue() const { return vscale; } + + /// Sets the value of vscale. Asserts if vscale has already been set. + void setVscale(vector::VectorScaleOp vscaleOp) { + assert(!vscale && "expected vscale to be unset"); + vscale = vscaleOp.getResult(); + } + + /// The minimum possible value of vscale. + unsigned getVscaleMin() const { return vscaleMin; } + + /// The maximum possible value of vscale. + unsigned getVscaleMax() const { return vscaleMax; } + + static char ID; + +private: + const unsigned vscaleMin; + const unsigned vscaleMax; + + // This will be set when the first `vector.vscale` operation is found within + // the `ValueBoundsOpInterface` implementation then reused from there on. + Value vscale = nullptr; +}; + +using ConstantOrScalableBound = + ScalableValueBoundsConstraintSet::ConstantOrScalableBound; + +} // namespace mlir::vector + +#endif // MLIR_DIALECT_VECTOR_IR_SCALABLEVALUEBOUNDSCONSTRAINTSET_H diff --git a/mlir/include/mlir/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.h b/mlir/include/mlir/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.h new file mode 100644 index 000000000000..4794bc9016c6 --- /dev/null +++ b/mlir/include/mlir/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.h @@ -0,0 +1,20 @@ +//===- ValueBoundsOpInterfaceImpl.h - Impl. of ValueBoundsOpInterface -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef MLIR_DIALECT_VECTOR_IR_VALUEBOUNDSOPINTERFACEIMPL_H +#define MLIR_DIALECT_VECTOR_IR_VALUEBOUNDSOPINTERFACEIMPL_H + +namespace mlir { +class DialectRegistry; + +namespace vector { +void registerValueBoundsOpInterfaceExternalModels(DialectRegistry ®istry); +} // namespace vector +} // namespace mlir + +#endif // MLIR_DIALECT_VECTOR_IR_VALUEBOUNDSOPINTERFACEIMPL_H diff --git a/mlir/include/mlir/InitAllDialects.h b/mlir/include/mlir/InitAllDialects.h index 21775e11e071..9bbf12d13254 100644 --- a/mlir/include/mlir/InitAllDialects.h +++ b/mlir/include/mlir/InitAllDialects.h @@ -82,6 +82,7 @@ #include "mlir/Dialect/Transform/IR/TransformDialect.h" #include "mlir/Dialect/Transform/PDLExtension/PDLExtension.h" #include "mlir/Dialect/UB/IR/UBOps.h" +#include "mlir/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.h" #include "mlir/Dialect/Vector/IR/VectorOps.h" #include "mlir/Dialect/Vector/Transforms/BufferizableOpInterfaceImpl.h" #include "mlir/Dialect/Vector/Transforms/SubsetOpInterfaceImpl.h" @@ -174,6 +175,7 @@ inline void registerAllDialects(DialectRegistry ®istry) { tosa::registerShardingInterfaceExternalModels(registry); vector::registerBufferizableOpInterfaceExternalModels(registry); vector::registerSubsetOpInterfaceExternalModels(registry); + vector::registerValueBoundsOpInterfaceExternalModels(registry); NVVM::registerNVVMTargetInterfaceExternalModels(registry); ROCDL::registerROCDLTargetInterfaceExternalModels(registry); spirv::registerSPIRVTargetInterfaceExternalModels(registry); diff --git a/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h b/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h index 28dadfb9ecf8..b4ed0967e63f 100644 --- a/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h +++ b/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h @@ -15,6 +15,7 @@ #include "mlir/IR/Value.h" #include "mlir/Interfaces/DestinationStyleOpInterface.h" #include "llvm/ADT/SetVector.h" +#include "llvm/Support/ExtensibleRTTI.h" #include @@ -63,7 +64,8 @@ using ValueDimList = SmallVector>>; /// /// Note: Any modification of existing IR invalides the data stored in this /// class. Adding new operations is allowed. -class ValueBoundsConstraintSet { +class ValueBoundsConstraintSet + : public llvm::RTTIExtends { protected: /// Helper class that builds a bound for a shaped value dimension or /// index-typed value. @@ -107,6 +109,8 @@ protected: }; public: + static char ID; + /// The stop condition when traversing the backward slice of a shaped value/ /// index-type value. The traversal continues until the stop condition /// evaluates to "true" for a value. @@ -265,6 +269,16 @@ protected: ValueBoundsConstraintSet(MLIRContext *ctx); + /// Populates the constraint set for a value/map without actually computing + /// the bound. Returns the position for the value/map (via the return value + /// and `posOut` output parameter). + int64_t populateConstraintsSet(Value value, + std::optional dim = std::nullopt, + StopConditionFn stopCondition = nullptr); + int64_t populateConstraintsSet(AffineMap map, ValueDimList mapOperands, + StopConditionFn stopCondition = nullptr, + int64_t *posOut = nullptr); + /// Iteratively process all elements on the worklist until an index-typed /// value or shaped value meets `stopCondition`. Such values are not processed /// any further. diff --git a/mlir/lib/Dialect/Vector/IR/CMakeLists.txt b/mlir/lib/Dialect/Vector/IR/CMakeLists.txt index 70f3fa8c297d..204462ffd047 100644 --- a/mlir/lib/Dialect/Vector/IR/CMakeLists.txt +++ b/mlir/lib/Dialect/Vector/IR/CMakeLists.txt @@ -1,5 +1,7 @@ add_mlir_dialect_library(MLIRVectorDialect VectorOps.cpp + ValueBoundsOpInterfaceImpl.cpp + ScalableValueBoundsConstraintSet.cpp ADDITIONAL_HEADER_DIRS ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/Vector/IR diff --git a/mlir/lib/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.cpp b/mlir/lib/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.cpp new file mode 100644 index 000000000000..6d7e3bc70f59 --- /dev/null +++ b/mlir/lib/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.cpp @@ -0,0 +1,103 @@ +//===- ScalableValueBoundsConstraintSet.cpp - Scalable Value Bounds -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h" + +#include "mlir/Dialect/Vector/IR/VectorOps.h" + +namespace mlir::vector { + +FailureOr +ConstantOrScalableBound::getSize() const { + if (map.isSingleConstant()) + return BoundSize{map.getSingleConstantResult(), /*scalable=*/false}; + if (map.getNumResults() != 1 || map.getNumInputs() != 1) + return failure(); + auto binop = dyn_cast(map.getResult(0)); + if (!binop || binop.getKind() != AffineExprKind::Mul) + return failure(); + auto matchConstant = [&](AffineExpr expr, int64_t &constant) -> bool { + if (auto cst = dyn_cast(expr)) { + constant = cst.getValue(); + return true; + } + return false; + }; + // Match `s0 * cst` or `cst * s0`: + int64_t cst = 0; + auto lhs = binop.getLHS(); + auto rhs = binop.getRHS(); + if ((matchConstant(lhs, cst) && isa(rhs)) || + (matchConstant(rhs, cst) && isa(lhs))) { + return BoundSize{cst, /*scalable=*/true}; + } + return failure(); +} + +char ScalableValueBoundsConstraintSet::ID = 0; + +FailureOr +ScalableValueBoundsConstraintSet::computeScalableBound( + Value value, std::optional dim, unsigned vscaleMin, + unsigned vscaleMax, presburger::BoundType boundType, bool closedUB, + StopConditionFn stopCondition) { + using namespace presburger; + + assert(vscaleMin <= vscaleMax); + ScalableValueBoundsConstraintSet scalableCstr(value.getContext(), vscaleMin, + vscaleMax); + + int64_t pos = scalableCstr.populateConstraintsSet(value, dim, stopCondition); + + // Project out all variables apart from vscale. + // This should result in constraints in terms of vscale only. + scalableCstr.projectOut( + [&](ValueDim p) { return p.first != scalableCstr.getVscaleValue(); }); + + assert(scalableCstr.cstr.getNumDimAndSymbolVars() == + scalableCstr.positionToValueDim.size() && + "inconsistent mapping state"); + + // Check that the only symbols left are vscale. + for (int64_t i = 0; i < scalableCstr.cstr.getNumDimAndSymbolVars(); ++i) { + if (i == pos) + continue; + if (scalableCstr.positionToValueDim[i] != + ValueDim(scalableCstr.getVscaleValue(), + ValueBoundsConstraintSet::kIndexValue)) { + return failure(); + } + } + + SmallVector lowerBound(1), upperBound(1); + scalableCstr.cstr.getSliceBounds(pos, 1, value.getContext(), &lowerBound, + &upperBound, closedUB); + + auto invalidBound = [](auto &bound) { + return !bound[0] || bound[0].getNumResults() != 1; + }; + + AffineMap bound = [&] { + if (boundType == BoundType::EQ && !invalidBound(lowerBound) && + lowerBound[0] == lowerBound[0]) { + return lowerBound[0]; + } else if (boundType == BoundType::LB && !invalidBound(lowerBound)) { + return lowerBound[0]; + } else if (boundType == BoundType::UB && !invalidBound(upperBound)) { + return upperBound[0]; + } + return AffineMap{}; + }(); + + if (!bound) + return failure(); + + return ConstantOrScalableBound{bound}; +} + +} // namespace mlir::vector diff --git a/mlir/lib/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.cpp b/mlir/lib/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.cpp new file mode 100644 index 000000000000..ca95072d9bb0 --- /dev/null +++ b/mlir/lib/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.cpp @@ -0,0 +1,51 @@ +//===- ValueBoundsOpInterfaceImpl.cpp - Impl. of ValueBoundsOpInterface ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mlir/Dialect/Vector/IR/ValueBoundsOpInterfaceImpl.h" + +#include "mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h" +#include "mlir/Dialect/Vector/IR/VectorOps.h" +#include "mlir/Interfaces/ValueBoundsOpInterface.h" + +using namespace mlir; + +namespace mlir::vector { +namespace { + +struct VectorScaleOpInterface + : public ValueBoundsOpInterface::ExternalModel { + void populateBoundsForIndexValue(Operation *op, Value value, + ValueBoundsConstraintSet &cstr) const { + auto *scalableCstr = dyn_cast(&cstr); + if (!scalableCstr) + return; + auto vscaleOp = cast(op); + assert(value == vscaleOp.getResult() && "invalid value"); + if (auto vscale = scalableCstr->getVscaleValue()) { + // All copies of vscale are equivalent. + scalableCstr->bound(value) == cstr.getExpr(vscale); + } else { + // We know vscale is confined to [vscaleMin, vscaleMax]. + scalableCstr->bound(value) >= scalableCstr->getVscaleMin(); + scalableCstr->bound(value) <= scalableCstr->getVscaleMax(); + scalableCstr->setVscale(vscaleOp); + } + } +}; + +} // namespace +} // namespace mlir::vector + +void mlir::vector::registerValueBoundsOpInterfaceExternalModels( + DialectRegistry ®istry) { + registry.addExtension(+[](MLIRContext *ctx, vector::VectorDialect *dialect) { + vector::VectorScaleOp::attachInterface( + *ctx); + }); +} diff --git a/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp b/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp index 85abc2df8947..06ec3f4e135e 100644 --- a/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp +++ b/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp @@ -70,6 +70,8 @@ static std::optional getConstantIntValue(OpFoldResult ofr) { ValueBoundsConstraintSet::ValueBoundsConstraintSet(MLIRContext *ctx) : builder(ctx) {} +char ValueBoundsConstraintSet::ID = 0; + #ifndef NDEBUG static void assertValidValueDim(Value value, std::optional dim) { if (value.getType().isIndex()) { @@ -471,55 +473,87 @@ FailureOr ValueBoundsConstraintSet::computeConstantBound( closedUB); } +FailureOr ValueBoundsConstraintSet::computeConstantBound( + presburger::BoundType type, AffineMap map, ArrayRef operands, + StopConditionFn stopCondition, bool closedUB) { + ValueDimList valueDims; + for (Value v : operands) { + assert(v.getType().isIndex() && "expected index type"); + valueDims.emplace_back(v, std::nullopt); + } + return computeConstantBound(type, map, valueDims, stopCondition, closedUB); +} + FailureOr ValueBoundsConstraintSet::computeConstantBound( presburger::BoundType type, AffineMap map, ValueDimList operands, StopConditionFn stopCondition, bool closedUB) { assert(map.getNumResults() == 1 && "expected affine map with one result"); ValueBoundsConstraintSet cstr(map.getContext()); - int64_t pos = cstr.insert(/*isSymbol=*/false); + + int64_t pos = 0; + if (stopCondition) { + cstr.populateConstraintsSet(map, operands, stopCondition, &pos); + } else { + // No stop condition specified: Keep adding constraints until a bound could + // be computed. + cstr.populateConstraintsSet( + map, operands, + [&](Value v, std::optional dim) { + return cstr.cstr.getConstantBound64(type, pos).has_value(); + }, + &pos); + } + // Compute constant bound for `valueDim`. + int64_t ubAdjustment = closedUB ? 0 : 1; + if (auto bound = cstr.cstr.getConstantBound64(type, pos)) + return type == BoundType::UB ? *bound + ubAdjustment : *bound; + return failure(); +} + +int64_t ValueBoundsConstraintSet::populateConstraintsSet( + Value value, std::optional dim, StopConditionFn stopCondition) { +#ifndef NDEBUG + assertValidValueDim(value, dim); +#endif // NDEBUG + + AffineMap map = + AffineMap::get(/*dimCount=*/1, /*symbolCount=*/0, + Builder(value.getContext()).getAffineDimExpr(0)); + return populateConstraintsSet(map, {{value, dim}}, stopCondition); +} + +int64_t ValueBoundsConstraintSet::populateConstraintsSet( + AffineMap map, ValueDimList operands, StopConditionFn stopCondition, + int64_t *posOut) { + assert(map.getNumResults() == 1 && "expected affine map with one result"); + int64_t pos = insert(/*isSymbol=*/false); + if (posOut) + *posOut = pos; // Add map and operands to the constraint set. Dimensions are converted to // symbols. All operands are added to the worklist. auto mapper = [&](std::pair> v) { - return cstr.getExpr(v.first, v.second); + return getExpr(v.first, v.second); }; SmallVector dimReplacements = llvm::to_vector( llvm::map_range(ArrayRef(operands).take_front(map.getNumDims()), mapper)); SmallVector symReplacements = llvm::to_vector( llvm::map_range(ArrayRef(operands).drop_front(map.getNumDims()), mapper)); - cstr.addBound( + addBound( presburger::BoundType::EQ, pos, map.getResult(0).replaceDimsAndSymbols(dimReplacements, symReplacements)); // Process the backward slice of `operands` (i.e., reverse use-def chain) // until `stopCondition` is met. if (stopCondition) { - cstr.processWorklist(stopCondition); + processWorklist(stopCondition); } else { - // No stop condition specified: Keep adding constraints until a bound could - // be computed. - cstr.processWorklist( - /*stopCondition=*/[&](Value v, std::optional dim) { - return cstr.cstr.getConstantBound64(type, pos).has_value(); - }); + // No stop condition specified: Keep adding constraints until the worklist + // is empty. + processWorklist([](Value v, std::optional dim) { return false; }); } - // Compute constant bound for `valueDim`. - int64_t ubAdjustment = closedUB ? 0 : 1; - if (auto bound = cstr.cstr.getConstantBound64(type, pos)) - return type == BoundType::UB ? *bound + ubAdjustment : *bound; - return failure(); -} - -FailureOr ValueBoundsConstraintSet::computeConstantBound( - presburger::BoundType type, AffineMap map, ArrayRef operands, - StopConditionFn stopCondition, bool closedUB) { - ValueDimList valueDims; - for (Value v : operands) { - assert(v.getType().isIndex() && "expected index type"); - valueDims.emplace_back(v, std::nullopt); - } - return computeConstantBound(type, map, valueDims, stopCondition, closedUB); + return pos; } FailureOr diff --git a/mlir/test/Dialect/Vector/test-scalable-bounds.mlir b/mlir/test/Dialect/Vector/test-scalable-bounds.mlir new file mode 100644 index 000000000000..245a6f5c13ac --- /dev/null +++ b/mlir/test/Dialect/Vector/test-scalable-bounds.mlir @@ -0,0 +1,161 @@ +// RUN: mlir-opt %s -test-affine-reify-value-bounds -cse -verify-diagnostics \ +// RUN: -verify-diagnostics -split-input-file | FileCheck %s + +#map_dim_i = affine_map<(d0)[s0] -> (-d0 + 32400, s0)> +#map_dim_j = affine_map<(d0)[s0] -> (-d0 + 16, s0)> + +// Here the upper bound for min_i is 4 x vscale, as we know 4 x vscale is +// always less than 32400. The bound for min_j is 16, as 16 is always less +// 4 x vscale_max (vscale_max is the UB for vscale). + +// CHECK: #[[$SCALABLE_BOUND_MAP_0:.*]] = affine_map<()[s0] -> (s0 * 4)> + +// CHECK-LABEL: @fixed_size_loop_nest +// CHECK-DAG: %[[VSCALE:.*]] = vector.vscale +// CHECK-DAG: %[[UB_i:.*]] = affine.apply #[[$SCALABLE_BOUND_MAP_0]]()[%[[VSCALE]]] +// CHECK-DAG: %[[UB_j:.*]] = arith.constant 16 : index +// CHECK: "test.some_use"(%[[UB_i]], %[[UB_j]]) : (index, index) -> () +func.func @fixed_size_loop_nest() { + %c16 = arith.constant 16 : index + %c32400 = arith.constant 32400 : index + %c4 = arith.constant 4 : index + %c0 = arith.constant 0 : index + %vscale = vector.vscale + %c4_vscale = arith.muli %vscale, %c4 : index + scf.for %i = %c0 to %c32400 step %c4_vscale { + %min_i = affine.min #map_dim_i(%i)[%c4_vscale] + scf.for %j = %c0 to %c16 step %c4_vscale { + %min_j = affine.min #map_dim_j(%j)[%c4_vscale] + %bound_i = "test.reify_scalable_bound"(%min_i) {type = "UB", vscale_min = 1, vscale_max = 16} : (index) -> index + %bound_j = "test.reify_scalable_bound"(%min_j) {type = "UB", vscale_min = 1, vscale_max = 16} : (index) -> index + "test.some_use"(%bound_i, %bound_j) : (index, index) -> () + } + } + return +} + +// ----- + +#map_dynamic_dim = affine_map<(d0)[s0, s1] -> (-d0 + s1, s0)> + +// Here upper bounds for both min_i and min_j are both (conservatively) +// 4 x vscale, as we know that is always the largest value they could take. As +// if `dim < 4 x vscale` then 4 x vscale is an overestimate, and if +// `dim > 4 x vscale` then the min will be clamped to 4 x vscale. + +// CHECK: #[[$SCALABLE_BOUND_MAP_1:.*]] = affine_map<()[s0] -> (s0 * 4)> + +// CHECK-LABEL: @dynamic_size_loop_nest +// CHECK: %[[VSCALE:.*]] = vector.vscale +// CHECK: %[[UB_ij:.*]] = affine.apply #[[$SCALABLE_BOUND_MAP_1]]()[%[[VSCALE]]] +// CHECK: "test.some_use"(%[[UB_ij]], %[[UB_ij]]) : (index, index) -> () +func.func @dynamic_size_loop_nest(%dim0: index, %dim1: index) { + %c4 = arith.constant 4 : index + %c0 = arith.constant 0 : index + %vscale = vector.vscale + %c4_vscale = arith.muli %vscale, %c4 : index + scf.for %i = %c0 to %dim0 step %c4_vscale { + %min_i = affine.min #map_dynamic_dim(%i)[%c4_vscale, %dim0] + scf.for %j = %c0 to %dim1 step %c4_vscale { + %min_j = affine.min #map_dynamic_dim(%j)[%c4_vscale, %dim1] + %bound_i = "test.reify_scalable_bound"(%min_i) {type = "UB", vscale_min = 1, vscale_max = 16} : (index) -> index + %bound_j = "test.reify_scalable_bound"(%min_j) {type = "UB", vscale_min = 1, vscale_max = 16} : (index) -> index + "test.some_use"(%bound_i, %bound_j) : (index, index) -> () + } + } + return +} + +// ----- + +// Here the bound is just a value + a constant. + +// CHECK: #[[$SCALABLE_BOUND_MAP_2:.*]] = affine_map<()[s0] -> (s0 + 8)> + +// CHECK-LABEL: @add_to_vscale +// CHECK: %[[VSCALE:.*]] = vector.vscale +// CHECK: %[[SCALABLE_BOUND:.*]] = affine.apply #[[$SCALABLE_BOUND_MAP_2]]()[%[[VSCALE]]] +// CHECK: "test.some_use"(%[[SCALABLE_BOUND]]) : (index) -> () +func.func @add_to_vscale() { + %vscale = vector.vscale + %c8 = arith.constant 8 : index + %vscale_plus_c8 = arith.addi %vscale, %c8 : index + %bound = "test.reify_scalable_bound"(%vscale_plus_c8) {type = "EQ", vscale_min = 1, vscale_max = 16} : (index) -> index + "test.some_use"(%bound) : (index) -> () + return +} + +// ----- + +// Here we know vscale is always 2 so we get a constant bound. + +// CHECK-LABEL: @vscale_fixed_size +// CHECK: %[[C2:.*]] = arith.constant 2 : index +// CHECK: "test.some_use"(%[[C2]]) : (index) -> () +func.func @vscale_fixed_size() { + %vscale = vector.vscale + %bound = "test.reify_scalable_bound"(%vscale) {type = "EQ", vscale_min = 2, vscale_max = 2} : (index) -> index + "test.some_use"(%bound) : (index) -> () + return +} + +// ----- + +// Here we don't know the upper bound (%a is underspecified) + +func.func @unknown_bound(%a: index) { + %vscale = vector.vscale + %vscale_plus_a = arith.muli %vscale, %a : index + // expected-error @below{{could not reify bound}} + %bound = "test.reify_scalable_bound"(%vscale_plus_a) {type = "UB", vscale_min = 1, vscale_max = 16} : (index) -> index + "test.some_use"(%bound) : (index) -> () + return +} + +// ----- + +// Here we have two vscale values (that have not been CSE'd), but they should +// still be treated as equivalent. + +// CHECK: #[[$SCALABLE_BOUND_MAP_3:.*]] = affine_map<()[s0] -> (s0 * 6)> + +// CHECK-LABEL: @duplicate_vscale_values +// CHECK: %[[VSCALE:.*]] = vector.vscale +// CHECK: %[[SCALABLE_BOUND:.*]] = affine.apply #[[$SCALABLE_BOUND_MAP_3]]()[%[[VSCALE]]] +// CHECK: "test.some_use"(%[[SCALABLE_BOUND]]) : (index) -> () +func.func @duplicate_vscale_values() { + %c4 = arith.constant 4 : index + %vscale_0 = vector.vscale + + %c2 = arith.constant 2 : index + %vscale_1 = vector.vscale + + %c4_vscale = arith.muli %vscale_0, %c4 : index + %c2_vscale = arith.muli %vscale_1, %c2 : index + %add = arith.addi %c2_vscale, %c4_vscale : index + + %bound = "test.reify_scalable_bound"(%add) {type = "EQ", vscale_min = 1, vscale_max = 16} : (index) -> index + "test.some_use"(%bound) : (index) -> () + return +} + +// ----- + +// Test some non-scalable code to ensure that works too: + +#map_dim_i = affine_map<(d0)[s0] -> (-d0 + 1024, s0)> + +// CHECK-LABEL: @non_scalable_code +// CHECK: %[[C4:.*]] = arith.constant 4 : index +// CHECK: "test.some_use"(%[[C4]]) : (index) -> () +func.func @non_scalable_code() { + %c1024 = arith.constant 1024 : index + %c4 = arith.constant 4 : index + %c0 = arith.constant 0 : index + scf.for %i = %c0 to %c1024 step %c4 { + %min_i = affine.min #map_dim_i(%i)[%c4] + %bound_i = "test.reify_scalable_bound"(%min_i) {type = "UB", vscale_min = 1, vscale_max = 16} : (index) -> index + "test.some_use"(%bound_i) : (index) -> () + } + return +} diff --git a/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp b/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp index 39671a930f2e..5e160b720db6 100644 --- a/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp +++ b/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp @@ -13,6 +13,7 @@ #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h" #include "mlir/IR/PatternMatch.h" #include "mlir/Interfaces/ValueBoundsOpInterface.h" #include "mlir/Pass/Pass.h" @@ -75,7 +76,8 @@ static LogicalResult testReifyValueBounds(func::FuncOp funcOp, WalkResult result = funcOp.walk([&](Operation *op) { // Look for test.reify_bound ops. if (op->getName().getStringRef() == "test.reify_bound" || - op->getName().getStringRef() == "test.reify_constant_bound") { + op->getName().getStringRef() == "test.reify_constant_bound" || + op->getName().getStringRef() == "test.reify_scalable_bound") { if (op->getNumOperands() != 1 || op->getNumResults() != 1 || !op->getResultTypes()[0].isIndex()) { op->emitOpError("invalid op"); @@ -110,6 +112,9 @@ static LogicalResult testReifyValueBounds(func::FuncOp funcOp, bool constant = op->getName().getStringRef() == "test.reify_constant_bound"; + bool scalable = !constant && op->getName().getStringRef() == + "test.reify_scalable_bound"; + // Prepare stop condition. By default, reify in terms of the op's // operands. No stop condition is used when a constant was requested. std::function)> stopCondition = @@ -137,6 +142,37 @@ static LogicalResult testReifyValueBounds(func::FuncOp funcOp, if (succeeded(reifiedConst)) reified = FailureOr(rewriter.getIndexAttr(*reifiedConst)); + } else if (scalable) { + unsigned vscaleMin = 0; + unsigned vscaleMax = 0; + if (auto attr = "vscale_min"; op->hasAttrOfType(attr)) { + vscaleMin = unsigned(op->getAttrOfType(attr).getInt()); + } else { + op->emitOpError("expected `vscale_min` to be provided"); + return WalkResult::skip(); + } + if (auto attr = "vscale_max"; op->hasAttrOfType(attr)) { + vscaleMax = unsigned(op->getAttrOfType(attr).getInt()); + } else { + op->emitOpError("expected `vscale_max` to be provided"); + return WalkResult::skip(); + } + + auto loc = op->getLoc(); + auto reifiedScalable = + vector::ScalableValueBoundsConstraintSet::computeScalableBound( + value, dim, vscaleMin, vscaleMax, *boundType); + if (succeeded(reifiedScalable)) { + SmallVector>, 1> + vscaleOperand; + if (reifiedScalable->map.getNumInputs() == 1) { + // The only possible input to the bound is vscale. + vscaleOperand.push_back(std::make_pair( + rewriter.create(loc), std::nullopt)); + } + reified = affine::materializeComputedBound( + rewriter, loc, reifiedScalable->map, vscaleOperand); + } } else { if (dim) { if (useArithOps) { -- GitLab From d1f182c895728d89c5c3d198b133e212a5d9d4a3 Mon Sep 17 00:00:00 2001 From: Joe Nash Date: Thu, 21 Mar 2024 10:42:39 -0400 Subject: [PATCH 150/296] [AMDGPU][MC][True16] Rename and combine VINTERP MC tests (#85949) NFC. gfx11_asm_vinterp.s already contained GFX12 run lines. Rename the assembler and disassembler tests to be sorted based on real16 or fake16 instead of gfxip. Note, both GFX11 and GFX12 currently only have fake16 (fake16 in encoding, but not by name) upstream, so that is why the test files have a -fake16 suffix. One test input is changed, and that is the disassembler test for unsupported bits in the instruction. It is now an input that is valid on both GFX11 and GFX12. This was necessary because the size of the opcode field changed. --- llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s | 278 ------------------ llvm/test/MC/AMDGPU/vinterp-fake16.s | 182 ++++++++++++ .../AMDGPU/gfx11_dasm_vinterp.txt | 251 ---------------- .../AMDGPU/gfx12_dasm_vinterp.txt | 251 ---------------- .../MC/Disassembler/AMDGPU/vinterp-fake16.txt | 252 ++++++++++++++++ 5 files changed, 434 insertions(+), 780 deletions(-) delete mode 100644 llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s create mode 100644 llvm/test/MC/AMDGPU/vinterp-fake16.s delete mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vinterp.txt delete mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt create mode 100644 llvm/test/MC/Disassembler/AMDGPU/vinterp-fake16.txt diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s b/llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s deleted file mode 100644 index fdfbf65c0e3c..000000000000 --- a/llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s +++ /dev/null @@ -1,278 +0,0 @@ -// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -show-encoding %s | FileCheck -check-prefix=GCN %s -// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck -check-prefix=GCN %s - -v_interp_p10_f32 v0, v1, v2, v3 -// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f32 v1, v10, v20, v30 -// GCN: v_interp_p10_f32 v1, v10, v20, v30 wait_exp:0 ; encoding: [0x01,0x00,0x00,0xcd,0x0a,0x29,0x7a,0x04] - -v_interp_p10_f32 v2, v11, v21, v31 -// GCN: v_interp_p10_f32 v2, v11, v21, v31 wait_exp:0 ; encoding: [0x02,0x00,0x00,0xcd,0x0b,0x2b,0x7e,0x04] - -v_interp_p10_f32 v3, v12, v22, v32 -// GCN: v_interp_p10_f32 v3, v12, v22, v32 wait_exp:0 ; encoding: [0x03,0x00,0x00,0xcd,0x0c,0x2d,0x82,0x04] - -v_interp_p10_f32 v0, v1, v2, v3 clamp -// GCN: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x00,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f32 v0, -v1, v2, v3 -// GCN: v_interp_p10_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24] - -v_interp_p10_f32 v0, v1, -v2, v3 -// GCN: v_interp_p10_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44] - -v_interp_p10_f32 v0, v1, v2, -v3 -// GCN: v_interp_p10_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84] - -v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 -// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1 -// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x00,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7 -// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x00,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7 -// GCN: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7 ; encoding: [0x00,0x87,0x00,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f32 v0, v1, v2, v3 -// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f32 v1, v10, v20, v30 -// GCN: v_interp_p2_f32 v1, v10, v20, v30 wait_exp:0 ; encoding: [0x01,0x00,0x01,0xcd,0x0a,0x29,0x7a,0x04] - -v_interp_p2_f32 v2, v11, v21, v31 -// GCN: v_interp_p2_f32 v2, v11, v21, v31 wait_exp:0 ; encoding: [0x02,0x00,0x01,0xcd,0x0b,0x2b,0x7e,0x04] - -v_interp_p2_f32 v3, v12, v22, v32 -// GCN: v_interp_p2_f32 v3, v12, v22, v32 wait_exp:0 ; encoding: [0x03,0x00,0x01,0xcd,0x0c,0x2d,0x82,0x04] - -v_interp_p2_f32 v0, v1, v2, v3 clamp -// GCN: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x01,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f32 v0, -v1, v2, v3 -// GCN: v_interp_p2_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24] - -v_interp_p2_f32 v0, v1, -v2, v3 -// GCN: v_interp_p2_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44] - -v_interp_p2_f32 v0, v1, v2, -v3 -// GCN: v_interp_p2_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84] - -v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 -// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1 -// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x01,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7 -// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x01,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7 -// GCN: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7 ; encoding: [0x00,0x87,0x01,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, -v1, v2, v3 -// GCN: v_interp_p10_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24] - -v_interp_p10_f16_f32 v0, v1, -v2, v3 -// GCN: v_interp_p10_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44] - -v_interp_p10_f16_f32 v0, v1, v2, -v3 -// GCN: v_interp_p10_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84] - -v_interp_p10_f16_f32 v0, v1, v2, v3 clamp -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1 -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7 -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,0] -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p10_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0xe4] - -v_interp_p2_f16_f32 v0, v1, v2, v3 -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, -v1, v2, v3 -// GCN: v_interp_p2_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24] - -v_interp_p2_f16_f32 v0, v1, -v2, v3 -// GCN: v_interp_p2_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44] - -v_interp_p2_f16_f32 v0, v1, v2, -v3 -// GCN: v_interp_p2_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84] - -v_interp_p2_f16_f32 v0, v1, v2, v3 clamp -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1 -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7 -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,0] -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p2_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0xe4] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 -// GCN: v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24] - -v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,0] -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p10_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p10_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0xe4] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 -// GCN: v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24] - -v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,0] -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0x04] - -v_interp_p2_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GCN: v_interp_p2_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0xe4] diff --git a/llvm/test/MC/AMDGPU/vinterp-fake16.s b/llvm/test/MC/AMDGPU/vinterp-fake16.s new file mode 100644 index 000000000000..33dacdd92c31 --- /dev/null +++ b/llvm/test/MC/AMDGPU/vinterp-fake16.s @@ -0,0 +1,182 @@ +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -show-encoding %s | FileCheck -check-prefix=GCN %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -show-encoding %s | FileCheck -check-prefix=GCN %s + +v_interp_p10_f32 v0, v1, v2, v3 +// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f32 v1, v10, v20, v30 +// GCN: v_interp_p10_f32 v1, v10, v20, v30 wait_exp:0 ; encoding: [0x01,0x00,0x00,0xcd,0x0a,0x29,0x7a,0x04] + +v_interp_p10_f32 v2, v11, v21, v31 +// GCN: v_interp_p10_f32 v2, v11, v21, v31 wait_exp:0 ; encoding: [0x02,0x00,0x00,0xcd,0x0b,0x2b,0x7e,0x04] + +v_interp_p10_f32 v3, v12, v22, v32 +// GCN: v_interp_p10_f32 v3, v12, v22, v32 wait_exp:0 ; encoding: [0x03,0x00,0x00,0xcd,0x0c,0x2d,0x82,0x04] + +v_interp_p10_f32 v0, v1, v2, v3 clamp +// GCN: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x00,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f32 v0, -v1, v2, v3 +// GCN: v_interp_p10_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24] + +v_interp_p10_f32 v0, v1, -v2, v3 +// GCN: v_interp_p10_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44] + +v_interp_p10_f32 v0, v1, v2, -v3 +// GCN: v_interp_p10_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84] + +v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 +// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1 +// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x00,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7 +// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x00,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7 +// GCN: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7 ; encoding: [0x00,0x87,0x00,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f32 v0, v1, v2, v3 +// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f32 v1, v10, v20, v30 +// GCN: v_interp_p2_f32 v1, v10, v20, v30 wait_exp:0 ; encoding: [0x01,0x00,0x01,0xcd,0x0a,0x29,0x7a,0x04] + +v_interp_p2_f32 v2, v11, v21, v31 +// GCN: v_interp_p2_f32 v2, v11, v21, v31 wait_exp:0 ; encoding: [0x02,0x00,0x01,0xcd,0x0b,0x2b,0x7e,0x04] + +v_interp_p2_f32 v3, v12, v22, v32 +// GCN: v_interp_p2_f32 v3, v12, v22, v32 wait_exp:0 ; encoding: [0x03,0x00,0x01,0xcd,0x0c,0x2d,0x82,0x04] + +v_interp_p2_f32 v0, v1, v2, v3 clamp +// GCN: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x01,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f32 v0, -v1, v2, v3 +// GCN: v_interp_p2_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24] + +v_interp_p2_f32 v0, v1, -v2, v3 +// GCN: v_interp_p2_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44] + +v_interp_p2_f32 v0, v1, v2, -v3 +// GCN: v_interp_p2_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84] + +v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 +// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1 +// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x01,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7 +// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x01,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7 +// GCN: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7 ; encoding: [0x00,0x87,0x01,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f16_f32 v0, v1, v2, v3 +// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f16_f32 v0, -v1, v2, v3 +// GFX11: v_interp_p10_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24] + +v_interp_p10_f16_f32 v0, v1, -v2, v3 +// GFX11: v_interp_p10_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44] + +v_interp_p10_f16_f32 v0, v1, v2, -v3 +// GFX11: v_interp_p10_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84] + +v_interp_p10_f16_f32 v0, v1, v2, v3 clamp +// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x02,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 +// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1 +// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x02,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7 +// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x02,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_f16_f32 v0, v1, v2, v3 +// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f16_f32 v0, v1, v2, v3 +// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f16_f32 v0, -v1, v2, v3 +// GFX11: v_interp_p2_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24] + +v_interp_p2_f16_f32 v0, v1, -v2, v3 +// GFX11: v_interp_p2_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44] + +v_interp_p2_f16_f32 v0, v1, v2, -v3 +// GFX11: v_interp_p2_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84] + +v_interp_p2_f16_f32 v0, v1, v2, v3 clamp +// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x03,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 +// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1 +// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x03,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7 +// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x03,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_f16_f32 v0, v1, v2, v3 +// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 +// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 +// GFX11: v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24] + +v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 +// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44] + +v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 +// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84] + +v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp +// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x04,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 +// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 +// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x04,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 +// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x04,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 +// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 +// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 +// GFX11: v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24] + +v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 +// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44] + +v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 +// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84] + +v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp +// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x05,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 +// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 +// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x05,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 +// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x05,0xcd,0x01,0x05,0x0e,0x04] + +v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 +// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vinterp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vinterp.txt deleted file mode 100644 index b22fd5e289fa..000000000000 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vinterp.txt +++ /dev/null @@ -1,251 +0,0 @@ -# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -disassemble %s | FileCheck -strict-whitespace -check-prefix=GFX11 %s - -# GFX11: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# Check that unused bits in the encoding are ignored. -# GFX11: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x80,0xcd,0x01,0x05,0x0e,0x1c - -# GFX11: v_interp_p10_f32 v1, v10, v20, v30 wait_exp:0{{$}} -0x01,0x00,0x00,0xcd,0x0a,0x29,0x7a,0x04 - -# GFX11: v_interp_p10_f32 v2, v11, v21, v31 wait_exp:0{{$}} -0x02,0x00,0x00,0xcd,0x0b,0x2b,0x7e,0x04 - -# GFX11: v_interp_p10_f32 v3, v12, v22, v32 wait_exp:0{{$}} -0x03,0x00,0x00,0xcd,0x0c,0x2d,0x82,0x04 - -# GFX11: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24 - -# GFX11: v_interp_p10_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44 - -# GFX11: v_interp_p10_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84 - -# GFX11: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7{{$}} -0x00,0x87,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f32 v1, v10, v20, v30 wait_exp:0{{$}} -0x01,0x00,0x01,0xcd,0x0a,0x29,0x7a,0x04 - -# GFX11: v_interp_p2_f32 v2, v11, v21, v31 wait_exp:0{{$}} -0x02,0x00,0x01,0xcd,0x0b,0x2b,0x7e,0x04 - -# GFX11: v_interp_p2_f32 v3, v12, v22, v32 wait_exp:0{{$}} -0x03,0x00,0x01,0xcd,0x0c,0x2d,0x82,0x04 - -# GFX11: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24 - -# GFX11: v_interp_p2_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44 - -# GFX11: v_interp_p2_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84 - -# GFX11: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7{{$}} -0x00,0x87,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24 - -# GFX11: v_interp_p10_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} -0x00,0x08,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} -0x00,0x10,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} -0x00,0x20,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} -0x00,0x40,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} -0x00,0x78,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0x4d,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0xe4 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24 - -# GFX11: v_interp_p2_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} -0x00,0x08,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} -0x00,0x10,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} -0x00,0x20,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} -0x00,0x40,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} -0x00,0x78,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0x4d,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0xe4 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} -0x00,0x08,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} -0x00,0x10,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} -0x00,0x20,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} -0x00,0x40,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} -0x00,0x78,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0x4d,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p10_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0xe4 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} -0x00,0x08,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} -0x00,0x10,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} -0x00,0x20,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} -0x00,0x40,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} -0x00,0x78,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0x4d,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX11: v_interp_p2_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0xe4 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt deleted file mode 100644 index 977cd732947c..000000000000 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt +++ /dev/null @@ -1,251 +0,0 @@ -# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s - -# GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# Check that unused bits in the encoding are ignored. -# GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0xe0,0xcd,0x01,0x05,0x0e,0x1c - -# GFX12: v_interp_p10_f32 v1, v10, v20, v30 wait_exp:0{{$}} -0x01,0x00,0x00,0xcd,0x0a,0x29,0x7a,0x04 - -# GFX12: v_interp_p10_f32 v2, v11, v21, v31 wait_exp:0{{$}} -0x02,0x00,0x00,0xcd,0x0b,0x2b,0x7e,0x04 - -# GFX12: v_interp_p10_f32 v3, v12, v22, v32 wait_exp:0{{$}} -0x03,0x00,0x00,0xcd,0x0c,0x2d,0x82,0x04 - -# GFX12: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24 - -# GFX12: v_interp_p10_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44 - -# GFX12: v_interp_p10_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84 - -# GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7{{$}} -0x00,0x87,0x00,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f32 v1, v10, v20, v30 wait_exp:0{{$}} -0x01,0x00,0x01,0xcd,0x0a,0x29,0x7a,0x04 - -# GFX12: v_interp_p2_f32 v2, v11, v21, v31 wait_exp:0{{$}} -0x02,0x00,0x01,0xcd,0x0b,0x2b,0x7e,0x04 - -# GFX12: v_interp_p2_f32 v3, v12, v22, v32 wait_exp:0{{$}} -0x03,0x00,0x01,0xcd,0x0c,0x2d,0x82,0x04 - -# GFX12: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24 - -# GFX12: v_interp_p2_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44 - -# GFX12: v_interp_p2_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84 - -# GFX12: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7{{$}} -0x00,0x87,0x01,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24 - -# GFX12: v_interp_p10_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} -0x00,0x08,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} -0x00,0x10,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} -0x00,0x20,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} -0x00,0x40,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} -0x00,0x78,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0x4d,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0xe4 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24 - -# GFX12: v_interp_p2_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} -0x00,0x08,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} -0x00,0x10,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} -0x00,0x20,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} -0x00,0x40,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} -0x00,0x78,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0x4d,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0xe4 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} -0x00,0x08,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} -0x00,0x10,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} -0x00,0x20,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} -0x00,0x40,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} -0x00,0x78,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0x4d,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p10_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0xe4 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} -0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} -0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} -0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} -0x00,0x80,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} -0x00,0x01,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} -0x00,0x07,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} -0x00,0x08,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} -0x00,0x10,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} -0x00,0x20,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} -0x00,0x40,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} -0x00,0x78,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0x4d,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0x04 - -# GFX12: v_interp_p2_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} -0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0xe4 diff --git a/llvm/test/MC/Disassembler/AMDGPU/vinterp-fake16.txt b/llvm/test/MC/Disassembler/AMDGPU/vinterp-fake16.txt new file mode 100644 index 000000000000..239f1d8b3058 --- /dev/null +++ b/llvm/test/MC/Disassembler/AMDGPU/vinterp-fake16.txt @@ -0,0 +1,252 @@ +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -disassemble %s | FileCheck -strict-whitespace -check-prefix=CHECK %s +# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -disassemble %s | FileCheck -strict-whitespace -check-prefix=CHECK %s + +# CHECK: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# Check that unused bits in the encoding are ignored. +# CHECK: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x80,0xcd,0x01,0x05,0x0e,0x1c + +# CHECK: v_interp_p10_f32 v1, v10, v20, v30 wait_exp:0{{$}} +0x01,0x00,0x00,0xcd,0x0a,0x29,0x7a,0x04 + +# CHECK: v_interp_p10_f32 v2, v11, v21, v31 wait_exp:0{{$}} +0x02,0x00,0x00,0xcd,0x0b,0x2b,0x7e,0x04 + +# CHECK: v_interp_p10_f32 v3, v12, v22, v32 wait_exp:0{{$}} +0x03,0x00,0x00,0xcd,0x0c,0x2d,0x82,0x04 + +# CHECK: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24 + +# CHECK: v_interp_p10_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44 + +# CHECK: v_interp_p10_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84 + +# CHECK: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7{{$}} +0x00,0x87,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f32 v1, v10, v20, v30 wait_exp:0{{$}} +0x01,0x00,0x01,0xcd,0x0a,0x29,0x7a,0x04 + +# CHECK: v_interp_p2_f32 v2, v11, v21, v31 wait_exp:0{{$}} +0x02,0x00,0x01,0xcd,0x0b,0x2b,0x7e,0x04 + +# CHECK: v_interp_p2_f32 v3, v12, v22, v32 wait_exp:0{{$}} +0x03,0x00,0x01,0xcd,0x0c,0x2d,0x82,0x04 + +# CHECK: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24 + +# CHECK: v_interp_p2_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44 + +# CHECK: v_interp_p2_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84 + +# CHECK: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7{{$}} +0x00,0x87,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24 + +# CHECK: v_interp_p10_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} +0x00,0x08,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} +0x00,0x10,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} +0x00,0x20,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} +0x00,0x40,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} +0x00,0x78,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0x4d,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0xe4 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24 + +# CHECK: v_interp_p2_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} +0x00,0x08,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} +0x00,0x10,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} +0x00,0x20,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} +0x00,0x40,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} +0x00,0x78,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0x4d,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0xe4 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} +0x00,0x08,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} +0x00,0x10,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} +0x00,0x20,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} +0x00,0x40,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} +0x00,0x78,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0x4d,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p10_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0xe4 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} +0x00,0x08,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} +0x00,0x10,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} +0x00,0x20,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} +0x00,0x40,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} +0x00,0x78,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0x4d,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# CHECK: v_interp_p2_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0xe4 -- GitLab From 44278f2326e0df463f6d5dd43820858ee4cffc6f Mon Sep 17 00:00:00 2001 From: Joe Nash Date: Thu, 21 Mar 2024 10:43:37 -0400 Subject: [PATCH 151/296] [AMDGPU][MC] Fix GFX12 check line typo and move test NFC. Fix CHECK lines that seem to have a copy paste error. Move the test that was formerly in gfx12_dasm_vinterp.txt (see #85949). --- llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_features.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_features.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_features.txt index 0c4427cff63e..1be97b242284 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_features.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_features.txt @@ -12,8 +12,13 @@ # GFX12: v_add3_u32_e64_dpp v5, v1, 42, v0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x54,0x01,0x04,0x01,0x77,0x39,0x05] 0x05,0x00,0x55,0xd6,0xe9,0x54,0x01,0x04,0x01,0x77,0x39,0x05 -# GFX1150: v_add3_u32_e64_dpp v5, v1, s2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0x0c,0x00,0x01,0x77,0x39,0x05] +# GFX12: v_add3_u32_e64_dpp v5, v1, s2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0x0c,0x00,0x01,0x77,0x39,0x05] 0x05,0x00,0x55,0xd6,0xe9,0x04,0x0c,0x00,0x01,0x77,0x39,0x05 -# GFX1150: v_cmp_ne_i32_e64_dpp vcc_lo, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x45,0xd4,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05] +# GFX12: v_cmp_ne_i32_e64_dpp vcc_lo, v1, s2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x6a,0x00,0x45,0xd4,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05] 0x6a,0x00,0x45,0xd4,0xe9,0x04,0x00,0x00,0x01,0x77,0x39,0x05 + +# Check that unused bits in the encoding are ignored. +# This is more strict than the check in vinterp-fake16.txt and is GFX12 specific. +# GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04] +0x00,0x00,0xe0,0xcd,0x01,0x05,0x0e,0x1c -- GitLab From b4b5e8277a86d441830dbba54917bf22b0ad8608 Mon Sep 17 00:00:00 2001 From: Jonas Paulsson Date: Thu, 21 Mar 2024 11:00:08 -0400 Subject: [PATCH 152/296] Check for all frame instructions in finalize isel. (#85945) Check for all frame instructions in finalize isel, not just for the frame setup opcode. This was proven necessary, see #78001 for discussion. --- llvm/lib/CodeGen/FinalizeISel.cpp | 3 +-- llvm/test/CodeGen/SystemZ/frame-adjstack.ll | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 llvm/test/CodeGen/SystemZ/frame-adjstack.ll diff --git a/llvm/lib/CodeGen/FinalizeISel.cpp b/llvm/lib/CodeGen/FinalizeISel.cpp index 978355f8eb1b..bf967eac22f1 100644 --- a/llvm/lib/CodeGen/FinalizeISel.cpp +++ b/llvm/lib/CodeGen/FinalizeISel.cpp @@ -59,8 +59,7 @@ bool FinalizeISel::runOnMachineFunction(MachineFunction &MF) { // Set AdjustsStack to true if the instruction selector emits a stack // frame setup instruction or a stack aligning inlineasm. - if (MI.getOpcode() == TII->getCallFrameSetupOpcode() || - MI.isStackAligningInlineAsm()) + if (TII->isFrameInstr(MI) || MI.isStackAligningInlineAsm()) MF.getFrameInfo().setAdjustsStack(true); // If MI is a pseudo, expand it. diff --git a/llvm/test/CodeGen/SystemZ/frame-adjstack.ll b/llvm/test/CodeGen/SystemZ/frame-adjstack.ll new file mode 100644 index 000000000000..7edacaa3d7d7 --- /dev/null +++ b/llvm/test/CodeGen/SystemZ/frame-adjstack.ll @@ -0,0 +1,16 @@ +; RUN: llc < %s -mtriple=s390x-linux-gnu -verify-machineinstrs | FileCheck %s +; +; Test that inserting a new MBB near a call during finalize isel custom +; insertion does not cause all frame instructions to be missed. That would +; result in a missing to set the AdjustsStack flag. + +; CHECK-LABEL: fun +define void @fun(i1 %cc) { + %sel = select i1 %cc, i32 5, i32 0 + tail call void @input_report_abs(i32 %sel) + %sel2 = select i1 %cc, i32 6, i32 1 + tail call void @input_report_abs(i32 %sel2) + ret void +} + +declare void @input_report_abs(i32) -- GitLab From 6898147b9f38c3bb46be5704bdad22abed7af339 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Thu, 21 Mar 2024 16:00:53 +0100 Subject: [PATCH 153/296] [InstCombine] Add contributor guide (#79007) Document expectations for contributions to InstCombine, especially regarding test coverage and alive2 proofs. --- llvm/docs/InstCombineContributorGuide.md | 556 +++++++++++++++++++++++ llvm/docs/UserGuides.rst | 5 + 2 files changed, 561 insertions(+) create mode 100644 llvm/docs/InstCombineContributorGuide.md diff --git a/llvm/docs/InstCombineContributorGuide.md b/llvm/docs/InstCombineContributorGuide.md new file mode 100644 index 000000000000..2416fd0920f6 --- /dev/null +++ b/llvm/docs/InstCombineContributorGuide.md @@ -0,0 +1,556 @@ +# InstCombine contributor guide + +This guide lays out a series of rules that contributions to InstCombine should +follow. **Following these rules will results in much faster PR approvals.** + +## Tests + +### Precommit tests + +Tests for new optimizations or miscompilation fixes should be pre-committed. +This means that you first commit the test with CHECK lines showing the behavior +*without* your change. Your actual change will then only contain CHECK line +diffs relative to that baseline. + +This means that pull requests should generally contain two commits: First, +one commit adding new tests with baseline check lines. Second, a commit with +functional changes and test diffs. + +If the second commit in your PR does not contain test diffs, you did something +wrong. Either you made a mistake when generating CHECK lines, or your tests are +not actually affected by your patch. + +Exceptions: When fixing assertion failures or infinite loops, do not pre-commit +tests. + +### Use `update_test_checks.py` + +CHECK lines should be generated using the `update_test_checks.py` script. Do +**not** manually edit check lines after using it. + +Be sure to use the correct opt binary when using the script. For example, if +your build directory is `build`, then you'll want to run: + +```sh +llvm/utils/update_test_checks.py --opt-binary build/bin/opt \ + llvm/test/Transforms/InstCombine/the_test.ll +``` + +Exceptions: Hand-written CHECK lines are allowed for debuginfo tests. + +### General testing considerations + +Place all tests relating to a transform into a single file. If you are adding +a regression test for a crash/miscompile in an existing transform, find the +file where the existing tests are located. A good way to do that is to comment +out the transform and see which tests fail. + +Make tests minimal. Only test exactly the pattern being transformed. If your +original motivating case is a larger pattern that your fold enables to +optimize in some non-trivial way, you may add it as well -- however, the bulk +of the test coverage should be minimal. + +Give tests short, but meaningful names. Don't call them `@test1`, `@test2` etc. +For example, a test checking multi-use behavior of a fold involving the +addition of two selects might be called `@add_of_selects_multi_use`. + +Add representative tests for each test category (discussed below), but don't +test all combinations of everything. If you have multi-use tests, and you have +commuted tests, you shouldn't also add commuted multi-use tests. + +Prefer to keep bit-widths for tests low to improve performance of proof checking using alive2. Using `i8` is better than `i128` where possible. + +### Add negative tests + +Make sure to add tests for which your transform does **not** apply. Start with +one of the test cases that succeeds and then create a sequence of negative +tests, such that **exactly one** different pre-condition of your transform is +not satisfied in each test. + +### Add multi-use tests + +Add multi-use tests that ensures your transform does not increase instruction +count if some instructions have additional uses. The standard pattern is to +introduce extra uses with function calls: + +```llvm +declare void @use(i8) + +define i8 @add_mul_const_multi_use(i8 %x) { + %add = add i8 %x, 1 + call void @use(i8 %add) + %mul = mul i8 %add, 3 + ret i8 %mul +} +``` + +Exceptions: For transform that only produce one instruction, multi-use tests +may be omitted. + +### Add commuted tests + +If the transform involves commutative operations, add tests with commuted +(swapped) operands. + +Make sure that the operand order stays intact in the CHECK lines of your +pre-commited tests. You should not see something like this: + +```llvm +; CHECK-NEXT: [[OR:%.*]] = or i8 [[X]], [[Y]] +; ... +%or = or i8 %y, %x +``` + +If this happens, you may need to change one of the operands to have higher +complexity (include the "thwart" comment in that case): + +```llvm +%y2 = mul i8 %y, %y ; thwart complexity-based canonicalization +%or = or i8 %y, %x +``` + +### Add vector tests + +When possible, it is recommended to add at least one test that uses vectors +instead of scalars. + +For patterns that include constants, we distinguish three kinds of tests. +The first are "splat" vectors, where all the vector elements are the same. +These tests *should* usually fold without additional effort. + +```llvm +define <2 x i8> @add_mul_const_vec_splat(<2 x i8> %x) { + %add = add <2 x i8> %x, + %mul = mul <2 x i8> %add, + ret <2 x i8> %mul +} +``` + +A minor variant is to replace some of the splat elements with poison. These +will often also fold without additional effort. + +```llvm +define <2 x i8> @add_mul_const_vec_splat_poison(<2 x i8> %x) { + %add = add <2 x i8> %x, + %mul = mul <2 x i8> %add, + ret <2 x i8> %mul +} +``` + +Finally, you can have non-splat vectors, where the vector elements are not +the same: + +```llvm +define <2 x i8> @add_mul_const_vec_non_splat(<2 x i8> %x) { + %add = add <2 x i8> %x, + %mul = mul <2 x i8> %add, + ret <2 x i8> %mul +} +``` + +Non-splat vectors will often not fold by default. You should **not** try to +make them fold, unless doing so does not add **any** additional complexity. +You should still add the test though, even if it does not fold. + +### Flag tests + +If your transform involves instructions that can have poison-generating flags, +such as `nuw` and `nsw` on `add`, you should test how these interact with the +transform. + +If your transform *requires* a certain flag for correctness, make sure to add +negative tests missing the required flag. + +If your transform doesn't require flags for correctness, you should have tests +for preservation behavior. If the input instructions have certain flags, are +they preserved in the output instructions, if it is valid to preserve them? +(This depends on the transform. Check with alive2.) + +The same also applies to fast-math-flags (FMF). In that case, please always +test specific flags like `nnan`, `nsz` or `reassoc`, rather than the umbrella +`fast` flag. + +### Other tests + +The test categories mentioned above are non-exhaustive. There may be more tests +to be added, depending on the instructions involved in the transform. Some +examples: + + * For folds involving memory accesses like load/store, check that scalable vectors and non-byte-size types (like i3) are handled correctly. Also check that volatile/atomic are handled. + * For folds that interact with the bitwidth in some non-trivial way, check an illegal type like i13. Also confirm that the transform is correct for i1. + * For folds that involve phis, you may want to check that the case of multiple incoming values from one block is handled correctly. + +## Proofs + +Your pull request description should contain one or more +[alive2 proofs](https://alive2.llvm.org/ce/) for the correctness of the +proposed transform. + +### Basics + +Proofs are written using LLVM IR, by specifying a `@src` and `@tgt` function. +It is possible to include multiple proofs in a single file by giving the src +and tgt functions matching suffixes. + +For example, here is a pair of proofs that both `(x-y)+y` and `(x+y)-y` can +be simplified to `x` ([online](https://alive2.llvm.org/ce/z/MsPPGz)): + +```llvm +define i8 @src_add_sub(i8 %x, i8 %y) { + %add = add i8 %x, %y + %sub = sub i8 %add, %y + ret i8 %sub +} + +define i8 @tgt_add_sub(i8 %x, i8 %y) { + ret i8 %x +} + + +define i8 @src_sub_add(i8 %x, i8 %y) { + %sub = sub i8 %x, %y + %add = add i8 %sub, %y + ret i8 %add +} + +define i8 @tgt_sub_add(i8 %x, i8 %y) { + ret i8 %x +} +``` + +### Use generic values in proofs + +Proofs should operate on generic values, rather than specific constants, to the degree that this is possible. + +For example, if we want to fold `X s/ C s< X` to `X s> 0`, the following would +be a *bad* proof: + +```llvm +; Don't do this! +define i1 @src(i8 %x) { + %div = sdiv i8 %x, 123 + %cmp = icmp slt i8 %div, %x + ret i1 %cmp +} + +define i1 @tgt(i8 %x) { + %cmp = icmp sgt i8 %x, 0 + ret i1 %cmp +} +``` + +This is because it only proves that the transform is correct for the specific +constant 123. Maybe there are some constants for which the transform is +incorrect? + +The correct way to write this proof is as follows +([online](https://alive2.llvm.org/ce/z/acjwb6)): + +```llvm +define i1 @src(i8 %x, i8 %C) { + %precond = icmp ne i8 %C, 1 + call void @llvm.assume(i1 %precond) + %div = sdiv i8 %x, %C + %cmp = icmp slt i8 %div, %x + ret i1 %cmp +} + +define i1 @tgt(i8 %x, i8 %C) { + %cmp = icmp sgt i8 %x, 0 + ret i1 %cmp +} +``` + +Note that the `@llvm.assume` intrinsic is used to specify pre-conditions for +the transform. In this case, the proof will fail unless we specify `C != 1` as +a pre-condition. + +It should be emphasized that there is, in general, no expectation that the +IR in the proofs will be transformed by the implemented fold. In the above +example, the transform would only apply if `%C` is actually a constant, but we +need to use non-constants in the proof. + +### Common pre-conditions + +Here are some examples of common preconditions. + +```llvm +; %x is non-negative: +%nonneg = icmp sgt i8 %x, -1 +call void @llvm.assume(i1 %nonneg) + +; %x is a power of two: +%ctpop = call i8 @llvm.ctpop.i8(i8 %x) +%pow2 = icmp eq i8 %x, 1 +call void @llvm.assume(i1 %pow2) + +; %x is a power of two or zero: +%ctpop = call i8 @llvm.ctpop.i8(i8 %x) +%pow2orzero = icmp ult i8 %x, 2 +call void @llvm.assume(i1 %pow2orzero) + +; Adding %x and %y does not overflow in a signed sense: +%wo = call { i8, i1 } @llvm.sadd.with.overflow(i8 %x, i8 %y) +%ov = extractvalue { i8, i1 } %wo, 1 +%ov.not = xor i1 %ov, true +call void @llvm.assume(i1 %ov.not) +``` + +### Timeouts + +Alive2 proofs will sometimes produce a timeout with the following message: + +``` +Alive2 timed out while processing your query. +There are a few things you can try: + +- remove extraneous instructions, if any + +- reduce variable widths, for example to i16, i8, or i4 + +- add the --disable-undef-input command line flag, which + allows Alive2 to assume that arguments to your IR are not + undef. This is, in general, unsound: it can cause Alive2 + to miss bugs. +``` + +This is good advice, follow it! + +Reducing the bitwidth usually helps. For floating point numbers, you can use +the `half` type for bitwidth reduction purposes. For pointers, you can reduce +the bitwidth by specifying a custom data layout: + +```llvm +; For 16-bit pointers +target datalayout = "p:16:16" +``` + +If reducing the bitwidth does not help, try `-disable-undef-input`. This will +often significantly improve performance, but also implies that the correctness +of the transform with `undef` values is no longer verified. This is usually +fine if the transform does not increase the number of uses of any value. + +Finally, it's possible to build alive2 locally and use `-smt-to=` to verify +the proof with a larger timeout. If you don't want to do this (or it still +does not work), please submit the proof you have despite the timeout. + +## Implementation + +### Real-world usefulness + +There is a very large number of transforms that *could* be implemented, but +only a tiny fraction of them are useful for real-world code. + +Transforms that do not have real-world usefulness provide *negative* value to +the LLVM project, by taking up valuable reviewer time, increasing code +complexity and increasing compile-time overhead. + +We do not require explicit proof of real-world usefulness for every transform +-- in most cases the usefulness is fairly "obvious". However, the question may +come up for complex or unusual folds. Keep this in mind when chosing what you +work on. + +In particular, fixes for fuzzer-generated missed optimization reports will +likely be rejected if there is no evidence of real-world usefulness. + +### Pick the correct optimization pass + +There are a number of passes and utilities in the InstCombine family, and it +is important to pick the right place when implementing a fold. + + * `ConstantFolding`: For folding instructions with constant arguments to a constant. (Mainly relevant for intrinsics.) + * `ValueTracking`: For analyzing instructions, e.g. for known bits, non-zero, etc. Tests should usually use `-passes=instsimplify`. + * `InstructionSimplify`: For folds that do not create new instructions (either fold to existing value or constant). + * `InstCombine`: For folds that create or modify instructions. + * `AggressiveInstCombine`: For folds that are expensive, or violate InstCombine requirements. + * `VectorCombine`: For folds of vector operations that require target-dependent cost-modelling. + +Sometimes, folds that logically belong in InstSimplify are placed in InstCombine instead, for example because they are too expensive, or because they are structurally simpler to implement in InstCombine. + +For example, if a fold produces new instructions in some cases but returns an existing value in others, it may be preferable to keep all cases in InstCombine, rather than trying to split them among InstCombine and InstSimplify. + +### Canonicalization and target-independence + +InstCombine is a target-independent canonicalization pass. This means that it +tries to bring IR into a "canonical form" that other optimizations (both inside +and outside of InstCombine) can rely on. For this reason, the chosen canonical +form needs to be the same for all targets, and not depend on target-specific +cost modelling. + +In many cases, "canonicalization" and "optimization" coincide. For example, if +we convert `x * 2` into `x << 1`, this both makes the IR more canonical +(because there is now only one way to express the same operation, rather than +two) and faster (because shifts will usually have lower latency than +multiplies). + +However, there are also canonicalizations that don't serve any direct +optimization purpose. For example, InstCombine will canonicalize non-strict +predicates like `ule` to strict predicates like `ult`. `icmp ule i8 %x, 7` +becomes `icmp ult i8 %x, 8`. This is not an optimization in any meaningful +sense, but it does reduce the number of cases that other transforms need to +handle. + +If some canonicalization is not profitable for a specific target, then a reverse +transform needs to be added in the backend. Patches to disable specific +InstCombine transforms on certain targets, or to drive them using +target-specific cost-modelling, **will not be accepted**. The only permitted +target-dependence is on DataLayout and TargetLibraryInfo. + +The use of TargetTransformInfo is only allowed for hooks for target-specific +intrinsics, such as `TargetTransformInfo::instCombineIntrinsic()`. These are +already inherently target-dependent anyway. + +For vector-specific transforms that require cost-modelling, the VectorCombine +pass can be used instead. In very rare circumstances, if there are no other +alternatives, target-dependent transforms may be accepted into +AggressiveInstCombine. + +### PatternMatch + +Many transforms make use of the matching infrastructure defined in +[PatternMatch.h](https://github.com/llvm/llvm-project/blame/main/llvm/include/llvm/IR/PatternMatch.h). + +Here is a typical usage example: + +``` +// Fold (A - B) + B and B + (A - B) to A. +Value *A, *B; +if (match(V, m_c_Add(m_Sub(m_Value(A), m_Value(B)), m_Deferred(B)))) + return A; +``` + +And another: + +``` +// Fold A + C1 == C2 to A == C1+C2 +Value *A; +if (match(V, m_ICmp(Pred, m_Add(m_Value(A), m_APInt(C1)), m_APInt(C2))) && + ICmpInst::isEquality(Pred)) + return Builder.CreateICmp(Pred, A, + ConstantInt::get(A->getType(), *C1 + *C2)); +``` + +Some common matchers are: + + * `m_Value(A)`: Match any value and write it into `Value *A`. + * `m_Specific(A)`: Check that the operand equals A. Use this if A is + assigned **outside** the pattern. + * `m_Deferred(A)`: Check that the operand equals A. Use this if A is + assigned **inside** the pattern, for example via `m_Value(A)`. + * `m_APInt(C)`: Match a scalar integer constant or splat vector constant into + `const APInt *C`. Does not permit undef/poison values. + * `m_ImmConstant(C)`: Match any non-constant-expression constant into + `Constant *C`. + * `m_Constant(C)`: Match any constant into `Constant *C`. Don't use this unless + you know what you're doing. + * `m_Add(M1, M2)`, `m_Sub(M1, M2)`, etc: Match an add/sub/etc where the first + operand matches M1 and the second M2. + * `m_c_Add(M1, M2)`, etc: Match an add commutatively. The operands must match + either M1 and M2 or M2 and M1. Most instruction matchers have a commutative + variant. + * `m_ICmp(Pred, M1, M2)` and `m_c_ICmp(Pred, M1, M2)`: Match an icmp, writing + the predicate into `IcmpInst::Predicate Pred`. If the commutative version + is used, and the operands match in order M2, M1, then `Pred` will be the + swapped predicate. + * `m_OneUse(M)`: Check that the value only has one use, and also matches M. + For example `m_OneUse(m_Add(...))`. See the next section for more + information. + +See the header for the full list of available matchers. + +### InstCombine APIs + +InstCombine transforms are handled by `visitXYZ()` methods, where XYZ +corresponds to the root instruction of your transform. If the outermost +instruction of the pattern you are matching is an icmp, the fold will be +located somewhere inside `visitICmpInst()`. + +The return value of the visit method is an instruction. You can either return +a new instruction, in which case it will be inserted before the old one, and +uses of the old one will be replaced by it. Or you can return the original +instruction to indicate that *some* kind of change has been made. Finally, a +nullptr return value indicates that no change occurred. + +For example, if your transform produces a single new icmp instruction, you could +write the following: + +``` +if (...) + return new ICmpInst(Pred, X, Y); +``` + +In this case the main InstCombine loop takes care of inserting the instruction +and replacing uses of the old instruction. + +Alternatively, you can also write it like this: + +``` +if (...) + return replaceInstUsesWith(OrigI, Builder.CreateICmp(Pred, X, Y)); +``` + +In this case `IRBuilder` will insert the instruction and `replaceInstUsesWith()` +will replace the uses of the old instruction, and return it to indicate that +a change occurred. + +Both forms are equivalent, and you can use whichever is more convenient in +context. For example, it's common that folds are inside helper functions that +return `Value *` and then `replaceInstUsesWith()` is invoked on the result of +that helper. + +InstCombine makes use of a worklist, which needs to be correctly updated during +transforms. This usually happens automatically, but there are some things to +keep in mind: + + * Don't use the `Value::replaceAllUsesWith()` API. Use InstCombine's + `replaceInstUsesWith()` helper instead. + * Don't use the `Instruction::eraseFromParent()` API. Use InstCombine's + `eraseInstFromFunction()` helper instead. (Explicitly erasing instruction + is usually not necessary, as side-effect free instructions without users + are automatically removed.) + * Apart from the "directly return an instruction" pattern above, use IRBUilder + to create all instruction. Do not manually create and insert them. + * When replacing operands or uses of instructions, use `replaceOperand()` + and `replaceUse()` instead of `setOperand()`. + +### Multi-use handling + +Transforms should usually not increase the total number of instructions. This +is not a hard requirement: For example, it is usually worthwhile to replace a +single division instruction with multiple other instructions. + +For example, if you have a transform that replaces two instructions, with two +other instructions, this is (usually) only profitable if *both* the original +instructions can be removed. To ensure that both instructions are removed, you +need to add a one-use check for the inner instruction. + +One-use checks can be performed using the `m_OneUse()` matcher, or the +`V->hasOneUse()` method. + +### Generalization + +Transforms can both be too specific (only handling some odd subset of patterns, +leading to unexpected optimization cliffs) and too general (introducing +complexity to handle cases with no real-world relevance). The right level of +generality is quite subjective, so this section only provides some broad +guidelines. + + * Avoid transforms that are hardcoded to specific constants. Try to figure + out what the general rule for arbitrary constants is. + * Add handling for conjugate patterns. For example, if you implement a fold + for `icmp eq`, you almost certainly also want to support `icmp ne`, with the + inverse result. Similarly, if you implement a pattern for `and` of `icmp`s, + you should also handle the de-Morgan conjugate using `or`. + * Handle non-splat vector constants if doing so is free, but do not add + handling for them if it adds any additional complexity to the code. + * Do not handle non-canonical patterns, unless there is a specific motivation + to do so. For example, it may sometimes be worthwhile to handle a pattern + that would normally be converted into a different canonical form, but can + still occur in multi-use scenarios. This is fine to do if there is specific + real-world motivation, but you should not go out of your way to do this + otherwise. + * Sometimes the motivating pattern uses a constant value with certain + properties, but the fold can be generalized to non-constant values by making + use of ValueTracking queries. Whether this makes sense depends on the case, + but it's usually a good idea to only handle the constant pattern first, and + then generalize later if it seems useful. diff --git a/llvm/docs/UserGuides.rst b/llvm/docs/UserGuides.rst index 155cb3361669..f40a04d414a2 100644 --- a/llvm/docs/UserGuides.rst +++ b/llvm/docs/UserGuides.rst @@ -43,6 +43,7 @@ intermediate LLVM representation. HowToCrossCompileBuiltinsOnArm HowToCrossCompileLLVM HowToUpdateDebugInfo + InstCombineContributorGuide InstrProfileFormat InstrRefDebugInfo LinkTimeOptimization @@ -186,6 +187,10 @@ Optimizations :doc:`InstrProfileFormat` This document explains two binary formats of instrumentation-based profiles. +:doc:`InstCombineContributorGuide` + This document specifies guidelines for contributions for InstCombine and + related passes. + Code Generation --------------- -- GitLab From 8d7a6e2fd8beadc54a2e54fa361d35c66fdb40a4 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Thu, 21 Mar 2024 08:00:03 -0700 Subject: [PATCH 154/296] [SLP]Fix a crash for gather node with instructions from different bbs, if cost threshold is very low. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 11 ++++++--- .../X86/gather-nodes-different-bb.ll | 24 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/gather-nodes-different-bb.ll diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 4853c2006fea..36b446962c4a 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -9275,11 +9275,16 @@ bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const { // Check if any of the gather node forms an insertelement buildvector // somewhere. - if (any_of(VectorizableTree, [](const std::unique_ptr &TE) { + bool IsAllowedSingleBVNode = + VectorizableTree.size() > 1 || + (VectorizableTree.size() == 1 && VectorizableTree.front()->getOpcode() && + allSameBlock(VectorizableTree.front()->Scalars)); + if (any_of(VectorizableTree, [&](const std::unique_ptr &TE) { return TE->State == TreeEntry::NeedToGather && - all_of(TE->Scalars, [](Value *V) { + all_of(TE->Scalars, [&](Value *V) { return isa(V) || - (!V->hasNUsesOrMore(UsesLimit) && + (IsAllowedSingleBVNode && + !V->hasNUsesOrMore(UsesLimit) && any_of(V->users(), [](User *U) { return isa(U); })); diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gather-nodes-different-bb.ll b/llvm/test/Transforms/SLPVectorizer/X86/gather-nodes-different-bb.ll new file mode 100644 index 000000000000..2acbe89b9775 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/X86/gather-nodes-different-bb.ll @@ -0,0 +1,24 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S -passes=slp-vectorizer -mtriple=x86_64-unknown-linux -mattr="-avx512pf,+avx512f,+avx512bw" -slp-threshold=-100 < %s | FileCheck %s + +define i1 @foo(i32 %a) { +; CHECK-LABEL: @foo( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = sub nsw i32 0, [[A:%.*]] +; CHECK-NEXT: br label [[BB4:%.*]] +; CHECK: bb1: +; CHECK-NEXT: [[LOCAL:%.*]] = sub nsw i32 0, 0 +; CHECK-NEXT: [[INS1:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i32 0 +; CHECK-NEXT: [[ADD:%.*]] = icmp eq i32 [[TMP0]], [[LOCAL]] +; CHECK-NEXT: ret i1 [[ADD]] +; +entry: + %0 = sub nsw i32 0, %a + br label %bb1 + +bb1: + %local = sub nsw i32 0, 0 + %ins1 = insertelement <2 x i32> poison, i32 %0, i32 0 + %add = icmp eq i32 %0, %local + ret i1 %add +} -- GitLab From e4fa2e3562f20106f0fe4a4d09df03a548db4eae Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 21 Mar 2024 15:11:59 +0000 Subject: [PATCH 155/296] [DAG] isGuaranteedNotToBeUndefOrPoisonForTargetNode - add fallback implementation (#86125) Allow targets to rely on TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode to test nodes for canCreateUndefOrPoisonForTargetNode + all arguments are isGuaranteedNotToBeUndefOrPoison. Targets can still perform this themselves for specific special case nodes (e.g. target shuffles). Matches the fallback in SelectionDAG::isGuaranteedNotToBeUndefOrPoison --- llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 5 +++-- llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp | 10 +++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 9d73a42df2a4..cd6f083243d0 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -5042,8 +5042,9 @@ bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, // If Op can't create undef/poison and none of its operands are undef/poison // then Op is never undef/poison. - // NOTE: TargetNodes should handle this in themselves in - // isGuaranteedNotToBeUndefOrPoisonForTargetNode. + // NOTE: TargetNodes can handle this in themselves in + // isGuaranteedNotToBeUndefOrPoisonForTargetNode or let + // TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it. return !canCreateUndefOrPoison(Op, PoisonOnly, /*ConsiderFlags*/ true, Depth) && all_of(Op->ops(), [&](SDValue V) { diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 57f8fc409de4..da29b1d5b312 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -3786,7 +3786,15 @@ bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode( Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op" " is a target node!"); - return false; + + // If Op can't create undef/poison and none of its operands are undef/poison + // then Op is never undef/poison. + return !canCreateUndefOrPoisonForTargetNode(Op, DemandedElts, DAG, PoisonOnly, + /*ConsiderFlags*/ true, Depth) && + all_of(Op->ops(), [&](SDValue V) { + return DAG.isGuaranteedNotToBeUndefOrPoison(V, PoisonOnly, + Depth + 1); + }); } bool TargetLowering::canCreateUndefOrPoisonForTargetNode( -- GitLab From 7678e6e562811688b472ad19900fa64cd00b7c06 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 21 Mar 2024 08:14:48 -0700 Subject: [PATCH 156/296] [RISCV] Lower the alignment requirement for a GPR pair spill for Zdinx on RV32. (#85871) I believe we can use XLen alignment as long as eliminateFrameIndex limits the maximum folded offset to 2043. This way when we split the load/store into two 2 instructions we'll be able to add 4 without overflowing simm12. --- llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp | 7 ++ llvm/lib/Target/RISCV/RISCVRegisterInfo.td | 2 +- llvm/test/CodeGen/RISCV/zdinx-large-spill.mir | 74 +++++++++++++++++++ 3 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 llvm/test/CodeGen/RISCV/zdinx-large-spill.mir diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp index 952d17468da5..74d65324b95d 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp @@ -446,6 +446,13 @@ bool RISCVRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, (Lo12 & 0b11111) != 0) { // Prefetch instructions require the offset to be 32 byte aligned. MI.getOperand(FIOperandNum + 1).ChangeToImmediate(0); + } else if ((Opc == RISCV::PseudoRV32ZdinxLD || + Opc == RISCV::PseudoRV32ZdinxSD) && + Lo12 >= 2044) { + // This instruction will be split into 2 instructions. The second + // instruction will add 4 to the immediate. If that would overflow 12 + // bits, we can't fold the offset. + MI.getOperand(FIOperandNum + 1).ChangeToImmediate(0); } else { // We can encode an add with 12 bit signed immediate in the immediate // operand of our user instruction. As a result, the remaining diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td index 225b57554c1d..9da1f73681c6 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td @@ -573,7 +573,7 @@ let RegAltNameIndices = [ABIRegAltName] in { } let RegInfos = RegInfoByHwMode<[RV32, RV64], - [RegInfo<64, 64, 64>, RegInfo<128, 128, 128>]>, + [RegInfo<64, 64, 32>, RegInfo<128, 128, 64>]>, DecoderMethod = "DecodeGPRPairRegisterClass" in def GPRPair : RegisterClass<"RISCV", [XLenPairFVT], 64, (add X10_X11, X12_X13, X14_X15, X16_X17, diff --git a/llvm/test/CodeGen/RISCV/zdinx-large-spill.mir b/llvm/test/CodeGen/RISCV/zdinx-large-spill.mir new file mode 100644 index 000000000000..a2a722a7f728 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/zdinx-large-spill.mir @@ -0,0 +1,74 @@ +# NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +# RUN: llc %s -mtriple=riscv32 -mattr=+zdinx -start-before=prologepilog -o - | FileCheck %s + +# We want to make sure eliminateFrameIndex doesn't fold sp+2044 as an offset in +# a GPR pair spill/reload instruction. When we split the pair spill, we would be +# unable to add 4 to the immediate without overflowing simm12. + +--- | + define void @foo() { + ; CHECK-LABEL: foo: + ; CHECK: # %bb.0: + ; CHECK-NEXT: addi sp, sp, -2048 + ; CHECK-NEXT: addi sp, sp, -16 + ; CHECK-NEXT: .cfi_def_cfa_offset 2064 + ; CHECK-NEXT: lui t0, 1 + ; CHECK-NEXT: add t0, sp, t0 + ; CHECK-NEXT: sw a0, -2040(t0) + ; CHECK-NEXT: sw a1, -2036(t0) + ; CHECK-NEXT: lui a0, 1 + ; CHECK-NEXT: add a0, sp, a0 + ; CHECK-NEXT: sw a2, -2048(a0) + ; CHECK-NEXT: sw a3, -2044(a0) + ; CHECK-NEXT: sw a4, 2040(sp) + ; CHECK-NEXT: sw a5, 2044(sp) + ; CHECK-NEXT: sw a6, 2032(sp) + ; CHECK-NEXT: sw a7, 2036(sp) + ; CHECK-NEXT: lui a0, 1 + ; CHECK-NEXT: add a0, sp, a0 + ; CHECK-NEXT: lw a1, -2036(a0) + ; CHECK-NEXT: lw a0, -2040(a0) + ; CHECK-NEXT: lui a0, 1 + ; CHECK-NEXT: add a0, sp, a0 + ; CHECK-NEXT: lw a2, -2048(a0) + ; CHECK-NEXT: lw a3, -2044(a0) + ; CHECK-NEXT: lw a4, 2040(sp) + ; CHECK-NEXT: lw a5, 2044(sp) + ; CHECK-NEXT: lw a6, 2032(sp) + ; CHECK-NEXT: lw a7, 2036(sp) + ; CHECK-NEXT: addi sp, sp, 2032 + ; CHECK-NEXT: addi sp, sp, 32 + ; CHECK-NEXT: ret + ret void + } +... +--- +name: foo +tracksRegLiveness: true +tracksDebugUserValues: true +frameInfo: + maxAlignment: 4 +stack: + - { id: 0, type: spill-slot, size: 8, alignment: 4 } + - { id: 1, type: spill-slot, size: 8, alignment: 4 } + - { id: 2, type: spill-slot, size: 8, alignment: 4 } + - { id: 3, type: spill-slot, size: 8, alignment: 4 } + - { id: 4, type: spill-slot, size: 2024, alignment: 4 } +machineFunctionInfo: + varArgsFrameIndex: 0 + varArgsSaveSize: 0 +body: | + bb.0: + liveins: $x10_x11, $x12_x13, $x14_x15, $x16_x17 + + PseudoRV32ZdinxSD killed renamable $x10_x11, %stack.0, 0 :: (store (s64) into %stack.0, align 4) + PseudoRV32ZdinxSD killed renamable $x12_x13, %stack.1, 0 :: (store (s64) into %stack.1, align 4) + PseudoRV32ZdinxSD killed renamable $x14_x15, %stack.2, 0 :: (store (s64) into %stack.2, align 4) + PseudoRV32ZdinxSD killed renamable $x16_x17, %stack.3, 0 :: (store (s64) into %stack.3, align 4) + renamable $x10_x11 = PseudoRV32ZdinxLD %stack.0, 0 :: (load (s64) from %stack.0, align 4) + renamable $x12_x13 = PseudoRV32ZdinxLD %stack.1, 0 :: (load (s64) from %stack.1, align 4) + renamable $x14_x15 = PseudoRV32ZdinxLD %stack.2, 0 :: (load (s64) from %stack.2, align 4) + renamable $x16_x17 = PseudoRV32ZdinxLD %stack.3, 0 :: (load (s64) from %stack.3, align 4) + PseudoRET + +... -- GitLab From c04807c84e2a2653ab325f1b8ec73916565e6c54 Mon Sep 17 00:00:00 2001 From: aniplcc Date: Thu, 21 Mar 2024 20:51:06 +0530 Subject: [PATCH 157/296] [libc][c11] Add stdio.h's rename() function (#85068) Adds stdio.h's rename() function as defined in n3096. Fixes #84980. --- libc/config/linux/aarch64/entrypoints.txt | 1 + libc/config/linux/riscv/entrypoints.txt | 1 + libc/config/linux/x86_64/entrypoints.txt | 1 + libc/docs/stdio.rst | 2 +- libc/spec/stdc.td | 5 ++ libc/src/stdio/CMakeLists.txt | 7 +++ libc/src/stdio/linux/CMakeLists.txt | 12 +++++ libc/src/stdio/linux/rename.cpp | 26 ++++++++++ libc/src/stdio/rename.h | 18 +++++++ libc/test/src/stdio/CMakeLists.txt | 15 ++++++ libc/test/src/stdio/rename_test.cpp | 49 +++++++++++++++++++ .../llvm-project-overlay/libc/BUILD.bazel | 11 +++++ 12 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 libc/src/stdio/linux/rename.cpp create mode 100644 libc/src/stdio/rename.h create mode 100644 libc/test/src/stdio/rename_test.cpp diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt index dbf81c284e78..6abb35ab0ead 100644 --- a/libc/config/linux/aarch64/entrypoints.txt +++ b/libc/config/linux/aarch64/entrypoints.txt @@ -195,6 +195,7 @@ set(TARGET_LIBC_ENTRYPOINTS # stdio.h entrypoints libc.src.stdio.remove + libc.src.stdio.rename libc.src.stdio.sprintf libc.src.stdio.snprintf libc.src.stdio.vsprintf diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt index b42a55a4d712..e34c87ec6b5d 100644 --- a/libc/config/linux/riscv/entrypoints.txt +++ b/libc/config/linux/riscv/entrypoints.txt @@ -196,6 +196,7 @@ set(TARGET_LIBC_ENTRYPOINTS # stdio.h entrypoints libc.src.stdio.remove + libc.src.stdio.rename libc.src.stdio.sprintf libc.src.stdio.snprintf libc.src.stdio.fprintf diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index c216f4349627..8e1ab5cd65f0 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -198,6 +198,7 @@ set(TARGET_LIBC_ENTRYPOINTS # stdio.h entrypoints libc.src.stdio.remove + libc.src.stdio.rename libc.src.stdio.sprintf libc.src.stdio.snprintf libc.src.stdio.fprintf diff --git a/libc/docs/stdio.rst b/libc/docs/stdio.rst index 4fd6b71a0917..d17821562c25 100644 --- a/libc/docs/stdio.rst +++ b/libc/docs/stdio.rst @@ -68,7 +68,7 @@ These functions operate on files on the host's system, without using the Function_Name Available ============= ========= remove |check| -rename +rename |check| tmpnam ============= ========= diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index 920036adfed5..76010a4b4533 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -706,6 +706,11 @@ def StdC : StandardSpec<"stdc"> { RetValSpec, [ArgSpec] >, + FunctionSpec< + "rename", + RetValSpec, + [ArgSpec, ArgSpec] + >, FunctionSpec< "setbuf", RetValSpec, diff --git a/libc/src/stdio/CMakeLists.txt b/libc/src/stdio/CMakeLists.txt index ece93fd56ef0..11e15c917351 100644 --- a/libc/src/stdio/CMakeLists.txt +++ b/libc/src/stdio/CMakeLists.txt @@ -256,6 +256,13 @@ add_entrypoint_object( .${LIBC_TARGET_OS}.remove ) +add_entrypoint_object( + rename + ALIAS + DEPENDS + .${LIBC_TARGET_OS}.rename +) + # These entrypoints have multiple potential implementations. add_stdio_entrypoint_object(feof) add_stdio_entrypoint_object(feof_unlocked) diff --git a/libc/src/stdio/linux/CMakeLists.txt b/libc/src/stdio/linux/CMakeLists.txt index 774f24b2db0b..a08ff0ba4832 100644 --- a/libc/src/stdio/linux/CMakeLists.txt +++ b/libc/src/stdio/linux/CMakeLists.txt @@ -12,3 +12,15 @@ add_entrypoint_object( libc.src.__support.OSUtil.osutil libc.src.errno.errno ) + +add_entrypoint_object( + rename + SRCS + rename.cpp + HDRS + ../rename.h + DEPENDS + libc.include.sys_syscall + libc.src.__support.OSUtil.osutil + libc.src.errno.errno +) diff --git a/libc/src/stdio/linux/rename.cpp b/libc/src/stdio/linux/rename.cpp new file mode 100644 index 000000000000..f3d684249ad2 --- /dev/null +++ b/libc/src/stdio/linux/rename.cpp @@ -0,0 +1,26 @@ +//===-- Linux implementation of rename ------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdio/rename.h" +#include "src/__support/OSUtil/syscall.h" // For internal syscall function. +#include "src/__support/common.h" +#include "src/errno/libc_errno.h" +#include // For syscall numbers. + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, rename, (const char *oldpath, const char *newpath)) { + int ret = LIBC_NAMESPACE::syscall_impl(SYS_rename, oldpath, newpath); + + if (ret >= 0) + return 0; + libc_errno = -ret; + return -1; +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdio/rename.h b/libc/src/stdio/rename.h new file mode 100644 index 000000000000..eadda7c3eac9 --- /dev/null +++ b/libc/src/stdio/rename.h @@ -0,0 +1,18 @@ +//===-- Implementation header of rename -------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDIO_RENAME_H +#define LLVM_LIBC_SRC_STDIO_RENAME_H + +namespace LIBC_NAMESPACE { + +int rename(const char *oldpath, const char *newpath); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDIO_RENAME_H diff --git a/libc/test/src/stdio/CMakeLists.txt b/libc/test/src/stdio/CMakeLists.txt index 3ccce16a76a2..a11a232c27e1 100644 --- a/libc/test/src/stdio/CMakeLists.txt +++ b/libc/test/src/stdio/CMakeLists.txt @@ -354,6 +354,21 @@ if(${LIBC_TARGET_OS} STREQUAL "linux") libc.src.unistd.access libc.src.unistd.close ) + + add_libc_test( + rename_test + SUITE + libc_stdio_unittests + SRCS + rename_test.cpp + DEPENDS + libc.src.errno.errno + libc.src.fcntl.open + libc.src.stdio.rename + libc.src.unistd.access + libc.src.unistd.close + libc.test.UnitTest.ErrnoSetterMatcher + ) endif() add_libc_test( diff --git a/libc/test/src/stdio/rename_test.cpp b/libc/test/src/stdio/rename_test.cpp new file mode 100644 index 000000000000..a9fbe24ded9c --- /dev/null +++ b/libc/test/src/stdio/rename_test.cpp @@ -0,0 +1,49 @@ +//===-- Unittests for rename ----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/errno/libc_errno.h" +#include "src/fcntl/open.h" +#include "src/stdio/rename.h" +#include "src/unistd/access.h" +#include "src/unistd/close.h" +#include "test/UnitTest/ErrnoSetterMatcher.h" +#include "test/UnitTest/Test.h" + +TEST(LlvmLibcRenameTest, CreateAndRenameFile) { + // The test strategy is to create a file and rename it, and also verify that + // it was renamed. + LIBC_NAMESPACE::libc_errno = 0; + using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Fails; + using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds; + + constexpr const char *FILENAME0 = "rename.test.file0"; + auto TEST_FILEPATH0 = libc_make_test_file_path(FILENAME0); + + int fd = LIBC_NAMESPACE::open(TEST_FILEPATH0, O_WRONLY | O_CREAT, S_IRWXU); + ASSERT_ERRNO_SUCCESS(); + ASSERT_GT(fd, 0); + ASSERT_THAT(LIBC_NAMESPACE::close(fd), Succeeds(0)); + ASSERT_THAT(LIBC_NAMESPACE::access(TEST_FILEPATH0, F_OK), Succeeds(0)); + + constexpr const char *FILENAME1 = "rename.test.file1"; + auto TEST_FILEPATH1 = libc_make_test_file_path(FILENAME1); + ASSERT_THAT(LIBC_NAMESPACE::rename(TEST_FILEPATH0, TEST_FILEPATH1), + Succeeds(0)); + ASSERT_THAT(LIBC_NAMESPACE::access(TEST_FILEPATH1, F_OK), Succeeds(0)); + ASSERT_THAT(LIBC_NAMESPACE::access(TEST_FILEPATH0, F_OK), Fails(ENOENT)); +} + +TEST(LlvmLibcRenameTest, RenameNonExistent) { + using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Fails; + + constexpr const char *FILENAME1 = "rename.test.file1"; + auto TEST_FILEPATH1 = libc_make_test_file_path(FILENAME1); + + ASSERT_THAT(LIBC_NAMESPACE::rename("non-existent", TEST_FILEPATH1), + Fails(ENOENT)); +} diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 40cfb1f470db..fe4b8e2de14e 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -3262,6 +3262,17 @@ libc_function( ], ) +libc_function( + name = "rename", + srcs = ["src/stdio/linux/rename.cpp"], + hdrs = ["src/stdio/rename.h"], + deps = [ + ":__support_common", + ":__support_osutil_syscall", + ":errno", + ], +) + ############################### sys/stat targets ############################### libc_function( -- GitLab From 50801f1095d33e712c3a51fdeef82569bd09007f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Thu, 21 Mar 2024 17:29:15 +0200 Subject: [PATCH 158/296] Reapply [libcxx] [modules] Fix relative paths with absolute LIBCXX_INSTALL_MODULES_DIR (#86020) This reapplies 272d1b44efdedb68c194970a610f0ca1b7b769c5 (from #85756), which was reverted in 407937036fa7640f61f225474b1ea6623a40dbdd. In the previous attempt, empty CMAKE_INSTALL_PREFIX was handled by quoting them, in d209d1340b99d4fbd325dffb5e13b757ab8264ea. That made the calls to cmake_path(ABSOLUTE_PATH) succeed, but the output paths of that weren't actually absolute, which was required by file(RELATIVE_PATH). Avoid this issue by constructing a non-empty base directory variable to use for calculating the relative path. --- libcxx/modules/CMakeLists.txt | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/libcxx/modules/CMakeLists.txt b/libcxx/modules/CMakeLists.txt index 0dea8cfca94a..d47d19a47553 100644 --- a/libcxx/modules/CMakeLists.txt +++ b/libcxx/modules/CMakeLists.txt @@ -206,9 +206,20 @@ add_custom_target(generate-cxx-modules # Configure the modules manifest. # Use the relative path between the installation and the module in the json # file. This allows moving the entire installation to a different location. +if("${CMAKE_INSTALL_PREFIX}" STREQUAL "") + set(BASE_DIRECTORY "/") +else() + set(BASE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +endif() +cmake_path(ABSOLUTE_PATH LIBCXX_INSTALL_LIBRARY_DIR + BASE_DIRECTORY ${BASE_DIRECTORY} + OUTPUT_VARIABLE ABS_LIBRARY_DIR) +cmake_path(ABSOLUTE_PATH LIBCXX_INSTALL_MODULES_DIR + BASE_DIRECTORY ${BASE_DIRECTORY} + OUTPUT_VARIABLE ABS_MODULES_DIR) file(RELATIVE_PATH LIBCXX_MODULE_RELATIVE_PATH - ${CMAKE_INSTALL_PREFIX}/${LIBCXX_INSTALL_LIBRARY_DIR} - ${CMAKE_INSTALL_PREFIX}/${LIBCXX_INSTALL_MODULES_DIR}) + ${ABS_LIBRARY_DIR} + ${ABS_MODULES_DIR}) configure_file( "modules.json.in" "${LIBCXX_LIBRARY_DIR}/libc++.modules.json" -- GitLab From 7650a01927b8488b1d6d0930109e78c695193faf Mon Sep 17 00:00:00 2001 From: timoh-ba Date: Thu, 21 Mar 2024 16:30:10 +0100 Subject: [PATCH 159/296] [DWARF5][COFF] Emit section-relative .debug_line_str relocations (#83773) Dwarf 5 allows separating filenames from .debug_line into a separate .debug_line_str section. The strings are referenced relative to the start of the .debug_line_str section. Previously, on COFF, the relocation information instead caused offsets to be relocated to the base address of the COFF-File. This lead to wrong offsets in linked COFF (PE) files which caused the debugger to be unable to find the correct source files. This patch fixes this problem by making the offsets relative to the start of the .debug_line_str section instead. There should be no changes for ELF-Files as everything seems to be working there. A test is also added to ensure that the correct relocation entries are emitted. --- llvm/lib/MC/MCDwarf.cpp | 7 ++++++- llvm/test/MC/COFF/dwarf5lineinfo.s | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 llvm/test/MC/COFF/dwarf5lineinfo.s diff --git a/llvm/lib/MC/MCDwarf.cpp b/llvm/lib/MC/MCDwarf.cpp index d0face9140de..2ee0c3eb27b9 100644 --- a/llvm/lib/MC/MCDwarf.cpp +++ b/llvm/lib/MC/MCDwarf.cpp @@ -360,7 +360,12 @@ void MCDwarfLineStr::emitRef(MCStreamer *MCOS, StringRef Path) { size_t Offset = addString(Path); if (UseRelocs) { MCContext &Ctx = MCOS->getContext(); - MCOS->emitValue(makeStartPlusIntExpr(Ctx, *LineStrLabel, Offset), RefSize); + if (Ctx.getAsmInfo()->needsDwarfSectionOffsetDirective()) { + MCOS->emitCOFFSecRel32(LineStrLabel, Offset); + } else { + MCOS->emitValue(makeStartPlusIntExpr(Ctx, *LineStrLabel, Offset), + RefSize); + } } else MCOS->emitIntValue(Offset, RefSize); } diff --git a/llvm/test/MC/COFF/dwarf5lineinfo.s b/llvm/test/MC/COFF/dwarf5lineinfo.s new file mode 100644 index 000000000000..f0789feb2085 --- /dev/null +++ b/llvm/test/MC/COFF/dwarf5lineinfo.s @@ -0,0 +1,13 @@ +// RUN: llvm-mc -filetype=obj -triple x86_64-pc-windows-gnu %s -o - | llvm-readobj -r - | FileCheck %s + +// CHECK: Relocations [ +// CHECK: Section (4) .debug_line { +// CHECK: 0x22 IMAGE_REL_AMD64_SECREL .debug_line_str (8) +// CHECK: 0x2C IMAGE_REL_AMD64_SECREL .debug_line_str (8) +// CHECK: 0x36 IMAGE_REL_AMD64_ADDR64 .text (0) +// CHECK: } + +main: + .file 0 "/" "test.c" + .loc 0 1 0 + retq -- GitLab From 0c8dfc85c3740bd8905e21642f616e6fd54854e0 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Thu, 21 Mar 2024 08:36:47 -0700 Subject: [PATCH 160/296] [libc][stdio][test] fixup rename test (#86136) Link: #84980 Link: #85068 --- libc/test/src/stdio/CMakeLists.txt | 2 +- libc/test/src/stdio/rename_test.cpp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/libc/test/src/stdio/CMakeLists.txt b/libc/test/src/stdio/CMakeLists.txt index a11a232c27e1..4c38e8aba7d7 100644 --- a/libc/test/src/stdio/CMakeLists.txt +++ b/libc/test/src/stdio/CMakeLists.txt @@ -354,7 +354,7 @@ if(${LIBC_TARGET_OS} STREQUAL "linux") libc.src.unistd.access libc.src.unistd.close ) - + add_libc_test( rename_test SUITE diff --git a/libc/test/src/stdio/rename_test.cpp b/libc/test/src/stdio/rename_test.cpp index a9fbe24ded9c..3ed39fe8c0eb 100644 --- a/libc/test/src/stdio/rename_test.cpp +++ b/libc/test/src/stdio/rename_test.cpp @@ -6,6 +6,8 @@ // //===----------------------------------------------------------------------===// +#include "include/llvm-libc-macros/linux/unistd-macros.h" +#include "include/llvm-libc-macros/linux/sys-stat-macros.h" #include "src/errno/libc_errno.h" #include "src/fcntl/open.h" #include "src/stdio/rename.h" -- GitLab From 556fe5f290ea88dcbb7ced16b0f057dcebce1fd0 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Thu, 21 Mar 2024 08:40:08 -0700 Subject: [PATCH 161/296] [lldb] Reland: Store SupportFile in FileEntry (NFC) (#85892) This is another step towards supporting DWARF5 checksums and inline source code in LLDB. This is a reland of #85468 but without the functional change of storing the support file from the line table (yet). --- lldb/include/lldb/Core/Disassembler.h | 2 +- lldb/include/lldb/Symbol/LineEntry.h | 5 ++++- lldb/include/lldb/Utility/SupportFile.h | 3 +++ lldb/source/API/SBLineEntry.cpp | 10 +++++----- lldb/source/API/SBThread.cpp | 2 +- lldb/source/Breakpoint/BreakpointResolver.cpp | 2 +- .../Breakpoint/BreakpointResolverFileLine.cpp | 7 ++++--- lldb/source/Commands/CommandObjectBreakpoint.cpp | 4 ++-- lldb/source/Commands/CommandObjectSource.cpp | 14 +++++++------- lldb/source/Commands/CommandObjectThread.cpp | 2 +- lldb/source/Core/Address.cpp | 2 +- lldb/source/Core/Disassembler.cpp | 8 ++++---- lldb/source/Core/FormatEntity.cpp | 2 +- lldb/source/Core/IOHandlerCursesGUI.cpp | 11 ++++++----- lldb/source/Core/SourceManager.cpp | 2 +- .../Clang/ClangExpressionSourceCode.cpp | 2 +- lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp | 4 ++-- lldb/source/Symbol/CompileUnit.cpp | 2 +- lldb/source/Symbol/Function.cpp | 4 ++-- lldb/source/Symbol/LineEntry.cpp | 13 +++++++------ lldb/source/Symbol/LineTable.cpp | 4 ++-- lldb/source/Symbol/SymbolContext.cpp | 4 ++-- lldb/source/Target/StackFrame.cpp | 2 +- lldb/source/Target/StackFrameList.cpp | 4 ++-- lldb/source/Target/Thread.cpp | 8 ++++---- lldb/source/Target/TraceDumper.cpp | 4 ++-- .../SymbolFile/PDB/SymbolFilePDBTests.cpp | 2 +- 27 files changed, 69 insertions(+), 60 deletions(-) diff --git a/lldb/include/lldb/Core/Disassembler.h b/lldb/include/lldb/Core/Disassembler.h index 885ac1bb4a7e..e037a49f152c 100644 --- a/lldb/include/lldb/Core/Disassembler.h +++ b/lldb/include/lldb/Core/Disassembler.h @@ -538,7 +538,7 @@ protected: ElideMixedSourceAndDisassemblyLine(const ExecutionContext &exe_ctx, const SymbolContext &sc, LineEntry &line) { SourceLine sl; - sl.file = line.file; + sl.file = line.GetFile(); sl.line = line.line; sl.column = line.column; return ElideMixedSourceAndDisassemblyLine(exe_ctx, sc, sl); diff --git a/lldb/include/lldb/Symbol/LineEntry.h b/lldb/include/lldb/Symbol/LineEntry.h index 31e1cd0b36f9..8da59cf0bd24 100644 --- a/lldb/include/lldb/Symbol/LineEntry.h +++ b/lldb/include/lldb/Symbol/LineEntry.h @@ -130,11 +130,14 @@ struct LineEntry { /// Shared pointer to the target this LineEntry belongs to. void ApplyFileMappings(lldb::TargetSP target_sp); + /// Helper to access the file. + const FileSpec &GetFile() const { return file_sp->GetSpecOnly(); } + /// The section offset address range for this line entry. AddressRange range; /// The source file, possibly mapped by the target.source-map setting. - FileSpec file; + lldb::SupportFileSP file_sp; /// The original source file, from debug info. lldb::SupportFileSP original_file_sp; diff --git a/lldb/include/lldb/Utility/SupportFile.h b/lldb/include/lldb/Utility/SupportFile.h index 0ea0ca4e7c97..7505d7f345c5 100644 --- a/lldb/include/lldb/Utility/SupportFile.h +++ b/lldb/include/lldb/Utility/SupportFile.h @@ -45,6 +45,9 @@ public: /// Materialize the file to disk and return the path to that temporary file. virtual const FileSpec &Materialize() { return m_file_spec; } + /// Change the file name. + void Update(const FileSpec &file_spec) { m_file_spec = file_spec; } + protected: FileSpec m_file_spec; Checksum m_checksum; diff --git a/lldb/source/API/SBLineEntry.cpp b/lldb/source/API/SBLineEntry.cpp index 28d12e65fdaf..99a7b8fe644c 100644 --- a/lldb/source/API/SBLineEntry.cpp +++ b/lldb/source/API/SBLineEntry.cpp @@ -81,8 +81,8 @@ SBFileSpec SBLineEntry::GetFileSpec() const { LLDB_INSTRUMENT_VA(this); SBFileSpec sb_file_spec; - if (m_opaque_up.get() && m_opaque_up->file) - sb_file_spec.SetFileSpec(m_opaque_up->file); + if (m_opaque_up.get() && m_opaque_up->GetFile()) + sb_file_spec.SetFileSpec(m_opaque_up->GetFile()); return sb_file_spec; } @@ -109,9 +109,9 @@ void SBLineEntry::SetFileSpec(lldb::SBFileSpec filespec) { LLDB_INSTRUMENT_VA(this, filespec); if (filespec.IsValid()) - ref().file = filespec.ref(); + ref().file_sp = std::make_shared(filespec.ref()); else - ref().file.Clear(); + ref().file_sp = std::make_shared(); } void SBLineEntry::SetLine(uint32_t line) { LLDB_INSTRUMENT_VA(this, line); @@ -168,7 +168,7 @@ bool SBLineEntry::GetDescription(SBStream &description) { if (m_opaque_up) { char file_path[PATH_MAX * 2]; - m_opaque_up->file.GetPath(file_path, sizeof(file_path)); + m_opaque_up->GetFile().GetPath(file_path, sizeof(file_path)); strm.Printf("%s:%u", file_path, GetLine()); if (GetColumn() > 0) strm.Printf(":%u", GetColumn()); diff --git a/lldb/source/API/SBThread.cpp b/lldb/source/API/SBThread.cpp index fa4c80e59d97..eb9cf063802c 100644 --- a/lldb/source/API/SBThread.cpp +++ b/lldb/source/API/SBThread.cpp @@ -819,7 +819,7 @@ SBError SBThread::StepOverUntil(lldb::SBFrame &sb_frame, step_file_spec = sb_file_spec.ref(); } else { if (frame_sc.line_entry.IsValid()) - step_file_spec = frame_sc.line_entry.file; + step_file_spec = frame_sc.line_entry.GetFile(); else { sb_error.SetErrorString("invalid file argument or no file for frame"); return sb_error; diff --git a/lldb/source/Breakpoint/BreakpointResolver.cpp b/lldb/source/Breakpoint/BreakpointResolver.cpp index 1861a0fe7c4f..ff4e2a998519 100644 --- a/lldb/source/Breakpoint/BreakpointResolver.cpp +++ b/lldb/source/Breakpoint/BreakpointResolver.cpp @@ -221,7 +221,7 @@ void BreakpointResolver::SetSCMatchesByLine( auto &match = all_scs[0]; auto worklist_begin = std::partition( all_scs.begin(), all_scs.end(), [&](const SymbolContext &sc) { - if (sc.line_entry.file == match.line_entry.file || + if (sc.line_entry.GetFile() == match.line_entry.GetFile() || *sc.line_entry.original_file_sp == *match.line_entry.original_file_sp) { // When a match is found, keep track of the smallest line number. diff --git a/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp b/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp index cc4e1d26724f..d7d8c714867e 100644 --- a/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp +++ b/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp @@ -147,8 +147,9 @@ void BreakpointResolverFileLine::FilterContexts(SymbolContextList &sc_list) { else continue; - if (file != sc.line_entry.file) { - LLDB_LOG(log, "unexpected symbol context file {0}", sc.line_entry.file); + if (file != sc.line_entry.GetFile()) { + LLDB_LOG(log, "unexpected symbol context file {0}", + sc.line_entry.GetFile()); continue; } @@ -223,7 +224,7 @@ void BreakpointResolverFileLine::DeduceSourceMapping( const bool case_sensitive = request_file.IsCaseSensitive(); for (const SymbolContext &sc : sc_list) { - FileSpec sc_file = sc.line_entry.file; + FileSpec sc_file = sc.line_entry.GetFile(); if (FileSpec::Equal(sc_file, request_file, /*full*/ true)) continue; diff --git a/lldb/source/Commands/CommandObjectBreakpoint.cpp b/lldb/source/Commands/CommandObjectBreakpoint.cpp index fbece865f113..cd4c7790f447 100644 --- a/lldb/source/Commands/CommandObjectBreakpoint.cpp +++ b/lldb/source/Commands/CommandObjectBreakpoint.cpp @@ -780,8 +780,8 @@ private: } else { const SymbolContext &sc = cur_frame->GetSymbolContext(eSymbolContextLineEntry); - if (sc.line_entry.file) { - file = sc.line_entry.file; + if (sc.line_entry.GetFile()) { + file = sc.line_entry.GetFile(); } else { result.AppendError("Can't find the file for the selected frame to " "use as the default file."); diff --git a/lldb/source/Commands/CommandObjectSource.cpp b/lldb/source/Commands/CommandObjectSource.cpp index fde74f02aea6..0c1267456a18 100644 --- a/lldb/source/Commands/CommandObjectSource.cpp +++ b/lldb/source/Commands/CommandObjectSource.cpp @@ -158,7 +158,7 @@ protected: if (module_list.GetSize() && module_list.GetIndexForModule(module) == LLDB_INVALID_INDEX32) continue; - if (!FileSpec::Match(file_spec, line_entry.file)) + if (!FileSpec::Match(file_spec, line_entry.GetFile())) continue; if (start_line > 0 && line_entry.line < start_line) continue; @@ -239,7 +239,7 @@ protected: num_matches++; if (num_lines > 0 && num_matches > num_lines) break; - assert(cu_file_spec == line_entry.file); + assert(cu_file_spec == line_entry.GetFile()); if (!cu_header_printed) { if (num_matches > 0) strm << "\n\n"; @@ -760,11 +760,11 @@ protected: bool operator<(const SourceInfo &rhs) const { if (function.GetCString() < rhs.function.GetCString()) return true; - if (line_entry.file.GetDirectory().GetCString() < - rhs.line_entry.file.GetDirectory().GetCString()) + if (line_entry.GetFile().GetDirectory().GetCString() < + rhs.line_entry.GetFile().GetDirectory().GetCString()) return true; - if (line_entry.file.GetFilename().GetCString() < - rhs.line_entry.file.GetFilename().GetCString()) + if (line_entry.GetFile().GetFilename().GetCString() < + rhs.line_entry.GetFile().GetFilename().GetCString()) return true; if (line_entry.line < rhs.line_entry.line) return true; @@ -799,7 +799,7 @@ protected: sc.function->GetEndLineSourceInfo(end_file, end_line); } else { // We have an inlined function - start_file = source_info.line_entry.file; + start_file = source_info.line_entry.GetFile(); start_line = source_info.line_entry.line; end_line = start_line + m_options.num_lines; } diff --git a/lldb/source/Commands/CommandObjectThread.cpp b/lldb/source/Commands/CommandObjectThread.cpp index cf4f8ccaa0c4..3dbbfd4f9d34 100644 --- a/lldb/source/Commands/CommandObjectThread.cpp +++ b/lldb/source/Commands/CommandObjectThread.cpp @@ -1705,7 +1705,7 @@ protected: line = sym_ctx.line_entry.line + m_options.m_line_offset; // Try the current file, but override if asked. - FileSpec file = sym_ctx.line_entry.file; + FileSpec file = sym_ctx.line_entry.GetFile(); if (m_options.m_filenames.GetSize() == 1) file = m_options.m_filenames.GetFileSpecAtIndex(0); diff --git a/lldb/source/Core/Address.cpp b/lldb/source/Core/Address.cpp index 6f5c366ab38a..b23398883fa5 100644 --- a/lldb/source/Core/Address.cpp +++ b/lldb/source/Core/Address.cpp @@ -398,7 +398,7 @@ bool Address::GetDescription(Stream &s, Target &target, "Non-brief descriptions not implemented"); LineEntry line_entry; if (CalculateSymbolContextLineEntry(line_entry)) { - s.Printf(" (%s:%u:%u)", line_entry.file.GetFilename().GetCString(), + s.Printf(" (%s:%u:%u)", line_entry.GetFile().GetFilename().GetCString(), line_entry.line, line_entry.column); return true; } diff --git a/lldb/source/Core/Disassembler.cpp b/lldb/source/Core/Disassembler.cpp index 7b07fcb26813..e31746fa0b8b 100644 --- a/lldb/source/Core/Disassembler.cpp +++ b/lldb/source/Core/Disassembler.cpp @@ -201,7 +201,7 @@ Disassembler::GetFunctionDeclLineEntry(const SymbolContext &sc) { uint32_t func_decl_line; sc.function->GetStartLineSourceInfo(func_decl_file, func_decl_line); - if (func_decl_file != prologue_end_line.file && + if (func_decl_file != prologue_end_line.GetFile() && func_decl_file != prologue_end_line.original_file_sp->GetSpecOnly()) return {}; @@ -354,7 +354,7 @@ void Disassembler::PrintInstructions(Debugger &debugger, const ArchSpec &arch, } if (sc.line_entry.IsValid()) { SourceLine this_line; - this_line.file = sc.line_entry.file; + this_line.file = sc.line_entry.GetFile(); this_line.line = sc.line_entry.line; this_line.column = sc.line_entry.column; if (!ElideMixedSourceAndDisassemblyLine(exe_ctx, sc, this_line)) @@ -406,7 +406,7 @@ void Disassembler::PrintInstructions(Debugger &debugger, const ArchSpec &arch, uint32_t func_decl_line; sc.function->GetStartLineSourceInfo(func_decl_file, func_decl_line); - if (func_decl_file == prologue_end_line.file || + if (func_decl_file == prologue_end_line.GetFile() || func_decl_file == prologue_end_line.original_file_sp->GetSpecOnly()) { // Add all the lines between the function declaration and @@ -439,7 +439,7 @@ void Disassembler::PrintInstructions(Debugger &debugger, const ArchSpec &arch, if (sc != prev_sc && sc.comp_unit && sc.line_entry.IsValid()) { SourceLine this_line; - this_line.file = sc.line_entry.file; + this_line.file = sc.line_entry.GetFile(); this_line.line = sc.line_entry.line; if (!ElideMixedSourceAndDisassemblyLine(exe_ctx, sc, diff --git a/lldb/source/Core/FormatEntity.cpp b/lldb/source/Core/FormatEntity.cpp index cf82676bedda..ba62e2625259 100644 --- a/lldb/source/Core/FormatEntity.cpp +++ b/lldb/source/Core/FormatEntity.cpp @@ -1792,7 +1792,7 @@ bool FormatEntity::Format(const Entry &entry, Stream &s, if (sc && sc->line_entry.IsValid()) { Module *module = sc->module_sp.get(); if (module) { - if (DumpFile(s, sc->line_entry.file, (FileKind)entry.number)) + if (DumpFile(s, sc->line_entry.GetFile(), (FileKind)entry.number)) return true; } } diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp index f86dce247135..d922d32f9105 100644 --- a/lldb/source/Core/IOHandlerCursesGUI.cpp +++ b/lldb/source/Core/IOHandlerCursesGUI.cpp @@ -6894,7 +6894,8 @@ public: if (context_changed) m_selected_line = m_pc_line; - if (m_file_sp && m_file_sp->GetFileSpec() == m_sc.line_entry.file) { + if (m_file_sp && + m_file_sp->GetFileSpec() == m_sc.line_entry.GetFile()) { // Same file, nothing to do, we should either have the lines or // not (source file missing) if (m_selected_line >= static_cast(m_first_visible_line)) { @@ -6909,8 +6910,8 @@ public: } else { // File changed, set selected line to the line with the PC m_selected_line = m_pc_line; - m_file_sp = - m_debugger.GetSourceManager().GetFile(m_sc.line_entry.file); + m_file_sp = m_debugger.GetSourceManager().GetFile( + m_sc.line_entry.GetFile()); if (m_file_sp) { const size_t num_lines = m_file_sp->GetNumLines(); m_line_width = 1; @@ -7000,7 +7001,7 @@ public: LineEntry bp_loc_line_entry; if (bp_loc_sp->GetAddress().CalculateSymbolContextLineEntry( bp_loc_line_entry)) { - if (m_file_sp->GetFileSpec() == bp_loc_line_entry.file) { + if (m_file_sp->GetFileSpec() == bp_loc_line_entry.GetFile()) { bp_lines.insert(bp_loc_line_entry.line); } } @@ -7477,7 +7478,7 @@ public: LineEntry bp_loc_line_entry; if (bp_loc_sp->GetAddress().CalculateSymbolContextLineEntry( bp_loc_line_entry)) { - if (m_file_sp->GetFileSpec() == bp_loc_line_entry.file && + if (m_file_sp->GetFileSpec() == bp_loc_line_entry.GetFile() && m_selected_line + 1 == bp_loc_line_entry.line) { bool removed = exe_ctx.GetTargetRef().RemoveBreakpointByID(bp_sp->GetID()); diff --git a/lldb/source/Core/SourceManager.cpp b/lldb/source/Core/SourceManager.cpp index 517a4b0268d2..0d70c554e534 100644 --- a/lldb/source/Core/SourceManager.cpp +++ b/lldb/source/Core/SourceManager.cpp @@ -418,7 +418,7 @@ bool SourceManager::GetDefaultFileAndLine(FileSpec &file_spec, uint32_t &line) { if (sc.function->GetAddressRange() .GetBaseAddress() .CalculateSymbolContextLineEntry(line_entry)) { - SetDefaultFileAndLine(line_entry.file, line_entry.line); + SetDefaultFileAndLine(line_entry.GetFile(), line_entry.line); file_spec = m_last_file_spec; line = m_last_line; return true; diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp index 3d43ed3f99ff..3b601726388d 100644 --- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp +++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp @@ -417,7 +417,7 @@ bool ClangExpressionSourceCode::GetText( if (sc.comp_unit && sc.line_entry.IsValid()) { DebugMacros *dm = sc.comp_unit->GetDebugMacros(); if (dm) { - AddMacroState state(sc.line_entry.file, sc.line_entry.line); + AddMacroState state(sc.line_entry.GetFile(), sc.line_entry.line); AddMacros(dm, sc.comp_unit, state, debug_macros_stream); } } diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp index 10a1fe039189..bcb04fae15bd 100644 --- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp +++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp @@ -106,13 +106,13 @@ bool lldb_private::formatters::LibcxxFunctionSummaryProvider( case CPPLanguageRuntime::LibCppStdFunctionCallableCase::Lambda: stream.Printf( " Lambda in File %s at Line %u", - callable_info.callable_line_entry.file.GetFilename().GetCString(), + callable_info.callable_line_entry.GetFile().GetFilename().GetCString(), callable_info.callable_line_entry.line); break; case CPPLanguageRuntime::LibCppStdFunctionCallableCase::CallableObject: stream.Printf( " Function in File %s at Line %u", - callable_info.callable_line_entry.file.GetFilename().GetCString(), + callable_info.callable_line_entry.GetFile().GetFilename().GetCString(), callable_info.callable_line_entry.line); break; case CPPLanguageRuntime::LibCppStdFunctionCallableCase::FreeOrMemberFunction: diff --git a/lldb/source/Symbol/CompileUnit.cpp b/lldb/source/Symbol/CompileUnit.cpp index 1b3cd23d9400..ddeacf18e855 100644 --- a/lldb/source/Symbol/CompileUnit.cpp +++ b/lldb/source/Symbol/CompileUnit.cpp @@ -320,7 +320,7 @@ void CompileUnit::ResolveSymbolContext( src_location_spec.GetColumn() ? std::optional(line_entry.column) : std::nullopt; - SourceLocationSpec found_entry(line_entry.file, line_entry.line, column, + SourceLocationSpec found_entry(line_entry.GetFile(), line_entry.line, column, inlines, exact); while (line_idx != UINT32_MAX) { diff --git a/lldb/source/Symbol/Function.cpp b/lldb/source/Symbol/Function.cpp index fdc090355771..194f89bc51d8 100644 --- a/lldb/source/Symbol/Function.cpp +++ b/lldb/source/Symbol/Function.cpp @@ -289,7 +289,7 @@ void Function::GetStartLineSourceInfo(FileSpec &source_file, if (line_table->FindLineEntryByAddress(GetAddressRange().GetBaseAddress(), line_entry, nullptr)) { line_no = line_entry.line; - source_file = line_entry.file; + source_file = line_entry.GetFile(); } } } @@ -311,7 +311,7 @@ void Function::GetEndLineSourceInfo(FileSpec &source_file, uint32_t &line_no) { LineEntry line_entry; if (line_table->FindLineEntryByAddress(scratch_addr, line_entry, nullptr)) { line_no = line_entry.line; - source_file = line_entry.file; + source_file = line_entry.GetFile(); } } diff --git a/lldb/source/Symbol/LineEntry.cpp b/lldb/source/Symbol/LineEntry.cpp index 389f8dcb65d8..9e0c06b6ff73 100644 --- a/lldb/source/Symbol/LineEntry.cpp +++ b/lldb/source/Symbol/LineEntry.cpp @@ -14,12 +14,12 @@ using namespace lldb_private; LineEntry::LineEntry() - : range(), file(), is_start_of_statement(0), is_start_of_basic_block(0), + : range(), is_start_of_statement(0), is_start_of_basic_block(0), is_prologue_end(0), is_epilogue_begin(0), is_terminal_entry(0) {} void LineEntry::Clear() { range.Clear(); - file.Clear(); + file_sp = std::make_shared(); original_file_sp = std::make_shared(); line = LLDB_INVALID_LINE_NUMBER; column = 0; @@ -35,6 +35,7 @@ bool LineEntry::IsValid() const { } bool LineEntry::DumpStopContext(Stream *s, bool show_fullpaths) const { + const FileSpec &file = file_sp->GetSpecOnly(); if (file) { if (show_fullpaths) file.Dump(s->AsRawOstream()); @@ -67,7 +68,7 @@ bool LineEntry::Dump(Stream *s, Target *target, bool show_file, return false; } if (show_file) - *s << ", file = " << file; + *s << ", file = " << GetFile(); if (line) s->Printf(", line = %u", line); if (column) @@ -103,7 +104,7 @@ bool LineEntry::GetDescription(Stream *s, lldb::DescriptionLevel level, Address::DumpStyleFileAddress); } - *s << ": " << file; + *s << ": " << GetFile(); if (line) { s->Printf(":%u", line); @@ -173,7 +174,7 @@ int LineEntry::Compare(const LineEntry &a, const LineEntry &b) { if (a.column > b.column) return +1; - return FileSpec::Compare(a.file, b.file, true); + return FileSpec::Compare(a.GetFile(), b.GetFile(), true); } AddressRange LineEntry::GetSameLineContiguousAddressRange( @@ -242,6 +243,6 @@ void LineEntry::ApplyFileMappings(lldb::TargetSP target_sp) { // Apply any file remappings to our file. if (auto new_file_spec = target_sp->GetSourcePathMap().FindFile( original_file_sp->GetSpecOnly())) - file = *new_file_spec; + file_sp->Update(*new_file_spec); } } diff --git a/lldb/source/Symbol/LineTable.cpp b/lldb/source/Symbol/LineTable.cpp index 444135f63bc0..06cf4f698316 100644 --- a/lldb/source/Symbol/LineTable.cpp +++ b/lldb/source/Symbol/LineTable.cpp @@ -288,8 +288,8 @@ bool LineTable::ConvertEntryAtIndexToLineEntry(uint32_t idx, else line_entry.range.SetByteSize(0); - line_entry.file = - m_comp_unit->GetSupportFiles().GetFileSpecAtIndex(entry.file_idx); + line_entry.file_sp = std::make_shared( + m_comp_unit->GetSupportFiles().GetFileSpecAtIndex(entry.file_idx)); line_entry.original_file_sp = m_comp_unit->GetSupportFiles().GetSupportFileAtIndex(entry.file_idx); line_entry.line = entry.line; diff --git a/lldb/source/Symbol/SymbolContext.cpp b/lldb/source/Symbol/SymbolContext.cpp index 3c70b8d8743c..f368896fbad4 100644 --- a/lldb/source/Symbol/SymbolContext.cpp +++ b/lldb/source/Symbol/SymbolContext.cpp @@ -472,8 +472,8 @@ bool SymbolContext::GetParentOfInlinedScope(const Address &curr_frame_pc, curr_inlined_block->GetInlinedFunctionInfo(); next_frame_pc = range.GetBaseAddress(); next_frame_sc.line_entry.range.GetBaseAddress() = next_frame_pc; - next_frame_sc.line_entry.file = - curr_inlined_block_inlined_info->GetCallSite().GetFile(); + next_frame_sc.line_entry.file_sp = std::make_shared( + curr_inlined_block_inlined_info->GetCallSite().GetFile()); next_frame_sc.line_entry.original_file_sp = std::make_shared( curr_inlined_block_inlined_info->GetCallSite().GetFile()); diff --git a/lldb/source/Target/StackFrame.cpp b/lldb/source/Target/StackFrame.cpp index c29a71d92572..3af62f52d575 100644 --- a/lldb/source/Target/StackFrame.cpp +++ b/lldb/source/Target/StackFrame.cpp @@ -1922,7 +1922,7 @@ bool StackFrame::GetStatus(Stream &strm, bool show_frame_info, bool show_source, size_t num_lines = target->GetSourceManager().DisplaySourceLinesWithLineNumbers( - m_sc.line_entry.file, start_line, m_sc.line_entry.column, + m_sc.line_entry.GetFile(), start_line, m_sc.line_entry.column, source_lines_before, source_lines_after, "->", &strm); if (num_lines != 0) have_source = true; diff --git a/lldb/source/Target/StackFrameList.cpp b/lldb/source/Target/StackFrameList.cpp index 2273e52e2e04..314b5e39c716 100644 --- a/lldb/source/Target/StackFrameList.cpp +++ b/lldb/source/Target/StackFrameList.cpp @@ -884,9 +884,9 @@ void StackFrameList::SetDefaultFileAndLineToSelectedFrame() { GetFrameAtIndex(GetSelectedFrameIndex(DoNoSelectMostRelevantFrame))); if (frame_sp) { SymbolContext sc = frame_sp->GetSymbolContext(eSymbolContextLineEntry); - if (sc.line_entry.file) + if (sc.line_entry.GetFile()) m_thread.CalculateTarget()->GetSourceManager().SetDefaultFileAndLine( - sc.line_entry.file, sc.line_entry.line); + sc.line_entry.GetFile(), sc.line_entry.line); } } } diff --git a/lldb/source/Target/Thread.cpp b/lldb/source/Target/Thread.cpp index 4dfad23b56e2..412e44ede9c1 100644 --- a/lldb/source/Target/Thread.cpp +++ b/lldb/source/Target/Thread.cpp @@ -302,10 +302,10 @@ bool Thread::SetSelectedFrameByIndexNoisily(uint32_t frame_idx, SymbolContext frame_sc( frame_sp->GetSymbolContext(eSymbolContextLineEntry)); const Debugger &debugger = GetProcess()->GetTarget().GetDebugger(); - if (debugger.GetUseExternalEditor() && frame_sc.line_entry.file && + if (debugger.GetUseExternalEditor() && frame_sc.line_entry.GetFile() && frame_sc.line_entry.line != 0) { if (llvm::Error e = Host::OpenFileInExternalEditor( - debugger.GetExternalEditor(), frame_sc.line_entry.file, + debugger.GetExternalEditor(), frame_sc.line_entry.GetFile(), frame_sc.line_entry.line)) { LLDB_LOG_ERROR(GetLog(LLDBLog::Host), std::move(e), "OpenFileInExternalEditor failed: {0}"); @@ -1753,10 +1753,10 @@ size_t Thread::GetStatus(Stream &strm, uint32_t start_frame, if (frame_sp) { SymbolContext frame_sc( frame_sp->GetSymbolContext(eSymbolContextLineEntry)); - if (frame_sc.line_entry.line != 0 && frame_sc.line_entry.file) { + if (frame_sc.line_entry.line != 0 && frame_sc.line_entry.GetFile()) { if (llvm::Error e = Host::OpenFileInExternalEditor( target->GetDebugger().GetExternalEditor(), - frame_sc.line_entry.file, frame_sc.line_entry.line)) { + frame_sc.line_entry.GetFile(), frame_sc.line_entry.line)) { LLDB_LOG_ERROR(GetLog(LLDBLog::Host), std::move(e), "OpenFileInExternalEditor failed: {0}"); } diff --git a/lldb/source/Target/TraceDumper.cpp b/lldb/source/Target/TraceDumper.cpp index e92419e70b32..4ef8efc1a676 100644 --- a/lldb/source/Target/TraceDumper.cpp +++ b/lldb/source/Target/TraceDumper.cpp @@ -57,7 +57,7 @@ static bool FileLineAndColumnMatches(const LineEntry &a, const LineEntry &b) { return false; if (a.column != b.column) return false; - return a.file == b.file; + return a.GetFile() == b.GetFile(); } /// Compare the symbol contexts of the provided \a SymbolInfo @@ -396,7 +396,7 @@ public: m_j.attribute( "source", ToOptionalString( - item.symbol_info->sc.line_entry.file.GetPath().c_str())); + item.symbol_info->sc.line_entry.GetFile().GetPath().c_str())); m_j.attribute("line", item.symbol_info->sc.line_entry.line); m_j.attribute("column", item.symbol_info->sc.line_entry.column); } diff --git a/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp b/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp index f237dd63ab1c..4379ffac9d74 100644 --- a/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp +++ b/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp @@ -102,7 +102,7 @@ protected: EXPECT_EQ(line, entry.line); EXPECT_EQ(address, entry.range.GetBaseAddress()); - EXPECT_TRUE(FileSpecMatchesAsBaseOrFull(spec, entry.file)); + EXPECT_TRUE(FileSpecMatchesAsBaseOrFull(spec, entry.GetFile())); } bool ContainsCompileUnit(const SymbolContextList &sc_list, -- GitLab From 6295e677220bb6ec1fa8abe2f4a94b513b91b786 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Wed, 20 Mar 2024 17:05:17 -0700 Subject: [PATCH 162/296] [Float2Int] Pre-commit test for SIToFP/UIToFP ConstantRange bug. NFC The range for these operations is being constructed without the maximum value for the range due to an incorrect usage of the ConstantRange constructor. This causes Float2Int to think the range for 'uitofp i1' only contains 0 instead of 0 and 1. --- llvm/test/Transforms/Float2Int/pr79158.ll | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 llvm/test/Transforms/Float2Int/pr79158.ll diff --git a/llvm/test/Transforms/Float2Int/pr79158.ll b/llvm/test/Transforms/Float2Int/pr79158.ll new file mode 100644 index 000000000000..d041e01a4b59 --- /dev/null +++ b/llvm/test/Transforms/Float2Int/pr79158.ll @@ -0,0 +1,19 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -passes=float2int -S | FileCheck %s + +define i32 @pr79158(i32 %x) { +; CHECK-LABEL: define i32 @pr79158( +; CHECK-SAME: i32 [[X:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[X]], 0 +; CHECK-NEXT: [[TMP0:%.*]] = zext i1 [[CMP]] to i32 +; CHECK-NEXT: [[MUL1:%.*]] = mul i32 [[TMP0]], 2147483647 +; CHECK-NEXT: ret i32 [[MUL1]] +; +entry: + %cmp = icmp sgt i32 %x, 0 + %conv = uitofp i1 %cmp to double + %mul = fmul double %conv, 0x41EFFFFFFFE00000 + %conv1 = fptoui double %mul to i32 + ret i32 %conv1 +} -- GitLab From 38f8a3cf0d75cd25e13d3757027f7356e4466cb9 Mon Sep 17 00:00:00 2001 From: Finn Plummer <50529406+inbelic@users.noreply.github.com> Date: Thu, 21 Mar 2024 08:49:27 -0700 Subject: [PATCH 163/296] [mlir][spirv] Improve folding of MemRef to SPIRV Lowering (#85433) Investigate the lowering of MemRef Load/Store ops and implement additional folding of created ops Aims to improve readability of generated lowered SPIR-V code. Part of work llvm#70704 --- .../MemRefToSPIRV/MemRefToSPIRV.cpp | 52 +++--- .../SPIRV/Transforms/SPIRVConversion.cpp | 9 +- .../Conversion/GPUToSPIRV/load-store.mlir | 8 +- .../MemRefToSPIRV/bitwidth-emulation.mlir | 158 +++++------------- .../MemRefToSPIRV/memref-to-spirv.mlir | 34 +--- mlir/test/Conversion/SCFToSPIRV/for.mlir | 12 +- .../TensorToSPIRV/tensor-ops-to-spirv.mlir | 10 +- .../VectorToSPIRV/vector-to-spirv.mlir | 20 +-- 8 files changed, 93 insertions(+), 210 deletions(-) diff --git a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp index 0acb2142f3f6..81b9f55cac80 100644 --- a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp +++ b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp @@ -50,11 +50,12 @@ static Value getOffsetForBitwidth(Location loc, Value srcIdx, int sourceBits, assert(targetBits % sourceBits == 0); Type type = srcIdx.getType(); IntegerAttr idxAttr = builder.getIntegerAttr(type, targetBits / sourceBits); - auto idx = builder.create(loc, type, idxAttr); + auto idx = builder.createOrFold(loc, type, idxAttr); IntegerAttr srcBitsAttr = builder.getIntegerAttr(type, sourceBits); - auto srcBitsValue = builder.create(loc, type, srcBitsAttr); - auto m = builder.create(loc, srcIdx, idx); - return builder.create(loc, type, m, srcBitsValue); + auto srcBitsValue = + builder.createOrFold(loc, type, srcBitsAttr); + auto m = builder.createOrFold(loc, srcIdx, idx); + return builder.createOrFold(loc, type, m, srcBitsValue); } /// Returns an adjusted spirv::AccessChainOp. Based on the @@ -74,11 +75,11 @@ adjustAccessChainForBitwidth(const SPIRVTypeConverter &typeConverter, Value lastDim = op->getOperand(op.getNumOperands() - 1); Type type = lastDim.getType(); IntegerAttr attr = builder.getIntegerAttr(type, targetBits / sourceBits); - auto idx = builder.create(loc, type, attr); + auto idx = builder.createOrFold(loc, type, attr); auto indices = llvm::to_vector<4>(op.getIndices()); // There are two elements if this is a 1-D tensor. assert(indices.size() == 2); - indices.back() = builder.create(loc, lastDim, idx); + indices.back() = builder.createOrFold(loc, lastDim, idx); Type t = typeConverter.convertType(op.getComponentPtr().getType()); return builder.create(loc, t, op.getBasePtr(), indices); } @@ -91,7 +92,8 @@ static Value castBoolToIntN(Location loc, Value srcBool, Type dstType, return srcBool; Value zero = spirv::ConstantOp::getZero(dstType, loc, builder); Value one = spirv::ConstantOp::getOne(dstType, loc, builder); - return builder.create(loc, dstType, srcBool, one, zero); + return builder.createOrFold(loc, dstType, srcBool, one, + zero); } /// Returns the `targetBits`-bit value shifted by the given `offset`, and cast @@ -111,10 +113,10 @@ static Value shiftValue(Location loc, Value value, Value offset, Value mask, loc, builder.getIntegerType(targetBits), value); } - value = builder.create(loc, value, mask); + value = builder.createOrFold(loc, value, mask); } - return builder.create(loc, value.getType(), value, - offset); + return builder.createOrFold(loc, value.getType(), + value, offset); } /// Returns true if the allocations of memref `type` generated from `allocOp` @@ -165,7 +167,7 @@ static Value castIntNToBool(Location loc, Value srcInt, OpBuilder &builder) { return srcInt; auto one = spirv::ConstantOp::getOne(srcInt.getType(), loc, builder); - return builder.create(loc, srcInt, one); + return builder.createOrFold(loc, srcInt, one); } //===----------------------------------------------------------------------===// @@ -597,13 +599,14 @@ IntLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor, // ____XXXX________ -> ____________XXXX Value lastDim = accessChainOp->getOperand(accessChainOp.getNumOperands() - 1); Value offset = getOffsetForBitwidth(loc, lastDim, srcBits, dstBits, rewriter); - Value result = rewriter.create( + Value result = rewriter.createOrFold( loc, spvLoadOp.getType(), spvLoadOp, offset); // Apply the mask to extract corresponding bits. - Value mask = rewriter.create( + Value mask = rewriter.createOrFold( loc, dstType, rewriter.getIntegerAttr(dstType, (1 << srcBits) - 1)); - result = rewriter.create(loc, dstType, result, mask); + result = + rewriter.createOrFold(loc, dstType, result, mask); // Apply sign extension on the loading value unconditionally. The signedness // semantic is carried in the operator itself, we relies other pattern to @@ -611,11 +614,11 @@ IntLoadOpPattern::matchAndRewrite(memref::LoadOp loadOp, OpAdaptor adaptor, IntegerAttr shiftValueAttr = rewriter.getIntegerAttr(dstType, dstBits - srcBits); Value shiftValue = - rewriter.create(loc, dstType, shiftValueAttr); - result = rewriter.create(loc, dstType, result, - shiftValue); - result = rewriter.create(loc, dstType, result, - shiftValue); + rewriter.createOrFold(loc, dstType, shiftValueAttr); + result = rewriter.createOrFold(loc, dstType, + result, shiftValue); + result = rewriter.createOrFold( + loc, dstType, result, shiftValue); rewriter.replaceOp(loadOp, result); @@ -744,11 +747,12 @@ IntStoreOpPattern::matchAndRewrite(memref::StoreOp storeOp, OpAdaptor adaptor, // Create a mask to clear the destination. E.g., if it is the second i8 in // i32, 0xFFFF00FF is created. - Value mask = rewriter.create( + Value mask = rewriter.createOrFold( loc, dstType, rewriter.getIntegerAttr(dstType, (1 << srcBits) - 1)); - Value clearBitsMask = - rewriter.create(loc, dstType, mask, offset); - clearBitsMask = rewriter.create(loc, dstType, clearBitsMask); + Value clearBitsMask = rewriter.createOrFold( + loc, dstType, mask, offset); + clearBitsMask = + rewriter.createOrFold(loc, dstType, clearBitsMask); Value storeVal = shiftValue(loc, adaptor.getValue(), offset, mask, rewriter); Value adjustedPtr = adjustAccessChainForBitwidth(typeConverter, accessChainOp, @@ -910,7 +914,7 @@ LogicalResult ReinterpretCastPattern::matchAndRewrite( int64_t attrVal = cast(offset.get()).getInt(); Attribute attr = rewriter.getIntegerAttr(intType, attrVal); - return rewriter.create(loc, intType, attr); + return rewriter.createOrFold(loc, intType, attr); }(); rewriter.replaceOpWithNewOp( diff --git a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp index 2b79c8022b8e..4072608dc8f8 100644 --- a/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp +++ b/mlir/lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp @@ -991,15 +991,16 @@ Value mlir::spirv::linearizeIndex(ValueRange indices, ArrayRef strides, // broken down into progressive small steps so we can have intermediate steps // using other dialects. At the moment SPIR-V is the final sink. - Value linearizedIndex = builder.create( + Value linearizedIndex = builder.createOrFold( loc, integerType, IntegerAttr::get(integerType, offset)); for (const auto &index : llvm::enumerate(indices)) { - Value strideVal = builder.create( + Value strideVal = builder.createOrFold( loc, integerType, IntegerAttr::get(integerType, strides[index.index()])); - Value update = builder.create(loc, strideVal, index.value()); + Value update = + builder.createOrFold(loc, index.value(), strideVal); linearizedIndex = - builder.create(loc, linearizedIndex, update); + builder.createOrFold(loc, update, linearizedIndex); } return linearizedIndex; } diff --git a/mlir/test/Conversion/GPUToSPIRV/load-store.mlir b/mlir/test/Conversion/GPUToSPIRV/load-store.mlir index fa12da8ef9d4..4339799ccd5e 100644 --- a/mlir/test/Conversion/GPUToSPIRV/load-store.mlir +++ b/mlir/test/Conversion/GPUToSPIRV/load-store.mlir @@ -60,13 +60,9 @@ module attributes { // CHECK: %[[INDEX2:.*]] = spirv.IAdd %[[ARG4]], %[[LOCALINVOCATIONIDX]] %13 = arith.addi %arg4, %3 : index // CHECK: %[[ZERO:.*]] = spirv.Constant 0 : i32 - // CHECK: %[[OFFSET1_0:.*]] = spirv.Constant 0 : i32 // CHECK: %[[STRIDE1_1:.*]] = spirv.Constant 4 : i32 - // CHECK: %[[UPDATE1_1:.*]] = spirv.IMul %[[STRIDE1_1]], %[[INDEX1]] : i32 - // CHECK: %[[OFFSET1_1:.*]] = spirv.IAdd %[[OFFSET1_0]], %[[UPDATE1_1]] : i32 - // CHECK: %[[STRIDE1_2:.*]] = spirv.Constant 1 : i32 - // CHECK: %[[UPDATE1_2:.*]] = spirv.IMul %[[STRIDE1_2]], %[[INDEX2]] : i32 - // CHECK: %[[OFFSET1_2:.*]] = spirv.IAdd %[[OFFSET1_1]], %[[UPDATE1_2]] : i32 + // CHECK: %[[UPDATE1_1:.*]] = spirv.IMul %[[INDEX1]], %[[STRIDE1_1]] : i32 + // CHECK: %[[OFFSET1_2:.*]] = spirv.IAdd %[[INDEX2]], %[[UPDATE1_1]] : i32 // CHECK: %[[PTR1:.*]] = spirv.AccessChain %[[ARG0]]{{\[}}%[[ZERO]], %[[OFFSET1_2]]{{\]}} // CHECK-NEXT: %[[VAL1:.*]] = spirv.Load "StorageBuffer" %[[PTR1]] %14 = memref.load %arg0[%12, %13] : memref<12x4xf32, #spirv.storage_class> diff --git a/mlir/test/Conversion/MemRefToSPIRV/bitwidth-emulation.mlir b/mlir/test/Conversion/MemRefToSPIRV/bitwidth-emulation.mlir index 470c8531e2e0..52ed14e8cce2 100644 --- a/mlir/test/Conversion/MemRefToSPIRV/bitwidth-emulation.mlir +++ b/mlir/test/Conversion/MemRefToSPIRV/bitwidth-emulation.mlir @@ -12,16 +12,10 @@ module attributes { // CHECK-LABEL: @load_i1 func.func @load_i1(%arg0: memref>) -> i1 { // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[FOUR:.+]] = spirv.Constant 4 : i32 - // CHECK: %[[QUOTIENT:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[QUOTIENT]]] + // CHECK: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[ZERO]]] // CHECK: %[[LOAD:.+]] = spirv.Load "StorageBuffer" %[[PTR]] - // CHECK: %[[EIGHT:.+]] = spirv.Constant 8 : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[BITS:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i32 - // CHECK: %[[VALUE:.+]] = spirv.ShiftRightArithmetic %[[LOAD]], %[[BITS]] : i32, i32 // CHECK: %[[MASK:.+]] = spirv.Constant 255 : i32 - // CHECK: %[[T1:.+]] = spirv.BitwiseAnd %[[VALUE]], %[[MASK]] : i32 + // CHECK: %[[T1:.+]] = spirv.BitwiseAnd %[[LOAD]], %[[MASK]] : i32 // CHECK: %[[T2:.+]] = spirv.Constant 24 : i32 // CHECK: %[[T3:.+]] = spirv.ShiftLeftLogical %[[T1]], %[[T2]] : i32, i32 // CHECK: %[[T4:.+]] = spirv.ShiftRightArithmetic %[[T3]], %[[T2]] : i32, i32 @@ -37,32 +31,20 @@ func.func @load_i1(%arg0: memref>) -> i1 // INDEX64-LABEL: @load_i8 func.func @load_i8(%arg0: memref>) -> i8 { // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[FOUR:.+]] = spirv.Constant 4 : i32 - // CHECK: %[[QUOTIENT:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[QUOTIENT]]] + // CHECK: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[ZERO]]] // CHECK: %[[LOAD:.+]] = spirv.Load "StorageBuffer" %[[PTR]] - // CHECK: %[[EIGHT:.+]] = spirv.Constant 8 : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[BITS:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i32 - // CHECK: %[[VALUE:.+]] = spirv.ShiftRightArithmetic %[[LOAD]], %[[BITS]] : i32, i32 // CHECK: %[[MASK:.+]] = spirv.Constant 255 : i32 - // CHECK: %[[T1:.+]] = spirv.BitwiseAnd %[[VALUE]], %[[MASK]] : i32 + // CHECK: %[[T1:.+]] = spirv.BitwiseAnd %[[LOAD]], %[[MASK]] : i32 // CHECK: %[[T2:.+]] = spirv.Constant 24 : i32 // CHECK: %[[T3:.+]] = spirv.ShiftLeftLogical %[[T1]], %[[T2]] : i32, i32 // CHECK: %[[SR:.+]] = spirv.ShiftRightArithmetic %[[T3]], %[[T2]] : i32, i32 // CHECK: builtin.unrealized_conversion_cast %[[SR]] // INDEX64: %[[ZERO:.+]] = spirv.Constant 0 : i64 - // INDEX64: %[[FOUR:.+]] = spirv.Constant 4 : i64 - // INDEX64: %[[QUOTIENT:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i64 - // INDEX64: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[QUOTIENT]]] : {{.+}}, i64, i64 + // INDEX64: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[ZERO]]] : {{.+}}, i64, i64 // INDEX64: %[[LOAD:.+]] = spirv.Load "StorageBuffer" %[[PTR]] : i32 - // INDEX64: %[[EIGHT:.+]] = spirv.Constant 8 : i64 - // INDEX64: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i64 - // INDEX64: %[[BITS:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i64 - // INDEX64: %[[VALUE:.+]] = spirv.ShiftRightArithmetic %[[LOAD]], %[[BITS]] : i32, i64 // INDEX64: %[[MASK:.+]] = spirv.Constant 255 : i32 - // INDEX64: %[[T1:.+]] = spirv.BitwiseAnd %[[VALUE]], %[[MASK]] : i32 + // INDEX64: %[[T1:.+]] = spirv.BitwiseAnd %[[LOAD]], %[[MASK]] : i32 // INDEX64: %[[T2:.+]] = spirv.Constant 24 : i32 // INDEX64: %[[T3:.+]] = spirv.ShiftLeftLogical %[[T1]], %[[T2]] : i32, i32 // INDEX64: %[[SR:.+]] = spirv.ShiftRightArithmetic %[[T3]], %[[T2]] : i32, i32 @@ -76,15 +58,12 @@ func.func @load_i8(%arg0: memref>) -> i8 func.func @load_i16(%arg0: memref<10xi16, #spirv.storage_class>, %index : index) -> i16 { // CHECK: %[[ARG1_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG1]] : index to i32 // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[ONE:.+]] = spirv.Constant 1 : i32 - // CHECK: %[[UPDATE:.+]] = spirv.IMul %[[ONE]], %[[ARG1_CAST]] : i32 - // CHECK: %[[FLAT_IDX:.+]] = spirv.IAdd %[[ZERO]], %[[UPDATE]] : i32 // CHECK: %[[TWO:.+]] = spirv.Constant 2 : i32 - // CHECK: %[[QUOTIENT:.+]] = spirv.SDiv %[[FLAT_IDX]], %[[TWO]] : i32 + // CHECK: %[[QUOTIENT:.+]] = spirv.SDiv %[[ARG1_CAST]], %[[TWO]] : i32 // CHECK: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[QUOTIENT]]] // CHECK: %[[LOAD:.+]] = spirv.Load "StorageBuffer" %[[PTR]] // CHECK: %[[SIXTEEN:.+]] = spirv.Constant 16 : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[FLAT_IDX]], %[[TWO]] : i32 + // CHECK: %[[IDX:.+]] = spirv.UMod %[[ARG1_CAST]], %[[TWO]] : i32 // CHECK: %[[BITS:.+]] = spirv.IMul %[[IDX]], %[[SIXTEEN]] : i32 // CHECK: %[[VALUE:.+]] = spirv.ShiftRightArithmetic %[[LOAD]], %[[BITS]] : i32, i32 // CHECK: %[[MASK:.+]] = spirv.Constant 65535 : i32 @@ -110,20 +89,12 @@ func.func @load_f32(%arg0: memref>) { func.func @store_i1(%arg0: memref>, %value: i1) { // CHECK: %[[ARG0_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG0]] // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[FOUR:.+]] = spirv.Constant 4 : i32 - // CHECK: %[[EIGHT:.+]] = spirv.Constant 8 : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[OFFSET:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i32 - // CHECK: %[[MASK1:.+]] = spirv.Constant 255 : i32 - // CHECK: %[[TMP1:.+]] = spirv.ShiftLeftLogical %[[MASK1]], %[[OFFSET]] : i32, i32 - // CHECK: %[[MASK:.+]] = spirv.Not %[[TMP1]] : i32 + // CHECK: %[[MASK:.+]] = spirv.Constant -256 : i32 // CHECK: %[[ONE:.+]] = spirv.Constant 1 : i32 // CHECK: %[[CASTED_ARG1:.+]] = spirv.Select %[[ARG1]], %[[ONE]], %[[ZERO]] : i1, i32 - // CHECK: %[[STORE_VAL:.+]] = spirv.ShiftLeftLogical %[[CASTED_ARG1]], %[[OFFSET]] : i32, i32 - // CHECK: %[[ACCESS_IDX:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ACCESS_IDX]]] + // CHECK: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ZERO]]] // CHECK: spirv.AtomicAnd %[[PTR]], %[[MASK]] - // CHECK: spirv.AtomicOr %[[PTR]], %[[STORE_VAL]] + // CHECK: spirv.AtomicOr %[[PTR]], %[[CASTED_ARG1]] memref.store %value, %arg0[] : memref> return } @@ -136,36 +107,22 @@ func.func @store_i8(%arg0: memref>, %val // CHECK-DAG: %[[ARG1_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG1]] : i8 to i32 // CHECK-DAG: %[[ARG0_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG0]] // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[FOUR:.+]] = spirv.Constant 4 : i32 - // CHECK: %[[EIGHT:.+]] = spirv.Constant 8 : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[OFFSET:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i32 // CHECK: %[[MASK1:.+]] = spirv.Constant 255 : i32 - // CHECK: %[[TMP1:.+]] = spirv.ShiftLeftLogical %[[MASK1]], %[[OFFSET]] : i32, i32 - // CHECK: %[[MASK:.+]] = spirv.Not %[[TMP1]] : i32 + // CHECK: %[[MASK2:.+]] = spirv.Constant -256 : i32 // CHECK: %[[CLAMPED_VAL:.+]] = spirv.BitwiseAnd %[[ARG1_CAST]], %[[MASK1]] : i32 - // CHECK: %[[STORE_VAL:.+]] = spirv.ShiftLeftLogical %[[CLAMPED_VAL]], %[[OFFSET]] : i32, i32 - // CHECK: %[[ACCESS_IDX:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ACCESS_IDX]]] - // CHECK: spirv.AtomicAnd %[[PTR]], %[[MASK]] - // CHECK: spirv.AtomicOr %[[PTR]], %[[STORE_VAL]] + // CHECK: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ZERO]]] + // CHECK: spirv.AtomicAnd %[[PTR]], %[[MASK2]] + // CHECK: spirv.AtomicOr %[[PTR]], %[[CLAMPED_VAL]] // INDEX64-DAG: %[[ARG1_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG1]] : i8 to i32 // INDEX64-DAG: %[[ARG0_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG0]] // INDEX64: %[[ZERO:.+]] = spirv.Constant 0 : i64 - // INDEX64: %[[FOUR:.+]] = spirv.Constant 4 : i64 - // INDEX64: %[[EIGHT:.+]] = spirv.Constant 8 : i64 - // INDEX64: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i64 - // INDEX64: %[[OFFSET:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i64 // INDEX64: %[[MASK1:.+]] = spirv.Constant 255 : i32 - // INDEX64: %[[TMP1:.+]] = spirv.ShiftLeftLogical %[[MASK1]], %[[OFFSET]] : i32, i64 - // INDEX64: %[[MASK:.+]] = spirv.Not %[[TMP1]] : i32 + // INDEX64: %[[MASK2:.+]] = spirv.Constant -256 : i32 // INDEX64: %[[CLAMPED_VAL:.+]] = spirv.BitwiseAnd %[[ARG1_CAST]], %[[MASK1]] : i32 - // INDEX64: %[[STORE_VAL:.+]] = spirv.ShiftLeftLogical %[[CLAMPED_VAL]], %[[OFFSET]] : i32, i64 - // INDEX64: %[[ACCESS_IDX:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i64 - // INDEX64: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ACCESS_IDX]]] : {{.+}}, i64, i64 - // INDEX64: spirv.AtomicAnd %[[PTR]], %[[MASK]] - // INDEX64: spirv.AtomicOr %[[PTR]], %[[STORE_VAL]] + // INDEX64: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ZERO]]] : {{.+}}, i64, i64 + // INDEX64: spirv.AtomicAnd %[[PTR]], %[[MASK2]] + // INDEX64: spirv.AtomicOr %[[PTR]], %[[CLAMPED_VAL]] memref.store %value, %arg0[] : memref> return } @@ -177,19 +134,16 @@ func.func @store_i16(%arg0: memref<10xi16, #spirv.storage_class>, // CHECK-DAG: %[[ARG0_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG0]] // CHECK-DAG: %[[ARG1_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG1]] : index to i32 // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[ONE:.+]] = spirv.Constant 1 : i32 - // CHECK: %[[UPDATE:.+]] = spirv.IMul %[[ONE]], %[[ARG1_CAST]] : i32 - // CHECK: %[[FLAT_IDX:.+]] = spirv.IAdd %[[ZERO]], %[[UPDATE]] : i32 // CHECK: %[[TWO:.+]] = spirv.Constant 2 : i32 // CHECK: %[[SIXTEEN:.+]] = spirv.Constant 16 : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[FLAT_IDX]], %[[TWO]] : i32 + // CHECK: %[[IDX:.+]] = spirv.UMod %[[ARG1_CAST]], %[[TWO]] : i32 // CHECK: %[[OFFSET:.+]] = spirv.IMul %[[IDX]], %[[SIXTEEN]] : i32 // CHECK: %[[MASK1:.+]] = spirv.Constant 65535 : i32 // CHECK: %[[TMP1:.+]] = spirv.ShiftLeftLogical %[[MASK1]], %[[OFFSET]] : i32, i32 // CHECK: %[[MASK:.+]] = spirv.Not %[[TMP1]] : i32 // CHECK: %[[CLAMPED_VAL:.+]] = spirv.BitwiseAnd %[[ARG2_CAST]], %[[MASK1]] : i32 // CHECK: %[[STORE_VAL:.+]] = spirv.ShiftLeftLogical %[[CLAMPED_VAL]], %[[OFFSET]] : i32, i32 - // CHECK: %[[ACCESS_IDX:.+]] = spirv.SDiv %[[FLAT_IDX]], %[[TWO]] : i32 + // CHECK: %[[ACCESS_IDX:.+]] = spirv.SDiv %[[ARG1_CAST]], %[[TWO]] : i32 // CHECK: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ACCESS_IDX]]] // CHECK: spirv.AtomicAnd %[[PTR]], %[[MASK]] // CHECK: spirv.AtomicOr %[[PTR]], %[[STORE_VAL]] @@ -222,15 +176,12 @@ module attributes { func.func @load_i4(%arg0: memref>, %i: index) -> i4 { // CHECK: %[[INDEX:.+]] = builtin.unrealized_conversion_cast %{{.+}} : index to i32 // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[ONE:.+]] = spirv.Constant 1 : i32 - // CHECK: %[[MUL:.+]] = spirv.IMul %[[ONE]], %[[INDEX]] : i32 - // CHECK: %[[OFFSET:.+]] = spirv.IAdd %[[ZERO]], %[[MUL]] : i32 // CHECK: %[[EIGHT:.+]] = spirv.Constant 8 : i32 - // CHECK: %[[QUOTIENT:.+]] = spirv.SDiv %[[OFFSET]], %[[EIGHT]] : i32 + // CHECK: %[[QUOTIENT:.+]] = spirv.SDiv %[[INDEX]], %[[EIGHT]] : i32 // CHECK: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[QUOTIENT]]] // CHECK: %[[LOAD:.+]] = spirv.Load "StorageBuffer" %[[PTR]] : i32 // CHECK: %[[FOUR:.+]] = spirv.Constant 4 : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[OFFSET]], %[[EIGHT]] : i32 + // CHECK: %[[IDX:.+]] = spirv.UMod %[[INDEX]], %[[EIGHT]] : i32 // CHECK: %[[BITS:.+]] = spirv.IMul %[[IDX]], %[[FOUR]] : i32 // CHECK: %[[VALUE:.+]] = spirv.ShiftRightArithmetic %[[LOAD]], %[[BITS]] : i32, i32 // CHECK: %[[MASK:.+]] = spirv.Constant 15 : i32 @@ -248,19 +199,16 @@ func.func @store_i4(%arg0: memref>, %v // CHECK: %[[VAL:.+]] = builtin.unrealized_conversion_cast %{{.+}} : i4 to i32 // CHECK: %[[INDEX:.+]] = builtin.unrealized_conversion_cast %{{.+}} : index to i32 // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[ONE:.+]] = spirv.Constant 1 : i32 - // CHECK: %[[MUL:.+]] = spirv.IMul %[[ONE]], %[[INDEX]] : i32 - // CHECK: %[[OFFSET:.+]] = spirv.IAdd %[[ZERO]], %[[MUL]] : i32 // CHECK: %[[EIGHT:.+]] = spirv.Constant 8 : i32 - // CHECK: %[[FOUR:.+]] = spirv.Constant [[OFFSET]] : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[OFFSET]], %[[EIGHT]] : i32 + // CHECK: %[[FOUR:.+]] = spirv.Constant 4 : i32 + // CHECK: %[[IDX:.+]] = spirv.UMod %[[INDEX]], %[[EIGHT]] : i32 // CHECK: %[[BITS:.+]] = spirv.IMul %[[IDX]], %[[FOUR]] : i32 // CHECK: %[[MASK1:.+]] = spirv.Constant 15 : i32 // CHECK: %[[SL:.+]] = spirv.ShiftLeftLogical %[[MASK1]], %[[BITS]] : i32, i32 // CHECK: %[[MASK2:.+]] = spirv.Not %[[SL]] : i32 // CHECK: %[[CLAMPED_VAL:.+]] = spirv.BitwiseAnd %[[VAL]], %[[MASK1]] : i32 // CHECK: %[[STORE_VAL:.+]] = spirv.ShiftLeftLogical %[[CLAMPED_VAL]], %[[BITS]] : i32, i32 - // CHECK: %[[ACCESS_INDEX:.+]] = spirv.SDiv %[[OFFSET]], %[[EIGHT]] : i32 + // CHECK: %[[ACCESS_INDEX:.+]] = spirv.SDiv %[[INDEX]], %[[EIGHT]] : i32 // CHECK: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[ACCESS_INDEX]]] // CHECK: spirv.AtomicAnd %[[PTR]], %[[MASK2]] // CHECK: spirv.AtomicOr %[[PTR]], %[[STORE_VAL]] @@ -283,16 +231,10 @@ module attributes { // INDEX64-LABEL: @load_i8 func.func @load_i8(%arg0: memref>) -> i8 { // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[FOUR:.+]] = spirv.Constant 4 : i32 - // CHECK: %[[QUOTIENT:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[QUOTIENT]]] + // CHECK: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[ZERO]]] // CHECK: %[[LOAD:.+]] = spirv.Load "StorageBuffer" %[[PTR]] - // CHECK: %[[EIGHT:.+]] = spirv.Constant 8 : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[BITS:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i32 - // CHECK: %[[VALUE:.+]] = spirv.ShiftRightArithmetic %[[LOAD]], %[[BITS]] : i32, i32 // CHECK: %[[MASK:.+]] = spirv.Constant 255 : i32 - // CHECK: %[[T1:.+]] = spirv.BitwiseAnd %[[VALUE]], %[[MASK]] : i32 + // CHECK: %[[T1:.+]] = spirv.BitwiseAnd %[[LOAD]], %[[MASK]] : i32 // CHECK: %[[T2:.+]] = spirv.Constant 24 : i32 // CHECK: %[[T3:.+]] = spirv.ShiftLeftLogical %[[T1]], %[[T2]] : i32, i32 // CHECK: %[[SR:.+]] = spirv.ShiftRightArithmetic %[[T3]], %[[T2]] : i32, i32 @@ -300,16 +242,10 @@ func.func @load_i8(%arg0: memref>) -> i8 // CHECK: return %[[CAST]] : i8 // INDEX64: %[[ZERO:.+]] = spirv.Constant 0 : i64 - // INDEX64: %[[FOUR:.+]] = spirv.Constant 4 : i64 - // INDEX64: %[[QUOTIENT:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i64 - // INDEX64: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[QUOTIENT]]] : {{.+}}, i64, i64 + // INDEX64: %[[PTR:.+]] = spirv.AccessChain %{{.+}}[%[[ZERO]], %[[ZERO]]] : {{.+}}, i64, i64 // INDEX64: %[[LOAD:.+]] = spirv.Load "StorageBuffer" %[[PTR]] : i32 - // INDEX64: %[[EIGHT:.+]] = spirv.Constant 8 : i64 - // INDEX64: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i64 - // INDEX64: %[[BITS:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i64 - // INDEX64: %[[VALUE:.+]] = spirv.ShiftRightArithmetic %[[LOAD]], %[[BITS]] : i32, i64 // INDEX64: %[[MASK:.+]] = spirv.Constant 255 : i32 - // INDEX64: %[[T1:.+]] = spirv.BitwiseAnd %[[VALUE]], %[[MASK]] : i32 + // INDEX64: %[[T1:.+]] = spirv.BitwiseAnd %[[LOAD]], %[[MASK]] : i32 // INDEX64: %[[T2:.+]] = spirv.Constant 24 : i32 // INDEX64: %[[T3:.+]] = spirv.ShiftLeftLogical %[[T1]], %[[T2]] : i32, i32 // INDEX64: %[[SR:.+]] = spirv.ShiftRightArithmetic %[[T3]], %[[T2]] : i32, i32 @@ -326,37 +262,19 @@ func.func @load_i8(%arg0: memref>) -> i8 func.func @store_i8(%arg0: memref>, %value: i8) { // CHECK-DAG: %[[ARG0_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG0]] // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[FOUR:.+]] = spirv.Constant 4 : i32 - // CHECK: %[[EIGHT:.+]] = spirv.Constant 8 : i32 - // CHECK: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[OFFSET:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i32 - // CHECK: %[[MASK1:.+]] = spirv.Constant 255 : i32 - // CHECK: %[[TMP1:.+]] = spirv.ShiftLeftLogical %[[MASK1]], %[[OFFSET]] : i32, i32 - // CHECK: %[[MASK:.+]] = spirv.Not %[[TMP1]] : i32 + // CHECK: %[[MASK1:.+]] = spirv.Constant -256 : i32 // CHECK: %[[ARG1_CAST:.+]] = spirv.UConvert %[[ARG1]] : i8 to i32 - // CHECK: %[[CLAMPED_VAL:.+]] = spirv.BitwiseAnd %[[ARG1_CAST]], %[[MASK1]] : i32 - // CHECK: %[[STORE_VAL:.+]] = spirv.ShiftLeftLogical %[[CLAMPED_VAL]], %[[OFFSET]] : i32, i32 - // CHECK: %[[ACCESS_IDX:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i32 - // CHECK: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ACCESS_IDX]]] - // CHECK: spirv.AtomicAnd %[[PTR]], %[[MASK]] - // CHECK: spirv.AtomicOr %[[PTR]], %[[STORE_VAL]] + // CHECK: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ZERO]]] + // CHECK: spirv.AtomicAnd %[[PTR]], %[[MASK1]] + // CHECK: spirv.AtomicOr %[[PTR]], %[[ARG1_CAST]] // INDEX64-DAG: %[[ARG0_CAST:.+]] = builtin.unrealized_conversion_cast %[[ARG0]] // INDEX64: %[[ZERO:.+]] = spirv.Constant 0 : i64 - // INDEX64: %[[FOUR:.+]] = spirv.Constant 4 : i64 - // INDEX64: %[[EIGHT:.+]] = spirv.Constant 8 : i64 - // INDEX64: %[[IDX:.+]] = spirv.UMod %[[ZERO]], %[[FOUR]] : i64 - // INDEX64: %[[OFFSET:.+]] = spirv.IMul %[[IDX]], %[[EIGHT]] : i64 - // INDEX64: %[[MASK1:.+]] = spirv.Constant 255 : i32 - // INDEX64: %[[TMP1:.+]] = spirv.ShiftLeftLogical %[[MASK1]], %[[OFFSET]] : i32, i64 - // INDEX64: %[[MASK:.+]] = spirv.Not %[[TMP1]] : i32 + // INDEX64: %[[MASK1:.+]] = spirv.Constant -256 : i32 // INDEX64: %[[ARG1_CAST:.+]] = spirv.UConvert %[[ARG1]] : i8 to i32 - // INDEX64: %[[CLAMPED_VAL:.+]] = spirv.BitwiseAnd %[[ARG1_CAST]], %[[MASK1]] : i32 - // INDEX64: %[[STORE_VAL:.+]] = spirv.ShiftLeftLogical %[[CLAMPED_VAL]], %[[OFFSET]] : i32, i64 - // INDEX64: %[[ACCESS_IDX:.+]] = spirv.SDiv %[[ZERO]], %[[FOUR]] : i64 - // INDEX64: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ACCESS_IDX]]] : {{.+}}, i64, i64 - // INDEX64: spirv.AtomicAnd %[[PTR]], %[[MASK]] - // INDEX64: spirv.AtomicOr %[[PTR]], %[[STORE_VAL]] + // INDEX64: %[[PTR:.+]] = spirv.AccessChain %[[ARG0_CAST]][%[[ZERO]], %[[ZERO]]] : {{.+}}, i64, i64 + // INDEX64: spirv.AtomicAnd %[[PTR]], %[[MASK1]] + // INDEX64: spirv.AtomicOr %[[PTR]], %[[ARG1_CAST]] memref.store %value, %arg0[] : memref> return } diff --git a/mlir/test/Conversion/MemRefToSPIRV/memref-to-spirv.mlir b/mlir/test/Conversion/MemRefToSPIRV/memref-to-spirv.mlir index feb6d4e92401..10c03a270005 100644 --- a/mlir/test/Conversion/MemRefToSPIRV/memref-to-spirv.mlir +++ b/mlir/test/Conversion/MemRefToSPIRV/memref-to-spirv.mlir @@ -70,11 +70,8 @@ func.func @load_store_unknown_dim(%i: index, %source: memref>, %i : index) -> i1 { // CHECK-DAG: %[[SRC_CAST:.+]] = builtin.unrealized_conversion_cast %[[SRC]] : memref<4xi1, #spirv.storage_class> to !spirv.ptr [0])>, StorageBuffer> // CHECK-DAG: %[[IDX_CAST:.+]] = builtin.unrealized_conversion_cast %[[IDX]] - // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[ONE:.+]] = spirv.Constant 1 : i32 - // CHECK: %[[MUL:.+]] = spirv.IMul %[[ONE]], %[[IDX_CAST]] : i32 - // CHECK: %[[ADD:.+]] = spirv.IAdd %[[ZERO]], %[[MUL]] : i32 - // CHECK: %[[ADDR:.+]] = spirv.AccessChain %[[SRC_CAST]][%[[ZERO]], %[[ADD]]] + // CHECK: %[[ZERO:.*]] = spirv.Constant 0 : i32 + // CHECK: %[[ADDR:.+]] = spirv.AccessChain %[[SRC_CAST]][%[[ZERO]], %[[IDX_CAST]]] // CHECK: %[[VAL:.+]] = spirv.Load "StorageBuffer" %[[ADDR]] : i8 // CHECK: %[[ONE_I8:.+]] = spirv.Constant 1 : i8 // CHECK: %[[BOOL:.+]] = spirv.IEqual %[[VAL]], %[[ONE_I8]] : i8 @@ -90,15 +87,10 @@ func.func @store_i1(%dst: memref<4xi1, #spirv.storage_class>, %i: %true = arith.constant true // CHECK-DAG: %[[DST_CAST:.+]] = builtin.unrealized_conversion_cast %[[DST]] : memref<4xi1, #spirv.storage_class> to !spirv.ptr [0])>, StorageBuffer> // CHECK-DAG: %[[IDX_CAST:.+]] = builtin.unrealized_conversion_cast %[[IDX]] - // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[ONE:.+]] = spirv.Constant 1 : i32 - // CHECK: %[[MUL:.+]] = spirv.IMul %[[ONE]], %[[IDX_CAST]] : i32 - // CHECK: %[[ADD:.+]] = spirv.IAdd %[[ZERO]], %[[MUL]] : i32 - // CHECK: %[[ADDR:.+]] = spirv.AccessChain %[[DST_CAST]][%[[ZERO]], %[[ADD]]] - // CHECK: %[[ZERO_I8:.+]] = spirv.Constant 0 : i8 + // CHECK: %[[ZERO:.*]] = spirv.Constant 0 : i32 + // CHECK: %[[ADDR:.+]] = spirv.AccessChain %[[DST_CAST]][%[[ZERO]], %[[IDX_CAST]]] // CHECK: %[[ONE_I8:.+]] = spirv.Constant 1 : i8 - // CHECK: %[[RES:.+]] = spirv.Select %{{.+}}, %[[ONE_I8]], %[[ZERO_I8]] : i1, i8 - // CHECK: spirv.Store "StorageBuffer" %[[ADDR]], %[[RES]] : i8 + // CHECK: spirv.Store "StorageBuffer" %[[ADDR]], %[[ONE_I8]] : i8 memref.store %true, %dst[%i]: memref<4xi1, #spirv.storage_class> return } @@ -234,11 +226,7 @@ func.func @load_store_unknown_dim(%i: index, %source: memref>, %i : index) -> i1 { // CHECK-DAG: %[[SRC_CAST:.+]] = builtin.unrealized_conversion_cast %[[SRC]] : memref<4xi1, #spirv.storage_class> to !spirv.ptr, CrossWorkgroup> // CHECK-DAG: %[[IDX_CAST:.+]] = builtin.unrealized_conversion_cast %[[IDX]] - // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[ONE:.+]] = spirv.Constant 1 : i32 - // CHECK: %[[MUL:.+]] = spirv.IMul %[[ONE]], %[[IDX_CAST]] : i32 - // CHECK: %[[ADD:.+]] = spirv.IAdd %[[ZERO]], %[[MUL]] : i32 - // CHECK: %[[ADDR:.+]] = spirv.AccessChain %[[SRC_CAST]][%[[ADD]]] + // CHECK: %[[ADDR:.+]] = spirv.AccessChain %[[SRC_CAST]][%[[IDX_CAST]]] // CHECK: %[[VAL:.+]] = spirv.Load "CrossWorkgroup" %[[ADDR]] : i8 // CHECK: %[[ONE_I8:.+]] = spirv.Constant 1 : i8 // CHECK: %[[BOOL:.+]] = spirv.IEqual %[[VAL]], %[[ONE_I8]] : i8 @@ -254,15 +242,9 @@ func.func @store_i1(%dst: memref<4xi1, #spirv.storage_class>, %i %true = arith.constant true // CHECK-DAG: %[[DST_CAST:.+]] = builtin.unrealized_conversion_cast %[[DST]] : memref<4xi1, #spirv.storage_class> to !spirv.ptr, CrossWorkgroup> // CHECK-DAG: %[[IDX_CAST:.+]] = builtin.unrealized_conversion_cast %[[IDX]] - // CHECK: %[[ZERO:.+]] = spirv.Constant 0 : i32 - // CHECK: %[[ONE:.+]] = spirv.Constant 1 : i32 - // CHECK: %[[MUL:.+]] = spirv.IMul %[[ONE]], %[[IDX_CAST]] : i32 - // CHECK: %[[ADD:.+]] = spirv.IAdd %[[ZERO]], %[[MUL]] : i32 - // CHECK: %[[ADDR:.+]] = spirv.AccessChain %[[DST_CAST]][%[[ADD]]] - // CHECK: %[[ZERO_I8:.+]] = spirv.Constant 0 : i8 + // CHECK: %[[ADDR:.+]] = spirv.AccessChain %[[DST_CAST]][%[[IDX_CAST]]] // CHECK: %[[ONE_I8:.+]] = spirv.Constant 1 : i8 - // CHECK: %[[RES:.+]] = spirv.Select %{{.+}}, %[[ONE_I8]], %[[ZERO_I8]] : i1, i8 - // CHECK: spirv.Store "CrossWorkgroup" %[[ADDR]], %[[RES]] : i8 + // CHECK: spirv.Store "CrossWorkgroup" %[[ADDR]], %[[ONE_I8]] : i8 memref.store %true, %dst[%i]: memref<4xi1, #spirv.storage_class> return } diff --git a/mlir/test/Conversion/SCFToSPIRV/for.mlir b/mlir/test/Conversion/SCFToSPIRV/for.mlir index 02558463b866..81661ec7a3a0 100644 --- a/mlir/test/Conversion/SCFToSPIRV/for.mlir +++ b/mlir/test/Conversion/SCFToSPIRV/for.mlir @@ -19,17 +19,9 @@ func.func @loop_kernel(%arg2 : memref<10xf32, #spirv.storage_class i32 { // CHECK: spirv.Store "Function" %[[VAR]], %[[CST]] : !spirv.array<12 x i32> // CHECK: %[[C0:.+]] = spirv.Constant 0 : i32 // CHECK: %[[C6:.+]] = spirv.Constant 6 : i32 - // CHECK: %[[MUL0:.+]] = spirv.IMul %[[C6]], %[[A]] : i32 - // CHECK: %[[ADD0:.+]] = spirv.IAdd %[[C0]], %[[MUL0]] : i32 + // CHECK: %[[MUL0:.+]] = spirv.IMul %[[A]], %[[C6]] : i32 // CHECK: %[[C3:.+]] = spirv.Constant 3 : i32 - // CHECK: %[[MUL1:.+]] = spirv.IMul %[[C3]], %[[B]] : i32 - // CHECK: %[[ADD1:.+]] = spirv.IAdd %[[ADD0]], %[[MUL1]] : i32 + // CHECK: %[[MUL1:.+]] = spirv.IMul %[[B]], %[[C3]] : i32 + // CHECK: %[[ADD1:.+]] = spirv.IAdd %[[MUL1]], %[[MUL0]] : i32 // CHECK: %[[C1:.+]] = spirv.Constant 1 : i32 - // CHECK: %[[MUL2:.+]] = spirv.IMul %[[C1]], %[[C]] : i32 - // CHECK: %[[ADD2:.+]] = spirv.IAdd %[[ADD1]], %[[MUL2]] : i32 + // CHECK: %[[ADD2:.+]] = spirv.IAdd %[[C]], %[[ADD1]] : i32 // CHECK: %[[AC:.+]] = spirv.AccessChain %[[VAR]][%[[ADD2]]] // CHECK: %[[VAL:.+]] = spirv.Load "Function" %[[AC]] : i32 %extract = tensor.extract %cst[%a, %b, %c] : tensor<2x2x3xi32> diff --git a/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir b/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir index c9984091d5ac..cddc4ee38535 100644 --- a/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir +++ b/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir @@ -720,9 +720,7 @@ module attributes { // CHECK: %[[CST1:.+]] = spirv.Constant 0 : i32 // CHECK: %[[CST2:.+]] = spirv.Constant 0 : i32 // CHECK: %[[CST3:.+]] = spirv.Constant 1 : i32 -// CHECK: %[[S2:.+]] = spirv.IMul %[[CST3]], %[[S1]] : i32 -// CHECK: %[[S3:.+]] = spirv.IAdd %[[CST2]], %[[S2]] : i32 -// CHECK: %[[S4:.+]] = spirv.AccessChain %[[S0]][%[[CST1]], %[[S3]]] : !spirv.ptr [0])>, StorageBuffer>, i32, i32 +// CHECK: %[[S4:.+]] = spirv.AccessChain %[[S0]][%[[CST1]], %[[S1]]] : !spirv.ptr [0])>, StorageBuffer>, i32, i32 // CHECK: %[[S5:.+]] = spirv.Bitcast %[[S4]] : !spirv.ptr to !spirv.ptr, StorageBuffer> // CHECK: %[[R0:.+]] = spirv.Load "StorageBuffer" %[[S5]] : vector<4xf32> // CHECK: return %[[R0]] : vector<4xf32> @@ -743,11 +741,9 @@ func.func @vector_load(%arg0 : memref<4xf32, #spirv.storage_class // CHECK: %[[CST0_1:.+]] = spirv.Constant 0 : i32 // CHECK: %[[CST0_2:.+]] = spirv.Constant 0 : i32 // CHECK: %[[CST4:.+]] = spirv.Constant 4 : i32 -// CHECK: %[[S3:.+]] = spirv.IMul %[[CST4]], %[[S1]] : i32 -// CHECK: %[[S4:.+]] = spirv.IAdd %[[CST0_2]], %[[S3]] : i32 +// CHECK: %[[S3:.+]] = spirv.IMul %[[S1]], %[[CST4]] : i32 // CHECK: %[[CST1:.+]] = spirv.Constant 1 : i32 -// CHECK: %[[S5:.+]] = spirv.IMul %[[CST1]], %[[S2]] : i32 -// CHECK: %[[S6:.+]] = spirv.IAdd %[[S4]], %[[S5]] : i32 +// CHECK: %[[S6:.+]] = spirv.IAdd %[[S2]], %[[S3]] : i32 // CHECK: %[[S7:.+]] = spirv.AccessChain %[[S0]][%[[CST0_1]], %[[S6]]] : !spirv.ptr [0])>, StorageBuffer>, i32, i32 // CHECK: %[[S8:.+]] = spirv.Bitcast %[[S7]] : !spirv.ptr to !spirv.ptr, StorageBuffer> // CHECK: %[[R0:.+]] = spirv.Load "StorageBuffer" %[[S8]] : vector<4xf32> @@ -768,9 +764,7 @@ func.func @vector_load_2d(%arg0 : memref<4x4xf32, #spirv.storage_class [0])>, StorageBuffer>, i32, i32 +// CHECK: %[[S4:.+]] = spirv.AccessChain %[[S0]][%[[CST1]], %[[S1]]] : !spirv.ptr [0])>, StorageBuffer>, i32, i32 // CHECK: %[[S5:.+]] = spirv.Bitcast %[[S4]] : !spirv.ptr to !spirv.ptr, StorageBuffer> // CHECK: spirv.Store "StorageBuffer" %[[S5]], %[[ARG1]] : vector<4xf32> func.func @vector_store(%arg0 : memref<4xf32, #spirv.storage_class>, %arg1 : vector<4xf32>) { @@ -790,11 +784,9 @@ func.func @vector_store(%arg0 : memref<4xf32, #spirv.storage_class [0])>, StorageBuffer>, i32, i32 // CHECK: %[[S8:.+]] = spirv.Bitcast %[[S7]] : !spirv.ptr to !spirv.ptr, StorageBuffer> // CHECK: spirv.Store "StorageBuffer" %[[S8]], %[[ARG1]] : vector<4xf32> -- GitLab From 6317c780d81327bd06701a6aa374fc92aa3d73ad Mon Sep 17 00:00:00 2001 From: Chelsea Cassanova Date: Thu, 21 Mar 2024 08:49:43 -0700 Subject: [PATCH 164/296] [lldb][progress][NFC] Clarify Doxygen comments for `details` field (#86002) The Doxygen comments for the `details` field of a progress report currently does not specify that this field will act as the initial set of details for a progress report that gets updated with `Progress::Increment()`. This commit clarifies this. --- lldb/include/lldb/Core/Progress.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lldb/include/lldb/Core/Progress.h b/lldb/include/lldb/Core/Progress.h index c38f6dd0a140..eb3af9816dc6 100644 --- a/lldb/include/lldb/Core/Progress.h +++ b/lldb/include/lldb/Core/Progress.h @@ -66,7 +66,11 @@ public: /// @param [in] title The title of this progress activity. /// /// @param [in] details Specific information about what the progress report - /// is currently working on. + /// is currently working on. Although not required, if the progress report is + /// updated with Progress::Increment() then this field will be overwritten + /// with the new set of details passed into that function, and the details + /// passed initially will act as an "item 0" for the total set of + /// items being reported on. /// /// @param [in] total The total units of work to be done if specified, if /// set to std::nullopt then an indeterminate progress indicator should be -- GitLab From f5c90f3000bc75a344bf01bd4e0401e3fb7f9453 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 21 Mar 2024 08:52:51 -0700 Subject: [PATCH 165/296] [RISCV] Use BuildPairF64 and SplitF64 for bitcast i64<->f64 on rv32 regardless of Zfa. (#85982) Previously we used BuildPairF64 and SplitF64 only if Zfa was supported since they will select register file moves that are only available with Zfa. We recently changed the handling of BuildPairF64/SplitF64 for Zdinx to not go through memory so we should use that for bitcast. That leaves the D without Zfa case that does need to go through memory. Previously we let type legalization expand to loads and stores using a new stack temporary created for each bitcast. After this patch we will create the loads ands stores in the custom inserter and share the same stack slot for all. This also allows DAGCombiner to optimize when bitcast is mixed with BuildPairF64/SplitF64. --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 11 ++++---- llvm/test/CodeGen/RISCV/double-convert.ll | 28 ++++--------------- .../RISCV/inline-asm-d-constraint-f.ll | 16 +---------- .../RISCV/rvv/fixed-vectors-bitcast.ll | 2 +- llvm/test/CodeGen/RISCV/spill-fill-fold.ll | 14 ++++++---- 5 files changed, 21 insertions(+), 50 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 3aa28215efc2..a3ebfb34ad7a 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -559,11 +559,12 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, if (Subtarget.hasStdExtDOrZdinx()) { setOperationAction(FPLegalNodeTypes, MVT::f64, Legal); + if (!Subtarget.is64Bit()) + setOperationAction(ISD::BITCAST, MVT::i64, Custom); + if (Subtarget.hasStdExtZfa()) { setOperationAction(FPRndMode, MVT::f64, Legal); setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal); - if (!Subtarget.is64Bit()) - setOperationAction(ISD::BITCAST, MVT::i64, Custom); } else { if (Subtarget.is64Bit()) setOperationAction(FPRndMode, MVT::f64, Custom); @@ -6071,8 +6072,7 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op, DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0); return FPConv; } - if (VT == MVT::f64 && Op0VT == MVT::i64 && XLenVT == MVT::i32 && - Subtarget.hasStdExtZfa()) { + if (VT == MVT::f64 && Op0VT == MVT::i64 && XLenVT == MVT::i32) { SDValue Lo, Hi; std::tie(Lo, Hi) = DAG.SplitScalar(Op0, DL, MVT::i32, MVT::i32); SDValue RetReg = @@ -12157,8 +12157,7 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N, SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0); Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv)); - } else if (VT == MVT::i64 && Op0VT == MVT::f64 && XLenVT == MVT::i32 && - Subtarget.hasStdExtZfa()) { + } else if (VT == MVT::i64 && Op0VT == MVT::f64 && XLenVT == MVT::i32) { SDValue NewReg = DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), Op0); SDValue RetReg = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, diff --git a/llvm/test/CodeGen/RISCV/double-convert.ll b/llvm/test/CodeGen/RISCV/double-convert.ll index 7a9439e5b322..c1429642962e 100644 --- a/llvm/test/CodeGen/RISCV/double-convert.ll +++ b/llvm/test/CodeGen/RISCV/double-convert.ll @@ -1116,13 +1116,7 @@ define i64 @fmv_x_d(double %a, double %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fmv_x_d: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 ; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a2 -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmv_x_d: @@ -1257,13 +1251,13 @@ define double @fmv_d_x(i64 %a, i64 %b) nounwind { ; RV32IFD-LABEL: fmv_d_x: ; RV32IFD: # %bb.0: ; RV32IFD-NEXT: addi sp, sp, -16 -; RV32IFD-NEXT: sw a3, 4(sp) -; RV32IFD-NEXT: sw a2, 0(sp) -; RV32IFD-NEXT: sw a1, 12(sp) ; RV32IFD-NEXT: sw a0, 8(sp) -; RV32IFD-NEXT: fld fa5, 0(sp) +; RV32IFD-NEXT: sw a1, 12(sp) +; RV32IFD-NEXT: fld fa5, 8(sp) +; RV32IFD-NEXT: sw a2, 8(sp) +; RV32IFD-NEXT: sw a3, 12(sp) ; RV32IFD-NEXT: fld fa4, 8(sp) -; RV32IFD-NEXT: fadd.d fa0, fa4, fa5 +; RV32IFD-NEXT: fadd.d fa0, fa5, fa4 ; RV32IFD-NEXT: addi sp, sp, 16 ; RV32IFD-NEXT: ret ; @@ -1276,17 +1270,7 @@ define double @fmv_d_x(i64 %a, i64 %b) nounwind { ; ; RV32IZFINXZDINX-LABEL: fmv_d_x: ; RV32IZFINXZDINX: # %bb.0: -; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 -; RV32IZFINXZDINX-NEXT: sw a3, 4(sp) -; RV32IZFINXZDINX-NEXT: sw a2, 0(sp) -; RV32IZFINXZDINX-NEXT: sw a1, 12(sp) -; RV32IZFINXZDINX-NEXT: sw a0, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a0, 0(sp) -; RV32IZFINXZDINX-NEXT: lw a1, 4(sp) -; RV32IZFINXZDINX-NEXT: lw a2, 8(sp) -; RV32IZFINXZDINX-NEXT: lw a3, 12(sp) -; RV32IZFINXZDINX-NEXT: fadd.d a0, a2, a0 -; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: fadd.d a0, a0, a2 ; RV32IZFINXZDINX-NEXT: ret ; ; RV64IZFINXZDINX-LABEL: fmv_d_x: diff --git a/llvm/test/CodeGen/RISCV/inline-asm-d-constraint-f.ll b/llvm/test/CodeGen/RISCV/inline-asm-d-constraint-f.ll index 71769a800c06..c480ba800c69 100644 --- a/llvm/test/CodeGen/RISCV/inline-asm-d-constraint-f.ll +++ b/llvm/test/CodeGen/RISCV/inline-asm-d-constraint-f.ll @@ -75,24 +75,10 @@ define double @constraint_f_double_abi_name(double %a) nounwind { define double @constraint_gpr(double %x) { ; RV32F-LABEL: constraint_gpr: ; RV32F: # %bb.0: -; RV32F-NEXT: addi sp, sp, -32 -; RV32F-NEXT: .cfi_def_cfa_offset 32 -; RV32F-NEXT: sw a0, 8(sp) -; RV32F-NEXT: sw a1, 12(sp) -; RV32F-NEXT: fld fa5, 8(sp) -; RV32F-NEXT: fsd fa5, 24(sp) -; RV32F-NEXT: lw a0, 24(sp) -; RV32F-NEXT: lw a1, 28(sp) +; RV32F-NEXT: .cfi_def_cfa_offset 0 ; RV32F-NEXT: #APP ; RV32F-NEXT: mv a0, a0 ; RV32F-NEXT: #NO_APP -; RV32F-NEXT: sw a1, 20(sp) -; RV32F-NEXT: sw a0, 16(sp) -; RV32F-NEXT: fld fa5, 16(sp) -; RV32F-NEXT: fsd fa5, 8(sp) -; RV32F-NEXT: lw a0, 8(sp) -; RV32F-NEXT: lw a1, 12(sp) -; RV32F-NEXT: addi sp, sp, 32 ; RV32F-NEXT: ret ; ; RV64F-LABEL: constraint_gpr: diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll index b7afee754f68..5252eb71c383 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-bitcast.ll @@ -416,8 +416,8 @@ define double @bitcast_v1i64_f64(<1 x i64> %a) { ; RV32ELEN32: # %bb.0: ; RV32ELEN32-NEXT: addi sp, sp, -16 ; RV32ELEN32-NEXT: .cfi_def_cfa_offset 16 -; RV32ELEN32-NEXT: sw a1, 12(sp) ; RV32ELEN32-NEXT: sw a0, 8(sp) +; RV32ELEN32-NEXT: sw a1, 12(sp) ; RV32ELEN32-NEXT: fld fa0, 8(sp) ; RV32ELEN32-NEXT: addi sp, sp, 16 ; RV32ELEN32-NEXT: ret diff --git a/llvm/test/CodeGen/RISCV/spill-fill-fold.ll b/llvm/test/CodeGen/RISCV/spill-fill-fold.ll index a9a0cc5cf94d..8cf5f55ad5c5 100644 --- a/llvm/test/CodeGen/RISCV/spill-fill-fold.ll +++ b/llvm/test/CodeGen/RISCV/spill-fill-fold.ll @@ -290,8 +290,8 @@ define double @spill_i64_to_double(i64 %a) nounwind { ; RV32ID-NEXT: fsd fs9, 40(sp) # 8-byte Folded Spill ; RV32ID-NEXT: fsd fs10, 32(sp) # 8-byte Folded Spill ; RV32ID-NEXT: fsd fs11, 24(sp) # 8-byte Folded Spill -; RV32ID-NEXT: sw a1, 20(sp) ; RV32ID-NEXT: sw a0, 16(sp) +; RV32ID-NEXT: sw a1, 20(sp) ; RV32ID-NEXT: fld fa5, 16(sp) ; RV32ID-NEXT: fsd fa5, 8(sp) # 8-byte Folded Spill ; RV32ID-NEXT: #APP @@ -804,13 +804,15 @@ define double @fill_i64_to_double(i64 %a) nounwind { ; RV32ID-NEXT: fsd fs9, 40(sp) # 8-byte Folded Spill ; RV32ID-NEXT: fsd fs10, 32(sp) # 8-byte Folded Spill ; RV32ID-NEXT: fsd fs11, 24(sp) # 8-byte Folded Spill -; RV32ID-NEXT: sw a1, 20(sp) -; RV32ID-NEXT: sw a0, 16(sp) -; RV32ID-NEXT: fld fa5, 16(sp) -; RV32ID-NEXT: fsd fa5, 8(sp) # 8-byte Folded Spill +; RV32ID-NEXT: sw a1, 12(sp) # 4-byte Folded Spill +; RV32ID-NEXT: sw a0, 8(sp) # 4-byte Folded Spill ; RV32ID-NEXT: #APP ; RV32ID-NEXT: #NO_APP -; RV32ID-NEXT: fld fa0, 8(sp) # 8-byte Folded Reload +; RV32ID-NEXT: lw a0, 8(sp) # 4-byte Folded Reload +; RV32ID-NEXT: sw a0, 16(sp) +; RV32ID-NEXT: lw a0, 12(sp) # 4-byte Folded Reload +; RV32ID-NEXT: sw a0, 20(sp) +; RV32ID-NEXT: fld fa0, 16(sp) ; RV32ID-NEXT: lw ra, 172(sp) # 4-byte Folded Reload ; RV32ID-NEXT: lw s0, 168(sp) # 4-byte Folded Reload ; RV32ID-NEXT: lw s1, 164(sp) # 4-byte Folded Reload -- GitLab From 81bd799819f498a55e32599bce51fa98b2e73238 Mon Sep 17 00:00:00 2001 From: Jonas Devlieghere Date: Thu, 21 Mar 2024 09:06:02 -0700 Subject: [PATCH 166/296] [lldb] Add missing initialization in LineEntry ctor --- lldb/source/Symbol/LineEntry.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lldb/source/Symbol/LineEntry.cpp b/lldb/source/Symbol/LineEntry.cpp index 9e0c06b6ff73..461399e0326e 100644 --- a/lldb/source/Symbol/LineEntry.cpp +++ b/lldb/source/Symbol/LineEntry.cpp @@ -14,8 +14,10 @@ using namespace lldb_private; LineEntry::LineEntry() - : range(), is_start_of_statement(0), is_start_of_basic_block(0), - is_prologue_end(0), is_epilogue_begin(0), is_terminal_entry(0) {} + : range(), file_sp(std::make_shared()), + original_file_sp(std::make_shared()), + is_start_of_statement(0), is_start_of_basic_block(0), is_prologue_end(0), + is_epilogue_begin(0), is_terminal_entry(0) {} void LineEntry::Clear() { range.Clear(); -- GitLab From d8b0d8d6713d474cfd622e03090a7ad5206ee574 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Thu, 21 Mar 2024 21:49:57 +0530 Subject: [PATCH 167/296] AMDGPU: Use defset to cleanup marking MFMA intrinsics as divergent (#85915) --- llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 86 ++++++++++-------- .../Target/AMDGPU/AMDGPUSearchableTables.td | 87 +++---------------- 2 files changed, 59 insertions(+), 114 deletions(-) diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td index 051e603c0819..fff03dee20a1 100644 --- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td @@ -2653,6 +2653,8 @@ class AMDGPUWmmaIntrinsicIU : // The OPSEL intrinsics read from and write to one half of the registers, selected by the op_sel bit. // The tied versions of the f16/bf16 wmma intrinsics tie the destination matrix registers to the input accumulator registers. // The content of the other 16-bit half is preserved from the input. + +defset list AMDGPUWMMAIntrinsicsGFX11 = { def int_amdgcn_wmma_f16_16x16x16_f16_tied : AMDGPUWmmaIntrinsicOPSEL; def int_amdgcn_wmma_bf16_16x16x16_bf16_tied : AMDGPUWmmaIntrinsicOPSEL; @@ -2668,6 +2670,7 @@ def int_amdgcn_wmma_i32_16x16x16_iu4 : AMDGPUWmmaIntrinsicIU; def int_amdgcn_wmma_bf16_16x16x16_bf16 : AMDGPUWmmaIntrinsicOPSEL; +} //===----------------------------------------------------------------------===// // GFX12 Intrinsics @@ -2687,20 +2690,6 @@ def int_amdgcn_permlanex16_var : ClangBuiltin<"__builtin_amdgcn_permlanex16_var" [IntrNoMem, IntrConvergent, IntrWillReturn, ImmArg>, ImmArg>, IntrNoCallback, IntrNoFree]>; - -// WMMA (Wave Matrix Multiply-Accumulate) intrinsics -// -// These operations perform a matrix multiplication and accumulation of -// the form: D = A * B + C . - -// A and B are <8 x fp8> or <8 x bf8>, but since fp8 and bf8 are not supported by llvm we use <2 x i32>. -def int_amdgcn_wmma_f32_16x16x16_fp8_fp8 : AMDGPUWmmaIntrinsic; -def int_amdgcn_wmma_f32_16x16x16_fp8_bf8 : AMDGPUWmmaIntrinsic; -def int_amdgcn_wmma_f32_16x16x16_bf8_fp8 : AMDGPUWmmaIntrinsic; -def int_amdgcn_wmma_f32_16x16x16_bf8_bf8 : AMDGPUWmmaIntrinsic; -// A and B are <16 x iu4>. -def int_amdgcn_wmma_i32_16x16x32_iu4 : AMDGPUWmmaIntrinsicIU; - // SWMMAC (Wave Matrix(sparse) Multiply-Accumulate) intrinsics // // These operations perform a sparse matrix multiplication and accumulation of @@ -2734,6 +2723,20 @@ class AMDGPUSWmmacIntrinsicIUIdx>, ImmArg>, ImmArg>] >; +defset list AMDGPUWMMAIntrinsicsGFX12 = { +// WMMA (Wave Matrix Multiply-Accumulate) intrinsics +// +// These operations perform a matrix multiplication and accumulation of +// the form: D = A * B + C . + +// A and B are <8 x fp8> or <8 x bf8>, but since fp8 and bf8 are not supported by llvm we use <2 x i32>. +def int_amdgcn_wmma_f32_16x16x16_fp8_fp8 : AMDGPUWmmaIntrinsic; +def int_amdgcn_wmma_f32_16x16x16_fp8_bf8 : AMDGPUWmmaIntrinsic; +def int_amdgcn_wmma_f32_16x16x16_bf8_fp8 : AMDGPUWmmaIntrinsic; +def int_amdgcn_wmma_f32_16x16x16_bf8_bf8 : AMDGPUWmmaIntrinsic; +// A and B are <16 x iu4>. +def int_amdgcn_wmma_i32_16x16x32_iu4 : AMDGPUWmmaIntrinsicIU; + def int_amdgcn_swmmac_f32_16x16x32_f16 : AMDGPUSWmmacIntrinsicIdx; def int_amdgcn_swmmac_f32_16x16x32_bf16 : AMDGPUSWmmacIntrinsicIdx; def int_amdgcn_swmmac_f16_16x16x32_f16 : AMDGPUSWmmacIntrinsicIdx; @@ -2745,6 +2748,7 @@ def int_amdgcn_swmmac_f32_16x16x32_fp8_fp8 : AMDGPUSWmmacIntrinsicIdx; def int_amdgcn_swmmac_f32_16x16x32_bf8_fp8 : AMDGPUSWmmacIntrinsicIdx; def int_amdgcn_swmmac_f32_16x16x32_bf8_bf8 : AMDGPUSWmmacIntrinsicIdx; +} def int_amdgcn_global_atomic_ordered_add_b64 : AMDGPUAtomicRtn; @@ -3012,6 +3016,7 @@ class AMDGPUMfmaIntrinsic : [IntrConvergent, IntrNoMem, ImmArg>, ImmArg>, ImmArg>]>; +defset list AMDGPUMFMAIntrinsics908 = { def int_amdgcn_mfma_f32_32x32x1f32 : AMDGPUMfmaIntrinsic; def int_amdgcn_mfma_f32_16x16x1f32 : AMDGPUMfmaIntrinsic; def int_amdgcn_mfma_f32_4x4x1f32 : AMDGPUMfmaIntrinsic; @@ -3032,6 +3037,7 @@ def int_amdgcn_mfma_f32_16x16x2bf16 : AMDGPUMfmaIntrinsic; def int_amdgcn_mfma_f32_32x32x4bf16 : AMDGPUMfmaIntrinsic; def int_amdgcn_mfma_f32_16x16x8bf16 : AMDGPUMfmaIntrinsic; +} //===----------------------------------------------------------------------===// // gfx90a intrinsics @@ -3043,6 +3049,7 @@ def int_amdgcn_flat_atomic_fadd : AMDGPUAtomicRtn; def int_amdgcn_flat_atomic_fmin : AMDGPUAtomicRtn; def int_amdgcn_flat_atomic_fmax : AMDGPUAtomicRtn; +defset list AMDGPUMFMAIntrinsics90A = { def int_amdgcn_mfma_f32_32x32x4bf16_1k : AMDGPUMfmaIntrinsic; def int_amdgcn_mfma_f32_16x16x4bf16_1k : AMDGPUMfmaIntrinsic; def int_amdgcn_mfma_f32_4x4x4bf16_1k : AMDGPUMfmaIntrinsic; @@ -3054,25 +3061,12 @@ def int_amdgcn_mfma_f32_16x16x16bf16_1k : AMDGPUMfmaIntrinsic; def int_amdgcn_mfma_f64_4x4x4f64 : AMDGPUMfmaIntrinsic; +} //===----------------------------------------------------------------------===// // gfx940 intrinsics // ===----------------------------------------------------------------------===// -// bf16 atomics use v2i16 argument since there is no bf16 data type in the llvm. -def int_amdgcn_global_atomic_fadd_v2bf16 : AMDGPUAtomicRtn; -def int_amdgcn_flat_atomic_fadd_v2bf16 : AMDGPUAtomicRtn; -def int_amdgcn_ds_fadd_v2bf16 : DefaultAttrsIntrinsic< - [llvm_v2i16_ty], - [LLVMQualPointerType<3>, llvm_v2i16_ty], - [IntrArgMemOnly, NoCapture>]>, - ClangBuiltin<"__builtin_amdgcn_ds_atomic_fadd_v2bf16">; - -def int_amdgcn_mfma_i32_16x16x32_i8 : AMDGPUMfmaIntrinsic; -def int_amdgcn_mfma_i32_32x32x16_i8 : AMDGPUMfmaIntrinsic; -def int_amdgcn_mfma_f32_16x16x8_xf32 : AMDGPUMfmaIntrinsic; -def int_amdgcn_mfma_f32_32x32x4_xf32 : AMDGPUMfmaIntrinsic; - class AMDGPUMFp8MfmaIntrinsic : AMDGPUMfmaIntrinsic; @@ -3081,9 +3075,6 @@ multiclass AMDGPUMFp8MfmaIntrinsic { def NAME#"_"#kind : AMDGPUMFp8MfmaIntrinsic; } -defm int_amdgcn_mfma_f32_16x16x32 : AMDGPUMFp8MfmaIntrinsic; -defm int_amdgcn_mfma_f32_32x32x16 : AMDGPUMFp8MfmaIntrinsic; - // llvm.amdgcn.smfmac.?32.* vdst, srcA, srcB, srcC, index, cbsz, abid class AMDGPUMSmfmacIntrinsic : ClangBuiltin, @@ -3093,13 +3084,6 @@ class AMDGPUMSmfmacIntrinsic : [IntrConvergent, IntrNoMem, ImmArg>, ImmArg>]>; -def int_amdgcn_smfmac_f32_16x16x32_f16 : AMDGPUMSmfmacIntrinsic; -def int_amdgcn_smfmac_f32_32x32x16_f16 : AMDGPUMSmfmacIntrinsic; -def int_amdgcn_smfmac_f32_16x16x32_bf16 : AMDGPUMSmfmacIntrinsic; -def int_amdgcn_smfmac_f32_32x32x16_bf16 : AMDGPUMSmfmacIntrinsic; -def int_amdgcn_smfmac_i32_16x16x64_i8 : AMDGPUMSmfmacIntrinsic; -def int_amdgcn_smfmac_i32_32x32x32_i8 : AMDGPUMSmfmacIntrinsic; - class AMDGPUMFp8SmfmacIntrinsic : AMDGPUMSmfmacIntrinsic; @@ -3108,8 +3092,34 @@ multiclass AMDGPUMFp8SmfmacIntrinsic { def NAME#"_"#kind : AMDGPUMFp8SmfmacIntrinsic; } +// bf16 atomics use v2i16 argument since there is no bf16 data type in the llvm. +def int_amdgcn_global_atomic_fadd_v2bf16 : AMDGPUAtomicRtn; +def int_amdgcn_flat_atomic_fadd_v2bf16 : AMDGPUAtomicRtn; +def int_amdgcn_ds_fadd_v2bf16 : DefaultAttrsIntrinsic< + [llvm_v2i16_ty], + [LLVMQualPointerType<3>, llvm_v2i16_ty], + [IntrArgMemOnly, NoCapture>]>, + ClangBuiltin<"__builtin_amdgcn_ds_atomic_fadd_v2bf16">; + +defset list AMDGPUMFMAIntrinsics940 = { +def int_amdgcn_mfma_i32_16x16x32_i8 : AMDGPUMfmaIntrinsic; +def int_amdgcn_mfma_i32_32x32x16_i8 : AMDGPUMfmaIntrinsic; +def int_amdgcn_mfma_f32_16x16x8_xf32 : AMDGPUMfmaIntrinsic; +def int_amdgcn_mfma_f32_32x32x4_xf32 : AMDGPUMfmaIntrinsic; + +defm int_amdgcn_mfma_f32_16x16x32 : AMDGPUMFp8MfmaIntrinsic; +defm int_amdgcn_mfma_f32_32x32x16 : AMDGPUMFp8MfmaIntrinsic; + +def int_amdgcn_smfmac_f32_16x16x32_f16 : AMDGPUMSmfmacIntrinsic; +def int_amdgcn_smfmac_f32_32x32x16_f16 : AMDGPUMSmfmacIntrinsic; +def int_amdgcn_smfmac_f32_16x16x32_bf16 : AMDGPUMSmfmacIntrinsic; +def int_amdgcn_smfmac_f32_32x32x16_bf16 : AMDGPUMSmfmacIntrinsic; +def int_amdgcn_smfmac_i32_16x16x64_i8 : AMDGPUMSmfmacIntrinsic; +def int_amdgcn_smfmac_i32_32x32x32_i8 : AMDGPUMSmfmacIntrinsic; + defm int_amdgcn_smfmac_f32_16x16x64 : AMDGPUMFp8SmfmacIntrinsic; defm int_amdgcn_smfmac_f32_32x32x32 : AMDGPUMFp8SmfmacIntrinsic; +} // llvm.amdgcn.cvt.f32.bf8 float vdst, int srcA, imm byte_sel [0..3] // byte_sel selects byte from srcA. diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td index bb1c6b733729..8eb46a980148 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td @@ -354,82 +354,17 @@ def : SourceOfDivergence; def : SourceOfDivergence; def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; -def : SourceOfDivergence; +foreach intr = AMDGPUMFMAIntrinsics908 in +def : SourceOfDivergence; +foreach intr = AMDGPUMFMAIntrinsics90A in +def : SourceOfDivergence; +foreach intr = AMDGPUMFMAIntrinsics940 in +def : SourceOfDivergence; +foreach intr = AMDGPUWMMAIntrinsicsGFX11 in +def : SourceOfDivergence; +foreach intr = AMDGPUWMMAIntrinsicsGFX12 in +def : SourceOfDivergence; + def : SourceOfDivergence; // The dummy boolean output is divergent from the IR's perspective, -- GitLab From d9f0d9a1452ed78e943423c9fbbd63674625f7f5 Mon Sep 17 00:00:00 2001 From: Tarun Prabhu Date: Thu, 21 Mar 2024 10:22:33 -0600 Subject: [PATCH 168/296] [flang][NFC] Fix header guards Some header guards conflicted with clang. Fix a few others to follow the convention in the rest of the headers in flang. --- flang/include/flang/Common/Version.h | 6 +++--- flang/include/flang/Frontend/CodeGenOptions.h | 6 +++--- flang/include/flang/Frontend/LangOptions.h | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/flang/include/flang/Common/Version.h b/flang/include/flang/Common/Version.h index b1bd2416a618..3257d4a4f645 100644 --- a/flang/include/flang/Common/Version.h +++ b/flang/include/flang/Common/Version.h @@ -12,8 +12,8 @@ /// //===----------------------------------------------------------------------===// -#ifndef LLVM_FLANG_COMMON_VERSION_H -#define LLVM_FLANG_COMMON_VERSION_H +#ifndef FORTRAN_COMMON_VERSION_H +#define FORTRAN_COMMON_VERSION_H #include "flang/Version.inc" #include "llvm/ADT/StringRef.h" @@ -53,4 +53,4 @@ std::string getFlangFullVersion(); std::string getFlangToolFullVersion(llvm::StringRef ToolName); } // namespace Fortran::common -#endif // LLVM_FLANG_COMMON_VERSION_H +#endif // FORTRAN_COMMON_VERSION_H diff --git a/flang/include/flang/Frontend/CodeGenOptions.h b/flang/include/flang/Frontend/CodeGenOptions.h index 0c318e4023af..b0bbace82c04 100644 --- a/flang/include/flang/Frontend/CodeGenOptions.h +++ b/flang/include/flang/Frontend/CodeGenOptions.h @@ -12,8 +12,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CLANG_BASIC_CODEGENOPTIONS_H -#define LLVM_CLANG_BASIC_CODEGENOPTIONS_H +#ifndef FORTRAN_FRONTEND_CODEGENOPTIONS_H +#define FORTRAN_FRONTEND_CODEGENOPTIONS_H #include "llvm/Frontend/Debug/Options.h" #include "llvm/Frontend/Driver/CodeGenOptions.h" @@ -141,4 +141,4 @@ public: } // end namespace Fortran::frontend -#endif +#endif // FORTRAN_FRONTEND_CODEGENOPTIONS_H diff --git a/flang/include/flang/Frontend/LangOptions.h b/flang/include/flang/Frontend/LangOptions.h index 7adf2eec9ca3..7ab219581886 100644 --- a/flang/include/flang/Frontend/LangOptions.h +++ b/flang/include/flang/Frontend/LangOptions.h @@ -12,8 +12,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_FLANG_FRONTEND_LANGOPTIONS_H -#define LLVM_FLANG_FRONTEND_LANGOPTIONS_H +#ifndef FORTRAN_FRONTEND_LANGOPTIONS_H +#define FORTRAN_FRONTEND_LANGOPTIONS_H #include @@ -63,4 +63,4 @@ public: } // end namespace Fortran::frontend -#endif +#endif // FORTRAN_FRONTEND_LANGOPTIONS_H -- GitLab From 12836467b76c56872b4c22a6fd44bcda696ea720 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 21 Mar 2024 09:25:13 -0700 Subject: [PATCH 169/296] [ConstantRange] Fix off by 1 bugs in UIToFP and SIToFP handling. (#86041) We were passing the min and max values of the range to the ConstantRange constructor, but the constructor expects the upper bound to 1 more than the max value so we need to add 1. We also need to use getNonEmpty so that passing 0, 0 to the constructor creates a full range rather than an empty range. And passing smin, smax+1 doesn't cause an assertion. I believe this fixes at least some of the reason #79158 was reverted. --- llvm/lib/IR/ConstantRange.cpp | 4 ++-- llvm/test/Transforms/Float2Int/pr79158.ll | 7 ++++--- llvm/unittests/IR/ConstantRangeTest.cpp | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/llvm/lib/IR/ConstantRange.cpp b/llvm/lib/IR/ConstantRange.cpp index 3394a1ec8dc4..59e7a9f5eb11 100644 --- a/llvm/lib/IR/ConstantRange.cpp +++ b/llvm/lib/IR/ConstantRange.cpp @@ -746,7 +746,7 @@ ConstantRange ConstantRange::castOp(Instruction::CastOps CastOp, Min = Min.zext(ResultBitWidth); Max = Max.zext(ResultBitWidth); } - return ConstantRange(std::move(Min), std::move(Max)); + return getNonEmpty(std::move(Min), std::move(Max) + 1); } case Instruction::SIToFP: { // TODO: use input range if available @@ -757,7 +757,7 @@ ConstantRange ConstantRange::castOp(Instruction::CastOps CastOp, SMin = SMin.sext(ResultBitWidth); SMax = SMax.sext(ResultBitWidth); } - return ConstantRange(std::move(SMin), std::move(SMax)); + return getNonEmpty(std::move(SMin), std::move(SMax) + 1); } case Instruction::FPTrunc: case Instruction::FPExt: diff --git a/llvm/test/Transforms/Float2Int/pr79158.ll b/llvm/test/Transforms/Float2Int/pr79158.ll index d041e01a4b59..5e78cc0bc66f 100644 --- a/llvm/test/Transforms/Float2Int/pr79158.ll +++ b/llvm/test/Transforms/Float2Int/pr79158.ll @@ -6,9 +6,10 @@ define i32 @pr79158(i32 %x) { ; CHECK-SAME: i32 [[X:%.*]]) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[X]], 0 -; CHECK-NEXT: [[TMP0:%.*]] = zext i1 [[CMP]] to i32 -; CHECK-NEXT: [[MUL1:%.*]] = mul i32 [[TMP0]], 2147483647 -; CHECK-NEXT: ret i32 [[MUL1]] +; CHECK-NEXT: [[TMP0:%.*]] = zext i1 [[CMP]] to i64 +; CHECK-NEXT: [[MUL1:%.*]] = mul i64 [[TMP0]], 4294967295 +; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[MUL1]] to i32 +; CHECK-NEXT: ret i32 [[TMP1]] ; entry: %cmp = icmp sgt i32 %x, 0 diff --git a/llvm/unittests/IR/ConstantRangeTest.cpp b/llvm/unittests/IR/ConstantRangeTest.cpp index 34a162a5514e..8ec120d70e99 100644 --- a/llvm/unittests/IR/ConstantRangeTest.cpp +++ b/llvm/unittests/IR/ConstantRangeTest.cpp @@ -2479,6 +2479,24 @@ TEST_F(ConstantRangeTest, castOps) { ConstantRange IntToPtr = A.castOp(Instruction::IntToPtr, 64); EXPECT_EQ(64u, IntToPtr.getBitWidth()); EXPECT_TRUE(IntToPtr.isFullSet()); + + ConstantRange UIToFP = A.castOp(Instruction::UIToFP, 16); + EXPECT_EQ(16u, UIToFP.getBitWidth()); + EXPECT_TRUE(UIToFP.isFullSet()); + + ConstantRange UIToFP2 = A.castOp(Instruction::UIToFP, 64); + ConstantRange B(APInt(64, 0), APInt(64, 65536)); + EXPECT_EQ(64u, UIToFP2.getBitWidth()); + EXPECT_EQ(B, UIToFP2); + + ConstantRange SIToFP = A.castOp(Instruction::SIToFP, 16); + EXPECT_EQ(16u, SIToFP.getBitWidth()); + EXPECT_TRUE(SIToFP.isFullSet()); + + ConstantRange SIToFP2 = A.castOp(Instruction::SIToFP, 64); + ConstantRange C(APInt(64, -32768), APInt(64, 32768)); + EXPECT_EQ(64u, SIToFP2.getBitWidth()); + EXPECT_EQ(C, SIToFP2); } TEST_F(ConstantRangeTest, binaryAnd) { -- GitLab From 9a87d4d546a4382879b1beb96687acbad0ef4cc0 Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Thu, 21 Mar 2024 17:25:37 +0100 Subject: [PATCH 170/296] [libc] Add `is_constant_evaluated` type_traits (#86139) This will replace `__builtin_is_constant_evaluated` in math_extras.h. --- libc/src/__support/CPP/CMakeLists.txt | 5 +++-- libc/src/__support/CPP/type_traits.h | 1 + .../CPP/type_traits/is_constant_evaluated.h | 21 +++++++++++++++++++ .../llvm-project-overlay/libc/BUILD.bazel | 3 ++- 4 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 libc/src/__support/CPP/type_traits/is_constant_evaluated.h diff --git a/libc/src/__support/CPP/CMakeLists.txt b/libc/src/__support/CPP/CMakeLists.txt index 6216505eae23..f76285be5219 100644 --- a/libc/src/__support/CPP/CMakeLists.txt +++ b/libc/src/__support/CPP/CMakeLists.txt @@ -103,23 +103,24 @@ add_header_library( type_traits HDRS type_traits.h - type_traits/always_false.h type_traits/add_lvalue_reference.h type_traits/add_pointer.h type_traits/add_rvalue_reference.h + type_traits/always_false.h type_traits/bool_constant.h type_traits/conditional.h type_traits/decay.h type_traits/enable_if.h type_traits/false_type.h type_traits/integral_constant.h - type_traits/invoke.h type_traits/invoke_result.h + type_traits/invoke.h type_traits/is_arithmetic.h type_traits/is_array.h type_traits/is_base_of.h type_traits/is_class.h type_traits/is_const.h + type_traits/is_constant_evaluated.h type_traits/is_convertible.h type_traits/is_destructible.h type_traits/is_enum.h diff --git a/libc/src/__support/CPP/type_traits.h b/libc/src/__support/CPP/type_traits.h index 697cf79d6ccc..1494aeb905e0 100644 --- a/libc/src/__support/CPP/type_traits.h +++ b/libc/src/__support/CPP/type_traits.h @@ -25,6 +25,7 @@ #include "src/__support/CPP/type_traits/is_base_of.h" #include "src/__support/CPP/type_traits/is_class.h" #include "src/__support/CPP/type_traits/is_const.h" +#include "src/__support/CPP/type_traits/is_constant_evaluated.h" #include "src/__support/CPP/type_traits/is_convertible.h" #include "src/__support/CPP/type_traits/is_destructible.h" #include "src/__support/CPP/type_traits/is_enum.h" diff --git a/libc/src/__support/CPP/type_traits/is_constant_evaluated.h b/libc/src/__support/CPP/type_traits/is_constant_evaluated.h new file mode 100644 index 000000000000..93cfd07b567f --- /dev/null +++ b/libc/src/__support/CPP/type_traits/is_constant_evaluated.h @@ -0,0 +1,21 @@ +//===-- is_constant_evaluated type_traits -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_CONSTANT_EVALUATED_H +#define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_CONSTANT_EVALUATED_H + +#include "src/__support/macros/attributes.h" + +namespace LIBC_NAMESPACE::cpp { + +LIBC_INLINE constexpr bool is_constant_evaluated() { + return __builtin_is_constant_evaluated(); +} + +} // namespace LIBC_NAMESPACE::cpp + +#endif // LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_IS_CONSTANT_EVALUATED_H diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index fe4b8e2de14e..2e8d475f196e 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -339,6 +339,7 @@ libc_support_library( "src/__support/CPP/type_traits/is_base_of.h", "src/__support/CPP/type_traits/is_class.h", "src/__support/CPP/type_traits/is_const.h", + "src/__support/CPP/type_traits/is_constant_evaluated.h", "src/__support/CPP/type_traits/is_convertible.h", "src/__support/CPP/type_traits/is_destructible.h", "src/__support/CPP/type_traits/is_enum.h", @@ -743,12 +744,12 @@ libc_support_library( deps = [ ":__support_common", ":__support_cpp_bit", - ":__support_sign", ":__support_cpp_type_traits", ":__support_libc_assert", ":__support_macros_attributes", ":__support_macros_properties_types", ":__support_math_extras", + ":__support_sign", ":__support_uint128", ], ) -- GitLab From c1c2551a2876f536b5a06f48fa809aeedbc3d7ba Mon Sep 17 00:00:00 2001 From: OverMighty Date: Thu, 21 Mar 2024 16:33:16 +0000 Subject: [PATCH 171/296] [clang] Implement __builtin_{clzg,ctzg} (#83431) Fixes #83075, fixes #83076. --- clang/docs/LanguageExtensions.rst | 41 ++++ clang/include/clang/Basic/Builtins.td | 12 +- .../clang/Basic/DiagnosticSemaKinds.td | 15 +- clang/lib/CodeGen/CGBuiltin.cpp | 46 +++- clang/lib/Sema/SemaChecking.cpp | 47 ++++ clang/test/CodeGen/builtins.c | 204 ++++++++++++++++++ clang/test/CodeGen/ubsan-builtin-checks.c | 6 + clang/test/Sema/builtin-popcountg.c | 23 -- clang/test/Sema/count-builtins.c | 87 ++++++++ 9 files changed, 441 insertions(+), 40 deletions(-) delete mode 100644 clang/test/Sema/builtin-popcountg.c create mode 100644 clang/test/Sema/count-builtins.c diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst index 201a4c27f7dd..5711972b55e6 100644 --- a/clang/docs/LanguageExtensions.rst +++ b/clang/docs/LanguageExtensions.rst @@ -3553,6 +3553,47 @@ argument can be of any unsigned integer type. ``__builtin_popcount{,l,ll}`` builtins, with support for other integer types, such as ``unsigned __int128`` and C23 ``unsigned _BitInt(N)``. +``__builtin_clzg`` and ``__builtin_ctzg`` +----------------------------------------- + +``__builtin_clzg`` (respectively ``__builtin_ctzg``) returns the number of +leading (respectively trailing) 0 bits in the first argument. The first argument +can be of any unsigned integer type. + +If the first argument is 0 and an optional second argument of ``int`` type is +provided, then the second argument is returned. If the first argument is 0, but +only one argument is provided, then the behavior is undefined. + +**Syntax**: + +.. code-block:: c++ + + int __builtin_clzg(type x[, int fallback]) + int __builtin_ctzg(type x[, int fallback]) + +**Examples**: + +.. code-block:: c++ + + unsigned int x = 1; + int x_lz = __builtin_clzg(x); + int x_tz = __builtin_ctzg(x); + + unsigned long y = 2; + int y_lz = __builtin_clzg(y); + int y_tz = __builtin_ctzg(y); + + unsigned _BitInt(128) z = 4; + int z_lz = __builtin_clzg(z); + int z_tz = __builtin_ctzg(z); + +**Description**: + +``__builtin_clzg`` (respectively ``__builtin_ctzg``) is meant to be a +type-generic alternative to the ``__builtin_clz{,l,ll}`` (respectively +``__builtin_ctz{,l,ll}``) builtins, with support for other integer types, such +as ``unsigned __int128`` and C23 ``unsigned _BitInt(N)``. + Multiprecision Arithmetic Builtins ---------------------------------- diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td index 491c9d895413..21ab9bb86d1b 100644 --- a/clang/include/clang/Basic/Builtins.td +++ b/clang/include/clang/Basic/Builtins.td @@ -676,7 +676,11 @@ def Clz : Builtin, BitShort_Int_Long_LongLongTemplate { let Prototype = "int(unsigned T)"; } -// FIXME: Add int clzimax(uintmax_t) +def Clzg : Builtin { + let Spellings = ["__builtin_clzg"]; + let Attributes = [NoThrow, Const, CustomTypeChecking]; + let Prototype = "int(...)"; +} def Ctz : Builtin, BitShort_Int_Long_LongLongTemplate { let Spellings = ["__builtin_ctz"]; @@ -684,7 +688,11 @@ def Ctz : Builtin, BitShort_Int_Long_LongLongTemplate { let Prototype = "int(unsigned T)"; } -// FIXME: Add int ctzimax(uintmax_t) +def Ctzg : Builtin { + let Spellings = ["__builtin_ctzg"]; + let Attributes = [NoThrow, Const, CustomTypeChecking]; + let Prototype = "int(...)"; +} def FFS : Builtin, BitInt_Long_LongLongTemplate { let Spellings = ["__builtin_ffs"]; diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 2646942a53e3..270af5d24611 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -12020,13 +12020,14 @@ def err_builtin_launder_invalid_arg : Error< "'__builtin_launder' is not allowed">; def err_builtin_invalid_arg_type: Error < - "%ordinal0 argument must be a " - "%select{vector, integer or floating point type|matrix|" - "pointer to a valid matrix element type|" - "signed integer or floating point type|vector type|" - "floating point type|" - "vector of integers|" - "type of unsigned integer}1 (was %2)">; + "%ordinal0 argument must be " + "%select{a vector, integer or floating point type|a matrix|" + "a pointer to a valid matrix element type|" + "a signed integer or floating point type|a vector type|" + "a floating point type|" + "a vector of integers|" + "an unsigned integer|" + "an 'int'}1 (was %2)">; def err_builtin_matrix_disabled: Error< "matrix types extension is disabled. Pass -fenable-matrix to enable it">; diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index 77cb269d43c5..e14e89088282 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -3128,36 +3128,66 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI__builtin_ctzs: case Builtin::BI__builtin_ctz: case Builtin::BI__builtin_ctzl: - case Builtin::BI__builtin_ctzll: { - Value *ArgValue = EmitCheckedArgForBuiltin(E->getArg(0), BCK_CTZPassedZero); + case Builtin::BI__builtin_ctzll: + case Builtin::BI__builtin_ctzg: { + bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg && + E->getNumArgs() > 1; + + Value *ArgValue = + HasFallback ? EmitScalarExpr(E->getArg(0)) + : EmitCheckedArgForBuiltin(E->getArg(0), BCK_CTZPassedZero); llvm::Type *ArgType = ArgValue->getType(); Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType); llvm::Type *ResultType = ConvertType(E->getType()); - Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); + Value *ZeroUndef = + Builder.getInt1(HasFallback || getTarget().isCLZForZeroUndef()); Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef}); if (Result->getType() != ResultType) Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, "cast"); - return RValue::get(Result); + if (!HasFallback) + return RValue::get(Result); + + Value *Zero = Constant::getNullValue(ArgType); + Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero"); + Value *FallbackValue = EmitScalarExpr(E->getArg(1)); + Value *ResultOrFallback = + Builder.CreateSelect(IsZero, FallbackValue, Result, "ctzg"); + return RValue::get(ResultOrFallback); } case Builtin::BI__builtin_clzs: case Builtin::BI__builtin_clz: case Builtin::BI__builtin_clzl: - case Builtin::BI__builtin_clzll: { - Value *ArgValue = EmitCheckedArgForBuiltin(E->getArg(0), BCK_CLZPassedZero); + case Builtin::BI__builtin_clzll: + case Builtin::BI__builtin_clzg: { + bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg && + E->getNumArgs() > 1; + + Value *ArgValue = + HasFallback ? EmitScalarExpr(E->getArg(0)) + : EmitCheckedArgForBuiltin(E->getArg(0), BCK_CLZPassedZero); llvm::Type *ArgType = ArgValue->getType(); Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType); llvm::Type *ResultType = ConvertType(E->getType()); - Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef()); + Value *ZeroUndef = + Builder.getInt1(HasFallback || getTarget().isCLZForZeroUndef()); Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef}); if (Result->getType() != ResultType) Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true, "cast"); - return RValue::get(Result); + if (!HasFallback) + return RValue::get(Result); + + Value *Zero = Constant::getNullValue(ArgType); + Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero"); + Value *FallbackValue = EmitScalarExpr(E->getArg(1)); + Value *ResultOrFallback = + Builder.CreateSelect(IsZero, FallbackValue, Result, "clzg"); + return RValue::get(ResultOrFallback); } case Builtin::BI__builtin_ffs: case Builtin::BI__builtin_ffsl: diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index ef3ab16ba29b..246e3577809a 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -2399,6 +2399,48 @@ static bool SemaBuiltinPopcountg(Sema &S, CallExpr *TheCall) { return false; } +/// Checks that __builtin_{clzg,ctzg} was called with a first argument, which is +/// an unsigned integer, and an optional second argument, which is promoted to +/// an 'int'. +static bool SemaBuiltinCountZeroBitsGeneric(Sema &S, CallExpr *TheCall) { + if (checkArgCountRange(S, TheCall, 1, 2)) + return true; + + ExprResult Arg0Res = S.DefaultLvalueConversion(TheCall->getArg(0)); + if (Arg0Res.isInvalid()) + return true; + + Expr *Arg0 = Arg0Res.get(); + TheCall->setArg(0, Arg0); + + QualType Arg0Ty = Arg0->getType(); + + if (!Arg0Ty->isUnsignedIntegerType()) { + S.Diag(Arg0->getBeginLoc(), diag::err_builtin_invalid_arg_type) + << 1 << /*unsigned integer ty*/ 7 << Arg0Ty; + return true; + } + + if (TheCall->getNumArgs() > 1) { + ExprResult Arg1Res = S.UsualUnaryConversions(TheCall->getArg(1)); + if (Arg1Res.isInvalid()) + return true; + + Expr *Arg1 = Arg1Res.get(); + TheCall->setArg(1, Arg1); + + QualType Arg1Ty = Arg1->getType(); + + if (!Arg1Ty->isSpecificBuiltinType(BuiltinType::Int)) { + S.Diag(Arg1->getBeginLoc(), diag::err_builtin_invalid_arg_type) + << 2 << /*'int' ty*/ 8 << Arg1Ty; + return true; + } + } + + return false; +} + ExprResult Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID, CallExpr *TheCall) { @@ -3187,6 +3229,11 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID, if (SemaBuiltinPopcountg(*this, TheCall)) return ExprError(); break; + case Builtin::BI__builtin_clzg: + case Builtin::BI__builtin_ctzg: + if (SemaBuiltinCountZeroBitsGeneric(*this, TheCall)) + return ExprError(); + break; } if (getLangOpts().HLSL && CheckHLSLBuiltinFunctionCall(BuiltinID, TheCall)) diff --git a/clang/test/CodeGen/builtins.c b/clang/test/CodeGen/builtins.c index 4f9641d357b7..407e0857d223 100644 --- a/clang/test/CodeGen/builtins.c +++ b/clang/test/CodeGen/builtins.c @@ -983,4 +983,208 @@ void test_builtin_popcountg(unsigned char uc, unsigned short us, // CHECK-NEXT: ret void } +// CHECK-LABEL: define{{.*}} void @test_builtin_clzg +void test_builtin_clzg(unsigned char uc, unsigned short us, unsigned int ui, + unsigned long ul, unsigned long long ull, + unsigned __int128 ui128, unsigned _BitInt(128) ubi128, + signed char sc, short s, int i) { + volatile int lz; + lz = __builtin_clzg(uc); + // CHECK: %1 = load i8, ptr %uc.addr, align 1 + // CHECK-NEXT: %2 = call i8 @llvm.ctlz.i8(i8 %1, i1 true) + // CHECK-NEXT: %cast = sext i8 %2 to i32 + // CHECK-NEXT: store volatile i32 %cast, ptr %lz, align 4 + lz = __builtin_clzg(us); + // CHECK-NEXT: %3 = load i16, ptr %us.addr, align 2 + // CHECK-NEXT: %4 = call i16 @llvm.ctlz.i16(i16 %3, i1 true) + // CHECK-NEXT: %cast1 = sext i16 %4 to i32 + // CHECK-NEXT: store volatile i32 %cast1, ptr %lz, align 4 + lz = __builtin_clzg(ui); + // CHECK-NEXT: %5 = load i32, ptr %ui.addr, align 4 + // CHECK-NEXT: %6 = call i32 @llvm.ctlz.i32(i32 %5, i1 true) + // CHECK-NEXT: store volatile i32 %6, ptr %lz, align 4 + lz = __builtin_clzg(ul); + // CHECK-NEXT: %7 = load i64, ptr %ul.addr, align 8 + // CHECK-NEXT: %8 = call i64 @llvm.ctlz.i64(i64 %7, i1 true) + // CHECK-NEXT: %cast2 = trunc i64 %8 to i32 + // CHECK-NEXT: store volatile i32 %cast2, ptr %lz, align 4 + lz = __builtin_clzg(ull); + // CHECK-NEXT: %9 = load i64, ptr %ull.addr, align 8 + // CHECK-NEXT: %10 = call i64 @llvm.ctlz.i64(i64 %9, i1 true) + // CHECK-NEXT: %cast3 = trunc i64 %10 to i32 + // CHECK-NEXT: store volatile i32 %cast3, ptr %lz, align 4 + lz = __builtin_clzg(ui128); + // CHECK-NEXT: %11 = load i128, ptr %ui128.addr, align 16 + // CHECK-NEXT: %12 = call i128 @llvm.ctlz.i128(i128 %11, i1 true) + // CHECK-NEXT: %cast4 = trunc i128 %12 to i32 + // CHECK-NEXT: store volatile i32 %cast4, ptr %lz, align 4 + lz = __builtin_clzg(ubi128); + // CHECK-NEXT: %13 = load i128, ptr %ubi128.addr, align 8 + // CHECK-NEXT: %14 = call i128 @llvm.ctlz.i128(i128 %13, i1 true) + // CHECK-NEXT: %cast5 = trunc i128 %14 to i32 + // CHECK-NEXT: store volatile i32 %cast5, ptr %lz, align 4 + lz = __builtin_clzg(uc, sc); + // CHECK-NEXT: %15 = load i8, ptr %uc.addr, align 1 + // CHECK-NEXT: %16 = call i8 @llvm.ctlz.i8(i8 %15, i1 true) + // CHECK-NEXT: %cast6 = sext i8 %16 to i32 + // CHECK-NEXT: %iszero = icmp eq i8 %15, 0 + // CHECK-NEXT: %17 = load i8, ptr %sc.addr, align 1 + // CHECK-NEXT: %conv = sext i8 %17 to i32 + // CHECK-NEXT: %clzg = select i1 %iszero, i32 %conv, i32 %cast6 + // CHECK-NEXT: store volatile i32 %clzg, ptr %lz, align 4 + lz = __builtin_clzg(us, uc); + // CHECK-NEXT: %18 = load i16, ptr %us.addr, align 2 + // CHECK-NEXT: %19 = call i16 @llvm.ctlz.i16(i16 %18, i1 true) + // CHECK-NEXT: %cast7 = sext i16 %19 to i32 + // CHECK-NEXT: %iszero8 = icmp eq i16 %18, 0 + // CHECK-NEXT: %20 = load i8, ptr %uc.addr, align 1 + // CHECK-NEXT: %conv9 = zext i8 %20 to i32 + // CHECK-NEXT: %clzg10 = select i1 %iszero8, i32 %conv9, i32 %cast7 + // CHECK-NEXT: store volatile i32 %clzg10, ptr %lz, align 4 + lz = __builtin_clzg(ui, s); + // CHECK-NEXT: %21 = load i32, ptr %ui.addr, align 4 + // CHECK-NEXT: %22 = call i32 @llvm.ctlz.i32(i32 %21, i1 true) + // CHECK-NEXT: %iszero11 = icmp eq i32 %21, 0 + // CHECK-NEXT: %23 = load i16, ptr %s.addr, align 2 + // CHECK-NEXT: %conv12 = sext i16 %23 to i32 + // CHECK-NEXT: %clzg13 = select i1 %iszero11, i32 %conv12, i32 %22 + // CHECK-NEXT: store volatile i32 %clzg13, ptr %lz, align 4 + lz = __builtin_clzg(ul, us); + // CHECK-NEXT: %24 = load i64, ptr %ul.addr, align 8 + // CHECK-NEXT: %25 = call i64 @llvm.ctlz.i64(i64 %24, i1 true) + // CHECK-NEXT: %cast14 = trunc i64 %25 to i32 + // CHECK-NEXT: %iszero15 = icmp eq i64 %24, 0 + // CHECK-NEXT: %26 = load i16, ptr %us.addr, align 2 + // CHECK-NEXT: %conv16 = zext i16 %26 to i32 + // CHECK-NEXT: %clzg17 = select i1 %iszero15, i32 %conv16, i32 %cast14 + // CHECK-NEXT: store volatile i32 %clzg17, ptr %lz, align 4 + lz = __builtin_clzg(ull, i); + // CHECK-NEXT: %27 = load i64, ptr %ull.addr, align 8 + // CHECK-NEXT: %28 = call i64 @llvm.ctlz.i64(i64 %27, i1 true) + // CHECK-NEXT: %cast18 = trunc i64 %28 to i32 + // CHECK-NEXT: %iszero19 = icmp eq i64 %27, 0 + // CHECK-NEXT: %29 = load i32, ptr %i.addr, align 4 + // CHECK-NEXT: %clzg20 = select i1 %iszero19, i32 %29, i32 %cast18 + // CHECK-NEXT: store volatile i32 %clzg20, ptr %lz, align 4 + lz = __builtin_clzg(ui128, i); + // CHECK-NEXT: %30 = load i128, ptr %ui128.addr, align 16 + // CHECK-NEXT: %31 = call i128 @llvm.ctlz.i128(i128 %30, i1 true) + // CHECK-NEXT: %cast21 = trunc i128 %31 to i32 + // CHECK-NEXT: %iszero22 = icmp eq i128 %30, 0 + // CHECK-NEXT: %32 = load i32, ptr %i.addr, align 4 + // CHECK-NEXT: %clzg23 = select i1 %iszero22, i32 %32, i32 %cast21 + // CHECK-NEXT: store volatile i32 %clzg23, ptr %lz, align 4 + lz = __builtin_clzg(ubi128, i); + // CHECK-NEXT: %33 = load i128, ptr %ubi128.addr, align 8 + // CHECK-NEXT: %34 = call i128 @llvm.ctlz.i128(i128 %33, i1 true) + // CHECK-NEXT: %cast24 = trunc i128 %34 to i32 + // CHECK-NEXT: %iszero25 = icmp eq i128 %33, 0 + // CHECK-NEXT: %35 = load i32, ptr %i.addr, align 4 + // CHECK-NEXT: %clzg26 = select i1 %iszero25, i32 %35, i32 %cast24 + // CHECK-NEXT: store volatile i32 %clzg26, ptr %lz, align 4 + // CHECK-NEXT: ret void +} + +// CHECK-LABEL: define{{.*}} void @test_builtin_ctzg +void test_builtin_ctzg(unsigned char uc, unsigned short us, unsigned int ui, + unsigned long ul, unsigned long long ull, + unsigned __int128 ui128, unsigned _BitInt(128) ubi128, + signed char sc, short s, int i) { + volatile int tz; + tz = __builtin_ctzg(uc); + // CHECK: %1 = load i8, ptr %uc.addr, align 1 + // CHECK-NEXT: %2 = call i8 @llvm.cttz.i8(i8 %1, i1 true) + // CHECK-NEXT: %cast = sext i8 %2 to i32 + // CHECK-NEXT: store volatile i32 %cast, ptr %tz, align 4 + tz = __builtin_ctzg(us); + // CHECK-NEXT: %3 = load i16, ptr %us.addr, align 2 + // CHECK-NEXT: %4 = call i16 @llvm.cttz.i16(i16 %3, i1 true) + // CHECK-NEXT: %cast1 = sext i16 %4 to i32 + // CHECK-NEXT: store volatile i32 %cast1, ptr %tz, align 4 + tz = __builtin_ctzg(ui); + // CHECK-NEXT: %5 = load i32, ptr %ui.addr, align 4 + // CHECK-NEXT: %6 = call i32 @llvm.cttz.i32(i32 %5, i1 true) + // CHECK-NEXT: store volatile i32 %6, ptr %tz, align 4 + tz = __builtin_ctzg(ul); + // CHECK-NEXT: %7 = load i64, ptr %ul.addr, align 8 + // CHECK-NEXT: %8 = call i64 @llvm.cttz.i64(i64 %7, i1 true) + // CHECK-NEXT: %cast2 = trunc i64 %8 to i32 + // CHECK-NEXT: store volatile i32 %cast2, ptr %tz, align 4 + tz = __builtin_ctzg(ull); + // CHECK-NEXT: %9 = load i64, ptr %ull.addr, align 8 + // CHECK-NEXT: %10 = call i64 @llvm.cttz.i64(i64 %9, i1 true) + // CHECK-NEXT: %cast3 = trunc i64 %10 to i32 + // CHECK-NEXT: store volatile i32 %cast3, ptr %tz, align 4 + tz = __builtin_ctzg(ui128); + // CHECK-NEXT: %11 = load i128, ptr %ui128.addr, align 16 + // CHECK-NEXT: %12 = call i128 @llvm.cttz.i128(i128 %11, i1 true) + // CHECK-NEXT: %cast4 = trunc i128 %12 to i32 + // CHECK-NEXT: store volatile i32 %cast4, ptr %tz, align 4 + tz = __builtin_ctzg(ubi128); + // CHECK-NEXT: %13 = load i128, ptr %ubi128.addr, align 8 + // CHECK-NEXT: %14 = call i128 @llvm.cttz.i128(i128 %13, i1 true) + // CHECK-NEXT: %cast5 = trunc i128 %14 to i32 + // CHECK-NEXT: store volatile i32 %cast5, ptr %tz, align 4 + tz = __builtin_ctzg(uc, sc); + // CHECK-NEXT: %15 = load i8, ptr %uc.addr, align 1 + // CHECK-NEXT: %16 = call i8 @llvm.cttz.i8(i8 %15, i1 true) + // CHECK-NEXT: %cast6 = sext i8 %16 to i32 + // CHECK-NEXT: %iszero = icmp eq i8 %15, 0 + // CHECK-NEXT: %17 = load i8, ptr %sc.addr, align 1 + // CHECK-NEXT: %conv = sext i8 %17 to i32 + // CHECK-NEXT: %ctzg = select i1 %iszero, i32 %conv, i32 %cast6 + // CHECK-NEXT: store volatile i32 %ctzg, ptr %tz, align 4 + tz = __builtin_ctzg(us, uc); + // CHECK-NEXT: %18 = load i16, ptr %us.addr, align 2 + // CHECK-NEXT: %19 = call i16 @llvm.cttz.i16(i16 %18, i1 true) + // CHECK-NEXT: %cast7 = sext i16 %19 to i32 + // CHECK-NEXT: %iszero8 = icmp eq i16 %18, 0 + // CHECK-NEXT: %20 = load i8, ptr %uc.addr, align 1 + // CHECK-NEXT: %conv9 = zext i8 %20 to i32 + // CHECK-NEXT: %ctzg10 = select i1 %iszero8, i32 %conv9, i32 %cast7 + // CHECK-NEXT: store volatile i32 %ctzg10, ptr %tz, align 4 + tz = __builtin_ctzg(ui, s); + // CHECK-NEXT: %21 = load i32, ptr %ui.addr, align 4 + // CHECK-NEXT: %22 = call i32 @llvm.cttz.i32(i32 %21, i1 true) + // CHECK-NEXT: %iszero11 = icmp eq i32 %21, 0 + // CHECK-NEXT: %23 = load i16, ptr %s.addr, align 2 + // CHECK-NEXT: %conv12 = sext i16 %23 to i32 + // CHECK-NEXT: %ctzg13 = select i1 %iszero11, i32 %conv12, i32 %22 + // CHECK-NEXT: store volatile i32 %ctzg13, ptr %tz, align 4 + tz = __builtin_ctzg(ul, us); + // CHECK-NEXT: %24 = load i64, ptr %ul.addr, align 8 + // CHECK-NEXT: %25 = call i64 @llvm.cttz.i64(i64 %24, i1 true) + // CHECK-NEXT: %cast14 = trunc i64 %25 to i32 + // CHECK-NEXT: %iszero15 = icmp eq i64 %24, 0 + // CHECK-NEXT: %26 = load i16, ptr %us.addr, align 2 + // CHECK-NEXT: %conv16 = zext i16 %26 to i32 + // CHECK-NEXT: %ctzg17 = select i1 %iszero15, i32 %conv16, i32 %cast14 + // CHECK-NEXT: store volatile i32 %ctzg17, ptr %tz, align 4 + tz = __builtin_ctzg(ull, i); + // CHECK-NEXT: %27 = load i64, ptr %ull.addr, align 8 + // CHECK-NEXT: %28 = call i64 @llvm.cttz.i64(i64 %27, i1 true) + // CHECK-NEXT: %cast18 = trunc i64 %28 to i32 + // CHECK-NEXT: %iszero19 = icmp eq i64 %27, 0 + // CHECK-NEXT: %29 = load i32, ptr %i.addr, align 4 + // CHECK-NEXT: %ctzg20 = select i1 %iszero19, i32 %29, i32 %cast18 + // CHECK-NEXT: store volatile i32 %ctzg20, ptr %tz, align 4 + tz = __builtin_ctzg(ui128, i); + // CHECK-NEXT: %30 = load i128, ptr %ui128.addr, align 16 + // CHECK-NEXT: %31 = call i128 @llvm.cttz.i128(i128 %30, i1 true) + // CHECK-NEXT: %cast21 = trunc i128 %31 to i32 + // CHECK-NEXT: %iszero22 = icmp eq i128 %30, 0 + // CHECK-NEXT: %32 = load i32, ptr %i.addr, align 4 + // CHECK-NEXT: %ctzg23 = select i1 %iszero22, i32 %32, i32 %cast21 + // CHECK-NEXT: store volatile i32 %ctzg23, ptr %tz, align 4 + tz = __builtin_ctzg(ubi128, i); + // CHECK-NEXT: %33 = load i128, ptr %ubi128.addr, align 8 + // CHECK-NEXT: %34 = call i128 @llvm.cttz.i128(i128 %33, i1 true) + // CHECK-NEXT: %cast24 = trunc i128 %34 to i32 + // CHECK-NEXT: %iszero25 = icmp eq i128 %33, 0 + // CHECK-NEXT: %35 = load i32, ptr %i.addr, align 4 + // CHECK-NEXT: %ctzg26 = select i1 %iszero25, i32 %35, i32 %cast24 + // CHECK-NEXT: store volatile i32 %ctzg26, ptr %tz, align 4 + // CHECK-NEXT: ret void +} + #endif diff --git a/clang/test/CodeGen/ubsan-builtin-checks.c b/clang/test/CodeGen/ubsan-builtin-checks.c index 2bc32d8df485..c7f6078f903b 100644 --- a/clang/test/CodeGen/ubsan-builtin-checks.c +++ b/clang/test/CodeGen/ubsan-builtin-checks.c @@ -23,6 +23,9 @@ void check_ctz(int n) { // CHECK: call void @__ubsan_handle_invalid_builtin __builtin_ctzll(n); + + // CHECK: call void @__ubsan_handle_invalid_builtin + __builtin_ctzg((unsigned int)n); } // CHECK: define{{.*}} void @check_clz @@ -44,4 +47,7 @@ void check_clz(int n) { // CHECK: call void @__ubsan_handle_invalid_builtin __builtin_clzll(n); + + // CHECK: call void @__ubsan_handle_invalid_builtin + __builtin_clzg((unsigned int)n); } diff --git a/clang/test/Sema/builtin-popcountg.c b/clang/test/Sema/builtin-popcountg.c deleted file mode 100644 index 9d095927d24e..000000000000 --- a/clang/test/Sema/builtin-popcountg.c +++ /dev/null @@ -1,23 +0,0 @@ -// RUN: %clang_cc1 -std=c23 -triple=x86_64-pc-linux-gnu -fsyntax-only -verify -Wpedantic %s - -typedef int int2 __attribute__((ext_vector_type(2))); - -void test_builtin_popcountg(short s, int i, __int128 i128, _BitInt(128) bi128, - double d, int2 i2) { - __builtin_popcountg(); - // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} - __builtin_popcountg(i, i); - // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} - __builtin_popcountg(s); - // expected-error@-1 {{1st argument must be a type of unsigned integer (was 'short')}} - __builtin_popcountg(i); - // expected-error@-1 {{1st argument must be a type of unsigned integer (was 'int')}} - __builtin_popcountg(i128); - // expected-error@-1 {{1st argument must be a type of unsigned integer (was '__int128')}} - __builtin_popcountg(bi128); - // expected-error@-1 {{1st argument must be a type of unsigned integer (was '_BitInt(128)')}} - __builtin_popcountg(d); - // expected-error@-1 {{1st argument must be a type of unsigned integer (was 'double')}} - __builtin_popcountg(i2); - // expected-error@-1 {{1st argument must be a type of unsigned integer (was 'int2' (vector of 2 'int' values))}} -} diff --git a/clang/test/Sema/count-builtins.c b/clang/test/Sema/count-builtins.c new file mode 100644 index 000000000000..79fa812f3f20 --- /dev/null +++ b/clang/test/Sema/count-builtins.c @@ -0,0 +1,87 @@ +// RUN: %clang_cc1 -std=c23 -triple=x86_64-pc-linux-gnu -fsyntax-only -verify -Wpedantic %s + +typedef int int2 __attribute__((ext_vector_type(2))); + +void test_builtin_popcountg(short s, int i, __int128 i128, _BitInt(128) bi128, + double d, int2 i2) { + __builtin_popcountg(); + // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} + __builtin_popcountg(i, i); + // expected-error@-1 {{too many arguments to function call, expected 1, have 2}} + __builtin_popcountg(s); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'short')}} + __builtin_popcountg(i); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'int')}} + __builtin_popcountg(i128); + // expected-error@-1 {{1st argument must be an unsigned integer (was '__int128')}} + __builtin_popcountg(bi128); + // expected-error@-1 {{1st argument must be an unsigned integer (was '_BitInt(128)')}} + __builtin_popcountg(d); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'double')}} + __builtin_popcountg(i2); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'int2' (vector of 2 'int' values))}} +} + +void test_builtin_clzg(short s, int i, unsigned int ui, __int128 i128, + _BitInt(128) bi128, double d, int2 i2) { + __builtin_clzg(); + // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} + __builtin_clzg(i, i, i); + // expected-error@-1 {{too many arguments to function call, expected at most 2, have 3}} + __builtin_clzg(s); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'short')}} + __builtin_clzg(i); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'int')}} + __builtin_clzg(i128); + // expected-error@-1 {{1st argument must be an unsigned integer (was '__int128')}} + __builtin_clzg(bi128); + // expected-error@-1 {{1st argument must be an unsigned integer (was '_BitInt(128)')}} + __builtin_clzg(d); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'double')}} + __builtin_clzg(i2); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'int2' (vector of 2 'int' values))}} + __builtin_clzg(i2); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'int2' (vector of 2 'int' values))}} + __builtin_clzg(ui, ui); + // expected-error@-1 {{2nd argument must be an 'int' (was 'unsigned int')}} + __builtin_clzg(ui, i128); + // expected-error@-1 {{2nd argument must be an 'int' (was '__int128')}} + __builtin_clzg(ui, bi128); + // expected-error@-1 {{2nd argument must be an 'int' (was '_BitInt(128)')}} + __builtin_clzg(ui, d); + // expected-error@-1 {{2nd argument must be an 'int' (was 'double')}} + __builtin_clzg(ui, i2); + // expected-error@-1 {{2nd argument must be an 'int' (was 'int2' (vector of 2 'int' values))}} +} + +void test_builtin_ctzg(short s, int i, unsigned int ui, __int128 i128, + _BitInt(128) bi128, double d, int2 i2) { + __builtin_ctzg(); + // expected-error@-1 {{too few arguments to function call, expected 1, have 0}} + __builtin_ctzg(i, i, i); + // expected-error@-1 {{too many arguments to function call, expected at most 2, have 3}} + __builtin_ctzg(s); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'short')}} + __builtin_ctzg(i); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'int')}} + __builtin_ctzg(i128); + // expected-error@-1 {{1st argument must be an unsigned integer (was '__int128')}} + __builtin_ctzg(bi128); + // expected-error@-1 {{1st argument must be an unsigned integer (was '_BitInt(128)')}} + __builtin_ctzg(d); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'double')}} + __builtin_ctzg(i2); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'int2' (vector of 2 'int' values))}} + __builtin_ctzg(i2); + // expected-error@-1 {{1st argument must be an unsigned integer (was 'int2' (vector of 2 'int' values))}} + __builtin_ctzg(ui, ui); + // expected-error@-1 {{2nd argument must be an 'int' (was 'unsigned int')}} + __builtin_ctzg(ui, i128); + // expected-error@-1 {{2nd argument must be an 'int' (was '__int128')}} + __builtin_ctzg(ui, bi128); + // expected-error@-1 {{2nd argument must be an 'int' (was '_BitInt(128)')}} + __builtin_ctzg(ui, d); + // expected-error@-1 {{2nd argument must be an 'int' (was 'double')}} + __builtin_ctzg(ui, i2); + // expected-error@-1 {{2nd argument must be an 'int' (was 'int2' (vector of 2 'int' values))}} +} -- GitLab From 6eff53b4f07c1d8f6ae271254499ec087f40cc83 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Thu, 21 Mar 2024 09:35:18 -0700 Subject: [PATCH 172/296] [libc][stdio] implement rename via SYS_renameat2 (#86140) SYS_rename may be unavailable on architectures such as aarch64 and riscv. rename can be implemented in terms of SYS_rename, SYS_renameat, or SYS_renameat2. I don't have a full picture of the history here, but it seems that SYS_renameat might also be unavailable on some platforms. `man 2 rename` mentions that SYS_renameat2 was added in Linux 3.15. We don't need to support such ancient kernel versions prior. Link: #84980 Link: #85068 --- libc/src/stdio/linux/rename.cpp | 4 +++- libc/test/src/stdio/rename_test.cpp | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libc/src/stdio/linux/rename.cpp b/libc/src/stdio/linux/rename.cpp index f3d684249ad2..379a6ef3c0c8 100644 --- a/libc/src/stdio/linux/rename.cpp +++ b/libc/src/stdio/linux/rename.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "src/stdio/rename.h" +#include "include/llvm-libc-macros/linux/fcntl-macros.h" #include "src/__support/OSUtil/syscall.h" // For internal syscall function. #include "src/__support/common.h" #include "src/errno/libc_errno.h" @@ -15,7 +16,8 @@ namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(int, rename, (const char *oldpath, const char *newpath)) { - int ret = LIBC_NAMESPACE::syscall_impl(SYS_rename, oldpath, newpath); + int ret = LIBC_NAMESPACE::syscall_impl(SYS_renameat2, AT_FDCWD, oldpath, + AT_FDCWD, newpath, 0); if (ret >= 0) return 0; diff --git a/libc/test/src/stdio/rename_test.cpp b/libc/test/src/stdio/rename_test.cpp index 3ed39fe8c0eb..a5dd734c6361 100644 --- a/libc/test/src/stdio/rename_test.cpp +++ b/libc/test/src/stdio/rename_test.cpp @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#include "include/llvm-libc-macros/linux/unistd-macros.h" #include "include/llvm-libc-macros/linux/sys-stat-macros.h" +#include "include/llvm-libc-macros/linux/unistd-macros.h" #include "src/errno/libc_errno.h" #include "src/fcntl/open.h" #include "src/stdio/rename.h" -- GitLab From 69429276098df2f2cf67dcab1c96ce8f56280c11 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 21 Mar 2024 16:37:27 +0000 Subject: [PATCH 173/296] [DAG] combineConcatVectorOfScalars - stop always creating UNDEF nodes. NFC. Noticed in debug logs - most calls to visitVECTOR_SHUFFLE resulted into wasteful UNDEF node creations, despite almost never being used. --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index c83793d15b2e..7009f375df11 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -23447,9 +23447,7 @@ static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG) { SDLoc DL(N); EVT VT = N->getValueType(0); SmallVector Ops; - EVT SVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits()); - SDValue ScalarUndef = DAG.getNode(ISD::UNDEF, DL, SVT); // Keep track of what we encounter. bool AnyInteger = false; @@ -23459,7 +23457,7 @@ static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG) { !Op.getOperand(0).getValueType().isVector()) Ops.push_back(Op.getOperand(0)); else if (ISD::UNDEF == Op.getOpcode()) - Ops.push_back(ScalarUndef); + Ops.push_back(DAG.getNode(ISD::UNDEF, DL, SVT)); else return SDValue(); @@ -23479,13 +23477,12 @@ static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG) { // Replace UNDEFs by another scalar UNDEF node, of the final desired type. if (AnyFP) { SVT = EVT::getFloatingPointVT(OpVT.getSizeInBits()); - ScalarUndef = DAG.getNode(ISD::UNDEF, DL, SVT); if (AnyInteger) { for (SDValue &Op : Ops) { if (Op.getValueType() == SVT) continue; if (Op.isUndef()) - Op = ScalarUndef; + Op = DAG.getNode(ISD::UNDEF, DL, SVT); else Op = DAG.getBitcast(SVT, Op); } -- GitLab From 4bf8dc1a0f9546afb2c13c121e34237ce16cfca6 Mon Sep 17 00:00:00 2001 From: Ilya Biryukov Date: Thu, 21 Mar 2024 17:48:13 +0100 Subject: [PATCH 174/296] [libc++] Remove macros for keeping std::allocator members and void specialization after C++20 (#85806) Fixes #75975. Remove `_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS` for the LLVM 19 release, it was previously marked as deprecated in LLVM 18. I believe that `_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION` was only used by Google in conjunction with `_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS`. Removing both macros together should not cause any issues in practice, even though we did not announce the removal of `_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION` before. --- libcxx/docs/ReleaseNotes/19.rst | 3 +- libcxx/docs/UsingLibcxx.rst | 12 --- libcxx/include/__config | 2 - libcxx/include/__memory/allocator.h | 16 ++-- ....cxx2a.pass.cpp => address.cxx20.pass.cpp} | 33 ++++---- .../address.cxx20.verify.cpp | 42 ++++++++++ .../address.depr_in_cxx17.verify.cpp | 4 +- ...cxx2a.pass.cpp => allocate.cxx20.pass.cpp} | 31 ++++---- .../allocate.cxx20.verify.cpp | 23 ++++++ .../allocate.cxx2a.verify.cpp | 28 ------- .../allocate.depr_in_cxx17.verify.cpp | 4 +- ...xx2a.pass.cpp => construct.cxx20.pass.cpp} | 53 ++++++------- .../construct.cxx20.verify.cpp | 77 +++++++++++++++++++ ...cxx2a.pass.cpp => max_size.cxx20.pass.cpp} | 15 ++-- .../max_size.cxx20.verify.cpp | 32 ++++++++ ...ass.cpp => allocator_types.cxx20.pass.cpp} | 24 +++--- ...ed_allocator_members.deprecated.verify.cpp | 20 ----- ...cxx20_allocator_void_no_members.verify.cpp | 25 ------ ...xx20_with_removed_members.compile.pass.cpp | 22 ------ .../containers/sequences/deque/types.pass.cpp | 25 +++--- .../containers/sequences/list/types.pass.cpp | 13 ++-- .../sequences/vector/types.pass.cpp | 25 +++--- 22 files changed, 291 insertions(+), 238 deletions(-) rename libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/{address.cxx2a.pass.cpp => address.cxx20.pass.cpp} (52%) create mode 100644 libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx20.verify.cpp rename libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/{allocate.cxx2a.pass.cpp => allocate.cxx20.pass.cpp} (82%) create mode 100644 libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx20.verify.cpp delete mode 100644 libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx2a.verify.cpp rename libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/{construct.cxx2a.pass.cpp => construct.cxx20.pass.cpp} (75%) create mode 100644 libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx20.verify.cpp rename libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/{max_size.cxx2a.pass.cpp => max_size.cxx20.pass.cpp} (56%) create mode 100644 libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx20.verify.cpp rename libcxx/test/libcxx/depr/depr.default.allocator/{allocator_types.cxx2a.pass.cpp => allocator_types.cxx20.pass.cpp} (59%) delete mode 100644 libcxx/test/libcxx/depr/depr.default.allocator/enable_removed_allocator_members.deprecated.verify.cpp delete mode 100644 libcxx/test/libcxx/utilities/memory/default.allocator/allocator_types.void.cxx20_allocator_void_no_members.verify.cpp delete mode 100644 libcxx/test/libcxx/utilities/memory/default.allocator/allocator_types.void.cxx20_with_removed_members.compile.pass.cpp diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst index c70ae477fafc..cac42f9c3c3f 100644 --- a/libcxx/docs/ReleaseNotes/19.rst +++ b/libcxx/docs/ReleaseNotes/19.rst @@ -70,7 +70,8 @@ Deprecations and Removals - The ``_LIBCPP_ENABLE_NARROWING_CONVERSIONS_IN_VARIANT`` macro that changed the behavior for narrowing conversions in ``std::variant`` has been removed in LLVM 19. -- TODO: The ``_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS`` macro has been removed in LLVM 19. +- The ``_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS`` and ``_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION`` + macros have been removed in LLVM 19. - TODO: The ``_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES`` and ``_LIBCPP_ENABLE_CXX20_REMOVED_FEATURES`` macros have been removed in LLVM 19. C++17 and C++20 removed features can still be re-enabled individually. diff --git a/libcxx/docs/UsingLibcxx.rst b/libcxx/docs/UsingLibcxx.rst index 3b1be286c169..ac12b0b96950 100644 --- a/libcxx/docs/UsingLibcxx.rst +++ b/libcxx/docs/UsingLibcxx.rst @@ -244,18 +244,6 @@ C++20 Specific Configuration Macros This macro is deprecated and will be removed in LLVM-19. Use the individual macros listed below. -**_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS**: - This macro is used to re-enable redundant members of `allocator`, - including `pointer`, `reference`, `rebind`, `address`, `max_size`, - `construct`, `destroy`, and the two-argument overload of `allocate`. - This macro has been deprecated and will be removed in LLVM-19. - -**_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION**: - This macro is used to re-enable the library-provided specializations of - `allocator` and `allocator`. - Use it in conjunction with `_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS` - to ensure that removed members of `allocator` can be accessed. - **_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS**: This macro is used to re-enable the `argument_type`, `result_type`, `first_argument_type`, and `second_argument_type` members of class diff --git a/libcxx/include/__config b/libcxx/include/__config index 11e13e0c2498..132cace3d5b2 100644 --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -1238,8 +1238,6 @@ __sanitizer_verify_double_ended_contiguous_container(const void*, const void*, c # endif // _LIBCPP_ENABLE_CXX17_REMOVED_FEATURES # if defined(_LIBCPP_ENABLE_CXX20_REMOVED_FEATURES) -# define _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -# define _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION # define _LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS # define _LIBCPP_ENABLE_CXX20_REMOVED_NEGATORS # define _LIBCPP_ENABLE_CXX20_REMOVED_RAW_STORAGE_ITERATOR diff --git a/libcxx/include/__memory/allocator.h b/libcxx/include/__memory/allocator.h index 4e6303914c38..26e5d4978b15 100644 --- a/libcxx/include/__memory/allocator.h +++ b/libcxx/include/__memory/allocator.h @@ -31,18 +31,12 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class allocator; -#if defined(_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS) && !defined(_LIBCPP_DISABLE_DEPRECATION_WARNINGS) -# pragma clang deprecated( \ - _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS, \ - "_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS is deprecated in LLVM 18 and will be removed in LLVM 19") -#endif - -#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION) +#if _LIBCPP_STD_VER <= 17 // These specializations shouldn't be marked _LIBCPP_DEPRECATED_IN_CXX17. // Specializing allocator is deprecated, but not using it. template <> class _LIBCPP_TEMPLATE_VIS allocator { -# if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS) +# if _LIBCPP_STD_VER <= 17 public: _LIBCPP_DEPRECATED_IN_CXX17 typedef void* pointer; @@ -58,7 +52,7 @@ public: template <> class _LIBCPP_TEMPLATE_VIS allocator { -# if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS) +# if _LIBCPP_STD_VER <= 17 public: _LIBCPP_DEPRECATED_IN_CXX17 typedef const void* pointer; @@ -141,7 +135,7 @@ public: } // C++20 Removed members -#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS) +#if _LIBCPP_STD_VER <= 17 _LIBCPP_DEPRECATED_IN_CXX17 typedef _Tp* pointer; _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp* const_pointer; _LIBCPP_DEPRECATED_IN_CXX17 typedef _Tp& reference; @@ -221,7 +215,7 @@ public: } // C++20 Removed members -#if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS) +#if _LIBCPP_STD_VER <= 17 _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp* pointer; _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp* const_pointer; _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp& reference; diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx2a.pass.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx20.pass.cpp similarity index 52% rename from libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx2a.pass.cpp rename to libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx20.pass.cpp index 59657ca46a14..d9a65eee4c13 100644 --- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx2a.pass.cpp +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx20.pass.cpp @@ -12,11 +12,9 @@ // pointer address(reference x) const; // const_pointer address(const_reference x) const; -// In C++20, parts of std::allocator have been removed. -// However, for backwards compatibility, if _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -// is defined before including , then removed members will be restored. +// Removed in C++20, deprecated in C++17. -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS +// REQUIRES: c++03 || c++11 || c++14 || c++17 // ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS #include @@ -25,25 +23,22 @@ #include "test_macros.h" template -void test_address() -{ - T* tp = new T(); - const T* ctp = tp; - const std::allocator a; - assert(a.address(*tp) == tp); - assert(a.address(*ctp) == tp); - delete tp; +void test_address() { + T* tp = new T(); + const T* ctp = tp; + const std::allocator a; + assert(a.address(*tp) == tp); + assert(a.address(*ctp) == tp); + delete tp; } -struct A -{ - void operator&() const {} +struct A { + void operator&() const {} }; -int main(int, char**) -{ - test_address(); - test_address(); +int main(int, char**) { + test_address(); + test_address(); return 0; } diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx20.verify.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx20.verify.cpp new file mode 100644 index 000000000000..21fd4d23449b --- /dev/null +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.cxx20.verify.cpp @@ -0,0 +1,42 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// + +// allocator: +// pointer address(reference x) const; +// const_pointer address(const_reference x) const; + +// In C++20, parts of std::allocator have been removed. +// UNSUPPORTED: c++03, c++11, c++14, c++17 + +#include +#include + +#include "test_macros.h" + +template +void test_address() { + T* tp = new T(); + const T* ctp = tp; + const std::allocator a; + assert(a.address(*tp) == tp); // expected-error 2 {{no member}} + assert(a.address(*ctp) == tp); // expected-error 2 {{no member}} + delete tp; +} + +struct A { + void operator&() const {} +}; + +int main(int, char**) { + test_address(); + test_address(); + + return 0; +} diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.depr_in_cxx17.verify.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.depr_in_cxx17.verify.cpp index 83d059a838ff..4098bdb2ee92 100644 --- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.depr_in_cxx17.verify.cpp +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/address.depr_in_cxx17.verify.cpp @@ -14,9 +14,7 @@ // Deprecated in C++17 -// UNSUPPORTED: c++03, c++11, c++14 - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -Wno-deprecated-pragma +// REQUIRES: c++17 #include diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx2a.pass.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx20.pass.cpp similarity index 82% rename from libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx2a.pass.cpp rename to libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx20.pass.cpp index f2fb606ee6db..8fc6628ebfba 100644 --- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx2a.pass.cpp +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx20.pass.cpp @@ -11,17 +11,18 @@ // allocator: // T* allocate(size_t n, const void* hint); -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS +// Removed in C++20, deprecated in C++17. + +// REQUIRES: c++03 || c++11 || c++14 || c++17 // ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS #include #include -#include // for std::max_align_t +#include // for std::max_align_t #include "test_macros.h" #include "count_new.h" - #ifdef TEST_HAS_NO_ALIGNED_ALLOCATION static const bool UsingAlignedNew = false; #else @@ -36,7 +37,6 @@ static const std::size_t MaxAligned = std::alignment_of::value static const std::size_t OverAligned = MaxAligned * 2; - template struct TEST_ALIGNAS(Align) AlignedType { char data; @@ -48,7 +48,6 @@ struct TEST_ALIGNAS(Align) AlignedType { template int AlignedType::constructed = 0; - template void test_aligned() { typedef AlignedType T; @@ -56,11 +55,11 @@ void test_aligned() { globalMemCounter.reset(); std::allocator a; const bool IsOverAlignedType = Align > MaxAligned; - const bool ExpectAligned = IsOverAlignedType && UsingAlignedNew; + const bool ExpectAligned = IsOverAlignedType && UsingAlignedNew; { - globalMemCounter.last_new_size = 0; + globalMemCounter.last_new_size = 0; globalMemCounter.last_new_align = 0; - T* ap2 = a.allocate(11, (const void*)5); + T* ap2 = a.allocate(11, (const void*)5); DoNotOptimize(ap2); assert(globalMemCounter.checkOutstandingNewEq(1)); assert(globalMemCounter.checkNewCalledEq(1)); @@ -80,14 +79,14 @@ void test_aligned() { } int main(int, char**) { - test_aligned<1>(); - test_aligned<2>(); - test_aligned<4>(); - test_aligned<8>(); - test_aligned<16>(); - test_aligned(); - test_aligned(); - test_aligned(); + test_aligned<1>(); + test_aligned<2>(); + test_aligned<4>(); + test_aligned<8>(); + test_aligned<16>(); + test_aligned(); + test_aligned(); + test_aligned(); return 0; } diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx20.verify.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx20.verify.cpp new file mode 100644 index 000000000000..bf02c7570d39 --- /dev/null +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx20.verify.cpp @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17 + +// + +// allocator: +// T* allocate(size_t n, const void* hint); + +// Removed in C++20. + +#include + +void f() { + std::allocator a; + a.allocate(3, nullptr); // expected-error {{too many arguments to function call}} +} diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx2a.verify.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx2a.verify.cpp deleted file mode 100644 index 2289cd6cd404..000000000000 --- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.cxx2a.verify.cpp +++ /dev/null @@ -1,28 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// UNSUPPORTED: c++03, c++11, c++14, c++17 - -// - -// allocator: -// T* allocate(size_t n, const void* hint); - -// In C++20, parts of std::allocator have been removed. -// However, for backwards compatibility, if _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -// is defined before including , then removed members will be restored. - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS - -#include - -void f() { - std::allocator a; - a.allocate(3, nullptr); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}} -} diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.depr_in_cxx17.verify.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.depr_in_cxx17.verify.cpp index 8b2e862e9503..8629df3c4164 100644 --- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.depr_in_cxx17.verify.cpp +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/allocate.depr_in_cxx17.verify.cpp @@ -13,9 +13,7 @@ // Deprecated in C++17 -// UNSUPPORTED: c++03, c++11, c++14 - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -Wno-deprecated-pragma +// REQUIRES: c++17 #include diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx2a.pass.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx20.pass.cpp similarity index 75% rename from libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx2a.pass.cpp rename to libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx20.pass.cpp index d3a7dadbbe11..9a37cf8af8e6 100644 --- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx2a.pass.cpp +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx20.pass.cpp @@ -11,12 +11,10 @@ // allocator: // template void construct(pointer p, Args&&... args); -// In C++20, parts of std::allocator have been removed. -// However, for backwards compatibility, if _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -// is defined before including , then removed members will be restored. - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS +// In C++20, parts of std::allocator have been removed. +// In C++17, they were deprecated. // ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS +// REQUIRES: c++03 || c++11 || c++14 || c++17 #include #include @@ -26,42 +24,39 @@ int A_constructed = 0; -struct A -{ - int data; - A() {++A_constructed;} +struct A { + int data; + A() { ++A_constructed; } - A(const A&) {++A_constructed;} + A(const A&) { ++A_constructed; } - explicit A(int) {++A_constructed;} - A(int, int*) {++A_constructed;} + explicit A(int) { ++A_constructed; } + A(int, int*) { ++A_constructed; } - ~A() {--A_constructed;} + ~A() { --A_constructed; } }; int move_only_constructed = 0; #if TEST_STD_VER >= 11 -class move_only -{ - move_only(const move_only&) = delete; - move_only& operator=(const move_only&)= delete; +class move_only { + move_only(const move_only&) = delete; + move_only& operator=(const move_only&) = delete; public: - move_only(move_only&&) {++move_only_constructed;} - move_only& operator=(move_only&&) {return *this;} + move_only(move_only&&) { ++move_only_constructed; } + move_only& operator=(move_only&&) { return *this; } - move_only() {++move_only_constructed;} - ~move_only() {--move_only_constructed;} + move_only() { ++move_only_constructed; } + ~move_only() { --move_only_constructed; } public: - int data; // unused other than to make sizeof(move_only) == sizeof(int). - // but public to suppress "-Wunused-private-field" + int data; // unused other than to make sizeof(move_only) == sizeof(int). + // but public to suppress "-Wunused-private-field" }; #endif // TEST_STD_VER >= 11 -int main(int, char**) -{ +int main(int, char**) { globalMemCounter.reset(); { std::allocator a; @@ -69,7 +64,7 @@ int main(int, char**) assert(A_constructed == 0); globalMemCounter.last_new_size = 0; - A* ap = a.allocate(3); + A* ap = a.allocate(3); DoNotOptimize(ap); assert(globalMemCounter.checkOutstandingNewEq(1)); assert(globalMemCounter.checkLastNewSizeEq(3 * sizeof(int))); @@ -113,13 +108,13 @@ int main(int, char**) assert(A_constructed == 0); } #if TEST_STD_VER >= 11 - { + { std::allocator a; assert(globalMemCounter.checkOutstandingNewEq(0)); assert(move_only_constructed == 0); globalMemCounter.last_new_size = 0; - move_only* ap = a.allocate(3); + move_only* ap = a.allocate(3); DoNotOptimize(ap); assert(globalMemCounter.checkOutstandingNewEq(1)); assert(globalMemCounter.checkLastNewSizeEq(3 * sizeof(int))); @@ -145,7 +140,7 @@ int main(int, char**) DoNotOptimize(ap); assert(globalMemCounter.checkOutstandingNewEq(0)); assert(move_only_constructed == 0); - } + } #endif return 0; diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx20.verify.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx20.verify.cpp new file mode 100644 index 000000000000..b39f9d918c95 --- /dev/null +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/construct.cxx20.verify.cpp @@ -0,0 +1,77 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// + +// allocator: +// template void construct(pointer p, Args&&... args); + +// Removed in C++20. + +// UNSUPPORTED: c++03, c++11, c++14, c++17 + +#include +#include + +int A_constructed = 0; + +struct A { + int data; + A() { ++A_constructed; } + + A(const A&) { ++A_constructed; } + + explicit A(int) { ++A_constructed; } + A(int, int*) { ++A_constructed; } + + ~A() { --A_constructed; } +}; + +int move_only_constructed = 0; + +class move_only { + move_only(const move_only&) = delete; + move_only& operator=(const move_only&) = delete; + +public: + move_only(move_only&&) { ++move_only_constructed; } + move_only& operator=(move_only&&) { return *this; } + + move_only() { ++move_only_constructed; } + ~move_only() { --move_only_constructed; } + +public: + int data; // unused other than to make sizeof(move_only) == sizeof(int). + // but public to suppress "-Wunused-private-field" +}; + +int main(int, char**) { + { + std::allocator a; + A* ap = a.allocate(3); + a.construct(ap); // expected-error {{no member}} + a.destroy(ap); // expected-error {{no member}} + a.construct(ap, A()); // expected-error {{no member}} + a.destroy(ap); // expected-error {{no member}} + a.construct(ap, 5); // expected-error {{no member}} + a.destroy(ap); // expected-error {{no member}} + a.construct(ap, 5, (int*)0); // expected-error {{no member}} + a.destroy(ap); // expected-error {{no member}} + a.deallocate(ap, 3); + } + { + std::allocator a; + move_only* ap = a.allocate(3); + a.construct(ap); // expected-error {{no member}} + a.destroy(ap); // expected-error {{no member}} + a.construct(ap, move_only()); // expected-error {{no member}} + a.destroy(ap); // expected-error {{no member}} + a.deallocate(ap, 3); + } + return 0; +} diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx2a.pass.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx20.pass.cpp similarity index 56% rename from libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx2a.pass.cpp rename to libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx20.pass.cpp index b07568355fee..92e3b919b0f7 100644 --- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx2a.pass.cpp +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx20.pass.cpp @@ -11,11 +11,9 @@ // allocator: // size_type max_size() const throw(); -// In C++20, parts of std::allocator have been removed. -// However, for backwards compatibility, if _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -// is defined before including , then removed members will be restored. +// Removed in C++20, deprecated in C++17. -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS +// REQUIRES: c++03 || c++11 || c++14 || c++17 // ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS #include @@ -27,11 +25,10 @@ int new_called = 0; -int main(int, char**) -{ - const std::allocator a; - std::size_t M = a.max_size(); - assert(M > 0xFFFF && M <= (std::numeric_limits::max() / sizeof(int))); +int main(int, char**) { + const std::allocator a; + std::size_t M = a.max_size(); + assert(M > 0xFFFF && M <= (std::numeric_limits::max() / sizeof(int))); return 0; } diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx20.verify.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx20.verify.cpp new file mode 100644 index 000000000000..0e0f3c3f4aa4 --- /dev/null +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator.members/max_size.cxx20.verify.cpp @@ -0,0 +1,32 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// + +// allocator: +// size_type max_size() const throw(); + +// In C++20, parts of std::allocator have been removed. +// UNSUPPORTED: c++03, c++11, c++14, c++17 + +#include +#include +#include +#include + +#include "test_macros.h" + +int new_called = 0; + +int main(int, char**) { + const std::allocator a; + std::size_t M = a.max_size(); // expected-error {{no member}} + assert(M > 0xFFFF && M <= (std::numeric_limits::max() / sizeof(int))); + + return 0; +} diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/allocator_types.cxx2a.pass.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/allocator_types.cxx20.pass.cpp similarity index 59% rename from libcxx/test/libcxx/depr/depr.default.allocator/allocator_types.cxx2a.pass.cpp rename to libcxx/test/libcxx/depr/depr.default.allocator/allocator_types.cxx20.pass.cpp index a6134b04a8f5..e462e07d896c 100644 --- a/libcxx/test/libcxx/depr/depr.default.allocator/allocator_types.cxx2a.pass.cpp +++ b/libcxx/test/libcxx/depr/depr.default.allocator/allocator_types.cxx20.pass.cpp @@ -26,7 +26,9 @@ // ... // }; -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS +// Removed in C++20, deprecated in C++17. + +// REQUIRES: c++03 || c++11 || c++14 || c++17 // ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS #include @@ -35,17 +37,17 @@ template void test() { - static_assert((std::is_same::size_type, std::size_t>::value), ""); - static_assert((std::is_same::difference_type, std::ptrdiff_t>::value), ""); - static_assert((std::is_same::pointer, T*>::value), ""); - static_assert((std::is_same::const_pointer, const T*>::value), ""); - static_assert((std::is_same::reference, T&>::value), ""); - static_assert((std::is_same::const_reference, const T&>::value), ""); - static_assert((std::is_same::template rebind::other, - std::allocator >::value), ""); + static_assert((std::is_same::size_type, std::size_t>::value), ""); + static_assert((std::is_same::difference_type, std::ptrdiff_t>::value), ""); + static_assert((std::is_same::pointer, T*>::value), ""); + static_assert((std::is_same::const_pointer, const T*>::value), ""); + static_assert((std::is_same::reference, T&>::value), ""); + static_assert((std::is_same::const_reference, const T&>::value), ""); + static_assert( + (std::is_same::template rebind::other, std::allocator >::value), ""); } int main(int, char**) { - test(); - return 0; + test(); + return 0; } diff --git a/libcxx/test/libcxx/depr/depr.default.allocator/enable_removed_allocator_members.deprecated.verify.cpp b/libcxx/test/libcxx/depr/depr.default.allocator/enable_removed_allocator_members.deprecated.verify.cpp deleted file mode 100644 index ab6495ea9db4..000000000000 --- a/libcxx/test/libcxx/depr/depr.default.allocator/enable_removed_allocator_members.deprecated.verify.cpp +++ /dev/null @@ -1,20 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// - -// Ensure that defining _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS yields a -// deprecation warning. We intend to issue a deprecation warning in LLVM 18 -// and remove the macro entirely in LLVM 19. As such, this test will be quite -// short lived. - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS - -// UNSUPPORTED: clang-modules-build - -#include // expected-warning@* 1+ {{macro '_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS' has been marked as deprecated}} diff --git a/libcxx/test/libcxx/utilities/memory/default.allocator/allocator_types.void.cxx20_allocator_void_no_members.verify.cpp b/libcxx/test/libcxx/utilities/memory/default.allocator/allocator_types.void.cxx20_allocator_void_no_members.verify.cpp deleted file mode 100644 index 8888683a044f..000000000000 --- a/libcxx/test/libcxx/utilities/memory/default.allocator/allocator_types.void.cxx20_allocator_void_no_members.verify.cpp +++ /dev/null @@ -1,25 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// Check that members of std::allocator are not provided in C++20 -// with _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION but without -// _LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS. - -// UNSUPPORTED: c++03, c++11, c++14, c++17 -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION -// -// Ignore any extra errors arising from typo correction. -// ADDITIONAL_COMPILE_FLAGS: -Xclang -verify-ignore-unexpected=error - -#include - -std::allocator::pointer x; // expected-error-re {{no {{(type|template)}} named 'pointer'}} -std::allocator::const_pointer y; // expected-error-re {{no {{(type|template)}} named 'const_pointer'}} -std::allocator::value_type z; // expected-error-re {{no {{(type|template)}} named 'value_type'}} -std::allocator::rebind::other t; // expected-error-re {{no {{(type|template)}} named 'rebind'}} diff --git a/libcxx/test/libcxx/utilities/memory/default.allocator/allocator_types.void.cxx20_with_removed_members.compile.pass.cpp b/libcxx/test/libcxx/utilities/memory/default.allocator/allocator_types.void.cxx20_with_removed_members.compile.pass.cpp deleted file mode 100644 index 3f151edefe1c..000000000000 --- a/libcxx/test/libcxx/utilities/memory/default.allocator/allocator_types.void.cxx20_with_removed_members.compile.pass.cpp +++ /dev/null @@ -1,22 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -// Check that the nested types of std::allocator are provided in C++20 -// with a flag that keeps the removed members. - -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION - -#include -#include - -static_assert((std::is_same::pointer, void*>::value), ""); -static_assert((std::is_same::const_pointer, const void*>::value), ""); -static_assert((std::is_same::value_type, void>::value), ""); -static_assert((std::is_same::rebind::other, std::allocator >::value), ""); diff --git a/libcxx/test/std/containers/sequences/deque/types.pass.cpp b/libcxx/test/std/containers/sequences/deque/types.pass.cpp index bfe4808f863d..8c14de0c7744 100644 --- a/libcxx/test/std/containers/sequences/deque/types.pass.cpp +++ b/libcxx/test/std/containers/sequences/deque/types.pass.cpp @@ -28,9 +28,6 @@ // typedef std::reverse_iterator const_reverse_iterator; // }; -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS - #include #include #include @@ -47,14 +44,22 @@ test() typedef std::deque C; static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); + static_assert( + (std::is_same::value_type>::value), ""); static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); + static_assert( + (std::is_same::size_type>::value), ""); + static_assert( + (std::is_same::difference_type>::value), + ""); + static_assert( + (std::is_same::value_type&>::value), ""); + static_assert((std::is_same::value_type&>::value), + ""); + static_assert((std::is_same::pointer>::value), ""); + static_assert( + (std::is_same::const_pointer>::value), ""); static_assert((std::is_same< typename std::iterator_traits::iterator_category, std::random_access_iterator_tag>::value), ""); diff --git a/libcxx/test/std/containers/sequences/list/types.pass.cpp b/libcxx/test/std/containers/sequences/list/types.pass.cpp index 0c1ca27745ce..8fe31e3949de 100644 --- a/libcxx/test/std/containers/sequences/list/types.pass.cpp +++ b/libcxx/test/std/containers/sequences/list/types.pass.cpp @@ -21,9 +21,6 @@ // typedef typename allocator_type::pointer pointer; // typedef typename allocator_type::const_pointer const_pointer; -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS - #include #include @@ -38,10 +35,12 @@ int main(int, char**) typedef std::list C; static_assert((std::is_same::value), ""); static_assert((std::is_same >::value), ""); - static_assert((std::is_same::reference>::value), ""); - static_assert((std::is_same::const_reference>::value), ""); - static_assert((std::is_same::pointer>::value), ""); - static_assert((std::is_same::const_pointer>::value), ""); + static_assert((std::is_same >::value_type&>::value), ""); + static_assert( + (std::is_same >::value_type&>::value), ""); + static_assert((std::is_same >::pointer>::value), ""); + static_assert( + (std::is_same >::const_pointer>::value), ""); static_assert((std::is_signed::value), ""); static_assert((std::is_unsigned::value), ""); diff --git a/libcxx/test/std/containers/sequences/vector/types.pass.cpp b/libcxx/test/std/containers/sequences/vector/types.pass.cpp index 4bcfbe7c3ea4..f4d7fa088842 100644 --- a/libcxx/test/std/containers/sequences/vector/types.pass.cpp +++ b/libcxx/test/std/containers/sequences/vector/types.pass.cpp @@ -28,9 +28,6 @@ // typedef std::reverse_iterator const_reverse_iterator; // }; -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS -// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS - #include #include #include @@ -52,14 +49,22 @@ test() // blindly pulling typedefs out of the allocator. This is why we can't call // test>() below. static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); + static_assert( + (std::is_same::value_type>::value), ""); static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); - static_assert((std::is_same::value), ""); + static_assert( + (std::is_same::size_type>::value), ""); + static_assert( + (std::is_same::difference_type>::value), + ""); + static_assert( + (std::is_same::value_type&>::value), ""); + static_assert((std::is_same::value_type&>::value), + ""); + static_assert((std::is_same::pointer>::value), ""); + static_assert( + (std::is_same::const_pointer>::value), ""); static_assert((std::is_signed::value), ""); static_assert((std::is_unsigned::value), ""); -- GitLab From 797336b1278cb7a8b788e467f8fbbc11939143a8 Mon Sep 17 00:00:00 2001 From: Janek van Oirschot <5994977+JanekvO@users.noreply.github.com> Date: Thu, 21 Mar 2024 17:19:54 +0000 Subject: [PATCH 175/296] Revert "[AMDGPU] MCExpr-ify MC layer kernel descriptor" (#86151) Reverts llvm/llvm-project#80855 --- llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp | 40 +- llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h | 11 +- .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 193 +++------ .../MCTargetDesc/AMDGPUMCKernelDescriptor.cpp | 32 -- .../MCTargetDesc/AMDGPUMCKernelDescriptor.h | 51 --- .../MCTargetDesc/AMDGPUTargetStreamer.cpp | 404 +++++++----------- .../MCTargetDesc/AMDGPUTargetStreamer.h | 33 +- .../Target/AMDGPU/MCTargetDesc/CMakeLists.txt | 1 - .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 88 ++-- llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 10 +- llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s | 27 -- llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s | 281 ------------ llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s | 190 -------- llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s | 186 -------- llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s | 184 -------- llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s | 168 -------- llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s | 171 -------- llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s | 148 ------- llvm/test/MC/AMDGPU/hsa-tg-split.s | 74 ---- 19 files changed, 304 insertions(+), 1988 deletions(-) delete mode 100644 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp delete mode 100644 llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h delete mode 100644 llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s delete mode 100644 llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s delete mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s delete mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s delete mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s delete mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s delete mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s delete mode 100644 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s delete mode 100644 llvm/test/MC/AMDGPU/hsa-tg-split.s diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp index 052b231d62a3..72e8b59e0a40 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp @@ -22,7 +22,6 @@ #include "AMDKernelCodeT.h" #include "GCNSubtarget.h" #include "MCTargetDesc/AMDGPUInstPrinter.h" -#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h" #include "MCTargetDesc/AMDGPUTargetStreamer.h" #include "R600AsmPrinter.h" #include "SIMachineFunctionInfo.h" @@ -429,43 +428,38 @@ uint16_t AMDGPUAsmPrinter::getAmdhsaKernelCodeProperties( return KernelCodeProperties; } -MCKernelDescriptor -AMDGPUAsmPrinter::getAmdhsaKernelDescriptor(const MachineFunction &MF, - const SIProgramInfo &PI) const { +amdhsa::kernel_descriptor_t AMDGPUAsmPrinter::getAmdhsaKernelDescriptor( + const MachineFunction &MF, + const SIProgramInfo &PI) const { const GCNSubtarget &STM = MF.getSubtarget(); const Function &F = MF.getFunction(); const SIMachineFunctionInfo *Info = MF.getInfo(); - MCContext &Ctx = MF.getContext(); - MCKernelDescriptor KernelDescriptor; + amdhsa::kernel_descriptor_t KernelDescriptor; + memset(&KernelDescriptor, 0x0, sizeof(KernelDescriptor)); assert(isUInt<32>(PI.ScratchSize)); assert(isUInt<32>(PI.getComputePGMRSrc1(STM))); assert(isUInt<32>(PI.getComputePGMRSrc2())); - KernelDescriptor.group_segment_fixed_size = - MCConstantExpr::create(PI.LDSSize, Ctx); - KernelDescriptor.private_segment_fixed_size = - MCConstantExpr::create(PI.ScratchSize, Ctx); + KernelDescriptor.group_segment_fixed_size = PI.LDSSize; + KernelDescriptor.private_segment_fixed_size = PI.ScratchSize; Align MaxKernArgAlign; - KernelDescriptor.kernarg_size = MCConstantExpr::create( - STM.getKernArgSegmentSize(F, MaxKernArgAlign), Ctx); + KernelDescriptor.kernarg_size = STM.getKernArgSegmentSize(F, MaxKernArgAlign); - KernelDescriptor.compute_pgm_rsrc1 = - MCConstantExpr::create(PI.getComputePGMRSrc1(STM), Ctx); - KernelDescriptor.compute_pgm_rsrc2 = - MCConstantExpr::create(PI.getComputePGMRSrc2(), Ctx); - KernelDescriptor.kernel_code_properties = - MCConstantExpr::create(getAmdhsaKernelCodeProperties(MF), Ctx); + KernelDescriptor.compute_pgm_rsrc1 = PI.getComputePGMRSrc1(STM); + KernelDescriptor.compute_pgm_rsrc2 = PI.getComputePGMRSrc2(); + KernelDescriptor.kernel_code_properties = getAmdhsaKernelCodeProperties(MF); assert(STM.hasGFX90AInsts() || CurrentProgramInfo.ComputePGMRSrc3GFX90A == 0); - KernelDescriptor.compute_pgm_rsrc3 = MCConstantExpr::create( - STM.hasGFX90AInsts() ? CurrentProgramInfo.ComputePGMRSrc3GFX90A : 0, Ctx); + if (STM.hasGFX90AInsts()) + KernelDescriptor.compute_pgm_rsrc3 = + CurrentProgramInfo.ComputePGMRSrc3GFX90A; - KernelDescriptor.kernarg_preload = MCConstantExpr::create( - AMDGPU::hasKernargPreload(STM) ? Info->getNumKernargPreloadedSGPRs() : 0, - Ctx); + if (AMDGPU::hasKernargPreload(STM)) + KernelDescriptor.kernarg_preload = + static_cast(Info->getNumKernargPreloadedSGPRs()); return KernelDescriptor; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h index b8b2718d293e..79326cd3d328 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h @@ -28,12 +28,15 @@ class MCCodeEmitter; class MCOperand; namespace AMDGPU { -struct MCKernelDescriptor; namespace HSAMD { class MetadataStreamer; } } // namespace AMDGPU +namespace amdhsa { +struct kernel_descriptor_t; +} + class AMDGPUAsmPrinter final : public AsmPrinter { private: unsigned CodeObjectVersion; @@ -72,9 +75,9 @@ private: uint16_t getAmdhsaKernelCodeProperties( const MachineFunction &MF) const; - AMDGPU::MCKernelDescriptor - getAmdhsaKernelDescriptor(const MachineFunction &MF, - const SIProgramInfo &PI) const; + amdhsa::kernel_descriptor_t getAmdhsaKernelDescriptor( + const MachineFunction &MF, + const SIProgramInfo &PI) const; void initTargetStreamer(Module &M); diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp index 38850f5acadd..529705479646 100644 --- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -8,7 +8,6 @@ #include "AMDKernelCodeT.h" #include "MCTargetDesc/AMDGPUMCExpr.h" -#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "MCTargetDesc/AMDGPUTargetStreamer.h" #include "SIDefines.h" @@ -5418,8 +5417,7 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { if (getParser().parseIdentifier(KernelName)) return true; - AMDGPU::MCKernelDescriptor KD = - getDefaultAmdhsaKernelDescriptor(&getSTI(), getContext()); + kernel_descriptor_t KD = getDefaultAmdhsaKernelDescriptor(&getSTI()); StringSet<> Seen; @@ -5459,111 +5457,89 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { return TokError(".amdhsa_ directives cannot be repeated"); SMLoc ValStart = getLoc(); - const MCExpr *ExprVal; - if (getParser().parseExpression(ExprVal)) + int64_t IVal; + if (getParser().parseAbsoluteExpression(IVal)) return true; SMLoc ValEnd = getLoc(); SMRange ValRange = SMRange(ValStart, ValEnd); - int64_t IVal = 0; + if (IVal < 0) + return OutOfRangeError(ValRange); + uint64_t Val = IVal; - bool EvaluatableExpr; - if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(IVal))) { - if (IVal < 0) - return OutOfRangeError(ValRange); - Val = IVal; - } #define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \ - if (!isUInt(Val)) \ + if (!isUInt(VALUE)) \ return OutOfRangeError(RANGE); \ - AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY, \ - getContext()); - -// Some fields use the parsed value immediately which requires the expression to -// be solvable. -#define EXPR_RESOLVE_OR_ERROR(RESOLVED) \ - if (!(RESOLVED)) \ - return Error(IDRange.Start, "directive should have resolvable expression", \ - IDRange); + AMDHSA_BITS_SET(FIELD, ENTRY, VALUE); if (ID == ".amdhsa_group_segment_fixed_size") { - if (!isUInt(Val)) + if (!isUInt(Val)) return OutOfRangeError(ValRange); - KD.group_segment_fixed_size = ExprVal; + KD.group_segment_fixed_size = Val; } else if (ID == ".amdhsa_private_segment_fixed_size") { - if (!isUInt(Val)) + if (!isUInt(Val)) return OutOfRangeError(ValRange); - KD.private_segment_fixed_size = ExprVal; + KD.private_segment_fixed_size = Val; } else if (ID == ".amdhsa_kernarg_size") { - if (!isUInt(Val)) + if (!isUInt(Val)) return OutOfRangeError(ValRange); - KD.kernarg_size = ExprVal; + KD.kernarg_size = Val; } else if (ID == ".amdhsa_user_sgpr_count") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); ExplicitUserSGPRCount = Val; } else if (ID == ".amdhsa_user_sgpr_private_segment_buffer") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (hasArchitectedFlatScratch()) return Error(IDRange.Start, "directive is not supported with architected flat scratch", IDRange); PARSE_BITS_ENTRY(KD.kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER, - ExprVal, ValRange); + Val, ValRange); if (Val) ImpliedUserSGPRCount += 4; } else if (ID == ".amdhsa_user_sgpr_kernarg_preload_length") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (!hasKernargPreload()) return Error(IDRange.Start, "directive requires gfx90a+", IDRange); if (Val > getMaxNumUserSGPRs()) return OutOfRangeError(ValRange); - PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_LENGTH, ExprVal, + PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_LENGTH, Val, ValRange); if (Val) { ImpliedUserSGPRCount += Val; PreloadLength = Val; } } else if (ID == ".amdhsa_user_sgpr_kernarg_preload_offset") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (!hasKernargPreload()) return Error(IDRange.Start, "directive requires gfx90a+", IDRange); if (Val >= 1024) return OutOfRangeError(ValRange); - PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_OFFSET, ExprVal, + PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_OFFSET, Val, ValRange); if (Val) PreloadOffset = Val; } else if (ID == ".amdhsa_user_sgpr_dispatch_ptr") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal, + KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, Val, ValRange); if (Val) ImpliedUserSGPRCount += 2; } else if (ID == ".amdhsa_user_sgpr_queue_ptr") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal, + KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, Val, ValRange); if (Val) ImpliedUserSGPRCount += 2; } else if (ID == ".amdhsa_user_sgpr_kernarg_segment_ptr") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR, - ExprVal, ValRange); + Val, ValRange); if (Val) ImpliedUserSGPRCount += 2; } else if (ID == ".amdhsa_user_sgpr_dispatch_id") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal, + KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, Val, ValRange); if (Val) ImpliedUserSGPRCount += 2; @@ -5572,39 +5548,34 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { return Error(IDRange.Start, "directive is not supported with architected flat scratch", IDRange); - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT, - ExprVal, ValRange); + KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT, Val, + ValRange); if (Val) ImpliedUserSGPRCount += 2; } else if (ID == ".amdhsa_user_sgpr_private_segment_size") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); PARSE_BITS_ENTRY(KD.kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE, - ExprVal, ValRange); + Val, ValRange); if (Val) ImpliedUserSGPRCount += 1; } else if (ID == ".amdhsa_wavefront_size32") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (IVersion.Major < 10) return Error(IDRange.Start, "directive requires gfx10+", IDRange); EnableWavefrontSize32 = Val; PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal, - ValRange); + KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, + Val, ValRange); } else if (ID == ".amdhsa_uses_dynamic_stack") { PARSE_BITS_ENTRY(KD.kernel_code_properties, - KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal, - ValRange); + KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, Val, ValRange); } else if (ID == ".amdhsa_system_sgpr_private_segment_wavefront_offset") { if (hasArchitectedFlatScratch()) return Error(IDRange.Start, "directive is not supported with architected flat scratch", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal, - ValRange); + COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, Val, ValRange); } else if (ID == ".amdhsa_enable_private_segment") { if (!hasArchitectedFlatScratch()) return Error( @@ -5612,48 +5583,42 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { "directive is not supported without architected flat scratch", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal, - ValRange); + COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, Val, ValRange); } else if (ID == ".amdhsa_system_sgpr_workgroup_id_x") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal, + COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, Val, ValRange); } else if (ID == ".amdhsa_system_sgpr_workgroup_id_y") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal, + COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, Val, ValRange); } else if (ID == ".amdhsa_system_sgpr_workgroup_id_z") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal, + COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, Val, ValRange); } else if (ID == ".amdhsa_system_sgpr_workgroup_info") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal, + COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, Val, ValRange); } else if (ID == ".amdhsa_system_vgpr_workitem_id") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal, + COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, Val, ValRange); } else if (ID == ".amdhsa_next_free_vgpr") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); VGPRRange = ValRange; NextFreeVGPR = Val; } else if (ID == ".amdhsa_next_free_sgpr") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); SGPRRange = ValRange; NextFreeSGPR = Val; } else if (ID == ".amdhsa_accum_offset") { if (!isGFX90A()) return Error(IDRange.Start, "directive requires gfx90a+", IDRange); - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); AccumOffset = Val; } else if (ID == ".amdhsa_reserve_vcc") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (!isUInt<1>(Val)) return OutOfRangeError(ValRange); ReserveVCC = Val; } else if (ID == ".amdhsa_reserve_flat_scratch") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (IVersion.Major < 7) return Error(IDRange.Start, "directive requires gfx7+", IDRange); if (hasArchitectedFlatScratch()) @@ -5673,105 +5638,97 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { IDRange); } else if (ID == ".amdhsa_float_round_mode_32") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal, - ValRange); + COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, Val, ValRange); } else if (ID == ".amdhsa_float_round_mode_16_64") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal, - ValRange); + COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, Val, ValRange); } else if (ID == ".amdhsa_float_denorm_mode_32") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal, - ValRange); + COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, Val, ValRange); } else if (ID == ".amdhsa_float_denorm_mode_16_64") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal, + COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, Val, ValRange); } else if (ID == ".amdhsa_dx10_clamp") { if (IVersion.Major >= 12) return Error(IDRange.Start, "directive unsupported on gfx12+", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal, + COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, Val, ValRange); } else if (ID == ".amdhsa_ieee_mode") { if (IVersion.Major >= 12) return Error(IDRange.Start, "directive unsupported on gfx12+", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal, + COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, Val, ValRange); } else if (ID == ".amdhsa_fp16_overflow") { if (IVersion.Major < 9) return Error(IDRange.Start, "directive requires gfx9+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal, + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, Val, ValRange); } else if (ID == ".amdhsa_tg_split") { if (!isGFX90A()) return Error(IDRange.Start, "directive requires gfx90a+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, - ExprVal, ValRange); + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Val, + ValRange); } else if (ID == ".amdhsa_workgroup_processor_mode") { if (IVersion.Major < 10) return Error(IDRange.Start, "directive requires gfx10+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal, + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, Val, ValRange); } else if (ID == ".amdhsa_memory_ordered") { if (IVersion.Major < 10) return Error(IDRange.Start, "directive requires gfx10+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal, + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, Val, ValRange); } else if (ID == ".amdhsa_forward_progress") { if (IVersion.Major < 10) return Error(IDRange.Start, "directive requires gfx10+", IDRange); - PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal, + PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, Val, ValRange); } else if (ID == ".amdhsa_shared_vgpr_count") { - EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); if (IVersion.Major < 10 || IVersion.Major >= 12) return Error(IDRange.Start, "directive requires gfx10 or gfx11", IDRange); SharedVGPRCount = Val; PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, - COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal, + COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, Val, ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_invalid_op") { PARSE_BITS_ENTRY( KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, - ExprVal, ValRange); + COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, Val, + ValRange); } else if (ID == ".amdhsa_exception_fp_denorm_src") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE, - ExprVal, ValRange); + Val, ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_div_zero") { PARSE_BITS_ENTRY( KD.compute_pgm_rsrc2, - COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, - ExprVal, ValRange); + COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, Val, + ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_overflow") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW, - ExprVal, ValRange); + Val, ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_underflow") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW, - ExprVal, ValRange); + Val, ValRange); } else if (ID == ".amdhsa_exception_fp_ieee_inexact") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT, - ExprVal, ValRange); + Val, ValRange); } else if (ID == ".amdhsa_exception_int_div_zero") { PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO, - ExprVal, ValRange); + Val, ValRange); } else if (ID == ".amdhsa_round_robin_scheduling") { if (IVersion.Major < 12) return Error(IDRange.Start, "directive requires gfx12+", IDRange); PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, - COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal, + COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, Val, ValRange); } else { return Error(IDRange.Start, "unknown .amdhsa_kernel directive", IDRange); @@ -5798,18 +5755,15 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { if (!isUInt( VGPRBlocks)) return OutOfRangeError(VGPRRange); - AMDGPU::MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc1, MCConstantExpr::create(VGPRBlocks, getContext()), - COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT, - COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, getContext()); + AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, + COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, VGPRBlocks); if (!isUInt( SGPRBlocks)) return OutOfRangeError(SGPRRange); - AMDGPU::MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc1, MCConstantExpr::create(SGPRBlocks, getContext()), - COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT, - COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, getContext()); + AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, + COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, + SGPRBlocks); if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount) return TokError("amdgpu_user_sgpr_count smaller than than implied by " @@ -5820,17 +5774,11 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { if (!isUInt(UserSGPRCount)) return TokError("too many user SGPRs enabled"); - AMDGPU::MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc2, MCConstantExpr::create(UserSGPRCount, getContext()), - COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_SHIFT, - COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, getContext()); - - int64_t IVal = 0; - if (!KD.kernarg_size->evaluateAsAbsolute(IVal)) - return TokError("Kernarg size should be resolvable"); - uint64_t kernarg_size = IVal; - if (PreloadLength && kernarg_size && - (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size)) + AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, + UserSGPRCount); + + if (PreloadLength && KD.kernarg_size && + (PreloadLength * 4 + PreloadOffset * 4 > KD.kernarg_size)) return TokError("Kernarg preload length + offset is larger than the " "kernarg segment size"); @@ -5842,11 +5790,8 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { "increments of 4"); if (AccumOffset > alignTo(std::max((uint64_t)1, NextFreeVGPR), 4)) return TokError("accum_offset exceeds total VGPR allocation"); - MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc3, - MCConstantExpr::create(AccumOffset / 4 - 1, getContext()), - COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT, - COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, getContext()); + AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, + (AccumOffset / 4 - 1)); } if (IVersion.Major >= 10 && IVersion.Major < 12) { diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp deleted file mode 100644 index 0179d575464d..000000000000 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.cpp +++ /dev/null @@ -1,32 +0,0 @@ -//===--- AMDHSAKernelDescriptor.h -----------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "AMDGPUMCKernelDescriptor.h" -#include "llvm/MC/MCContext.h" -#include "llvm/MC/MCExpr.h" - -using namespace llvm; -using namespace llvm::AMDGPU; - -void MCKernelDescriptor::bits_set(const MCExpr *&Dst, const MCExpr *Value, - uint32_t Shift, uint32_t Mask, - MCContext &Ctx) { - auto Sft = MCConstantExpr::create(Shift, Ctx); - auto Msk = MCConstantExpr::create(Mask, Ctx); - Dst = MCBinaryExpr::createAnd(Dst, MCUnaryExpr::createNot(Msk, Ctx), Ctx); - Dst = MCBinaryExpr::createOr(Dst, MCBinaryExpr::createShl(Value, Sft, Ctx), - Ctx); -} - -const MCExpr *MCKernelDescriptor::bits_get(const MCExpr *Src, uint32_t Shift, - uint32_t Mask, MCContext &Ctx) { - auto Sft = MCConstantExpr::create(Shift, Ctx); - auto Msk = MCConstantExpr::create(Mask, Ctx); - return MCBinaryExpr::createLShr(MCBinaryExpr::createAnd(Src, Msk, Ctx), Sft, - Ctx); -} diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h deleted file mode 100644 index 71659e642dd7..000000000000 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCKernelDescriptor.h +++ /dev/null @@ -1,51 +0,0 @@ -//===--- AMDGPUMCKernelDescriptor.h ---------------------------*- C++ -*---===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -/// \file -/// AMDHSA kernel descriptor MCExpr struct for use in MC layer. Uses -/// AMDHSAKernelDescriptor.h for sizes and constants. -/// -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCKERNELDESCRIPTOR_H -#define LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCKERNELDESCRIPTOR_H - -#include "llvm/Support/AMDHSAKernelDescriptor.h" - -namespace llvm { -class MCExpr; -class MCContext; -namespace AMDGPU { - -struct MCKernelDescriptor { - const MCExpr *group_segment_fixed_size = nullptr; - const MCExpr *private_segment_fixed_size = nullptr; - const MCExpr *kernarg_size = nullptr; - const MCExpr *compute_pgm_rsrc3 = nullptr; - const MCExpr *compute_pgm_rsrc1 = nullptr; - const MCExpr *compute_pgm_rsrc2 = nullptr; - const MCExpr *kernel_code_properties = nullptr; - const MCExpr *kernarg_preload = nullptr; - - // MCExpr for: - // Dst = Dst & ~Mask - // Dst = Dst | (Value << Shift) - static void bits_set(const MCExpr *&Dst, const MCExpr *Value, uint32_t Shift, - uint32_t Mask, MCContext &Ctx); - - // MCExpr for: - // return (Src & Mask) >> Shift - static const MCExpr *bits_get(const MCExpr *Src, uint32_t Shift, - uint32_t Mask, MCContext &Ctx); -}; - -} // end namespace AMDGPU -} // end namespace llvm - -#endif // LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCKERNELDESCRIPTOR_H diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp index 3006fcdb9282..4742b0b3e52e 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp @@ -11,7 +11,6 @@ //===----------------------------------------------------------------------===// #include "AMDGPUTargetStreamer.h" -#include "AMDGPUMCKernelDescriptor.h" #include "AMDGPUPTNote.h" #include "AMDKernelCodeT.h" #include "Utils/AMDGPUBaseInfo.h" @@ -308,142 +307,94 @@ bool AMDGPUTargetAsmStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) { void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor( const MCSubtargetInfo &STI, StringRef KernelName, - const MCKernelDescriptor &KD, uint64_t NextVGPR, uint64_t NextSGPR, + const amdhsa::kernel_descriptor_t &KD, uint64_t NextVGPR, uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) { IsaVersion IVersion = getIsaVersion(STI.getCPU()); - const MCAsmInfo *MAI = getContext().getAsmInfo(); OS << "\t.amdhsa_kernel " << KernelName << '\n'; - auto PrintField = [&](const MCExpr *Expr, uint32_t Shift, uint32_t Mask, - StringRef Directive) { - int64_t IVal; - OS << "\t\t" << Directive << ' '; - const MCExpr *pgm_rsrc1_bits = - MCKernelDescriptor::bits_get(Expr, Shift, Mask, getContext()); - if (pgm_rsrc1_bits->evaluateAsAbsolute(IVal)) - OS << static_cast(IVal); - else - pgm_rsrc1_bits->print(OS, MAI); - OS << '\n'; - }; - - OS << "\t\t.amdhsa_group_segment_fixed_size "; - KD.group_segment_fixed_size->print(OS, MAI); - OS << '\n'; - - OS << "\t\t.amdhsa_private_segment_fixed_size "; - KD.private_segment_fixed_size->print(OS, MAI); - OS << '\n'; - - OS << "\t\t.amdhsa_kernarg_size "; - KD.kernarg_size->print(OS, MAI); - OS << '\n'; - - PrintField( - KD.compute_pgm_rsrc2, amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, ".amdhsa_user_sgpr_count"); +#define PRINT_FIELD(STREAM, DIRECTIVE, KERNEL_DESC, MEMBER_NAME, FIELD_NAME) \ + STREAM << "\t\t" << DIRECTIVE << " " \ + << AMDHSA_BITS_GET(KERNEL_DESC.MEMBER_NAME, FIELD_NAME) << '\n'; + + OS << "\t\t.amdhsa_group_segment_fixed_size " << KD.group_segment_fixed_size + << '\n'; + OS << "\t\t.amdhsa_private_segment_fixed_size " + << KD.private_segment_fixed_size << '\n'; + OS << "\t\t.amdhsa_kernarg_size " << KD.kernarg_size << '\n'; + + PRINT_FIELD(OS, ".amdhsa_user_sgpr_count", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT); if (!hasArchitectedFlatScratch(STI)) - PrintField( - KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER, - ".amdhsa_user_sgpr_private_segment_buffer"); - PrintField(KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, - ".amdhsa_user_sgpr_dispatch_ptr"); - PrintField(KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, - ".amdhsa_user_sgpr_queue_ptr"); - PrintField(KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR, - ".amdhsa_user_sgpr_kernarg_segment_ptr"); - PrintField(KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, - ".amdhsa_user_sgpr_dispatch_id"); + PRINT_FIELD( + OS, ".amdhsa_user_sgpr_private_segment_buffer", KD, + kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER); + PRINT_FIELD(OS, ".amdhsa_user_sgpr_dispatch_ptr", KD, + kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR); + PRINT_FIELD(OS, ".amdhsa_user_sgpr_queue_ptr", KD, + kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR); + PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_segment_ptr", KD, + kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR); + PRINT_FIELD(OS, ".amdhsa_user_sgpr_dispatch_id", KD, + kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID); if (!hasArchitectedFlatScratch(STI)) - PrintField(KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT, - ".amdhsa_user_sgpr_flat_scratch_init"); + PRINT_FIELD(OS, ".amdhsa_user_sgpr_flat_scratch_init", KD, + kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT); if (hasKernargPreload(STI)) { - PrintField(KD.kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_LENGTH_SHIFT, - amdhsa::KERNARG_PRELOAD_SPEC_LENGTH, - ".amdhsa_user_sgpr_kernarg_preload_length"); - PrintField(KD.kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_OFFSET_SHIFT, - amdhsa::KERNARG_PRELOAD_SPEC_OFFSET, - ".amdhsa_user_sgpr_kernarg_preload_offset"); + PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_preload_length ", KD, + kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_LENGTH); + PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_preload_offset ", KD, + kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_OFFSET); } - PrintField( - KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE, - ".amdhsa_user_sgpr_private_segment_size"); + PRINT_FIELD(OS, ".amdhsa_user_sgpr_private_segment_size", KD, + kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE); if (IVersion.Major >= 10) - PrintField(KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, - ".amdhsa_wavefront_size32"); + PRINT_FIELD(OS, ".amdhsa_wavefront_size32", KD, + kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32); if (CodeObjectVersion >= AMDGPU::AMDHSA_COV5) - PrintField(KD.kernel_code_properties, - amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, - ".amdhsa_uses_dynamic_stack"); - PrintField(KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, - (hasArchitectedFlatScratch(STI) - ? ".amdhsa_enable_private_segment" - : ".amdhsa_system_sgpr_private_segment_wavefront_offset")); - PrintField(KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, - ".amdhsa_system_sgpr_workgroup_id_x"); - PrintField(KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, - ".amdhsa_system_sgpr_workgroup_id_y"); - PrintField(KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, - ".amdhsa_system_sgpr_workgroup_id_z"); - PrintField(KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, - ".amdhsa_system_sgpr_workgroup_info"); - PrintField(KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, - ".amdhsa_system_vgpr_workitem_id"); + PRINT_FIELD(OS, ".amdhsa_uses_dynamic_stack", KD, kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK); + PRINT_FIELD(OS, + (hasArchitectedFlatScratch(STI) + ? ".amdhsa_enable_private_segment" + : ".amdhsa_system_sgpr_private_segment_wavefront_offset"), + KD, compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT); + PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_x", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X); + PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_y", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y); + PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_z", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z); + PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_info", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO); + PRINT_FIELD(OS, ".amdhsa_system_vgpr_workitem_id", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID); // These directives are required. OS << "\t\t.amdhsa_next_free_vgpr " << NextVGPR << '\n'; OS << "\t\t.amdhsa_next_free_sgpr " << NextSGPR << '\n'; - if (AMDGPU::isGFX90A(STI)) { - // MCExpr equivalent of taking the (accum_offset + 1) * 4. - const MCExpr *accum_bits = MCKernelDescriptor::bits_get( - KD.compute_pgm_rsrc3, - amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT, - amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, getContext()); - accum_bits = MCBinaryExpr::createAdd( - accum_bits, MCConstantExpr::create(1, getContext()), getContext()); - accum_bits = MCBinaryExpr::createMul( - accum_bits, MCConstantExpr::create(4, getContext()), getContext()); - OS << "\t\t.amdhsa_accum_offset "; - int64_t IVal; - if (accum_bits->evaluateAsAbsolute(IVal)) { - OS << static_cast(IVal); - } else { - accum_bits->print(OS, MAI); - } - OS << '\n'; - } + if (AMDGPU::isGFX90A(STI)) + OS << "\t\t.amdhsa_accum_offset " << + (AMDHSA_BITS_GET(KD.compute_pgm_rsrc3, + amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET) + 1) * 4 + << '\n'; if (!ReserveVCC) OS << "\t\t.amdhsa_reserve_vcc " << ReserveVCC << '\n'; @@ -460,105 +411,74 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor( break; } - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, - ".amdhsa_float_round_mode_32"); - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, - ".amdhsa_float_round_mode_16_64"); - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, - ".amdhsa_float_denorm_mode_32"); - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, - ".amdhsa_float_denorm_mode_16_64"); + PRINT_FIELD(OS, ".amdhsa_float_round_mode_32", KD, + compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32); + PRINT_FIELD(OS, ".amdhsa_float_round_mode_16_64", KD, + compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64); + PRINT_FIELD(OS, ".amdhsa_float_denorm_mode_32", KD, + compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32); + PRINT_FIELD(OS, ".amdhsa_float_denorm_mode_16_64", KD, + compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64); if (IVersion.Major < 12) { - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, - ".amdhsa_dx10_clamp"); - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, - ".amdhsa_ieee_mode"); - } - if (IVersion.Major >= 9) { - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, - ".amdhsa_fp16_overflow"); + PRINT_FIELD(OS, ".amdhsa_dx10_clamp", KD, compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP); + PRINT_FIELD(OS, ".amdhsa_ieee_mode", KD, compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE); } + if (IVersion.Major >= 9) + PRINT_FIELD(OS, ".amdhsa_fp16_overflow", KD, + compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL); if (AMDGPU::isGFX90A(STI)) - PrintField(KD.compute_pgm_rsrc3, - amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT, - amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, ".amdhsa_tg_split"); + PRINT_FIELD(OS, ".amdhsa_tg_split", KD, + compute_pgm_rsrc3, + amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT); if (IVersion.Major >= 10) { - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, - ".amdhsa_workgroup_processor_mode"); - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, - ".amdhsa_memory_ordered"); - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, - ".amdhsa_forward_progress"); + PRINT_FIELD(OS, ".amdhsa_workgroup_processor_mode", KD, + compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE); + PRINT_FIELD(OS, ".amdhsa_memory_ordered", KD, + compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED); + PRINT_FIELD(OS, ".amdhsa_forward_progress", KD, + compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS); } if (IVersion.Major >= 10 && IVersion.Major < 12) { - PrintField(KD.compute_pgm_rsrc3, - amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT_SHIFT, - amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, - ".amdhsa_shared_vgpr_count"); + PRINT_FIELD(OS, ".amdhsa_shared_vgpr_count", KD, compute_pgm_rsrc3, + amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT); } - if (IVersion.Major >= 12) { - PrintField(KD.compute_pgm_rsrc1, - amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, - ".amdhsa_round_robin_scheduling"); - } - PrintField( - KD.compute_pgm_rsrc2, - amdhsa:: - COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, - ".amdhsa_exception_fp_ieee_invalid_op"); - PrintField( - KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE, - ".amdhsa_exception_fp_denorm_src"); - PrintField( - KD.compute_pgm_rsrc2, - amdhsa:: - COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, - ".amdhsa_exception_fp_ieee_div_zero"); - PrintField( - KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW, - ".amdhsa_exception_fp_ieee_overflow"); - PrintField( - KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW, - ".amdhsa_exception_fp_ieee_underflow"); - PrintField( - KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT, - ".amdhsa_exception_fp_ieee_inexact"); - PrintField( - KD.compute_pgm_rsrc2, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO, - ".amdhsa_exception_int_div_zero"); + if (IVersion.Major >= 12) + PRINT_FIELD(OS, ".amdhsa_round_robin_scheduling", KD, compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN); + PRINT_FIELD( + OS, ".amdhsa_exception_fp_ieee_invalid_op", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION); + PRINT_FIELD(OS, ".amdhsa_exception_fp_denorm_src", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE); + PRINT_FIELD( + OS, ".amdhsa_exception_fp_ieee_div_zero", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO); + PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_overflow", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW); + PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_underflow", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW); + PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_inexact", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT); + PRINT_FIELD(OS, ".amdhsa_exception_int_div_zero", KD, + compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO); +#undef PRINT_FIELD OS << "\t.end_amdhsa_kernel\n"; } @@ -915,7 +835,7 @@ bool AMDGPUTargetELFStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) { void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor( const MCSubtargetInfo &STI, StringRef KernelName, - const MCKernelDescriptor &KernelDescriptor, uint64_t NextVGPR, + const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) { auto &Streamer = getStreamer(); auto &Context = Streamer.getContext(); @@ -933,7 +853,7 @@ void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor( // Kernel descriptor symbol's type and size are fixed. KernelDescriptorSymbol->setType(ELF::STT_OBJECT); KernelDescriptorSymbol->setSize( - MCConstantExpr::create(sizeof(amdhsa::kernel_descriptor_t), Context)); + MCConstantExpr::create(sizeof(KernelDescriptor), Context)); // The visibility of the kernel code symbol must be protected or less to allow // static relocations from the kernel descriptor to be used. @@ -941,43 +861,31 @@ void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor( KernelCodeSymbol->setVisibility(ELF::STV_PROTECTED); Streamer.emitLabel(KernelDescriptorSymbol); - Streamer.emitValue( - KernelDescriptor.group_segment_fixed_size, - sizeof(amdhsa::kernel_descriptor_t::group_segment_fixed_size)); - Streamer.emitValue( - KernelDescriptor.private_segment_fixed_size, - sizeof(amdhsa::kernel_descriptor_t::private_segment_fixed_size)); - Streamer.emitValue(KernelDescriptor.kernarg_size, - sizeof(amdhsa::kernel_descriptor_t::kernarg_size)); - - for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved0); ++i) - Streamer.emitInt8(0u); + Streamer.emitInt32(KernelDescriptor.group_segment_fixed_size); + Streamer.emitInt32(KernelDescriptor.private_segment_fixed_size); + Streamer.emitInt32(KernelDescriptor.kernarg_size); + + for (uint8_t Res : KernelDescriptor.reserved0) + Streamer.emitInt8(Res); // FIXME: Remove the use of VK_AMDGPU_REL64 in the expression below. The // expression being created is: // (start of kernel code) - (start of kernel descriptor) // It implies R_AMDGPU_REL64, but ends up being R_AMDGPU_ABS64. - Streamer.emitValue( - MCBinaryExpr::createSub( - MCSymbolRefExpr::create(KernelCodeSymbol, - MCSymbolRefExpr::VK_AMDGPU_REL64, Context), - MCSymbolRefExpr::create(KernelDescriptorSymbol, - MCSymbolRefExpr::VK_None, Context), - Context), - sizeof(amdhsa::kernel_descriptor_t::kernel_code_entry_byte_offset)); - for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved1); ++i) - Streamer.emitInt8(0u); - Streamer.emitValue(KernelDescriptor.compute_pgm_rsrc3, - sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc3)); - Streamer.emitValue(KernelDescriptor.compute_pgm_rsrc1, - sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc1)); - Streamer.emitValue(KernelDescriptor.compute_pgm_rsrc2, - sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc2)); - Streamer.emitValue( - KernelDescriptor.kernel_code_properties, - sizeof(amdhsa::kernel_descriptor_t::kernel_code_properties)); - Streamer.emitValue(KernelDescriptor.kernarg_preload, - sizeof(amdhsa::kernel_descriptor_t::kernarg_preload)); - for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved3); ++i) - Streamer.emitInt8(0u); + Streamer.emitValue(MCBinaryExpr::createSub( + MCSymbolRefExpr::create( + KernelCodeSymbol, MCSymbolRefExpr::VK_AMDGPU_REL64, Context), + MCSymbolRefExpr::create( + KernelDescriptorSymbol, MCSymbolRefExpr::VK_None, Context), + Context), + sizeof(KernelDescriptor.kernel_code_entry_byte_offset)); + for (uint8_t Res : KernelDescriptor.reserved1) + Streamer.emitInt8(Res); + Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc3); + Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc1); + Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc2); + Streamer.emitInt16(KernelDescriptor.kernel_code_properties); + Streamer.emitInt16(KernelDescriptor.kernarg_preload); + for (uint8_t Res : KernelDescriptor.reserved3) + Streamer.emitInt8(Res); } diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h index 706897a5dc1f..5aa80ff578c6 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h @@ -22,13 +22,15 @@ class MCSymbol; class formatted_raw_ostream; namespace AMDGPU { - -struct MCKernelDescriptor; namespace HSAMD { struct Metadata; } } // namespace AMDGPU +namespace amdhsa { +struct kernel_descriptor_t; +} + class AMDGPUTargetStreamer : public MCTargetStreamer { AMDGPUPALMetadata PALMetadata; @@ -92,11 +94,10 @@ public: return true; } - virtual void - EmitAmdhsaKernelDescriptor(const MCSubtargetInfo &STI, StringRef KernelName, - const AMDGPU::MCKernelDescriptor &KernelDescriptor, - uint64_t NextVGPR, uint64_t NextSGPR, - bool ReserveVCC, bool ReserveFlatScr) {} + virtual void EmitAmdhsaKernelDescriptor( + const MCSubtargetInfo &STI, StringRef KernelName, + const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, + uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) {} static StringRef getArchNameFromElfMach(unsigned ElfMach); static unsigned getElfMach(StringRef GPU); @@ -149,11 +150,10 @@ public: bool EmitKernargPreloadHeader(const MCSubtargetInfo &STI, bool TrapEnabled) override; - void - EmitAmdhsaKernelDescriptor(const MCSubtargetInfo &STI, StringRef KernelName, - const AMDGPU::MCKernelDescriptor &KernelDescriptor, - uint64_t NextVGPR, uint64_t NextSGPR, - bool ReserveVCC, bool ReserveFlatScr) override; + void EmitAmdhsaKernelDescriptor( + const MCSubtargetInfo &STI, StringRef KernelName, + const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, + uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) override; }; class AMDGPUTargetELFStreamer final : public AMDGPUTargetStreamer { @@ -205,11 +205,10 @@ public: bool EmitKernargPreloadHeader(const MCSubtargetInfo &STI, bool TrapEnabled) override; - void - EmitAmdhsaKernelDescriptor(const MCSubtargetInfo &STI, StringRef KernelName, - const AMDGPU::MCKernelDescriptor &KernelDescriptor, - uint64_t NextVGPR, uint64_t NextSGPR, - bool ReserveVCC, bool ReserveFlatScr) override; + void EmitAmdhsaKernelDescriptor( + const MCSubtargetInfo &STI, StringRef KernelName, + const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, + uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) override; }; } #endif diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt index 14a02b6d8e36..0842a58f794b 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt @@ -8,7 +8,6 @@ add_llvm_component_library(LLVMAMDGPUDesc AMDGPUMCExpr.cpp AMDGPUMCTargetDesc.cpp AMDGPUTargetStreamer.cpp - AMDGPUMCKernelDescriptor.cpp R600InstPrinter.cpp R600MCCodeEmitter.cpp R600MCTargetDesc.cpp diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index 4970055c4bdb..6d53f68ace70 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -10,7 +10,6 @@ #include "AMDGPU.h" #include "AMDGPUAsmUtils.h" #include "AMDKernelCodeT.h" -#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "llvm/ADT/StringExtras.h" #include "llvm/BinaryFormat/ELF.h" @@ -21,7 +20,6 @@ #include "llvm/IR/IntrinsicsAMDGPU.h" #include "llvm/IR/IntrinsicsR600.h" #include "llvm/IR/LLVMContext.h" -#include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSubtargetInfo.h" @@ -1217,64 +1215,44 @@ void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, } } -MCKernelDescriptor getDefaultAmdhsaKernelDescriptor(const MCSubtargetInfo *STI, - MCContext &Ctx) { +amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor( + const MCSubtargetInfo *STI) { IsaVersion Version = getIsaVersion(STI->getCPU()); - MCKernelDescriptor KD; - const MCExpr *ZeroMCExpr = MCConstantExpr::create(0, Ctx); - const MCExpr *OneMCExpr = MCConstantExpr::create(1, Ctx); - - KD.group_segment_fixed_size = ZeroMCExpr; - KD.private_segment_fixed_size = ZeroMCExpr; - KD.compute_pgm_rsrc1 = ZeroMCExpr; - KD.compute_pgm_rsrc2 = ZeroMCExpr; - KD.compute_pgm_rsrc3 = ZeroMCExpr; - KD.kernarg_size = ZeroMCExpr; - KD.kernel_code_properties = ZeroMCExpr; - KD.kernarg_preload = ZeroMCExpr; - - MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc1, - MCConstantExpr::create(amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE, Ctx), - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, Ctx); - if (Version.Major < 12) { - MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc1, OneMCExpr, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, Ctx); - MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc1, OneMCExpr, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, Ctx); + amdhsa::kernel_descriptor_t KD; + memset(&KD, 0, sizeof(KD)); + + AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, + amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE); + if (Version.Major >= 12) { + AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, 0); + AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_DISABLE_PERF, 0); + } else { + AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, 1); + AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, 1); } - MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc2, OneMCExpr, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X_SHIFT, - amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, Ctx); + AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, + amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1); if (Version.Major >= 10) { - if (STI->getFeatureBits().test(FeatureWavefrontSize32)) - MCKernelDescriptor::bits_set( - KD.kernel_code_properties, OneMCExpr, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32_SHIFT, - amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, Ctx); - if (!STI->getFeatureBits().test(FeatureCuMode)) - MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc1, OneMCExpr, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, Ctx); - - MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc1, OneMCExpr, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED_SHIFT, - amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, Ctx); + AMDHSA_BITS_SET(KD.kernel_code_properties, + amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, + STI->getFeatureBits().test(FeatureWavefrontSize32) ? 1 : 0); + AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, + STI->getFeatureBits().test(FeatureCuMode) ? 0 : 1); + AMDHSA_BITS_SET(KD.compute_pgm_rsrc1, + amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, 1); + } + if (AMDGPU::isGFX90A(*STI)) { + AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, + amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, + STI->getFeatureBits().test(FeatureTgSplit) ? 1 : 0); } - if (AMDGPU::isGFX90A(*STI) && STI->getFeatureBits().test(FeatureTgSplit)) - MCKernelDescriptor::bits_set( - KD.compute_pgm_rsrc3, OneMCExpr, - amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT, - amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Ctx); return KD; } diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index 32b73f1d868d..29ac402d9535 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -26,7 +26,6 @@ struct Align; class Argument; class Function; class GlobalValue; -class MCContext; class MCInstrInfo; class MCRegisterClass; class MCRegisterInfo; @@ -35,9 +34,12 @@ class StringRef; class Triple; class raw_ostream; +namespace amdhsa { +struct kernel_descriptor_t; +} + namespace AMDGPU { -struct MCKernelDescriptor; struct IsaVersion; /// Generic target versions emitted by this version of LLVM. @@ -850,8 +852,8 @@ unsigned mapWMMA3AddrTo2AddrOpcode(unsigned Opc); void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, const MCSubtargetInfo *STI); -MCKernelDescriptor getDefaultAmdhsaKernelDescriptor(const MCSubtargetInfo *STI, - MCContext &Ctx); +amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor( + const MCSubtargetInfo *STI); bool isGroupSegment(const GlobalValue *GV); bool isGlobalSegment(const GlobalValue *GV); diff --git a/llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s b/llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s deleted file mode 100644 index 4623500987be..000000000000 --- a/llvm/test/MC/AMDGPU/hsa-amdgpu-exprs.s +++ /dev/null @@ -1,27 +0,0 @@ -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck --check-prefix=ASM %s -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -filetype=obj < %s > %t -// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s - -// OBJDUMP: 0000 00000000 0f000000 00000000 00000000 - -.text - -.p2align 8 -.type caller,@function -caller: - s_endpgm - -.rodata - -.p2align 6 -.amdhsa_kernel caller - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_private_segment_fixed_size max(7, callee1.private_seg_size, callee2.private_seg_size) -.end_amdhsa_kernel - -.set callee1.private_seg_size, 4 -.set callee2.private_seg_size, 15 - -// ASM: .amdhsa_private_segment_fixed_size max(7, callee1.private_seg_size, callee2.private_seg_size) diff --git a/llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s b/llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s deleted file mode 100644 index fab3e893352b..000000000000 --- a/llvm/test/MC/AMDGPU/hsa-sym-expr-failure.s +++ /dev/null @@ -1,281 +0,0 @@ -// RUN: not llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a %s 2>&1 | FileCheck --check-prefix=ASM %s - -// Some expression currently require (immediately) solvable expressions, i.e., -// they don't depend on yet-unknown symbolic values. - -.text -// ASM: .text - -.amdhsa_code_object_version 4 -// ASM: .amdhsa_code_object_version 4 - -.p2align 8 -.type user_sgpr_count,@function -user_sgpr_count: - s_endpgm - -.p2align 6 -.amdhsa_kernel user_sgpr_count - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_count defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_count - -.p2align 8 -.type user_sgpr_private_segment_buffer,@function -user_sgpr_private_segment_buffer: - s_endpgm - -.amdhsa_kernel user_sgpr_private_segment_buffer - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_private_segment_buffer defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer - -.p2align 8 -.type user_sgpr_kernarg_preload_length,@function -user_sgpr_kernarg_preload_length: - s_endpgm - -.amdhsa_kernel user_sgpr_kernarg_preload_length - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_kernarg_preload_length defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length defined_boolean - -.p2align 8 -.type user_sgpr_kernarg_preload_offset,@function -user_sgpr_kernarg_preload_offset: - s_endpgm - -.amdhsa_kernel user_sgpr_kernarg_preload_offset - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_kernarg_preload_offset defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset defined_boolean - -.p2align 8 -.type user_sgpr_dispatch_ptr,@function -user_sgpr_dispatch_ptr: - s_endpgm - -.p2align 6 -.amdhsa_kernel user_sgpr_dispatch_ptr - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_dispatch_ptr defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr - -.p2align 8 -.type user_sgpr_queue_ptr,@function -user_sgpr_queue_ptr: - s_endpgm - -.p2align 6 -.amdhsa_kernel user_sgpr_queue_ptr - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_queue_ptr defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr - -.p2align 8 -.type user_sgpr_kernarg_segment_ptr,@function -user_sgpr_kernarg_segment_ptr: - s_endpgm - -.p2align 6 -.amdhsa_kernel user_sgpr_kernarg_segment_ptr - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_kernarg_segment_ptr defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr - -.p2align 8 -.type user_sgpr_dispatch_id,@function -user_sgpr_dispatch_id: - s_endpgm - -.p2align 6 -.amdhsa_kernel user_sgpr_dispatch_id - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_dispatch_id defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id - -.p2align 8 -.type user_sgpr_flat_scratch_init,@function -user_sgpr_flat_scratch_init: - s_endpgm - -.p2align 6 -.amdhsa_kernel user_sgpr_flat_scratch_init - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_flat_scratch_init defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init - -.p2align 8 -.type user_sgpr_private_segment_size,@function -user_sgpr_private_segment_size: - s_endpgm - -.p2align 6 -.amdhsa_kernel user_sgpr_private_segment_size - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_user_sgpr_private_segment_size defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size - -.p2align 8 -.type wavefront_size32,@function -wavefront_size32: - s_endpgm - -.p2align 6 -.amdhsa_kernel wavefront_size32 - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_wavefront_size32 defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_wavefront_size32 - -.p2align 8 -.type next_free_vgpr,@function -next_free_vgpr: - s_endpgm - -.p2align 6 -.amdhsa_kernel next_free_vgpr - .amdhsa_next_free_vgpr defined_boolean - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_next_free_vgpr - -.p2align 8 -.type next_free_sgpr,@function -next_free_sgpr: - s_endpgm - -.p2align 6 -.amdhsa_kernel next_free_sgpr - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr defined_boolean - .amdhsa_accum_offset 4 -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_next_free_sgpr - -.p2align 8 -.type accum_offset,@function -accum_offset: - s_endpgm - -.p2align 6 -.amdhsa_kernel accum_offset - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_accum_offset - -.p2align 8 -.type reserve_vcc,@function -reserve_vcc: - s_endpgm - -.p2align 6 -.amdhsa_kernel reserve_vcc - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_reserve_vcc defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_reserve_vcc - -.p2align 8 -.type reserve_flat_scratch,@function -reserve_flat_scratch: - s_endpgm - -.p2align 6 -.amdhsa_kernel reserve_flat_scratch - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_reserve_flat_scratch defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_reserve_flat_scratch - -.p2align 8 -.type shared_vgpr_count,@function -shared_vgpr_count: - s_endpgm - -.p2align 6 -.amdhsa_kernel shared_vgpr_count - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 - .amdhsa_shared_vgpr_count defined_boolean -.end_amdhsa_kernel - -// ASM: error: directive should have resolvable expression -// ASM-NEXT: .amdhsa_shared_vgpr_count - -.set defined_boolean, 1 - -// ASM: .set defined_boolean, 1 -// ASM-NEXT: .no_dead_strip defined_boolean diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s deleted file mode 100644 index 95af59c413ae..000000000000 --- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s +++ /dev/null @@ -1,190 +0,0 @@ -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck --check-prefix=ASM %s -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1010 -filetype=obj < %s > %t -// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s - -// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). -// When going from asm -> obj, the expressions should get resolved (through fixups), - -// OBJDUMP: Contents of section .rodata -// expr_defined_later -// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 -// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0030 00f0afe4 801f007f 000c0000 00000000 -// expr_defined -// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 -// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0070 00f0afe4 801f007f 000c0000 00000000 - -.text -// ASM: .text - -.amdhsa_code_object_version 4 -// ASM: .amdhsa_code_object_version 4 - -.p2align 8 -.type expr_defined_later,@function -expr_defined_later: - s_endpgm - -.p2align 8 -.type expr_defined,@function -expr_defined: - s_endpgm - -.rodata -// ASM: .rodata - -.p2align 6 -.amdhsa_kernel expr_defined_later - .amdhsa_group_segment_fixed_size defined_value+2 - .amdhsa_private_segment_fixed_size defined_value+3 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_fp16_overflow defined_boolean - .amdhsa_workgroup_processor_mode defined_boolean - .amdhsa_memory_ordered defined_boolean - .amdhsa_forward_progress defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -.set defined_value, 41 -.set defined_2_bits, 3 -.set defined_boolean, 1 - -.p2align 6 -.amdhsa_kernel expr_defined - .amdhsa_group_segment_fixed_size defined_value+1 - .amdhsa_private_segment_fixed_size defined_value+2 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_fp16_overflow defined_boolean - .amdhsa_workgroup_processor_mode defined_boolean - .amdhsa_memory_ordered defined_boolean - .amdhsa_forward_progress defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -// ASM: .amdhsa_kernel expr_defined_later -// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 -// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1)>>0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4 -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&32)>>5 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6 -// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10 -// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 -// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 -// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 -// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 -// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26 -// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&536870912)>>29 -// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&1073741824)>>30 -// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2147483648)>>31 -// ASM-NEXT: .amdhsa_shared_vgpr_count 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29 -// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30 -// ASM-NEXT: .end_amdhsa_kernel - -// ASM: .set defined_value, 41 -// ASM-NEXT: .no_dead_strip defined_value -// ASM-NEXT: .set defined_2_bits, 3 -// ASM-NEXT: .no_dead_strip defined_2_bits -// ASM-NEXT: .set defined_boolean, 1 -// ASM-NEXT: .no_dead_strip defined_boolean - -// ASM: .amdhsa_kernel expr_defined -// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 -// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 -// ASM-NEXT: .amdhsa_wavefront_size32 1 -// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 -// ASM-NEXT: .amdhsa_float_round_mode_32 3 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 -// ASM-NEXT: .amdhsa_dx10_clamp 1 -// ASM-NEXT: .amdhsa_ieee_mode 1 -// ASM-NEXT: .amdhsa_fp16_overflow 1 -// ASM-NEXT: .amdhsa_workgroup_processor_mode 1 -// ASM-NEXT: .amdhsa_memory_ordered 1 -// ASM-NEXT: .amdhsa_forward_progress 1 -// ASM-NEXT: .amdhsa_shared_vgpr_count 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 -// ASM-NEXT: .amdhsa_exception_int_div_zero 1 -// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s deleted file mode 100644 index e1107fb69ba4..000000000000 --- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s +++ /dev/null @@ -1,186 +0,0 @@ -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1100 < %s | FileCheck --check-prefix=ASM %s -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1100 -filetype=obj < %s > %t -// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s - -// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). -// When going from asm -> obj, the expressions should get resolved (through fixups), - -// OBJDUMP: Contents of section .rodata -// expr_defined_later -// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 -// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0030 00f0afe4 811f007f 000c0000 00000000 -// expr_defined -// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 -// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0070 00f0afe4 811f007f 000c0000 00000000 - -.text -// ASM: .text - -.amdhsa_code_object_version 4 -// ASM: .amdhsa_code_object_version 4 - -.p2align 8 -.type expr_defined_later,@function -expr_defined_later: - s_endpgm - -.p2align 8 -.type expr_defined,@function -expr_defined: - s_endpgm - -.rodata -// ASM: .rodata - -.p2align 6 -.amdhsa_kernel expr_defined_later - .amdhsa_group_segment_fixed_size defined_value+2 - .amdhsa_private_segment_fixed_size defined_value+3 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_fp16_overflow defined_boolean - .amdhsa_workgroup_processor_mode defined_boolean - .amdhsa_memory_ordered defined_boolean - .amdhsa_forward_progress defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_enable_private_segment defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -.set defined_value, 41 -.set defined_2_bits, 3 -.set defined_boolean, 1 - -.p2align 6 -.amdhsa_kernel expr_defined - .amdhsa_group_segment_fixed_size defined_value+1 - .amdhsa_private_segment_fixed_size defined_value+2 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_fp16_overflow defined_boolean - .amdhsa_workgroup_processor_mode defined_boolean - .amdhsa_memory_ordered defined_boolean - .amdhsa_forward_progress defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_enable_private_segment defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -// ASM: .amdhsa_kernel expr_defined_later -// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 -// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6 -// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10 -// ASM-NEXT: .amdhsa_enable_private_segment (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 -// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 -// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 -// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26 -// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&536870912)>>29 -// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&1073741824)>>30 -// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(0<<0))&(~960))|(0<<6))&2147483648)>>31 -// ASM-NEXT: .amdhsa_shared_vgpr_count 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29 -// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30 -// ASM-NEXT: .end_amdhsa_kernel - -// ASM: .set defined_value, 41 -// ASM-NEXT: .no_dead_strip defined_value -// ASM-NEXT: .set defined_2_bits, 3 -// ASM-NEXT: .no_dead_strip defined_2_bits -// ASM-NEXT: .set defined_boolean, 1 -// ASM-NEXT: .no_dead_strip defined_boolean - -// ASM: .amdhsa_kernel expr_defined -// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 -// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 -// ASM-NEXT: .amdhsa_wavefront_size32 1 -// ASM-NEXT: .amdhsa_enable_private_segment 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_float_round_mode_32 3 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 -// ASM-NEXT: .amdhsa_dx10_clamp 1 -// ASM-NEXT: .amdhsa_ieee_mode 1 -// ASM-NEXT: .amdhsa_fp16_overflow 1 -// ASM-NEXT: .amdhsa_workgroup_processor_mode 1 -// ASM-NEXT: .amdhsa_memory_ordered 1 -// ASM-NEXT: .amdhsa_forward_progress 1 -// ASM-NEXT: .amdhsa_shared_vgpr_count 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 -// ASM-NEXT: .amdhsa_exception_int_div_zero 1 -// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s deleted file mode 100644 index 449616d35186..000000000000 --- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s +++ /dev/null @@ -1,184 +0,0 @@ -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefix=ASM %s -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1200 -filetype=obj < %s > %t -// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s - -// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). -// When going from asm -> obj, the expressions should get resolved (through fixups), - -// OBJDUMP: Contents of section .rodata -// expr_defined_later -// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 -// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0030 00f02fe4 811f007f 000c0000 00000000 -// expr_defined -// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 -// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0070 00f02fe4 811f007f 000c0000 00000000 - -.text -// ASM: .text - -.amdhsa_code_object_version 4 -// ASM: .amdhsa_code_object_version 4 - -.p2align 8 -.type expr_defined_later,@function -expr_defined_later: - s_endpgm - -.p2align 8 -.type expr_defined,@function -expr_defined: - s_endpgm - -.rodata -// ASM: .rodata - -.p2align 6 -.amdhsa_kernel expr_defined_later - .amdhsa_group_segment_fixed_size defined_value+2 - .amdhsa_private_segment_fixed_size defined_value+3 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_fp16_overflow defined_boolean - .amdhsa_workgroup_processor_mode defined_boolean - .amdhsa_memory_ordered defined_boolean - .amdhsa_forward_progress defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_round_robin_scheduling defined_boolean - .amdhsa_enable_private_segment defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -.set defined_value, 41 -.set defined_2_bits, 3 -.set defined_boolean, 1 - -.p2align 6 -.amdhsa_kernel expr_defined - .amdhsa_group_segment_fixed_size defined_value+1 - .amdhsa_private_segment_fixed_size defined_value+2 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_fp16_overflow defined_boolean - .amdhsa_workgroup_processor_mode defined_boolean - .amdhsa_memory_ordered defined_boolean - .amdhsa_forward_progress defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_round_robin_scheduling defined_boolean - .amdhsa_enable_private_segment defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -// ASM: .amdhsa_kernel expr_defined_later -// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 -// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6 -// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10 -// ASM-NEXT: .amdhsa_enable_private_segment (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 -// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26 -// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&536870912)>>29 -// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&1073741824)>>30 -// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&2147483648)>>31 -// ASM-NEXT: .amdhsa_round_robin_scheduling (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29 -// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30 -// ASM-NEXT: .end_amdhsa_kernel - -// ASM: .set defined_value, 41 -// ASM-NEXT: .no_dead_strip defined_value -// ASM-NEXT: .set defined_2_bits, 3 -// ASM-NEXT: .no_dead_strip defined_2_bits -// ASM-NEXT: .set defined_boolean, 1 -// ASM-NEXT: .no_dead_strip defined_boolean - -// ASM: .amdhsa_kernel expr_defined -// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 -// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 -// ASM-NEXT: .amdhsa_wavefront_size32 1 -// ASM-NEXT: .amdhsa_enable_private_segment 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_float_round_mode_32 3 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 -// ASM-NEXT: .amdhsa_fp16_overflow 1 -// ASM-NEXT: .amdhsa_workgroup_processor_mode 1 -// ASM-NEXT: .amdhsa_memory_ordered 1 -// ASM-NEXT: .amdhsa_forward_progress 1 -// ASM-NEXT: .amdhsa_round_robin_scheduling 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 -// ASM-NEXT: .amdhsa_exception_int_div_zero 1 -// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s deleted file mode 100644 index c7e05441b45f..000000000000 --- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s +++ /dev/null @@ -1,168 +0,0 @@ -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck --check-prefix=ASM %s -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj < %s > %t -// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s - -// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). -// When going from asm -> obj, the expressions should get resolved (through fixups), - -// OBJDUMP: Contents of section .rodata -// expr_defined_later -// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 -// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0030 00f0af00 801f007f 00080000 00000000 -// expr_defined -// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 -// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0070 00f0af00 801f007f 00080000 00000000 - -.text -// ASM: .text - -.amdhsa_code_object_version 4 -// ASM: .amdhsa_code_object_version 4 - -.p2align 8 -.type expr_defined_later,@function -expr_defined_later: - s_endpgm - -.p2align 8 -.type expr_defined,@function -expr_defined: - s_endpgm - -.rodata -// ASM: .rodata - -.p2align 6 -.amdhsa_kernel expr_defined_later - .amdhsa_group_segment_fixed_size defined_value+2 - .amdhsa_private_segment_fixed_size defined_value+3 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -.set defined_value, 41 -.set defined_2_bits, 3 -.set defined_boolean, 1 - -.p2align 6 -.amdhsa_kernel expr_defined - .amdhsa_group_segment_fixed_size defined_value+1 - .amdhsa_private_segment_fixed_size defined_value+2 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -// ASM: .amdhsa_kernel expr_defined_later -// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 -// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((0&(~2048))|(defined_boolean<<11))&1)>>0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((0&(~2048))|(defined_boolean<<11))&2)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((0&(~2048))|(defined_boolean<<11))&4)>>2 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((0&(~2048))|(defined_boolean<<11))&8)>>3 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((0&(~2048))|(defined_boolean<<11))&16)>>4 -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((0&(~2048))|(defined_boolean<<11))&32)>>5 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((0&(~2048))|(defined_boolean<<11))&64)>>6 -// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 -// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 -// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29 -// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30 -// ASM-NEXT: .end_amdhsa_kernel - -// ASM: .set defined_value, 41 -// ASM-NEXT: .no_dead_strip defined_value -// ASM-NEXT: .set defined_2_bits, 3 -// ASM-NEXT: .no_dead_strip defined_2_bits -// ASM-NEXT: .set defined_boolean, 1 -// ASM-NEXT: .no_dead_strip defined_boolean - -// ASM: .amdhsa_kernel expr_defined -// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 -// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 -// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_float_round_mode_32 3 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 -// ASM-NEXT: .amdhsa_dx10_clamp 1 -// ASM-NEXT: .amdhsa_ieee_mode 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 -// ASM-NEXT: .amdhsa_exception_int_div_zero 1 -// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s deleted file mode 100644 index 49a5015987a6..000000000000 --- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s +++ /dev/null @@ -1,171 +0,0 @@ -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx801 < %s | FileCheck --check-prefix=ASM %s - -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx801 -filetype=obj < %s > %t -// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s - -// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). -// When going from asm -> obj, the expressions should get resolved (through fixups), - -// OBJDUMP: Contents of section .rodata -// expr_defined_later -// OBJDUMP-NEXT: 0000 2b000000 2c000000 00000000 00000000 -// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0030 00f0af00 801f007f 00080000 00000000 -// expr_defined -// OBJDUMP-NEXT: 0040 2a000000 2b000000 00000000 00000000 -// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0070 00f0af00 801f007f 00080000 00000000 - -.text -// ASM: .text - -.amdhsa_code_object_version 4 -// ASM: .amdhsa_code_object_version 4 - -.p2align 8 -.type expr_defined_later,@function -expr_defined_later: - s_endpgm - -.p2align 8 -.type expr_defined,@function -expr_defined: - s_endpgm - -.rodata -// ASM: .rodata - -.p2align 6 -.amdhsa_kernel expr_defined_later - .amdhsa_group_segment_fixed_size defined_value+2 - .amdhsa_private_segment_fixed_size defined_value+3 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -.set defined_value, 41 -.set defined_2_bits, 3 -.set defined_boolean, 1 - -.p2align 6 -.amdhsa_kernel expr_defined - .amdhsa_group_segment_fixed_size defined_value+1 - .amdhsa_private_segment_fixed_size defined_value+2 - .amdhsa_system_vgpr_workitem_id defined_2_bits - .amdhsa_float_round_mode_32 defined_2_bits - .amdhsa_float_round_mode_16_64 defined_2_bits - .amdhsa_float_denorm_mode_32 defined_2_bits - .amdhsa_float_denorm_mode_16_64 defined_2_bits - .amdhsa_system_sgpr_workgroup_id_x defined_boolean - .amdhsa_system_sgpr_workgroup_id_y defined_boolean - .amdhsa_system_sgpr_workgroup_id_z defined_boolean - .amdhsa_system_sgpr_workgroup_info defined_boolean - .amdhsa_exception_fp_ieee_invalid_op defined_boolean - .amdhsa_exception_fp_denorm_src defined_boolean - .amdhsa_exception_fp_ieee_div_zero defined_boolean - .amdhsa_exception_fp_ieee_overflow defined_boolean - .amdhsa_exception_fp_ieee_underflow defined_boolean - .amdhsa_exception_fp_ieee_inexact defined_boolean - .amdhsa_exception_int_div_zero defined_boolean - .amdhsa_uses_dynamic_stack defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 -.end_amdhsa_kernel - -// ASM: .amdhsa_kernel expr_defined_later -// ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2 -// ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((0&(~2048))|(defined_boolean<<11))&1)>>0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((0&(~2048))|(defined_boolean<<11))&2)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((0&(~2048))|(defined_boolean<<11))&4)>>2 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((0&(~2048))|(defined_boolean<<11))&8)>>3 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((0&(~2048))|(defined_boolean<<11))&16)>>4 -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((0&(~2048))|(defined_boolean<<11))&32)>>5 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((0&(~2048))|(defined_boolean<<11))&64)>>6 -// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 -// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 -// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 -// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29 -// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30 -// ASM-NEXT: .end_amdhsa_kernel - -// ASM: .set defined_value, 41 -// ASM-NEXT: .no_dead_strip defined_value -// ASM-NEXT: .set defined_2_bits, 3 -// ASM-NEXT: .no_dead_strip defined_2_bits -// ASM-NEXT: .set defined_boolean, 1 -// ASM-NEXT: .no_dead_strip defined_boolean - -// ASM: .amdhsa_kernel expr_defined -// ASM-NEXT: .amdhsa_group_segment_fixed_size 42 -// ASM-NEXT: .amdhsa_private_segment_fixed_size 43 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 -// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 3 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 -// ASM-NEXT: .amdhsa_float_round_mode_32 3 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 3 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 -// ASM-NEXT: .amdhsa_dx10_clamp 1 -// ASM-NEXT: .amdhsa_ieee_mode 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1 -// ASM-NEXT: .amdhsa_exception_int_div_zero 1 -// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s deleted file mode 100644 index b7f89239160f..000000000000 --- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s +++ /dev/null @@ -1,148 +0,0 @@ -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck --check-prefix=ASM %s -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -filetype=obj < %s > %t -// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s - -// When going from asm -> asm, the expressions should remain the same (i.e., symbolic). -// When going from asm -> obj, the expressions should get resolved (through fixups), - -// OBJDUMP: Contents of section .rodata -// expr_defined_later -// OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000100 -// OBJDUMP-NEXT: 0030 0000ac04 81000000 00000000 00000000 -// expr_defined -// OBJDUMP-NEXT: 0040 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000100 -// OBJDUMP-NEXT: 0070 0000ac04 81000000 00000000 00000000 - -.text -// ASM: .text - -.amdhsa_code_object_version 4 -// ASM: .amdhsa_code_object_version 4 - -.p2align 8 -.type expr_defined_later,@function -expr_defined_later: - s_endpgm - -.p2align 8 -.type expr_defined,@function -expr_defined: - s_endpgm - -.rodata -// ASM: .rodata - -.p2align 6 -.amdhsa_kernel expr_defined_later - .amdhsa_system_sgpr_private_segment_wavefront_offset defined_boolean - .amdhsa_dx10_clamp defined_boolean - .amdhsa_ieee_mode defined_boolean - .amdhsa_fp16_overflow defined_boolean - .amdhsa_tg_split defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 -.end_amdhsa_kernel - -.set defined_boolean, 1 - -.p2align 6 -.amdhsa_kernel expr_defined - .amdhsa_system_sgpr_private_segment_wavefront_offset defined_boolean - .amdhsa_dx10_clamp defined_boolean - .amdhsa_ieee_mode defined_boolean - .amdhsa_fp16_overflow defined_boolean - .amdhsa_tg_split defined_boolean - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 -.end_amdhsa_kernel - -// ASM: .amdhsa_kernel expr_defined_later -// ASM-NEXT: .amdhsa_group_segment_fixed_size 0 -// ASM-NEXT: .amdhsa_private_segment_fixed_size 0 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 -// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_accum_offset (((((((0&(~65536))|(defined_boolean<<16))&(~63))|(0<<0))&63)>>0)+1)*4 -// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 -// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&12288)>>12 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&49152)>>14 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&196608)>>16 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&786432)>>18 -// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&2097152)>>21 -// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&8388608)>>23 -// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(0<<0))&(~960))|(0<<6))&67108864)>>26 -// ASM-NEXT: .amdhsa_tg_split (((((0&(~65536))|(defined_boolean<<16))&(~63))|(0<<0))&65536)>>16 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29 -// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30 -// ASM-NEXT: .end_amdhsa_kernel - -// ASM: .set defined_boolean, 1 -// ASM-NEXT: .no_dead_strip defined_boolean - -// ASM: .amdhsa_kernel expr_defined -// ASM-NEXT: .amdhsa_group_segment_fixed_size 0 -// ASM-NEXT: .amdhsa_private_segment_fixed_size 0 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 -// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 0 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 0 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_accum_offset 4 -// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 -// ASM-NEXT: .amdhsa_float_round_mode_32 0 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 0 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 0 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 -// ASM-NEXT: .amdhsa_dx10_clamp 1 -// ASM-NEXT: .amdhsa_ieee_mode 1 -// ASM-NEXT: .amdhsa_fp16_overflow 1 -// ASM-NEXT: .amdhsa_tg_split 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 0 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 0 -// ASM-NEXT: .amdhsa_exception_int_div_zero 0 -// ASM-NEXT: .end_amdhsa_kernel diff --git a/llvm/test/MC/AMDGPU/hsa-tg-split.s b/llvm/test/MC/AMDGPU/hsa-tg-split.s deleted file mode 100644 index 5a4d3e2c279c..000000000000 --- a/llvm/test/MC/AMDGPU/hsa-tg-split.s +++ /dev/null @@ -1,74 +0,0 @@ -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+xnack,+tgsplit < %s | FileCheck --check-prefix=ASM %s -// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+xnack,+tgsplit -filetype=obj < %s > %t -// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s - -// OBJDUMP: Contents of section .rodata -// OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000 -// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000100 -// OBJDUMP-NEXT: 0030 0000ac00 80000000 00000000 00000000 - -.text -// ASM: .text - -.amdgcn_target "amdgcn-amd-amdhsa--gfx90a:xnack+" -// ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx90a:xnack+" - -.amdhsa_code_object_version 4 -// ASM: .amdhsa_code_object_version 4 - -.p2align 8 -.type minimal,@function -minimal: - s_endpgm - -.rodata -// ASM: .rodata - -.p2align 6 -.amdhsa_kernel minimal - .amdhsa_next_free_vgpr 0 - .amdhsa_next_free_sgpr 0 - .amdhsa_accum_offset 4 -.end_amdhsa_kernel - -// ASM: .amdhsa_kernel minimal -// ASM-NEXT: .amdhsa_group_segment_fixed_size 0 -// ASM-NEXT: .amdhsa_private_segment_fixed_size 0 -// ASM-NEXT: .amdhsa_kernarg_size 0 -// ASM-NEXT: .amdhsa_user_sgpr_count 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0 -// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0 -// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length 0 -// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset 0 -// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0 -// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset 0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0 -// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 0 -// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 0 -// ASM-NEXT: .amdhsa_next_free_vgpr 0 -// ASM-NEXT: .amdhsa_next_free_sgpr 0 -// ASM-NEXT: .amdhsa_accum_offset 4 -// ASM-NEXT: .amdhsa_reserve_xnack_mask 1 -// ASM-NEXT: .amdhsa_float_round_mode_32 0 -// ASM-NEXT: .amdhsa_float_round_mode_16_64 0 -// ASM-NEXT: .amdhsa_float_denorm_mode_32 0 -// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3 -// ASM-NEXT: .amdhsa_dx10_clamp 1 -// ASM-NEXT: .amdhsa_ieee_mode 1 -// ASM-NEXT: .amdhsa_fp16_overflow 0 -// ASM-NEXT: .amdhsa_tg_split 1 -// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 0 -// ASM-NEXT: .amdhsa_exception_fp_denorm_src 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 0 -// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 0 -// ASM-NEXT: .amdhsa_exception_int_div_zero 0 -// ASM-NEXT: .end_amdhsa_kernel -- GitLab From c8772940ee4d85b1a4578b3faea7c825300ce59f Mon Sep 17 00:00:00 2001 From: Balazs Benics Date: Thu, 21 Mar 2024 18:22:22 +0100 Subject: [PATCH 176/296] [analyzer] Wrap SymbolicRegions by ElementRegions before getting a FieldRegion (#85211) Inside the ExprEngine when we process the initializers, we create a PostInitializer program-point, which will refer to the field being initialized, see `FieldLoc` inside `ExprEngine::ProcessInitializer`. When a constructor (of which we evaluate the initializer-list) is analyzed in top-level context, then the `this` pointer will be represented by a `SymbolicRegion`, (as it should be). This means that we will form a `FieldRegion{SymbolicRegion{.}}` as the initialized region. ```c++ class Bear { public: void brum() const; }; class Door { public: // PostInitializer would refer to "FieldRegion{SymRegion{this}}" // whereas in the store and everywhere else it would be: // "FieldRegion{ELementRegion{SymRegion{Ty*, this}, 0, Ty}". Door() : ptr(nullptr) { ptr->brum(); // Bug } private: Bear* ptr; }; ``` We (as CSA folks) decided to avoid the creation of FieldRegions directly of symbolic regions in the past: https://github.com/llvm/llvm-project/commit/f8643a9b31c4029942f67d4534c9139b45173504 --- In this patch, I propose to also canonicalize it as in the mentioned patch, into this: `FieldRegion{ElementRegion{SymbolicRegion{Ty*, .}, 0, Ty}` This would mean that FieldRegions will/should never simply wrap a SymbolicRegion directly, but rather an ElementRegion that is sitting in between. This patch should have practically no observable effects, as the store (due to the mentioned patch) was made resilient to this issue, but we use `PostInitializer::getLocationValue()` for an alternative reporting, where we faced this issue. Note that in really rare cases it suppresses now dereference bugs, as demonstrated in the test. It is because in the past we failed to follow the region of the PostInitializer inside the StoreSiteFinder visitor - because it was using this code: ```c++ // If this is a post initializer expression, initializing the region, we // should track the initializer expression. if (std::optional PIP = Pred->getLocationAs()) { const MemRegion *FieldReg = (const MemRegion *)PIP->getLocationValue(); if (FieldReg == R) { StoreSite = Pred; InitE = PIP->getInitializer()->getInit(); } } ``` Notice that the equality check didn't pass for the regions I'm canonicalizing in this patch. Given the nature of this change, we would rather upstream this patch. CPP-4954 --- .../Core/PathSensitive/ProgramState.h | 16 ++-------- .../lib/StaticAnalyzer/Core/ProgramState.cpp | 32 +++++++++++++++++++ .../inlining/false-positive-suppression.cpp | 17 ++++++++++ 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h index ca75c2a756a4..51d76dc257ee 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h @@ -494,6 +494,8 @@ private: InvalidatedSymbols *IS, RegionAndSymbolInvalidationTraits *HTraits, const CallEvent *Call) const; + + SVal wrapSymbolicRegion(SVal Base) const; }; //===----------------------------------------------------------------------===// @@ -782,20 +784,6 @@ inline SVal ProgramState::getLValue(const ObjCIvarDecl *D, SVal Base) const { return getStateManager().StoreMgr->getLValueIvar(D, Base); } -inline SVal ProgramState::getLValue(const FieldDecl *D, SVal Base) const { - return getStateManager().StoreMgr->getLValueField(D, Base); -} - -inline SVal ProgramState::getLValue(const IndirectFieldDecl *D, - SVal Base) const { - StoreManager &SM = *getStateManager().StoreMgr; - for (const auto *I : D->chain()) { - Base = SM.getLValueField(cast(I), Base); - } - - return Base; -} - inline SVal ProgramState::getLValue(QualType ElementType, SVal Idx, SVal Base) const{ if (std::optional N = Idx.getAs()) return getStateManager().StoreMgr->getLValueElement(ElementType, *N, Base); diff --git a/clang/lib/StaticAnalyzer/Core/ProgramState.cpp b/clang/lib/StaticAnalyzer/Core/ProgramState.cpp index f12f1a5ac970..f82cd944750a 100644 --- a/clang/lib/StaticAnalyzer/Core/ProgramState.cpp +++ b/clang/lib/StaticAnalyzer/Core/ProgramState.cpp @@ -226,6 +226,20 @@ ProgramStateRef ProgramState::killBinding(Loc LV) const { return makeWithStore(newStore); } +/// SymbolicRegions are expected to be wrapped by an ElementRegion as a +/// canonical representation. As a canonical representation, SymbolicRegions +/// should be wrapped by ElementRegions before getting a FieldRegion. +/// See f8643a9b31c4029942f67d4534c9139b45173504 why. +SVal ProgramState::wrapSymbolicRegion(SVal Val) const { + const auto *BaseReg = dyn_cast_or_null(Val.getAsRegion()); + if (!BaseReg) + return Val; + + StoreManager &SM = getStateManager().getStoreManager(); + QualType ElemTy = BaseReg->getPointeeStaticType(); + return loc::MemRegionVal{SM.GetElementZeroRegion(BaseReg, ElemTy)}; +} + ProgramStateRef ProgramState::enterStackFrame(const CallEvent &Call, const StackFrameContext *CalleeCtx) const { @@ -451,6 +465,24 @@ void ProgramState::setStore(const StoreRef &newStore) { store = newStoreStore; } +SVal ProgramState::getLValue(const FieldDecl *D, SVal Base) const { + Base = wrapSymbolicRegion(Base); + return getStateManager().StoreMgr->getLValueField(D, Base); +} + +SVal ProgramState::getLValue(const IndirectFieldDecl *D, SVal Base) const { + StoreManager &SM = *getStateManager().StoreMgr; + Base = wrapSymbolicRegion(Base); + + // FIXME: This should work with `SM.getLValueField(D->getAnonField(), Base)`, + // but that would break some tests. There is probably a bug somewhere that it + // would expose. + for (const auto *I : D->chain()) { + Base = SM.getLValueField(cast(I), Base); + } + return Base; +} + //===----------------------------------------------------------------------===// // State pretty-printing. //===----------------------------------------------------------------------===// diff --git a/clang/test/Analysis/inlining/false-positive-suppression.cpp b/clang/test/Analysis/inlining/false-positive-suppression.cpp index 56659b4a1941..2f9ed7f78b3f 100644 --- a/clang/test/Analysis/inlining/false-positive-suppression.cpp +++ b/clang/test/Analysis/inlining/false-positive-suppression.cpp @@ -210,3 +210,20 @@ namespace Cleanups { testArgumentHelper(NonTrivial().getNull()); } } + +class Bear *getNullBear() { return nullptr; } +class Bear { +public: + void brum() const; +}; +class Door { +public: + Door() : ptr(getNullBear()) { + ptr->brum(); +#ifndef SUPPRESSED + // expected-warning@-2 {{Called C++ object pointer is null}} +#endif + } +private: + Bear* ptr; +}; -- GitLab From 86d479fd7c837e97be116ffb9e4c92812b87360f Mon Sep 17 00:00:00 2001 From: T-Gruber <100079402+T-Gruber@users.noreply.github.com> Date: Thu, 21 Mar 2024 18:27:53 +0100 Subject: [PATCH 177/296] Adapted MemRegion::getDescriptiveName to handle ElementRegions (#85104) Fixes https://github.com/llvm/llvm-project/issues/84463 Changes: - Adapted MemRegion::getDescriptiveName - Added unittest to check name for a given clang::ento::ElementRegion - Some format changes due to clang-format --------- Co-authored-by: Andreas Steinhausen Co-authored-by: Balazs Benics --- clang/lib/StaticAnalyzer/Core/MemRegion.cpp | 20 ++- clang/unittests/StaticAnalyzer/CMakeLists.txt | 1 + .../MemRegionDescriptiveNameTest.cpp | 145 ++++++++++++++++++ .../clang/unittests/StaticAnalyzer/BUILD.gn | 1 + 4 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 clang/unittests/StaticAnalyzer/MemRegionDescriptiveNameTest.cpp diff --git a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp index 16db6b249dc9..a8e573f7982b 100644 --- a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp +++ b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp @@ -720,13 +720,21 @@ std::string MemRegion::getDescriptiveName(bool UseQuotes) const { CI->getValue().toString(Idx); ArrayIndices = (llvm::Twine("[") + Idx.str() + "]" + ArrayIndices).str(); } - // If not a ConcreteInt, try to obtain the variable - // name by calling 'getDescriptiveName' recursively. + // Index is symbolic, but may have a descriptive name. else { - std::string Idx = ER->getDescriptiveName(false); - if (!Idx.empty()) { - ArrayIndices = (llvm::Twine("[") + Idx + "]" + ArrayIndices).str(); - } + auto SI = ER->getIndex().getAs(); + if (!SI) + return ""; + + const MemRegion *OR = SI->getAsSymbol()->getOriginRegion(); + if (!OR) + return ""; + + std::string Idx = OR->getDescriptiveName(false); + if (Idx.empty()) + return ""; + + ArrayIndices = (llvm::Twine("[") + Idx + "]" + ArrayIndices).str(); } R = ER->getSuperRegion(); } diff --git a/clang/unittests/StaticAnalyzer/CMakeLists.txt b/clang/unittests/StaticAnalyzer/CMakeLists.txt index 775f0f8486b8..519be36fe0fa 100644 --- a/clang/unittests/StaticAnalyzer/CMakeLists.txt +++ b/clang/unittests/StaticAnalyzer/CMakeLists.txt @@ -11,6 +11,7 @@ add_clang_unittest(StaticAnalysisTests CallEventTest.cpp ConflictingEvalCallsTest.cpp FalsePositiveRefutationBRVisitorTest.cpp + MemRegionDescriptiveNameTest.cpp NoStateChangeFuncVisitorTest.cpp ParamRegionTest.cpp RangeSetTest.cpp diff --git a/clang/unittests/StaticAnalyzer/MemRegionDescriptiveNameTest.cpp b/clang/unittests/StaticAnalyzer/MemRegionDescriptiveNameTest.cpp new file mode 100644 index 000000000000..ba0c4d25e13b --- /dev/null +++ b/clang/unittests/StaticAnalyzer/MemRegionDescriptiveNameTest.cpp @@ -0,0 +1,145 @@ +//===- MemRegionDescriptiveNameTest.cpp -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "CheckerRegistration.h" +#include "clang/StaticAnalyzer/Core/Checker.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/CallDescription.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/MemRegion.h" +#include "gtest/gtest.h" +#include + +using namespace clang; +using namespace ento; + +namespace { + +class DescriptiveNameChecker : public Checker { +public: + void checkPreCall(const CallEvent &Call, CheckerContext &C) const { + if (!HandlerFn.matches(Call)) + return; + + const MemRegion *ArgReg = Call.getArgSVal(0).getAsRegion(); + assert(ArgReg && "expecting a location as the first argument"); + + auto DescriptiveName = ArgReg->getDescriptiveName(/*UseQuotes=*/false); + if (ExplodedNode *Node = C.generateNonFatalErrorNode(C.getState())) { + auto Report = + std::make_unique(Bug, DescriptiveName, Node); + C.emitReport(std::move(Report)); + } + } + +private: + const BugType Bug{this, "DescriptiveNameBug"}; + const CallDescription HandlerFn = {{"reportDescriptiveName"}, 1}; +}; + +void addDescriptiveNameChecker(AnalysisASTConsumer &AnalysisConsumer, + AnalyzerOptions &AnOpts) { + AnOpts.CheckersAndPackages = {{"DescriptiveNameChecker", true}}; + AnalysisConsumer.AddCheckerRegistrationFn([](CheckerRegistry &Registry) { + Registry.addChecker("DescriptiveNameChecker", + "Desc", "DocsURI"); + }); +} + +bool runChecker(StringRef Code, std::string &Output) { + return runCheckerOnCode(Code.str(), Output, + /*OnlyEmitWarnings=*/true); +} + +TEST(MemRegionDescriptiveNameTest, ConcreteIntElementRegionIndex) { + StringRef Code = R"cpp( +void reportDescriptiveName(int *p); +const unsigned int index = 1; +extern int array[3]; +void top() { + reportDescriptiveName(&array[index]); +})cpp"; + + std::string Output; + ASSERT_TRUE(runChecker(Code, Output)); + EXPECT_EQ(Output, "DescriptiveNameChecker: array[1]\n"); +} + +TEST(MemRegionDescriptiveNameTest, SymbolicElementRegionIndex) { + StringRef Code = R"cpp( +void reportDescriptiveName(int *p); +extern unsigned int index; +extern int array[3]; +void top() { + reportDescriptiveName(&array[index]); +})cpp"; + + std::string Output; + ASSERT_TRUE(runChecker(Code, Output)); + EXPECT_EQ(Output, "DescriptiveNameChecker: array[index]\n"); +} + +TEST(MemRegionDescriptiveNameTest, SymbolicElementRegionIndexSymbolValFails) { + StringRef Code = R"cpp( +void reportDescriptiveName(int *p); +extern int* ptr; +extern int array[3]; +void top() { + reportDescriptiveName(&array[(long)ptr]); +})cpp"; + + std::string Output; + ASSERT_TRUE(runChecker(Code, Output)); + EXPECT_EQ(Output, "DescriptiveNameChecker: \n"); +} + +TEST(MemRegionDescriptiveNameTest, SymbolicElementRegionIndexOrigRegionFails) { + StringRef Code = R"cpp( +void reportDescriptiveName(int *p); +extern int getInt(void); +extern int array[3]; +void top() { + reportDescriptiveName(&array[getInt()]); +})cpp"; + + std::string Output; + ASSERT_TRUE(runChecker(Code, Output)); + EXPECT_EQ(Output, "DescriptiveNameChecker: \n"); +} + +TEST(MemRegionDescriptiveNameTest, SymbolicElementRegionIndexDescrNameFails) { + StringRef Code = R"cpp( +void reportDescriptiveName(int *p); +extern int *ptr; +extern int array[3]; +void top() { + reportDescriptiveName(&array[*ptr]); +})cpp"; + + std::string Output; + ASSERT_TRUE(runChecker(Code, Output)); + EXPECT_EQ(Output, "DescriptiveNameChecker: \n"); +} + +TEST(MemRegionDescriptiveNameTest, + SymbolicElementRegionIndexIncorrectSymbolName) { + StringRef Code = R"cpp( +void reportDescriptiveName(int *p); +extern int x, y; +extern int array[3]; +void top() { + y = x; + reportDescriptiveName(&array[y]); +})cpp"; + + std::string Output; + ASSERT_TRUE(runChecker(Code, Output)); + // FIXME: Should return array[y], but returns array[x] (OriginRegion). + EXPECT_EQ(Output, "DescriptiveNameChecker: array[x]\n"); +} + +} // namespace diff --git a/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn index 01c2b6ced336..9230ac79a48a 100644 --- a/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn +++ b/llvm/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn @@ -19,6 +19,7 @@ unittest("StaticAnalysisTests") { "CallEventTest.cpp", "ConflictingEvalCallsTest.cpp", "FalsePositiveRefutationBRVisitorTest.cpp", + "MemRegionDescriptiveNameTest.cpp", "NoStateChangeFuncVisitorTest.cpp", "ParamRegionTest.cpp", "RangeSetTest.cpp", -- GitLab From aa571a1e8f03faa311fb135ea4b24b9414db1b74 Mon Sep 17 00:00:00 2001 From: LLVM GN Syncbot Date: Thu, 21 Mar 2024 17:34:43 +0000 Subject: [PATCH 178/296] [gn build] Port 797336b1278c --- .../gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn index 5ba91fcec83a..12d875cf40c9 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/MCTargetDesc/BUILD.gn @@ -104,7 +104,6 @@ static_library("MCTargetDesc") { "AMDGPUMCAsmInfo.cpp", "AMDGPUMCCodeEmitter.cpp", "AMDGPUMCExpr.cpp", - "AMDGPUMCKernelDescriptor.cpp", "AMDGPUMCTargetDesc.cpp", "AMDGPUTargetStreamer.cpp", "R600InstPrinter.cpp", -- GitLab From e84a985cbf4b26c10812e9bd339db9cfd037f581 Mon Sep 17 00:00:00 2001 From: Daniel Chen Date: Thu, 21 Mar 2024 13:50:40 -0400 Subject: [PATCH 179/296] [Flang] Support for NULL() and procedure in structure constructor for procedure pointer component. (#85991) This PR fixes a subset of procedure pointer component initialization in structure constructor. It covers 1. NULL() 2. procedure For example: ``` MODULE M TYPE :: DT !PROCEDURE(Fun), POINTER, NOPASS :: pp1 PROCEDURE(Fun), POINTER :: pp1 END TYPE CONTAINS INTEGER FUNCTION Fun(Arg) class(dt) :: arg END FUNCTION END MODULE PROGRAM MAIN USE M IMPLICIT NONE TYPE (DT), PARAMETER :: v1 = DT(NULL()) TYPE (DT) :: v2 v2 = DT(fun) END ``` Passing a procedure pointer itself or reference to a function that returns a procedure pointer is TODO. --- flang/lib/Lower/ConvertConstant.cpp | 22 +++++++-- ...ointer-component-structure-constructor.f90 | 48 +++++++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90 diff --git a/flang/lib/Lower/ConvertConstant.cpp b/flang/lib/Lower/ConvertConstant.cpp index 336944d35b7e..ed389bbe4ae5 100644 --- a/flang/lib/Lower/ConvertConstant.cpp +++ b/flang/lib/Lower/ConvertConstant.cpp @@ -14,9 +14,12 @@ #include "flang/Evaluate/expression.h" #include "flang/Lower/AbstractConverter.h" #include "flang/Lower/BuiltinModules.h" +#include "flang/Lower/ConvertExprToHLFIR.h" #include "flang/Lower/ConvertType.h" #include "flang/Lower/ConvertVariable.h" #include "flang/Lower/Mangler.h" +#include "flang/Lower/StatementContext.h" +#include "flang/Lower/SymbolMap.h" #include "flang/Optimizer/Builder/Complex.h" #include "flang/Optimizer/Builder/MutableBox.h" #include "flang/Optimizer/Builder/Todo.h" @@ -380,10 +383,21 @@ static mlir::Value genStructureComponentInit( } if (Fortran::semantics::IsPointer(sym)) { - if (Fortran::semantics::IsProcedure(sym)) - TODO(loc, "procedure pointer component initial value"); - mlir::Value initialTarget = - Fortran::lower::genInitialDataTarget(converter, loc, componentTy, expr); + mlir::Value initialTarget; + if (Fortran::semantics::IsProcedure(sym)) { + if (Fortran::evaluate::UnwrapExpr(expr)) + initialTarget = + fir::factory::createNullBoxProc(builder, loc, componentTy); + else { + Fortran::lower::SymMap globalOpSymMap; + Fortran::lower::StatementContext stmtCtx; + auto box{getBase(Fortran::lower::convertExprToAddress( + loc, converter, expr, globalOpSymMap, stmtCtx))}; + initialTarget = builder.createConvert(loc, componentTy, box); + } + } else + initialTarget = Fortran::lower::genInitialDataTarget(converter, loc, + componentTy, expr); res = builder.create( loc, recTy, res, initialTarget, builder.getArrayAttr(field.getAttributes())); diff --git a/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90 b/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90 new file mode 100644 index 000000000000..f41c832ee5ec --- /dev/null +++ b/flang/test/Lower/HLFIR/procedure-pointer-component-structure-constructor.f90 @@ -0,0 +1,48 @@ +! Test passing +! 1. NULL(), +! 2. procedure, +! 3. procedure pointer, (pending) +! 4. reference to a function that returns a procedure pointer (pending) +! to a derived type structure constructor. +! RUN: bbc -emit-hlfir -o - %s | FileCheck %s + + MODULE M + TYPE :: DT + PROCEDURE(Fun), POINTER, NOPASS :: pp1 + END TYPE + + CONTAINS + + INTEGER FUNCTION Fun(Arg) + INTEGER :: Arg + Fun = Arg + END FUNCTION + + END MODULE + + PROGRAM MAIN + USE M + IMPLICIT NONE + TYPE (DT), PARAMETER :: v1 = DT(NULL()) + TYPE (DT) :: v2 + v2 = DT(fun) + END + +! CDHECK-LABEL: fir.global internal @_QFECv1 constant : !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> { +! CHECK: %[[VAL_0:.*]] = fir.undefined !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> +! CHECK: %[[VAL_1:.*]] = fir.field_index pp1, !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> +! CHECK: %[[VAL_2:.*]] = fir.zero_bits (!fir.ref) -> i32 +! CHECK: %[[VAL_3:.*]] = fir.emboxproc %[[VAL_2]] : ((!fir.ref) -> i32) -> !fir.boxproc<(!fir.ref) -> i32> +! CHECK: %[[VAL_4:.*]] = fir.insert_value %[[VAL_0]], %[[VAL_3]], ["pp1", !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}>] : (!fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}>, !fir.boxproc<(!fir.ref) -> i32>) -> !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> +! CHECK: fir.has_value %[[VAL_4]] : !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> +! CHECK: } + +! CHECK-LABEL: fir.global internal @_QQro._QMmTdt.0 constant : !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> { +! CHECK: %[[VAL_0:.*]] = fir.undefined !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> +! CHECK: %[[VAL_1:.*]] = fir.field_index pp1, !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> +! CHECK: %[[VAL_2:.*]] = fir.address_of(@_QMmPfun) : (!fir.ref) -> i32 +! CHECK: %[[VAL_3:.*]] = fir.emboxproc %[[VAL_2]] : ((!fir.ref) -> i32) -> !fir.boxproc<() -> ()> +! CHECK: %[[VAL_4:.*]] = fir.convert %[[VAL_3]] : (!fir.boxproc<() -> ()>) -> !fir.boxproc<(!fir.ref) -> i32> +! CHECK: %[[VAL_5:.*]] = fir.insert_value %[[VAL_0]], %[[VAL_4]], ["pp1", !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}>] : (!fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}>, !fir.boxproc<(!fir.ref) -> i32>) -> !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> +! CHECK: fir.has_value %[[VAL_5]] : !fir.type<_QMmTdt{pp1:!fir.boxproc<(!fir.ref) -> i32>}> +! CHECK: } -- GitLab From 3218570620a320573dfd3315da8277600a010bc1 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 21 Mar 2024 17:52:53 +0000 Subject: [PATCH 180/296] [X86] Add shuffle test case for Issue #86068 --- llvm/test/CodeGen/X86/oddshuffles.ll | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/llvm/test/CodeGen/X86/oddshuffles.ll b/llvm/test/CodeGen/X86/oddshuffles.ll index 5da18ee6ad7c..01056a8b2c24 100644 --- a/llvm/test/CodeGen/X86/oddshuffles.ll +++ b/llvm/test/CodeGen/X86/oddshuffles.ll @@ -2369,6 +2369,31 @@ define void @PR41097() { ret void } +; FIXME - should use INSERTPS +define <2 x float> @PR86068(<2 x float> %0, <2 x float> %1) { +; SSE2-LABEL: PR86068: +; SSE2: # %bb.0: # %entry +; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[1,1] +; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm1[1,1] +; SSE2-NEXT: retq +; +; SSE42-LABEL: PR86068: +; SSE42: # %bb.0: # %entry +; SSE42-NEXT: movshdup {{.*#+}} xmm1 = xmm1[1,1,3,3] +; SSE42-NEXT: blendps {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3] +; SSE42-NEXT: retq +; +; AVX-LABEL: PR86068: +; AVX: # %bb.0: # %entry +; AVX-NEXT: vmovshdup {{.*#+}} xmm1 = xmm1[1,1,3,3] +; AVX-NEXT: vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3] +; AVX-NEXT: retq +entry: + %3 = shufflevector <2 x float> %1, <2 x float> poison, <2 x i32> + %4 = shufflevector <2 x float> %3, <2 x float> %0, <2 x i32> + ret <2 x float> %4 +} + define void @D107009(ptr %input, ptr %output) { ; SSE-LABEL: D107009: ; SSE: # %bb.0: -- GitLab From aa7f8200ac29823e6c662d1f41763c2509514161 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Wed, 20 Mar 2024 21:33:02 -0500 Subject: [PATCH 181/296] [IR] Add helpers for `NUWAddLike` and `NSWAddLike` to also match `or disjoint`; NFC `or disjoint` implies `add nuw nsw`: https://alive2.llvm.org/ce/z/VABhDA --- llvm/include/llvm/IR/PatternMatch.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h index 382009d9df78..04feb1fd17fd 100644 --- a/llvm/include/llvm/IR/PatternMatch.h +++ b/llvm/include/llvm/IR/PatternMatch.h @@ -1319,6 +1319,26 @@ m_AddLike(const LHS &L, const RHS &R) { return m_CombineOr(m_Add(L, R), m_DisjointOr(L, R)); } +/// Match either "add nsw" or "or disjoint" +template +inline match_combine_or< + OverflowingBinaryOp_match, + DisjointOr_match> +m_NSWAddLike(const LHS &L, const RHS &R) { + return m_CombineOr(m_NSWAdd(L, R), m_DisjointOr(L, R)); +} + +/// Match either "add nuw" or "or disjoint" +template +inline match_combine_or< + OverflowingBinaryOp_match, + DisjointOr_match> +m_NUWAddLike(const LHS &L, const RHS &R) { + return m_CombineOr(m_NUWAdd(L, R), m_DisjointOr(L, R)); +} + //===----------------------------------------------------------------------===// // Class that matches a group of binary opcodes. // -- GitLab From ac13e5c0f698c1e3f759e622d2d3cf36ab3b77f3 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Wed, 20 Mar 2024 22:09:23 -0500 Subject: [PATCH 182/296] [InstCombine] Add tests for integrating `N{U,S}WAddLike`; NFC --- llvm/test/Transforms/InstCombine/add.ll | 79 +++++++++++++++++++ llvm/test/Transforms/InstCombine/div.ll | 26 ++++++ .../InstCombine/sadd-with-overflow.ll | 33 ++++++++ llvm/test/Transforms/InstCombine/shift-add.ll | 33 ++++++++ .../InstCombine/uadd-with-overflow.ll | 24 ++++++ 5 files changed, 195 insertions(+) diff --git a/llvm/test/Transforms/InstCombine/add.ll b/llvm/test/Transforms/InstCombine/add.ll index 522dcf8db27f..846da7f760b1 100644 --- a/llvm/test/Transforms/InstCombine/add.ll +++ b/llvm/test/Transforms/InstCombine/add.ll @@ -3986,5 +3986,84 @@ define i32 @add_reduce_sqr_sum_varC_invalid2(i32 %a, i32 %b) { ret i32 %ab2 } +define i32 @fold_sext_addition_or_disjoint(i8 %x) { +; CHECK-LABEL: @fold_sext_addition_or_disjoint( +; CHECK-NEXT: [[XX:%.*]] = or disjoint i8 [[X:%.*]], 12 +; CHECK-NEXT: [[SE:%.*]] = sext i8 [[XX]] to i32 +; CHECK-NEXT: [[R:%.*]] = add nsw i32 [[SE]], 1234 +; CHECK-NEXT: ret i32 [[R]] +; + %xx = or disjoint i8 %x, 12 + %se = sext i8 %xx to i32 + %r = add i32 %se, 1234 + ret i32 %r +} + +define i32 @fold_sext_addition_fail(i8 %x) { +; CHECK-LABEL: @fold_sext_addition_fail( +; CHECK-NEXT: [[XX:%.*]] = or i8 [[X:%.*]], 12 +; CHECK-NEXT: [[SE:%.*]] = sext i8 [[XX]] to i32 +; CHECK-NEXT: [[R:%.*]] = add nsw i32 [[SE]], 1234 +; CHECK-NEXT: ret i32 [[R]] +; + %xx = or i8 %x, 12 + %se = sext i8 %xx to i32 + %r = add i32 %se, 1234 + ret i32 %r +} + +define i32 @fold_zext_addition_or_disjoint(i8 %x) { +; CHECK-LABEL: @fold_zext_addition_or_disjoint( +; CHECK-NEXT: [[XX:%.*]] = or disjoint i8 [[X:%.*]], 12 +; CHECK-NEXT: [[SE:%.*]] = zext i8 [[XX]] to i32 +; CHECK-NEXT: [[R:%.*]] = add nuw nsw i32 [[SE]], 1234 +; CHECK-NEXT: ret i32 [[R]] +; + %xx = or disjoint i8 %x, 12 + %se = zext i8 %xx to i32 + %r = add i32 %se, 1234 + ret i32 %r +} + +define i32 @fold_zext_addition_or_disjoint2(i8 %x) { +; CHECK-LABEL: @fold_zext_addition_or_disjoint2( +; CHECK-NEXT: [[XX:%.*]] = or disjoint i8 [[X:%.*]], 18 +; CHECK-NEXT: [[SE:%.*]] = zext i8 [[XX]] to i32 +; CHECK-NEXT: [[R:%.*]] = add nsw i32 [[SE]], -14 +; CHECK-NEXT: ret i32 [[R]] +; + %xx = or disjoint i8 %x, 18 + %se = zext i8 %xx to i32 + %r = add i32 %se, -14 + ret i32 %r +} + +define i32 @fold_zext_addition_fail(i8 %x) { +; CHECK-LABEL: @fold_zext_addition_fail( +; CHECK-NEXT: [[XX:%.*]] = or i8 [[X:%.*]], 12 +; CHECK-NEXT: [[SE:%.*]] = zext i8 [[XX]] to i32 +; CHECK-NEXT: [[R:%.*]] = add nuw nsw i32 [[SE]], 1234 +; CHECK-NEXT: ret i32 [[R]] +; + %xx = or i8 %x, 12 + %se = zext i8 %xx to i32 + %r = add i32 %se, 1234 + ret i32 %r +} + +define i32 @fold_zext_addition_fail2(i8 %x) { +; CHECK-LABEL: @fold_zext_addition_fail2( +; CHECK-NEXT: [[XX:%.*]] = or i8 [[X:%.*]], 18 +; CHECK-NEXT: [[SE:%.*]] = zext i8 [[XX]] to i32 +; CHECK-NEXT: [[R:%.*]] = add nsw i32 [[SE]], -14 +; CHECK-NEXT: ret i32 [[R]] +; + %xx = or i8 %x, 18 + %se = zext i8 %xx to i32 + %r = add i32 %se, -14 + ret i32 %r +} + + declare void @llvm.assume(i1) declare void @fake_func(i32) diff --git a/llvm/test/Transforms/InstCombine/div.ll b/llvm/test/Transforms/InstCombine/div.ll index 1309dee817cf..a4d604b329c3 100644 --- a/llvm/test/Transforms/InstCombine/div.ll +++ b/llvm/test/Transforms/InstCombine/div.ll @@ -1810,3 +1810,29 @@ define i6 @udiv_distribute_mul_nsw_add_nuw(i6 %x) { %div = udiv i6 %add, 3 ret i6 %div } + +define i32 @fold_disjoint_or_over_sdiv(i32 %x) { +; CHECK-LABEL: @fold_disjoint_or_over_sdiv( +; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[X:%.*]], 9 +; CHECK-NEXT: [[OR:%.*]] = or disjoint i32 [[MUL]], 81 +; CHECK-NEXT: [[R:%.*]] = sdiv i32 [[OR]], 9 +; CHECK-NEXT: ret i32 [[R]] +; + %mul = mul nsw i32 %x, 9 + %or = or disjoint i32 %mul, 81 + %r = sdiv i32 %or, 9 + ret i32 %r +} + +define i32 @fold_disjoint_or_over_udiv(i32 %x) { +; CHECK-LABEL: @fold_disjoint_or_over_udiv( +; CHECK-NEXT: [[MUL:%.*]] = mul nuw i32 [[X:%.*]], 9 +; CHECK-NEXT: [[OR:%.*]] = or disjoint i32 [[MUL]], 81 +; CHECK-NEXT: [[R:%.*]] = udiv i32 [[OR]], 9 +; CHECK-NEXT: ret i32 [[R]] +; + %mul = mul nuw i32 %x, 9 + %or = or disjoint i32 %mul, 81 + %r = udiv i32 %or, 9 + ret i32 %r +} diff --git a/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll b/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll index 4b37ccbe3370..904dd480caaf 100644 --- a/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll +++ b/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll @@ -122,3 +122,36 @@ define { i32, i1 } @fold_sub_simple(i32 %x) { %b = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 30) ret { i32, i1 } %b } + +define { i32, i1 } @fold_with_distjoin_or(i32 %x) { +; CHECK-LABEL: @fold_with_distjoin_or( +; CHECK-NEXT: [[B:%.*]] = add i32 [[X:%.*]], 6 +; CHECK-NEXT: [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 poison, i1 false }, i32 [[B]], 0 +; CHECK-NEXT: ret { i32, i1 } [[TMP1]] +; + %a = or disjoint i32 %x, 13 + %b = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 -7) + ret { i32, i1 } %b +} + +define { i32, i1 } @fold_with_disjoint_or2(i32 %x) { +; CHECK-LABEL: @fold_with_disjoint_or2( +; CHECK-NEXT: [[A:%.*]] = or disjoint i32 [[X:%.*]], 100 +; CHECK-NEXT: [[B:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A]], i32 27) +; CHECK-NEXT: ret { i32, i1 } [[B]] +; + %a = or disjoint i32 %x, 100 + %b = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 27) + ret { i32, i1 } %b +} + +define { i32, i1 } @fold_with_or_fail(i32 %x) { +; CHECK-LABEL: @fold_with_or_fail( +; CHECK-NEXT: [[A:%.*]] = or i32 [[X:%.*]], 100 +; CHECK-NEXT: [[B:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A]], i32 27) +; CHECK-NEXT: ret { i32, i1 } [[B]] +; + %a = or i32 %x, 100 + %b = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 27) + ret { i32, i1 } %b +} diff --git a/llvm/test/Transforms/InstCombine/shift-add.ll b/llvm/test/Transforms/InstCombine/shift-add.ll index 1b2567505993..2be6d9b29d52 100644 --- a/llvm/test/Transforms/InstCombine/shift-add.ll +++ b/llvm/test/Transforms/InstCombine/shift-add.ll @@ -775,3 +775,36 @@ define <3 x i32> @add3_i96(<3 x i32> %0, <3 x i32> %1) { %25 = insertelement <3 x i32> %24, i32 %20, i32 2 ret <3 x i32> %25 } + +define i8 @shl_fold_or_disjoint_cnt(i8 %x) { +; CHECK-LABEL: @shl_fold_or_disjoint_cnt( +; CHECK-NEXT: [[A:%.*]] = or disjoint i8 [[X:%.*]], 3 +; CHECK-NEXT: [[R:%.*]] = shl i8 2, [[A]] +; CHECK-NEXT: ret i8 [[R]] +; + %a = or disjoint i8 %x, 3 + %r = shl i8 2, %a + ret i8 %r +} + +define <2 x i8> @ashr_fold_or_disjoint_cnt(<2 x i8> %x) { +; CHECK-LABEL: @ashr_fold_or_disjoint_cnt( +; CHECK-NEXT: [[A:%.*]] = or disjoint <2 x i8> [[X:%.*]], +; CHECK-NEXT: [[R:%.*]] = lshr <2 x i8> , [[A]] +; CHECK-NEXT: ret <2 x i8> [[R]] +; + %a = or disjoint <2 x i8> %x, + %r = ashr <2 x i8> , %a + ret <2 x i8> %r +} + +define <2 x i8> @lshr_fold_or_disjoint_cnt_out_of_bounds(<2 x i8> %x) { +; CHECK-LABEL: @lshr_fold_or_disjoint_cnt_out_of_bounds( +; CHECK-NEXT: [[A:%.*]] = or disjoint <2 x i8> [[X:%.*]], +; CHECK-NEXT: [[R:%.*]] = lshr <2 x i8> , [[A]] +; CHECK-NEXT: ret <2 x i8> [[R]] +; + %a = or disjoint <2 x i8> %x, + %r = lshr <2 x i8> , %a + ret <2 x i8> %r +} diff --git a/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll b/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll index 28d309baaa41..b1819e6c3e2b 100644 --- a/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll +++ b/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll @@ -124,3 +124,27 @@ define { i32, i1 } @no_fold_wrapped_add(i32 %x) { %b = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 30, i32 %a) ret { i32, i1 } %b } + + +define { <2 x i32>, <2 x i1> } @fold_simple_splat_with_disjoint_or_constant(<2 x i32> %x) { +; CHECK-LABEL: @fold_simple_splat_with_disjoint_or_constant( +; CHECK-NEXT: [[A:%.*]] = or disjoint <2 x i32> [[X:%.*]], +; CHECK-NEXT: [[B:%.*]] = tail call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> [[A]], <2 x i32> ) +; CHECK-NEXT: ret { <2 x i32>, <2 x i1> } [[B]] +; + %a = or disjoint <2 x i32> %x, + %b = tail call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> ) + ret { <2 x i32>, <2 x i1> } %b +} + + +define { <2 x i32>, <2 x i1> } @fold_simple_splat_constant_with_or_fail(<2 x i32> %x) { +; CHECK-LABEL: @fold_simple_splat_constant_with_or_fail( +; CHECK-NEXT: [[A:%.*]] = or <2 x i32> [[X:%.*]], +; CHECK-NEXT: [[B:%.*]] = tail call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> [[A]], <2 x i32> ) +; CHECK-NEXT: ret { <2 x i32>, <2 x i1> } [[B]] +; + %a = or <2 x i32> %x, + %b = tail call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> ) + ret { <2 x i32>, <2 x i1> } %b +} -- GitLab From b3ee127e7dead25fa284b26d2c5a067671d0e896 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Wed, 20 Mar 2024 22:09:33 -0500 Subject: [PATCH 183/296] [InstCombine] integrate `N{U,S}WAddLike` into existing folds Just went a quick replacement of `N{U,S}WAdd` with the `Like` variant that old matches `or disjoint` Closes #86082 --- .../Transforms/InstCombine/InstCombineAddSub.cpp | 8 +++++--- .../Transforms/InstCombine/InstCombineCalls.cpp | 5 +++-- .../InstCombine/InstCombineMulDivRem.cpp | 8 ++++---- .../Transforms/InstCombine/InstCombineShifts.cpp | 2 +- llvm/test/Transforms/InstCombine/add.ll | 15 ++++++--------- llvm/test/Transforms/InstCombine/div.ll | 8 ++------ .../Transforms/InstCombine/sadd-with-overflow.ll | 3 +-- llvm/test/Transforms/InstCombine/shift-add.ll | 10 +++------- .../Transforms/InstCombine/uadd-with-overflow.ll | 3 +-- 9 files changed, 26 insertions(+), 36 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp index aaf7184a5562..a978e9a643f5 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp @@ -819,7 +819,7 @@ static Instruction *foldNoWrapAdd(BinaryOperator &Add, Value *X; const APInt *C1, *C2; if (match(Op1, m_APInt(C1)) && - match(Op0, m_OneUse(m_ZExt(m_NUWAdd(m_Value(X), m_APInt(C2))))) && + match(Op0, m_OneUse(m_ZExt(m_NUWAddLike(m_Value(X), m_APInt(C2))))) && C1->isNegative() && C1->sge(-C2->sext(C1->getBitWidth()))) { Constant *NewC = ConstantInt::get(X->getType(), *C2 + C1->trunc(C2->getBitWidth())); @@ -829,14 +829,16 @@ static Instruction *foldNoWrapAdd(BinaryOperator &Add, // More general combining of constants in the wide type. // (sext (X +nsw NarrowC)) + C --> (sext X) + (sext(NarrowC) + C) Constant *NarrowC; - if (match(Op0, m_OneUse(m_SExt(m_NSWAdd(m_Value(X), m_Constant(NarrowC)))))) { + if (match(Op0, + m_OneUse(m_SExt(m_NSWAddLike(m_Value(X), m_Constant(NarrowC)))))) { Value *WideC = Builder.CreateSExt(NarrowC, Ty); Value *NewC = Builder.CreateAdd(WideC, Op1C); Value *WideX = Builder.CreateSExt(X, Ty); return BinaryOperator::CreateAdd(WideX, NewC); } // (zext (X +nuw NarrowC)) + C --> (zext X) + (zext(NarrowC) + C) - if (match(Op0, m_OneUse(m_ZExt(m_NUWAdd(m_Value(X), m_Constant(NarrowC)))))) { + if (match(Op0, + m_OneUse(m_ZExt(m_NUWAddLike(m_Value(X), m_Constant(NarrowC)))))) { Value *WideC = Builder.CreateZExt(NarrowC, Ty); Value *NewC = Builder.CreateAdd(WideC, Op1C); Value *WideX = Builder.CreateZExt(X, Ty); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index 426b548c074a..526fe5d08059 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -2093,8 +2093,9 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { Value *Arg0 = II->getArgOperand(0); Value *Arg1 = II->getArgOperand(1); bool IsSigned = IID == Intrinsic::sadd_with_overflow; - bool HasNWAdd = IsSigned ? match(Arg0, m_NSWAdd(m_Value(X), m_APInt(C0))) - : match(Arg0, m_NUWAdd(m_Value(X), m_APInt(C0))); + bool HasNWAdd = IsSigned + ? match(Arg0, m_NSWAddLike(m_Value(X), m_APInt(C0))) + : match(Arg0, m_NUWAddLike(m_Value(X), m_APInt(C0))); if (HasNWAdd && match(Arg1, m_APInt(C1))) { bool Overflow; APInt NewC = diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp index 6e05fd8fb4d6..10a4b1c2060c 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp @@ -1171,14 +1171,14 @@ Instruction *InstCombinerImpl::commonIDivTransforms(BinaryOperator &I) { // We need a multiple of the divisor for a signed add constant, but // unsigned is fine with any constant pair. if (IsSigned && - match(Op0, m_NSWAdd(m_NSWMul(m_Value(X), m_SpecificInt(*C2)), - m_APInt(C1))) && + match(Op0, m_NSWAddLike(m_NSWMul(m_Value(X), m_SpecificInt(*C2)), + m_APInt(C1))) && isMultiple(*C1, *C2, Quotient, IsSigned)) { return BinaryOperator::CreateNSWAdd(X, ConstantInt::get(Ty, Quotient)); } if (!IsSigned && - match(Op0, m_NUWAdd(m_NUWMul(m_Value(X), m_SpecificInt(*C2)), - m_APInt(C1)))) { + match(Op0, m_NUWAddLike(m_NUWMul(m_Value(X), m_SpecificInt(*C2)), + m_APInt(C1)))) { return BinaryOperator::CreateNUWAdd(X, ConstantInt::get(Ty, C1->udiv(*C2))); } diff --git a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp index eafd2889ec50..95aa2119e2d8 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp @@ -437,7 +437,7 @@ Instruction *InstCombinerImpl::commonShiftTransforms(BinaryOperator &I) { Value *A; Constant *C, *C1; if (match(Op0, m_Constant(C)) && - match(Op1, m_NUWAdd(m_Value(A), m_Constant(C1)))) { + match(Op1, m_NUWAddLike(m_Value(A), m_Constant(C1)))) { Value *NewC = Builder.CreateBinOp(I.getOpcode(), C, C1); BinaryOperator *NewShiftOp = BinaryOperator::Create(I.getOpcode(), NewC, A); if (I.getOpcode() == Instruction::Shl) { diff --git a/llvm/test/Transforms/InstCombine/add.ll b/llvm/test/Transforms/InstCombine/add.ll index 846da7f760b1..ec3aca26514c 100644 --- a/llvm/test/Transforms/InstCombine/add.ll +++ b/llvm/test/Transforms/InstCombine/add.ll @@ -3988,9 +3988,8 @@ define i32 @add_reduce_sqr_sum_varC_invalid2(i32 %a, i32 %b) { define i32 @fold_sext_addition_or_disjoint(i8 %x) { ; CHECK-LABEL: @fold_sext_addition_or_disjoint( -; CHECK-NEXT: [[XX:%.*]] = or disjoint i8 [[X:%.*]], 12 -; CHECK-NEXT: [[SE:%.*]] = sext i8 [[XX]] to i32 -; CHECK-NEXT: [[R:%.*]] = add nsw i32 [[SE]], 1234 +; CHECK-NEXT: [[SE:%.*]] = sext i8 [[XX:%.*]] to i32 +; CHECK-NEXT: [[R:%.*]] = add nsw i32 [[SE]], 1246 ; CHECK-NEXT: ret i32 [[R]] ; %xx = or disjoint i8 %x, 12 @@ -4014,9 +4013,8 @@ define i32 @fold_sext_addition_fail(i8 %x) { define i32 @fold_zext_addition_or_disjoint(i8 %x) { ; CHECK-LABEL: @fold_zext_addition_or_disjoint( -; CHECK-NEXT: [[XX:%.*]] = or disjoint i8 [[X:%.*]], 12 -; CHECK-NEXT: [[SE:%.*]] = zext i8 [[XX]] to i32 -; CHECK-NEXT: [[R:%.*]] = add nuw nsw i32 [[SE]], 1234 +; CHECK-NEXT: [[SE:%.*]] = zext i8 [[XX:%.*]] to i32 +; CHECK-NEXT: [[R:%.*]] = add nuw nsw i32 [[SE]], 1246 ; CHECK-NEXT: ret i32 [[R]] ; %xx = or disjoint i8 %x, 12 @@ -4027,10 +4025,9 @@ define i32 @fold_zext_addition_or_disjoint(i8 %x) { define i32 @fold_zext_addition_or_disjoint2(i8 %x) { ; CHECK-LABEL: @fold_zext_addition_or_disjoint2( -; CHECK-NEXT: [[XX:%.*]] = or disjoint i8 [[X:%.*]], 18 +; CHECK-NEXT: [[XX:%.*]] = add nuw i8 [[X:%.*]], 4 ; CHECK-NEXT: [[SE:%.*]] = zext i8 [[XX]] to i32 -; CHECK-NEXT: [[R:%.*]] = add nsw i32 [[SE]], -14 -; CHECK-NEXT: ret i32 [[R]] +; CHECK-NEXT: ret i32 [[SE]] ; %xx = or disjoint i8 %x, 18 %se = zext i8 %xx to i32 diff --git a/llvm/test/Transforms/InstCombine/div.ll b/llvm/test/Transforms/InstCombine/div.ll index a4d604b329c3..e8a25ff44d02 100644 --- a/llvm/test/Transforms/InstCombine/div.ll +++ b/llvm/test/Transforms/InstCombine/div.ll @@ -1813,9 +1813,7 @@ define i6 @udiv_distribute_mul_nsw_add_nuw(i6 %x) { define i32 @fold_disjoint_or_over_sdiv(i32 %x) { ; CHECK-LABEL: @fold_disjoint_or_over_sdiv( -; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[X:%.*]], 9 -; CHECK-NEXT: [[OR:%.*]] = or disjoint i32 [[MUL]], 81 -; CHECK-NEXT: [[R:%.*]] = sdiv i32 [[OR]], 9 +; CHECK-NEXT: [[R:%.*]] = add nsw i32 [[X:%.*]], 9 ; CHECK-NEXT: ret i32 [[R]] ; %mul = mul nsw i32 %x, 9 @@ -1826,9 +1824,7 @@ define i32 @fold_disjoint_or_over_sdiv(i32 %x) { define i32 @fold_disjoint_or_over_udiv(i32 %x) { ; CHECK-LABEL: @fold_disjoint_or_over_udiv( -; CHECK-NEXT: [[MUL:%.*]] = mul nuw i32 [[X:%.*]], 9 -; CHECK-NEXT: [[OR:%.*]] = or disjoint i32 [[MUL]], 81 -; CHECK-NEXT: [[R:%.*]] = udiv i32 [[OR]], 9 +; CHECK-NEXT: [[R:%.*]] = add nuw i32 [[X:%.*]], 9 ; CHECK-NEXT: ret i32 [[R]] ; %mul = mul nuw i32 %x, 9 diff --git a/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll b/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll index 904dd480caaf..729ca03ddfd1 100644 --- a/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll +++ b/llvm/test/Transforms/InstCombine/sadd-with-overflow.ll @@ -136,8 +136,7 @@ define { i32, i1 } @fold_with_distjoin_or(i32 %x) { define { i32, i1 } @fold_with_disjoint_or2(i32 %x) { ; CHECK-LABEL: @fold_with_disjoint_or2( -; CHECK-NEXT: [[A:%.*]] = or disjoint i32 [[X:%.*]], 100 -; CHECK-NEXT: [[B:%.*]] = tail call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[A]], i32 27) +; CHECK-NEXT: [[B:%.*]] = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 [[X:%.*]], i32 127) ; CHECK-NEXT: ret { i32, i1 } [[B]] ; %a = or disjoint i32 %x, 100 diff --git a/llvm/test/Transforms/InstCombine/shift-add.ll b/llvm/test/Transforms/InstCombine/shift-add.ll index 2be6d9b29d52..aa3a238e0949 100644 --- a/llvm/test/Transforms/InstCombine/shift-add.ll +++ b/llvm/test/Transforms/InstCombine/shift-add.ll @@ -778,8 +778,7 @@ define <3 x i32> @add3_i96(<3 x i32> %0, <3 x i32> %1) { define i8 @shl_fold_or_disjoint_cnt(i8 %x) { ; CHECK-LABEL: @shl_fold_or_disjoint_cnt( -; CHECK-NEXT: [[A:%.*]] = or disjoint i8 [[X:%.*]], 3 -; CHECK-NEXT: [[R:%.*]] = shl i8 2, [[A]] +; CHECK-NEXT: [[R:%.*]] = shl i8 16, [[X:%.*]] ; CHECK-NEXT: ret i8 [[R]] ; %a = or disjoint i8 %x, 3 @@ -789,8 +788,7 @@ define i8 @shl_fold_or_disjoint_cnt(i8 %x) { define <2 x i8> @ashr_fold_or_disjoint_cnt(<2 x i8> %x) { ; CHECK-LABEL: @ashr_fold_or_disjoint_cnt( -; CHECK-NEXT: [[A:%.*]] = or disjoint <2 x i8> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = lshr <2 x i8> , [[A]] +; CHECK-NEXT: [[R:%.*]] = lshr <2 x i8> , [[X:%.*]] ; CHECK-NEXT: ret <2 x i8> [[R]] ; %a = or disjoint <2 x i8> %x, @@ -800,9 +798,7 @@ define <2 x i8> @ashr_fold_or_disjoint_cnt(<2 x i8> %x) { define <2 x i8> @lshr_fold_or_disjoint_cnt_out_of_bounds(<2 x i8> %x) { ; CHECK-LABEL: @lshr_fold_or_disjoint_cnt_out_of_bounds( -; CHECK-NEXT: [[A:%.*]] = or disjoint <2 x i8> [[X:%.*]], -; CHECK-NEXT: [[R:%.*]] = lshr <2 x i8> , [[A]] -; CHECK-NEXT: ret <2 x i8> [[R]] +; CHECK-NEXT: ret <2 x i8> zeroinitializer ; %a = or disjoint <2 x i8> %x, %r = lshr <2 x i8> , %a diff --git a/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll b/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll index b1819e6c3e2b..fd5d38bb38dd 100644 --- a/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll +++ b/llvm/test/Transforms/InstCombine/uadd-with-overflow.ll @@ -128,8 +128,7 @@ define { i32, i1 } @no_fold_wrapped_add(i32 %x) { define { <2 x i32>, <2 x i1> } @fold_simple_splat_with_disjoint_or_constant(<2 x i32> %x) { ; CHECK-LABEL: @fold_simple_splat_with_disjoint_or_constant( -; CHECK-NEXT: [[A:%.*]] = or disjoint <2 x i32> [[X:%.*]], -; CHECK-NEXT: [[B:%.*]] = tail call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> [[A]], <2 x i32> ) +; CHECK-NEXT: [[B:%.*]] = call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> [[X:%.*]], <2 x i32> ) ; CHECK-NEXT: ret { <2 x i32>, <2 x i1> } [[B]] ; %a = or disjoint <2 x i32> %x, -- GitLab From 796efa8cd5800a42eb8362564be64f3d72512a05 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 21 Mar 2024 11:09:40 -0700 Subject: [PATCH 184/296] [Float2Int] Fix pessimization in the MinBW calculation. (#86051) The MinBW was being calculated using the significant bits of the upper and lower bounds. The upper bound is 1 past the last value in the range so I don't think it should be included. Instead use ConstantRange::getMinSignedBits. I'm still not sure if the +1 is needed after the getMinSignedBits call. --- llvm/lib/Transforms/Scalar/Float2Int.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/Float2Int.cpp b/llvm/lib/Transforms/Scalar/Float2Int.cpp index ccca8bcc1a56..de8c05d5689f 100644 --- a/llvm/lib/Transforms/Scalar/Float2Int.cpp +++ b/llvm/lib/Transforms/Scalar/Float2Int.cpp @@ -359,9 +359,7 @@ bool Float2IntPass::validateAndTransform() { // The number of bits required is the maximum of the upper and // lower limits, plus one so it can be signed. - unsigned MinBW = std::max(R.getLower().getSignificantBits(), - R.getUpper().getSignificantBits()) + - 1; + unsigned MinBW = R.getMinSignedBits() + 1; LLVM_DEBUG(dbgs() << "F2I: MinBitwidth=" << MinBW << ", R: " << R << "\n"); // If we've run off the realms of the exactly representable integers, -- GitLab From f5ef9bd26d531d104f44f9e5b283bd2f80c024be Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Thu, 21 Mar 2024 11:17:52 -0700 Subject: [PATCH 185/296] [memprof] Call SmallVector::reserve (#86055) With one raw memprof file I have, NumPCs averages about 41. Given the default number of inline elements being 8 for SmallVector, we should reserve the storage in advance. --- llvm/lib/ProfileData/RawMemProfReader.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/ProfileData/RawMemProfReader.cpp b/llvm/lib/ProfileData/RawMemProfReader.cpp index 0e2b8668bab7..60c37c417aa0 100644 --- a/llvm/lib/ProfileData/RawMemProfReader.cpp +++ b/llvm/lib/ProfileData/RawMemProfReader.cpp @@ -127,6 +127,7 @@ CallStackMap readStackInfo(const char *Ptr) { endian::readNext(Ptr); SmallVector CallStack; + CallStack.reserve(NumPCs); for (uint64_t J = 0; J < NumPCs; J++) { CallStack.push_back( endian::readNext(Ptr)); -- GitLab From 999d4f840777bf8de26d45947192aa0728edc0fb Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Thu, 21 Mar 2024 14:18:43 -0400 Subject: [PATCH 186/296] Split -Wcast-function-type into a separate group (#86131) We want to add -Wcast-function-type to -Wextra (as done in 1de7e6c8cba27296f3fc16d107822ea0ee856759), but we do not want to add -Wcast-function-type-strict in at the same time (https://lab.llvm.org/buildbot/#/builders/57/builds/33601/steps/5/logs/stdio). This moves the existing warning to a new group (-Wcast-function-type-mismatch), puts the new group under the existing -Wcast-function-type warning group, and adds -Wcast-function-type-mismatch to -Wextra. --- clang/docs/ReleaseNotes.rst | 20 ++++++++++++++++++- clang/include/clang/Basic/DiagnosticGroups.td | 7 +++++-- .../clang/Basic/DiagnosticSemaKinds.td | 2 +- .../Sema/warn-cast-function-type-strict.c | 12 +++++------ .../warn-cast-function-type-strict.cpp | 10 +++++----- 5 files changed, 36 insertions(+), 15 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 50990140a53a..fd12bb41be47 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -198,7 +198,25 @@ Modified Compiler Flags ``-Wreturn-type``, and moved some of the diagnostics previously controlled by ``-Wreturn-type`` under this new flag. Fixes #GH72116. -- Added ``-Wcast-function-type`` as a warning enabled by ``-Wextra``. #GH76872 +- Added ``-Wcast-function-type-mismatch`` under the ``-Wcast-function-type`` + warning group. Moved the diagnostic previously controlled by + ``-Wcast-function-type`` to the new warning group and added + ``-Wcast-function-type-mismatch`` to ``-Wextra``. #GH76872 + + .. code-block:: c + + int x(long); + typedef int (f2)(void*); + typedef int (f3)(); + + void func(void) { + // Diagnoses under -Wcast-function-type, -Wcast-function-type-mismatch, + // -Wcast-function-type-strict, -Wextra + f2 *b = (f2 *)x; + // Diagnoses under -Wcast-function-type, -Wcast-function-type-strict + f3 *c = (f3 *)x; + } + Removed Compiler Flags ------------------------- diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td index bf03d4e8f67e..44035e2fd16f 100644 --- a/clang/include/clang/Basic/DiagnosticGroups.td +++ b/clang/include/clang/Basic/DiagnosticGroups.td @@ -573,7 +573,10 @@ def SelTypeCast : DiagGroup<"cast-of-sel-type">; def FunctionDefInObjCContainer : DiagGroup<"function-def-in-objc-container">; def BadFunctionCast : DiagGroup<"bad-function-cast">; def CastFunctionTypeStrict : DiagGroup<"cast-function-type-strict">; -def CastFunctionType : DiagGroup<"cast-function-type", [CastFunctionTypeStrict]>; +def CastFunctionTypeMismatch : DiagGroup<"cast-function-type-mismatch">; +def CastFunctionType : DiagGroup<"cast-function-type", + [CastFunctionTypeStrict, + CastFunctionTypeMismatch]>; def ObjCPropertyImpl : DiagGroup<"objc-property-implementation">; def ObjCPropertyNoAttribute : DiagGroup<"objc-property-no-attribute">; def ObjCPropertyAssignOnObjectType : DiagGroup<"objc-property-assign-on-object-type">; @@ -1038,7 +1041,7 @@ def Extra : DiagGroup<"extra", [ EmptyInitStatement, StringConcatation, FUseLdPath, - CastFunctionType, + CastFunctionTypeMismatch, ]>; def Most : DiagGroup<"most", [ diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 270af5d24611..fc727cef9cd8 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -9058,7 +9058,7 @@ def warn_bad_function_cast : Warning< InGroup, DefaultIgnore; def warn_cast_function_type : Warning< "cast %diff{from $ to $ |}0,1converts to incompatible function type">, - InGroup, DefaultIgnore; + InGroup, DefaultIgnore; def warn_cast_function_type_strict : Warning, InGroup, DefaultIgnore; def err_cast_pointer_to_non_pointer_int : Error< diff --git a/clang/test/Sema/warn-cast-function-type-strict.c b/clang/test/Sema/warn-cast-function-type-strict.c index 8c88f275d2b3..b0a70cf324b7 100644 --- a/clang/test/Sema/warn-cast-function-type-strict.c +++ b/clang/test/Sema/warn-cast-function-type-strict.c @@ -1,5 +1,5 @@ -// RUN: %clang_cc1 %s -fsyntax-only -Wcast-function-type -verify -// RUN: %clang_cc1 %s -fsyntax-only -Wcast-function-type-strict -verify +// RUN: %clang_cc1 %s -fsyntax-only -Wcast-function-type -verify=expected,strict +// RUN: %clang_cc1 %s -fsyntax-only -Wcast-function-type-strict -verify=expected,strict // RUN: %clang_cc1 %s -fsyntax-only -Wextra -Wno-ignored-qualifiers -verify int t(int array[static 12]); @@ -32,13 +32,13 @@ f10 *j; void foo(void) { a = (f1 *)x; b = (f2 *)x; /* expected-warning {{cast from 'int (*)(long)' to 'f2 *' (aka 'int (*)(void *)') converts to incompatible function type}} */ - c = (f3 *)x; /* expected-warning {{cast from 'int (*)(long)' to 'f3 *' (aka 'int (*)()') converts to incompatible function type}} */ + c = (f3 *)x; /* strict-warning {{cast from 'int (*)(long)' to 'f3 *' (aka 'int (*)()') converts to incompatible function type}} */ d = (f4 *)x; /* expected-warning {{cast from 'int (*)(long)' to 'f4 *' (aka 'void (*)()') converts to incompatible function type}} */ - e = (f5 *)x; /* expected-warning {{cast from 'int (*)(long)' to 'f5 *' (aka 'void (*)(void)') converts to incompatible function type}} */ + e = (f5 *)x; /* strict-warning {{cast from 'int (*)(long)' to 'f5 *' (aka 'void (*)(void)') converts to incompatible function type}} */ f = (f6 *)x; /* expected-warning {{cast from 'int (*)(long)' to 'f6 *' (aka 'int (*)(long, int)') converts to incompatible function type}} */ - g = (f7 *)x; /* expected-warning {{cast from 'int (*)(long)' to 'f7 *' (aka 'int (*)(long, ...)') converts to incompatible function type}} */ + g = (f7 *)x; /* strict-warning {{cast from 'int (*)(long)' to 'f7 *' (aka 'int (*)(long, ...)') converts to incompatible function type}} */ h = (f8 *)t; i = (f9 *)u; // FIXME: return type qualifier should not be included in the function type . Warning should be absent after this issue is fixed. https://github.com/llvm/llvm-project/issues/39494 . - j = (f10 *)v; /* expected-warning {{cast from 'const int (*)(int)' to 'f10 *' (aka 'int (*)(int)') converts to incompatible function type}} */ + j = (f10 *)v; /* strict-warning {{cast from 'const int (*)(int)' to 'f10 *' (aka 'int (*)(int)') converts to incompatible function type}} */ } diff --git a/clang/test/SemaCXX/warn-cast-function-type-strict.cpp b/clang/test/SemaCXX/warn-cast-function-type-strict.cpp index b3164afde5a0..8887b3c4c5d5 100644 --- a/clang/test/SemaCXX/warn-cast-function-type-strict.cpp +++ b/clang/test/SemaCXX/warn-cast-function-type-strict.cpp @@ -1,5 +1,5 @@ -// RUN: %clang_cc1 %s -fblocks -fsyntax-only -Wcast-function-type -verify -// RUN: %clang_cc1 %s -fblocks -fsyntax-only -Wcast-function-type-strict -verify +// RUN: %clang_cc1 %s -fblocks -fsyntax-only -Wcast-function-type -verify=expected,strict +// RUN: %clang_cc1 %s -fblocks -fsyntax-only -Wcast-function-type-strict -verify=expected,strict // RUN: %clang_cc1 %s -fblocks -fsyntax-only -Wextra -verify int x(long); @@ -33,11 +33,11 @@ void foo() { a = (f1 *)x; b = (f2 *)x; // expected-warning {{cast from 'int (*)(long)' to 'f2 *' (aka 'int (*)(void *)') converts to incompatible function type}} b = reinterpret_cast(x); // expected-warning {{cast from 'int (*)(long)' to 'f2 *' (aka 'int (*)(void *)') converts to incompatible function type}} - c = (f3 *)x; // expected-warning {{cast from 'int (*)(long)' to 'f3 *' (aka 'int (*)(...)') converts to incompatible function type}} + c = (f3 *)x; // strict-warning {{cast from 'int (*)(long)' to 'f3 *' (aka 'int (*)(...)') converts to incompatible function type}} d = (f4 *)x; // expected-warning {{cast from 'int (*)(long)' to 'f4 *' (aka 'void (*)(...)') converts to incompatible function type}} - e = (f5 *)x; // expected-warning {{cast from 'int (*)(long)' to 'f5 *' (aka 'void (*)()') converts to incompatible function type}} + e = (f5 *)x; // strict-warning {{cast from 'int (*)(long)' to 'f5 *' (aka 'void (*)()') converts to incompatible function type}} f = (f6 *)x; // expected-warning {{cast from 'int (*)(long)' to 'f6 *' (aka 'int (*)(long, int)') converts to incompatible function type}} - g = (f7 *)x; // expected-warning {{cast from 'int (*)(long)' to 'f7 *' (aka 'int (*)(long, ...)') converts to incompatible function type}} + g = (f7 *)x; // strict-warning {{cast from 'int (*)(long)' to 'f7 *' (aka 'int (*)(long, ...)') converts to incompatible function type}} mf p1 = (mf)&S::foo; // expected-warning {{cast from 'void (S::*)(int *)' to 'mf' (aka 'void (S::*)(int)') converts to incompatible function type}} -- GitLab From 3fefeafa49299ef924414bfa1b678e0f656b3618 Mon Sep 17 00:00:00 2001 From: dmaclach Date: Thu, 21 Mar 2024 11:22:35 -0700 Subject: [PATCH 187/296] [OBJC] Allow __attribute__((NSObject)) types be used as lightweight generic specifiers (#84593) As per https://clang.llvm.org/docs/AutomaticReferenceCounting.html#retainable-object-pointers, types with `__attribute__((NSObject))` are retainable, and thus should be eligible to be used as lightweight generic specifiers. Fix for #84592 84592 --- clang/lib/Sema/SemaType.cpp | 5 +++++ clang/test/SemaObjC/attr-objc-NSObject.m | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 clang/test/SemaObjC/attr-objc-NSObject.m diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp index 7b14323b0674..d7521a5363a3 100644 --- a/clang/lib/Sema/SemaType.cpp +++ b/clang/lib/Sema/SemaType.cpp @@ -1018,6 +1018,11 @@ static QualType applyObjCTypeArgs(Sema &S, SourceLocation loc, QualType type, return type; } + // Types that have __attribute__((NSObject)) are permitted. + if (typeArg->isObjCNSObjectType()) { + continue; + } + // Dependent types will be checked at instantiation time. if (typeArg->isDependentType()) { continue; diff --git a/clang/test/SemaObjC/attr-objc-NSObject.m b/clang/test/SemaObjC/attr-objc-NSObject.m new file mode 100644 index 000000000000..76a01dcef016 --- /dev/null +++ b/clang/test/SemaObjC/attr-objc-NSObject.m @@ -0,0 +1,23 @@ +// RUN: %clang_cc1 -verify -Wno-objc-root-class -fsyntax-only %s + +@interface NSArray<__covariant ObjectType> +- (void)containsObject:(ObjectType)anObject; // expected-note {{passing argument to parameter 'anObject' here}} +- (void)description; +@end + +typedef __attribute__((NSObject)) struct Foo *FooRef; +typedef struct Bar *BarRef; + +void good() { + FooRef object; + NSArray *array; + [array containsObject:object]; + [object description]; +} + +void bad() { + BarRef object; + NSArray *array; // expected-error {{type argument 'BarRef' (aka 'struct Bar *') is neither an Objective-C object nor a block type}} + [array containsObject:object]; // expected-warning {{incompatible pointer types sending 'BarRef' (aka 'struct Bar *') to parameter of type 'id'}} + [object description]; // expected-warning {{receiver type 'BarRef' (aka 'struct Bar *') is not 'id' or interface pointer, consider casting it to 'id'}} +} -- GitLab From 22e7e68a40b8b1aac8b44137685d21ac4b98bd17 Mon Sep 17 00:00:00 2001 From: lorenzo chelini Date: Thu, 21 Mar 2024 19:40:26 +0100 Subject: [PATCH 188/296] [mlir][Affine] Fix unused variable warning (NFC) --- mlir/lib/Dialect/Affine/Utils/Utils.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp index 3dc5539cde3d..8b8ed2578ca5 100644 --- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp +++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp @@ -1792,8 +1792,7 @@ MemRefType mlir::affine::normalizeMemRefType(MemRefType memrefType) { MLIRContext *context = memrefType.getContext(); for (unsigned d = 0; d < newRank; ++d) { // Check if this dimension is dynamic. - if (bool isDynDim = - isNormalizedMemRefDynamicDim(d, layoutMap, memrefTypeDynDims)) { + if (isNormalizedMemRefDynamicDim(d, layoutMap, memrefTypeDynDims)) { newShape[d] = ShapedType::kDynamic; continue; } -- GitLab From c96b61adc33b9d4ab26e2d0e4bce929b31c48768 Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Thu, 21 Mar 2024 20:33:05 +0100 Subject: [PATCH 189/296] [libc] Add reverse_iterator comparisons (#86147) https://en.cppreference.com/w/cpp/iterator/reverse_iterator/operator_cmp --- libc/src/__support/CPP/iterator.h | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/libc/src/__support/CPP/iterator.h b/libc/src/__support/CPP/iterator.h index c5bfb1912c7b..4d06e181bcf0 100644 --- a/libc/src/__support/CPP/iterator.h +++ b/libc/src/__support/CPP/iterator.h @@ -20,6 +20,7 @@ namespace cpp { template struct iterator_traits; template struct iterator_traits { using reference = T &; + using value_type = T; }; template class reverse_iterator { @@ -27,6 +28,8 @@ template class reverse_iterator { public: using reference = typename iterator_traits::reference; + using value_type = typename iterator_traits::value_type; + using iterator_type = Iter; LIBC_INLINE reverse_iterator() : current() {} LIBC_INLINE constexpr explicit reverse_iterator(Iter it) : current(it) {} @@ -38,6 +41,38 @@ public: LIBC_INLINE constexpr explicit reverse_iterator(const Other &it) : current(it) {} + LIBC_INLINE friend constexpr bool operator==(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() == rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator!=(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() != rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator<(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() > rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator<=(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() >= rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator>(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() < rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator>=(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() <= rhs.base(); + } + + LIBC_INLINE constexpr iterator_type base() const { current; } + LIBC_INLINE constexpr reference operator*() const { Iter tmp = current; return *--tmp; -- GitLab From 3eb58d15b353534fd42a3a0d1eeb7cd33d128b34 Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Thu, 21 Mar 2024 20:37:04 +0100 Subject: [PATCH 190/296] Revert "[libc] Add reverse_iterator comparisons" (#86186) Reverts llvm/llvm-project#86147 --- libc/src/__support/CPP/iterator.h | 35 ------------------------------- 1 file changed, 35 deletions(-) diff --git a/libc/src/__support/CPP/iterator.h b/libc/src/__support/CPP/iterator.h index 4d06e181bcf0..c5bfb1912c7b 100644 --- a/libc/src/__support/CPP/iterator.h +++ b/libc/src/__support/CPP/iterator.h @@ -20,7 +20,6 @@ namespace cpp { template struct iterator_traits; template struct iterator_traits { using reference = T &; - using value_type = T; }; template class reverse_iterator { @@ -28,8 +27,6 @@ template class reverse_iterator { public: using reference = typename iterator_traits::reference; - using value_type = typename iterator_traits::value_type; - using iterator_type = Iter; LIBC_INLINE reverse_iterator() : current() {} LIBC_INLINE constexpr explicit reverse_iterator(Iter it) : current(it) {} @@ -41,38 +38,6 @@ public: LIBC_INLINE constexpr explicit reverse_iterator(const Other &it) : current(it) {} - LIBC_INLINE friend constexpr bool operator==(const reverse_iterator &lhs, - const reverse_iterator &rhs) { - return lhs.base() == rhs.base(); - } - - LIBC_INLINE friend constexpr bool operator!=(const reverse_iterator &lhs, - const reverse_iterator &rhs) { - return lhs.base() != rhs.base(); - } - - LIBC_INLINE friend constexpr bool operator<(const reverse_iterator &lhs, - const reverse_iterator &rhs) { - return lhs.base() > rhs.base(); - } - - LIBC_INLINE friend constexpr bool operator<=(const reverse_iterator &lhs, - const reverse_iterator &rhs) { - return lhs.base() >= rhs.base(); - } - - LIBC_INLINE friend constexpr bool operator>(const reverse_iterator &lhs, - const reverse_iterator &rhs) { - return lhs.base() < rhs.base(); - } - - LIBC_INLINE friend constexpr bool operator>=(const reverse_iterator &lhs, - const reverse_iterator &rhs) { - return lhs.base() <= rhs.base(); - } - - LIBC_INLINE constexpr iterator_type base() const { current; } - LIBC_INLINE constexpr reference operator*() const { Iter tmp = current; return *--tmp; -- GitLab From 536cb1fad3ea3edaba8264992c8de2f4b07abc84 Mon Sep 17 00:00:00 2001 From: Michele Scandale Date: Thu, 21 Mar 2024 12:40:18 -0700 Subject: [PATCH 191/296] [InstCombine] Fix for folding select-like `shufflevector` into floating point binary operators. (#85452) Folding a select-like `shufflevector` into a floating point binary operators can only be done if the result is preserved for both case. In particular, if the common operand of the `shufflevector` and the floating point binary operator can be a NaN, then the transformation won't preserve the result value. --- .../InstCombine/InstCombineVectorOps.cpp | 20 ++++++++++++++++--- .../shuffle_select-inseltpoison.ll | 17 +++++++++++++--- .../Transforms/InstCombine/shuffle_select.ll | 17 +++++++++++++--- 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp index 3c4c0f35eb6d..c7f4fb17648c 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -2135,7 +2135,8 @@ static Instruction *foldSelectShuffleOfSelectShuffle(ShuffleVectorInst &Shuf) { return new ShuffleVectorInst(X, Y, NewMask); } -static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) { +static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf, + const SimplifyQuery &SQ) { assert(Shuf.isSelect() && "Must have select-equivalent shuffle"); // Are we shuffling together some value and that same value after it has been @@ -2159,6 +2160,19 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) { if (!IdC) return nullptr; + Value *X = Op0IsBinop ? Op1 : Op0; + + // Prevent folding in the case the non-binop operand might have NaN values. + // If X can have NaN elements then we have that the floating point math + // operation in the transformed code may not preserve the exact NaN + // bit-pattern -- e.g. `fadd sNaN, 0.0 -> qNaN`. + // This makes the transformation incorrect since the original program would + // have preserved the exact NaN bit-pattern. + // Avoid the folding if X can have NaN elements. + if (Shuf.getType()->getElementType()->isFloatingPointTy() && + !isKnownNeverNaN(X, 0, SQ)) + return nullptr; + // Shuffle identity constants into the lanes that return the original value. // Example: shuf (mul X, {-1,-2,-3,-4}), X, {0,5,6,3} --> mul X, {-1,1,1,-4} // Example: shuf X, (add X, {-1,-2,-3,-4}), {0,1,6,7} --> add X, {0,0,-3,-4} @@ -2175,7 +2189,6 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) { // shuf (bop X, C), X, M --> bop X, C' // shuf X, (bop X, C), M --> bop X, C' - Value *X = Op0IsBinop ? Op1 : Op0; Instruction *NewBO = BinaryOperator::Create(BOpcode, X, NewC); NewBO->copyIRFlags(BO); @@ -2241,7 +2254,8 @@ Instruction *InstCombinerImpl::foldSelectShuffle(ShuffleVectorInst &Shuf) { if (Instruction *I = foldSelectShuffleOfSelectShuffle(Shuf)) return I; - if (Instruction *I = foldSelectShuffleWith1Binop(Shuf)) + if (Instruction *I = foldSelectShuffleWith1Binop( + Shuf, getSimplifyQuery().getWithInstruction(&Shuf))) return I; BinaryOperator *B0, *B1; diff --git a/llvm/test/Transforms/InstCombine/shuffle_select-inseltpoison.ll b/llvm/test/Transforms/InstCombine/shuffle_select-inseltpoison.ll index 44ec77e471bb..f573ff36d2ce 100644 --- a/llvm/test/Transforms/InstCombine/shuffle_select-inseltpoison.ll +++ b/llvm/test/Transforms/InstCombine/shuffle_select-inseltpoison.ll @@ -336,7 +336,18 @@ define <4 x i32> @srem(<4 x i32> %v) { ; Try FP ops/types. -define <4 x float> @fadd(<4 x float> %v) { +define <4 x float> @fadd_maybe_nan(<4 x float> %v) { +; CHECK-LABEL: @fadd_maybe_nan( +; CHECK-NEXT: [[B:%.*]] = fadd <4 x float> [[V:%.*]], +; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x float> [[B]], <4 x float> [[V]], <4 x i32> +; CHECK-NEXT: ret <4 x float> [[S]] +; + %b = fadd <4 x float> %v, + %s = shufflevector <4 x float> %b, <4 x float> %v, <4 x i32> + ret <4 x float> %s +} + +define <4 x float> @fadd(<4 x float> nofpclass(nan) %v) { ; CHECK-LABEL: @fadd( ; CHECK-NEXT: [[S:%.*]] = fadd <4 x float> [[V:%.*]], ; CHECK-NEXT: ret <4 x float> [[S]] @@ -359,7 +370,7 @@ define <4 x double> @fsub(<4 x double> %v) { ; Propagate any FMF. -define <4 x float> @fmul(<4 x float> %v) { +define <4 x float> @fmul(<4 x float> nofpclass(nan) %v) { ; CHECK-LABEL: @fmul( ; CHECK-NEXT: [[S:%.*]] = fmul nnan ninf <4 x float> [[V:%.*]], ; CHECK-NEXT: ret <4 x float> [[S]] @@ -380,7 +391,7 @@ define <4 x double> @fdiv_constant_op0(<4 x double> %v) { ret <4 x double> %s } -define <4 x double> @fdiv_constant_op1(<4 x double> %v) { +define <4 x double> @fdiv_constant_op1(<4 x double> nofpclass(nan) %v) { ; CHECK-LABEL: @fdiv_constant_op1( ; CHECK-NEXT: [[S:%.*]] = fdiv reassoc <4 x double> [[V:%.*]], ; CHECK-NEXT: ret <4 x double> [[S]] diff --git a/llvm/test/Transforms/InstCombine/shuffle_select.ll b/llvm/test/Transforms/InstCombine/shuffle_select.ll index a1b0d782b554..efadb5c3c109 100644 --- a/llvm/test/Transforms/InstCombine/shuffle_select.ll +++ b/llvm/test/Transforms/InstCombine/shuffle_select.ll @@ -336,7 +336,18 @@ define <4 x i32> @srem(<4 x i32> %v) { ; Try FP ops/types. -define <4 x float> @fadd(<4 x float> %v) { +define <4 x float> @fadd_maybe_nan(<4 x float> %v) { +; CHECK-LABEL: @fadd_maybe_nan( +; CHECK-NEXT: [[B:%.*]] = fadd <4 x float> [[V:%.*]], +; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x float> [[B]], <4 x float> [[V]], <4 x i32> +; CHECK-NEXT: ret <4 x float> [[S]] +; + %b = fadd <4 x float> %v, + %s = shufflevector <4 x float> %b, <4 x float> %v, <4 x i32> + ret <4 x float> %s +} + +define <4 x float> @fadd(<4 x float> nofpclass(nan) %v) { ; CHECK-LABEL: @fadd( ; CHECK-NEXT: [[S:%.*]] = fadd <4 x float> [[V:%.*]], ; CHECK-NEXT: ret <4 x float> [[S]] @@ -359,7 +370,7 @@ define <4 x double> @fsub(<4 x double> %v) { ; Propagate any FMF. -define <4 x float> @fmul(<4 x float> %v) { +define <4 x float> @fmul(<4 x float> nofpclass(nan) %v) { ; CHECK-LABEL: @fmul( ; CHECK-NEXT: [[S:%.*]] = fmul nnan ninf <4 x float> [[V:%.*]], ; CHECK-NEXT: ret <4 x float> [[S]] @@ -380,7 +391,7 @@ define <4 x double> @fdiv_constant_op0(<4 x double> %v) { ret <4 x double> %s } -define <4 x double> @fdiv_constant_op1(<4 x double> %v) { +define <4 x double> @fdiv_constant_op1(<4 x double> nofpclass(nan) %v) { ; CHECK-LABEL: @fdiv_constant_op1( ; CHECK-NEXT: [[S:%.*]] = fdiv reassoc <4 x double> [[V:%.*]], ; CHECK-NEXT: ret <4 x double> [[S]] -- GitLab From 85ccfb5ed5389a5fb2d58eab12a9266e7ea064ce Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Thu, 21 Mar 2024 12:42:55 -0700 Subject: [PATCH 192/296] [HWASan] [NFC] pull logic to get sanitizer ptr out of hwasan (#86024) Also some drive by cleanup removing an unnnecessary argument and a redundant condition. --- .../Transforms/Utils/MemoryTaggingSupport.h | 1 + .../Instrumentation/HWAddressSanitizer.cpp | 22 +++++-------------- .../Transforms/Utils/MemoryTaggingSupport.cpp | 10 +++++++++ 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h b/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h index 0a0e16d2a9e6..158c358a9e46 100644 --- a/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h +++ b/llvm/include/llvm/Transforms/Utils/MemoryTaggingSupport.h @@ -84,6 +84,7 @@ bool isLifetimeIntrinsic(Value *V); Value *readRegister(IRBuilder<> &IRB, StringRef Name); Value *getFP(IRBuilder<> &IRB); Value *getPC(const Triple &TargetTriple, IRBuilder<> &IRB); +Value *getAndroidSanitizerSlotPtr(IRBuilder<> &IRB); } // namespace memtag } // namespace llvm diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp index 4bdeb6bbab85..3c95610fa3e8 100644 --- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp @@ -363,7 +363,7 @@ private: Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, unsigned AllocaNo); Value *getUARTag(IRBuilder<> &IRB); - Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty); + Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB); Value *applyTagMask(IRBuilder<> &IRB, Value *OldTag); unsigned retagMask(unsigned AllocaNo); @@ -1219,20 +1219,10 @@ Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) { return UntaggedPtrLong; } -Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) { - Module *M = IRB.GetInsertBlock()->getParent()->getParent(); - if (TargetTriple.isAArch64() && TargetTriple.isAndroid()) { - // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER - // in Bionic's libc/private/bionic_tls.h. - Function *ThreadPointerFunc = - Intrinsic::getDeclaration(M, Intrinsic::thread_pointer); - return IRB.CreateConstGEP1_32(Int8Ty, IRB.CreateCall(ThreadPointerFunc), - 0x30); - } - if (ThreadPtrGlobal) - return ThreadPtrGlobal; - - return nullptr; +Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB) { + if (TargetTriple.isAArch64() && TargetTriple.isAndroid()) + return memtag::getAndroidSanitizerSlotPtr(IRB); + return ThreadPtrGlobal; } Value *HWAddressSanitizer::getCachedFP(IRBuilder<> &IRB) { @@ -1271,7 +1261,7 @@ void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) { auto getThreadLongMaybeUntagged = [&]() { if (!SlotPtr) - SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy); + SlotPtr = getHwasanThreadSlotPtr(IRB); if (!ThreadLong) ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr); // Extract the address field from ThreadLong. Unnecessary on AArch64 with diff --git a/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp b/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp index 8dd1002a6e4a..fd94a120bc66 100644 --- a/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp +++ b/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp @@ -273,5 +273,15 @@ Value *getFP(IRBuilder<> &IRB) { IRB.getIntPtrTy(M->getDataLayout())); } +Value *getAndroidSanitizerSlotPtr(IRBuilder<> &IRB) { + Module *M = IRB.GetInsertBlock()->getParent()->getParent(); + // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER + // in Bionic's libc/private/bionic_tls.h. + Function *ThreadPointerFunc = + Intrinsic::getDeclaration(M, Intrinsic::thread_pointer); + return IRB.CreateConstGEP1_32(IRB.getInt8Ty(), + IRB.CreateCall(ThreadPointerFunc), 0x30); +} + } // namespace memtag } // namespace llvm -- GitLab From 6d939a6ec69adf284cdbef2034b49fd02ba503fc Mon Sep 17 00:00:00 2001 From: Kevin Frei Date: Thu, 21 Mar 2024 13:09:04 -0700 Subject: [PATCH 193/296] DebugInfoD tests + fixing issues exposed by tests (#85693) Finally getting back to Debuginfod tests: I've migrated the tests in my [earlier PR](https://github.com/llvm/llvm-project/pull/79181) from shell to API (at @JDevlieghere's suggestion) and addressed a couple issues that came about during testing. The tests first test the "normal" situation (no DebugInfoD involvement, just normal debug files sitting around), then the "no debug info" situation (to make sure the test is seeing failure properly), then it tests to validate that when Debuginfod returns the symbols, things work properly. This is duplicated for DWP/split-dwarf scenarios. --------- Co-authored-by: Kevin Frei --- .../Python/lldbsuite/test/make/Makefile.rules | 33 ++- .../SymbolFile/DWARF/SymbolFileDWARF.cpp | 38 ++-- .../Plugins/SymbolLocator/CMakeLists.txt | 7 +- .../SymbolVendor/ELF/SymbolVendorELF.cpp | 30 ++- lldb/test/API/debuginfod/Normal/Makefile | 25 +++ .../API/debuginfod/Normal/TestDebuginfod.py | 185 +++++++++++++++++ lldb/test/API/debuginfod/Normal/main.c | 7 + lldb/test/API/debuginfod/SplitDWARF/Makefile | 28 +++ .../SplitDWARF/TestDebuginfodDWP.py | 194 ++++++++++++++++++ lldb/test/API/debuginfod/SplitDWARF/main.c | 7 + 10 files changed, 537 insertions(+), 17 deletions(-) create mode 100644 lldb/test/API/debuginfod/Normal/Makefile create mode 100644 lldb/test/API/debuginfod/Normal/TestDebuginfod.py create mode 100644 lldb/test/API/debuginfod/Normal/main.c create mode 100644 lldb/test/API/debuginfod/SplitDWARF/Makefile create mode 100644 lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py create mode 100644 lldb/test/API/debuginfod/SplitDWARF/main.c diff --git a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules index bfd249ccd43f..75efcde1f040 100644 --- a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules +++ b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules @@ -51,7 +51,7 @@ LLDB_BASE_DIR := $(THIS_FILE_DIR)/../../../../../ # # GNUWin32 uname gives "windows32" or "server version windows32" while # some versions of MSYS uname return "MSYS_NT*", but most environments -# standardize on "Windows_NT", so we'll make it consistent here. +# standardize on "Windows_NT", so we'll make it consistent here. # When running tests from Visual Studio, the environment variable isn't # inherited all the way down to the process spawned for make. #---------------------------------------------------------------------- @@ -210,6 +210,12 @@ else ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES" DSYM = $(EXE).debug endif + + ifeq "$(MAKE_DWP)" "YES" + MAKE_DWO := YES + DWP_NAME = $(EXE).dwp + DYLIB_DWP_NAME = $(DYLIB_NAME).dwp + endif endif LIMIT_DEBUG_INFO_FLAGS = @@ -357,6 +363,7 @@ ifneq "$(OS)" "Darwin" OBJCOPY ?= $(call replace_cc_with,objcopy) ARCHIVER ?= $(call replace_cc_with,ar) + DWP ?= $(call replace_cc_with,dwp) override AR = $(ARCHIVER) endif @@ -527,6 +534,10 @@ ifneq "$(CXX)" "" endif endif +ifeq "$(GEN_GNU_BUILD_ID)" "YES" + LDFLAGS += -Wl,--build-id +endif + #---------------------------------------------------------------------- # DYLIB_ONLY variable can be used to skip the building of a.out. # See the sections below regarding dSYM file as well as the building of @@ -565,11 +576,25 @@ else endif else ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES" +ifeq "$(SAVE_FULL_DEBUG_BINARY)" "YES" + cp "$(EXE)" "$(EXE).full" +endif $(OBJCOPY) --only-keep-debug "$(EXE)" "$(DSYM)" $(OBJCOPY) --strip-debug --add-gnu-debuglink="$(DSYM)" "$(EXE)" "$(EXE)" endif +ifeq "$(MAKE_DWP)" "YES" + $(DWP) -o "$(DWP_NAME)" $(DWOS) +endif endif + +#---------------------------------------------------------------------- +# Support emitting the content of the GNU build-id into a file +# This needs to be used in conjunction with GEN_GNU_BUILD_ID := YES +#---------------------------------------------------------------------- +$(EXE).uuid : $(EXE) + $(OBJCOPY) --dump-section=.note.gnu.build-id=$@ $< + #---------------------------------------------------------------------- # Make the dylib #---------------------------------------------------------------------- @@ -610,9 +635,15 @@ endif else $(LD) $(DYLIB_OBJECTS) $(LDFLAGS) -shared -o "$(DYLIB_FILENAME)" ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES" + ifeq "$(SAVE_FULL_DEBUG_BINARY)" "YES" + cp "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME).full" + endif $(OBJCOPY) --only-keep-debug "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME).debug" $(OBJCOPY) --strip-debug --add-gnu-debuglink="$(DYLIB_FILENAME).debug" "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME)" endif +ifeq "$(MAKE_DWP)" "YES" + $(DWP) -o $(DYLIB_DWP_FILE) $(DYLIB_DWOS) +endif endif #---------------------------------------------------------------------- diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp index 5f67658f86ea..08ce7b82b0c1 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp @@ -4377,26 +4377,38 @@ const std::shared_ptr &SymbolFileDWARF::GetDwpSymbolFile() { FileSpecList search_paths = Target::GetDefaultDebugFileSearchPaths(); ModuleSpec module_spec; module_spec.GetFileSpec() = m_objfile_sp->GetFileSpec(); + FileSpec dwp_filespec; for (const auto &symfile : symfiles.files()) { module_spec.GetSymbolFileSpec() = FileSpec(symfile.GetPath() + ".dwp", symfile.GetPathStyle()); LLDB_LOG(log, "Searching for DWP using: \"{0}\"", module_spec.GetSymbolFileSpec()); - FileSpec dwp_filespec = + dwp_filespec = PluginManager::LocateExecutableSymbolFile(module_spec, search_paths); if (FileSystem::Instance().Exists(dwp_filespec)) { - LLDB_LOG(log, "Found DWP file: \"{0}\"", dwp_filespec); - DataBufferSP dwp_file_data_sp; - lldb::offset_t dwp_file_data_offset = 0; - ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin( - GetObjectFile()->GetModule(), &dwp_filespec, 0, - FileSystem::Instance().GetByteSize(dwp_filespec), dwp_file_data_sp, - dwp_file_data_offset); - if (dwp_obj_file) { - m_dwp_symfile = std::make_shared( - *this, dwp_obj_file, DIERef::k_file_index_mask); - break; - } + break; + } + } + if (!FileSystem::Instance().Exists(dwp_filespec)) { + LLDB_LOG(log, "No DWP file found locally"); + // Fill in the UUID for the module we're trying to match for, so we can + // find the correct DWP file, as the Debuginfod plugin uses *only* this + // data to correctly match the DWP file with the binary. + module_spec.GetUUID() = m_objfile_sp->GetUUID(); + dwp_filespec = + PluginManager::LocateExecutableSymbolFile(module_spec, search_paths); + } + if (FileSystem::Instance().Exists(dwp_filespec)) { + LLDB_LOG(log, "Found DWP file: \"{0}\"", dwp_filespec); + DataBufferSP dwp_file_data_sp; + lldb::offset_t dwp_file_data_offset = 0; + ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin( + GetObjectFile()->GetModule(), &dwp_filespec, 0, + FileSystem::Instance().GetByteSize(dwp_filespec), dwp_file_data_sp, + dwp_file_data_offset); + if (dwp_obj_file) { + m_dwp_symfile = std::make_shared( + *this, dwp_obj_file, DIERef::k_file_index_mask); } } if (!m_dwp_symfile) { diff --git a/lldb/source/Plugins/SymbolLocator/CMakeLists.txt b/lldb/source/Plugins/SymbolLocator/CMakeLists.txt index ca969626f4ff..3367022639ab 100644 --- a/lldb/source/Plugins/SymbolLocator/CMakeLists.txt +++ b/lldb/source/Plugins/SymbolLocator/CMakeLists.txt @@ -1,5 +1,10 @@ +# Order matters here: the first symbol locator prevents further searching. +# For DWARF binaries that are both stripped and split, the Default plugin +# will return the stripped binary when asked for the ObjectFile, which then +# prevents an unstripped binary from being requested from the Debuginfod +# provider. +add_subdirectory(Debuginfod) add_subdirectory(Default) if (CMAKE_SYSTEM_NAME MATCHES "Darwin") add_subdirectory(DebugSymbols) endif() -add_subdirectory(Debuginfod) diff --git a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp index b5fe35d71032..91b8b4a979e0 100644 --- a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp +++ b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp @@ -44,6 +44,25 @@ llvm::StringRef SymbolVendorELF::GetPluginDescriptionStatic() { "executables."; } +// If this is needed elsewhere, it can be exported/moved. +static bool IsDwpSymbolFile(const lldb::ModuleSP &module_sp, + const FileSpec &file_spec) { + DataBufferSP dwp_file_data_sp; + lldb::offset_t dwp_file_data_offset = 0; + // Try to create an ObjectFile from the file_spec. + ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin( + module_sp, &file_spec, 0, FileSystem::Instance().GetByteSize(file_spec), + dwp_file_data_sp, dwp_file_data_offset); + if (!ObjectFileELF::classof(dwp_obj_file.get())) + return false; + // The presence of a debug_cu_index section is the key identifying feature of + // a DWP file. Make sure we don't fill in the section list on dwp_obj_file + // (by calling GetSectionList(false)) as this is invoked before we may have + // all the symbol files collected and available. + return dwp_obj_file && dwp_obj_file->GetSectionList(false)->FindSectionByType( + eSectionTypeDWARFDebugCuIndex, false); +} + // CreateInstance // // Platforms can register a callback to use when creating symbol vendors to @@ -87,8 +106,15 @@ SymbolVendorELF::CreateInstance(const lldb::ModuleSP &module_sp, FileSpecList search_paths = Target::GetDefaultDebugFileSearchPaths(); FileSpec dsym_fspec = PluginManager::LocateExecutableSymbolFile(module_spec, search_paths); - if (!dsym_fspec) - return nullptr; + if (!dsym_fspec || IsDwpSymbolFile(module_sp, dsym_fspec)) { + // If we have a stripped binary or if we got a DWP file, we should prefer + // symbols in the executable acquired through a plugin. + ModuleSpec unstripped_spec = + PluginManager::LocateExecutableObjectFile(module_spec); + if (!unstripped_spec) + return nullptr; + dsym_fspec = unstripped_spec.GetFileSpec(); + } DataBufferSP dsym_file_data_sp; lldb::offset_t dsym_file_data_offset = 0; diff --git a/lldb/test/API/debuginfod/Normal/Makefile b/lldb/test/API/debuginfod/Normal/Makefile new file mode 100644 index 000000000000..bd2fa623df47 --- /dev/null +++ b/lldb/test/API/debuginfod/Normal/Makefile @@ -0,0 +1,25 @@ +C_SOURCES := main.c + +# For normal (non DWP) Debuginfod tests, we need: + +# * The "full" binary: a.out.debug +# Produced by Makefile.rules with KEEP_FULL_DEBUG_BINARY set to YES and +# SPLIT_DEBUG_SYMBOLS set to YES + +# * The stripped binary (a.out) +# Produced by Makefile.rules with SPLIT_DEBUG_SYMBOLS set to YES + +# * The 'only-keep-debug' binary (a.out.dbg) +# Produced below + +# * The .uuid file (for a little easier testing code) +# Produced below + +# Don't strip the debug info from a.out: +SPLIT_DEBUG_SYMBOLS := YES +SAVE_FULL_DEBUG_BINARY := YES +GEN_GNU_BUILD_ID := YES + +all: a.out.uuid a.out + +include Makefile.rules diff --git a/lldb/test/API/debuginfod/Normal/TestDebuginfod.py b/lldb/test/API/debuginfod/Normal/TestDebuginfod.py new file mode 100644 index 000000000000..eb5efe83c17a --- /dev/null +++ b/lldb/test/API/debuginfod/Normal/TestDebuginfod.py @@ -0,0 +1,185 @@ +import os +import shutil +import tempfile +import struct + +import lldb +from lldbsuite.test.decorators import * +import lldbsuite.test.lldbutil as lldbutil +from lldbsuite.test.lldbtest import * + + +def getUUID(aoutuuid): + """ + Pull the 20 byte UUID out of the .note.gnu.build-id section that was dumped + to a file already, as part of the build. + """ + with open(aoutuuid, "rb") as f: + data = f.read(36) + if len(data) != 36: + return None + header = struct.unpack_from("<4I", data) + if len(header) != 4: + return None + # 4 element 'prefix', 20 bytes of uuid, 3 byte long string: 'GNU': + if header[0] != 4 or header[1] != 20 or header[2] != 3 or header[3] != 0x554E47: + return None + return data[16:].hex() + + +""" +Test support for the DebugInfoD network symbol acquisition protocol. +This one is for simple / no split-dwarf scenarios. + +For no-split-dwarf scenarios, there are 2 variations: +1 - A stripped binary with it's corresponding unstripped binary: +2 - A stripped binary with a corresponding --only-keep-debug symbols file +""" + + +@skipUnlessPlatform(["linux", "freebsd"]) +class DebugInfodTests(TestBase): + # No need to try every flavor of debug inf. + NO_DEBUG_INFO_TESTCASE = True + + def test_normal_no_symbols(self): + """ + Validate behavior with no symbols or symbol locator. + ('baseline negative' behavior) + """ + test_root = self.config_test(["a.out"]) + self.try_breakpoint(False) + + def test_normal_default(self): + """ + Validate behavior with symbols, but no symbol locator. + ('baseline positive' behavior) + """ + test_root = self.config_test(["a.out", "a.out.debug"]) + self.try_breakpoint(True) + + def test_debuginfod_symbols(self): + """ + Test behavior with the full binary available from Debuginfod as + 'debuginfo' from the plug-in. + """ + test_root = self.config_test(["a.out"], "a.out.full") + self.try_breakpoint(True) + + def test_debuginfod_executable(self): + """ + Test behavior with the full binary available from Debuginfod as + 'executable' from the plug-in. + """ + test_root = self.config_test(["a.out"], None, "a.out.full") + self.try_breakpoint(True) + + def test_debuginfod_okd_symbols(self): + """ + Test behavior with the 'only-keep-debug' symbols available from Debuginfod. + """ + test_root = self.config_test(["a.out"], "a.out.debug") + self.try_breakpoint(True) + + def try_breakpoint(self, should_have_loc): + """ + This function creates a target from self.aout, sets a function-name + breakpoint, and checks to see if we have a file/line location, + as a way to validate that the symbols have been loaded. + should_have_loc specifies if we're testing that symbols have or + haven't been loaded. + """ + target = self.dbg.CreateTarget(self.aout) + self.assertTrue(target and target.IsValid(), "Target is valid") + + bp = target.BreakpointCreateByName("func") + self.assertTrue(bp and bp.IsValid(), "Breakpoint is valid") + self.assertEqual(bp.GetNumLocations(), 1) + + loc = bp.GetLocationAtIndex(0) + self.assertTrue(loc and loc.IsValid(), "Location is valid") + addr = loc.GetAddress() + self.assertTrue(addr and addr.IsValid(), "Loc address is valid") + line_entry = addr.GetLineEntry() + self.assertEqual( + should_have_loc, + line_entry != None and line_entry.IsValid(), + "Loc line entry is valid", + ) + if should_have_loc: + self.assertEqual(line_entry.GetLine(), 4) + self.assertEqual( + line_entry.GetFileSpec().GetFilename(), + self.main_source_file.GetFilename(), + ) + self.dbg.DeleteTarget(target) + shutil.rmtree(self.tmp_dir) + + def config_test(self, local_files, debuginfo=None, executable=None): + """ + Set up a test with local_files[] copied to a different location + so that we control which files are, or are not, found in the file system. + Also, create a stand-alone file-system 'hosted' debuginfod server with the + provided debuginfo and executable files (if they exist) + + Make the filesystem look like: + + /tmp//test/[local_files] + + /tmp//cache (for lldb to use as a temp cache) + + /tmp//buildid//executable -> + /tmp//buildid//debuginfo -> + Returns the /tmp/ path + """ + + self.build() + + uuid = getUUID(self.getBuildArtifact("a.out.uuid")) + + self.main_source_file = lldb.SBFileSpec("main.c") + self.tmp_dir = tempfile.mkdtemp() + test_dir = os.path.join(self.tmp_dir, "test") + os.makedirs(test_dir) + + self.aout = "" + # Copy the files used by the test: + for f in local_files: + shutil.copy(self.getBuildArtifact(f), test_dir) + # The first item is the binary to be used for the test + if self.aout == "": + self.aout = os.path.join(test_dir, f) + + use_debuginfod = debuginfo != None or executable != None + + # Populated the 'file://... mocked' Debuginfod server: + if use_debuginfod: + os.makedirs(os.path.join(self.tmp_dir, "cache")) + uuid_dir = os.path.join(self.tmp_dir, "buildid", uuid) + os.makedirs(uuid_dir) + if debuginfo: + shutil.copy( + self.getBuildArtifact(debuginfo), + os.path.join(uuid_dir, "debuginfo"), + ) + if executable: + shutil.copy( + self.getBuildArtifact(executable), + os.path.join(uuid_dir, "executable"), + ) + + # Configure LLDB for the test: + self.runCmd( + "settings set symbols.enable-external-lookup %s" + % str(use_debuginfod).lower() + ) + self.runCmd("settings clear plugin.symbol-locator.debuginfod.server-urls") + if use_debuginfod: + self.runCmd( + "settings set plugin.symbol-locator.debuginfod.cache-path %s/cache" + % self.tmp_dir + ) + self.runCmd( + "settings insert-before plugin.symbol-locator.debuginfod.server-urls 0 file://%s" + % self.tmp_dir + ) diff --git a/lldb/test/API/debuginfod/Normal/main.c b/lldb/test/API/debuginfod/Normal/main.c new file mode 100644 index 000000000000..4c7184609b45 --- /dev/null +++ b/lldb/test/API/debuginfod/Normal/main.c @@ -0,0 +1,7 @@ +// This is a dump little pair of test files + +int func(int argc, const char *argv[]) { + return (argc + 1) * (argv[argc][0] + 2); +} + +int main(int argc, const char *argv[]) { return func(0, argv); } diff --git a/lldb/test/API/debuginfod/SplitDWARF/Makefile b/lldb/test/API/debuginfod/SplitDWARF/Makefile new file mode 100644 index 000000000000..266d74cf9062 --- /dev/null +++ b/lldb/test/API/debuginfod/SplitDWARF/Makefile @@ -0,0 +1,28 @@ +C_SOURCES := main.c + +# For split-dwarf Debuginfod tests, we need: + +# * A .DWP file (a.out.dwp) +# Produced by Makefile.rules with MAKE_DWO and MERGE_DWOS both set to YES + +# * The "full" binary: it's missing things that live in .dwo's (a.out.debug) +# Produced by Makefile.rules with KEEP_FULL_DEBUG_BINARY set to YES and +# SPLIT_DEBUG_SYMBOLS set to YES + +# * The stripped binary (a.out) +# Produced by Makefile.rules + +# * The 'only-keep-debug' binary (a.out.dbg) +# Produced below + +# * The .uuid file (for a little easier testing code) +# Produced here in the rule below + +MAKE_DWP := YES +SPLIT_DEBUG_SYMBOLS := YES +SAVE_FULL_DEBUG_BINARY := YES +GEN_GNU_BUILD_ID := YES + +all: a.out.uuid a.out + +include Makefile.rules diff --git a/lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py b/lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py new file mode 100644 index 000000000000..09f91b6f1c6c --- /dev/null +++ b/lldb/test/API/debuginfod/SplitDWARF/TestDebuginfodDWP.py @@ -0,0 +1,194 @@ +""" +Test support for the DebugInfoD network symbol acquisition protocol. +""" +import os +import shutil +import tempfile +import struct + +import lldb +from lldbsuite.test.decorators import * +import lldbsuite.test.lldbutil as lldbutil +from lldbsuite.test.lldbtest import * + + +def getUUID(aoutuuid): + """ + Pull the 20 byte UUID out of the .note.gnu.build-id section that was dumped + to a file already, as part of the build. + """ + with open(aoutuuid, "rb") as f: + data = f.read(36) + if len(data) != 36: + return None + header = struct.unpack_from("<4I", data) + if len(header) != 4: + return None + # 4 element 'prefix', 20 bytes of uuid, 3 byte long string: 'GNU': + if header[0] != 4 or header[1] != 20 or header[2] != 3 or header[3] != 0x554E47: + return None + return data[16:].hex() + + +""" +Test support for the DebugInfoD network symbol acquisition protocol. +This file is for split-dwarf (dwp) scenarios. + +1 - A split binary target with it's corresponding DWP file +2 - A stripped, split binary target with an unstripped binary and a DWP file +3 - A stripped, split binary target with an --only-keep-debug symbols file and a DWP file +""" + + +@skipUnlessPlatform(["linux", "freebsd"]) +class DebugInfodDWPTests(TestBase): + # No need to try every flavor of debug inf. + NO_DEBUG_INFO_TESTCASE = True + + def test_normal_stripped(self): + """ + Validate behavior with a stripped binary, no symbols or symbol locator. + """ + self.config_test(["a.out"]) + self.try_breakpoint(False) + + def test_normal_stripped_split_with_dwp(self): + """ + Validate behavior with symbols, but no symbol locator. + """ + self.config_test(["a.out", "a.out.debug", "a.out.dwp"]) + self.try_breakpoint(True) + + def test_normal_stripped_only_dwp(self): + """ + Validate behavior *with* dwp symbols only, but missing other symbols, + but no symbol locator. This shouldn't work: without the other symbols + DWO's appear mostly useless. + """ + self.config_test(["a.out", "a.out.dwp"]) + self.try_breakpoint(False) + + def test_debuginfod_dwp_from_service(self): + """ + Test behavior with the unstripped binary, and DWP from the service. + """ + self.config_test(["a.out.debug"], "a.out.dwp") + self.try_breakpoint(True) + + def test_debuginfod_both_symfiles_from_service(self): + """ + Test behavior with a stripped binary, with the unstripped binary and + dwp symbols from Debuginfod. + """ + self.config_test(["a.out"], "a.out.dwp", "a.out.full") + self.try_breakpoint(True) + + def test_debuginfod_both_okd_symfiles_from_service(self): + """ + Test behavior with both the only-keep-debug symbols and the dwp symbols + from Debuginfod. + """ + self.config_test(["a.out"], "a.out.dwp", "a.out.debug") + self.try_breakpoint(True) + + def try_breakpoint(self, should_have_loc): + """ + This function creates a target from self.aout, sets a function-name + breakpoint, and checks to see if we have a file/line location, + as a way to validate that the symbols have been loaded. + should_have_loc specifies if we're testing that symbols have or + haven't been loaded. + """ + target = self.dbg.CreateTarget(self.aout) + self.assertTrue(target and target.IsValid(), "Target is valid") + + bp = target.BreakpointCreateByName("func") + self.assertTrue(bp and bp.IsValid(), "Breakpoint is valid") + self.assertEqual(bp.GetNumLocations(), 1) + + loc = bp.GetLocationAtIndex(0) + self.assertTrue(loc and loc.IsValid(), "Location is valid") + addr = loc.GetAddress() + self.assertTrue(addr and addr.IsValid(), "Loc address is valid") + line_entry = addr.GetLineEntry() + self.assertEqual( + should_have_loc, + line_entry != None and line_entry.IsValid(), + "Loc line entry is valid", + ) + if should_have_loc: + self.assertEqual(line_entry.GetLine(), 4) + self.assertEqual( + line_entry.GetFileSpec().GetFilename(), + self.main_source_file.GetFilename(), + ) + self.dbg.DeleteTarget(target) + shutil.rmtree(self.tmp_dir) + + def config_test(self, local_files, debuginfo=None, executable=None): + """ + Set up a test with local_files[] copied to a different location + so that we control which files are, or are not, found in the file system. + Also, create a stand-alone file-system 'hosted' debuginfod server with the + provided debuginfo and executable files (if they exist) + + Make the filesystem look like: + + /tmp//test/[local_files] + + /tmp//cache (for lldb to use as a temp cache) + + /tmp//buildid//executable -> + /tmp//buildid//debuginfo -> + Returns the /tmp/ path + """ + + self.build() + + uuid = getUUID(self.getBuildArtifact("a.out.uuid")) + + self.main_source_file = lldb.SBFileSpec("main.c") + self.tmp_dir = tempfile.mkdtemp() + self.test_dir = os.path.join(self.tmp_dir, "test") + os.makedirs(self.test_dir) + + self.aout = "" + # Copy the files used by the test: + for f in local_files: + shutil.copy(self.getBuildArtifact(f), self.test_dir) + if self.aout == "": + self.aout = os.path.join(self.test_dir, f) + + use_debuginfod = debuginfo != None or executable != None + + # Populated the 'file://... mocked' Debuginfod server: + if use_debuginfod: + os.makedirs(os.path.join(self.tmp_dir, "cache")) + uuid_dir = os.path.join(self.tmp_dir, "buildid", uuid) + os.makedirs(uuid_dir) + if debuginfo: + shutil.copy( + self.getBuildArtifact(debuginfo), + os.path.join(uuid_dir, "debuginfo"), + ) + if executable: + shutil.copy( + self.getBuildArtifact(executable), + os.path.join(uuid_dir, "executable"), + ) + os.remove(self.getBuildArtifact("main.dwo")) + # Configure LLDB for the test: + self.runCmd( + "settings set symbols.enable-external-lookup %s" + % str(use_debuginfod).lower() + ) + self.runCmd("settings clear plugin.symbol-locator.debuginfod.server-urls") + if use_debuginfod: + self.runCmd( + "settings set plugin.symbol-locator.debuginfod.cache-path %s/cache" + % self.tmp_dir + ) + self.runCmd( + "settings insert-before plugin.symbol-locator.debuginfod.server-urls 0 file://%s" + % self.tmp_dir + ) diff --git a/lldb/test/API/debuginfod/SplitDWARF/main.c b/lldb/test/API/debuginfod/SplitDWARF/main.c new file mode 100644 index 000000000000..4c7184609b45 --- /dev/null +++ b/lldb/test/API/debuginfod/SplitDWARF/main.c @@ -0,0 +1,7 @@ +// This is a dump little pair of test files + +int func(int argc, const char *argv[]) { + return (argc + 1) * (argv[argc][0] + 2); +} + +int main(int argc, const char *argv[]) { return func(0, argv); } -- GitLab From 2ab106cbd428984df3dda2f6983d5f956917cb69 Mon Sep 17 00:00:00 2001 From: Krzysztof Parzyszek Date: Thu, 21 Mar 2024 15:12:43 -0500 Subject: [PATCH 194/296] [flang][OpenMP] Convert processTODO and remove unused objects (#81627) Remove `ClauseIterator2` and `clauses2` from ClauseProcessor. [Clause representation 5/6] --- flang/lib/Lower/OpenMP/ClauseProcessor.h | 16 +++----- flang/lib/Lower/OpenMP/OpenMP.cpp | 51 +++++++++--------------- 2 files changed, 23 insertions(+), 44 deletions(-) diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h index 1b76eb97e823..8582716e6b9a 100644 --- a/flang/lib/Lower/OpenMP/ClauseProcessor.h +++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h @@ -46,13 +46,11 @@ namespace omp { /// methods that relate to clauses that can impact the lowering of that /// construct. class ClauseProcessor { - using ClauseTy = Fortran::parser::OmpClause; - public: ClauseProcessor(Fortran::lower::AbstractConverter &converter, Fortran::semantics::SemanticsContext &semaCtx, const Fortran::parser::OmpClauseList &clauses) - : converter(converter), semaCtx(semaCtx), clauses2(clauses), + : converter(converter), semaCtx(semaCtx), clauses(makeList(clauses, semaCtx)) {} // 'Unique' clauses: They can appear at most once in the clause list. @@ -156,7 +154,6 @@ public: private: using ClauseIterator = List::const_iterator; - using ClauseIterator2 = std::list::const_iterator; /// Utility to find a clause within a range in the clause list. template @@ -182,7 +179,6 @@ private: Fortran::lower::AbstractConverter &converter; Fortran::semantics::SemanticsContext &semaCtx; - const Fortran::parser::OmpClauseList &clauses2; List clauses; }; @@ -238,19 +234,17 @@ bool ClauseProcessor::processMotionClauses( template void ClauseProcessor::processTODO(mlir::Location currentLocation, llvm::omp::Directive directive) const { - auto checkUnhandledClause = [&](const auto *x) { + auto checkUnhandledClause = [&](llvm::omp::Clause id, const auto *x) { if (!x) return; TODO(currentLocation, - "Unhandled clause " + - llvm::StringRef(Fortran::parser::ParseTreeDumper::GetNodeName(*x)) - .upper() + + "Unhandled clause " + llvm::omp::getOpenMPClauseName(id).upper() + " in " + llvm::omp::getOpenMPDirectiveName(directive).upper() + " construct"); }; - for (ClauseIterator2 it = clauses2.v.begin(); it != clauses2.v.end(); ++it) - (checkUnhandledClause(std::get_if(&it->u)), ...); + for (ClauseIterator it = clauses.begin(); it != clauses.end(); ++it) + (checkUnhandledClause(it->id, std::get_if(&it->u)), ...); } template diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp index 160ada379a08..d91694c4f639 100644 --- a/flang/lib/Lower/OpenMP/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP/OpenMP.cpp @@ -734,9 +734,7 @@ genTaskOp(Fortran::lower::AbstractConverter &converter, cp.processMergeable(mergeableAttr); cp.processPriority(stmtCtx, priorityClauseOperand); cp.processDepend(dependTypeOperands, dependOperands); - cp.processTODO( + cp.processTODO( currentLocation, llvm::omp::Directive::OMPD_task); return genOpWithBody( @@ -762,8 +760,8 @@ genTaskgroupOp(Fortran::lower::AbstractConverter &converter, llvm::SmallVector allocateOperands, allocatorOperands; ClauseProcessor cp(converter, semaCtx, clauseList); cp.processAllocate(allocatorOperands, allocateOperands); - cp.processTODO( - currentLocation, llvm::omp::Directive::OMPD_taskgroup); + cp.processTODO(currentLocation, + llvm::omp::Directive::OMPD_taskgroup); return genOpWithBody( OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval) .setGenNested(genNested) @@ -1102,16 +1100,11 @@ genTargetOp(Fortran::lower::AbstractConverter &converter, cp.processMap(currentLocation, directive, stmtCtx, mapOperands, &mapSymTypes, &mapSymLocs, &mapSymbols); - cp.processTODO( + cp.processTODO( currentLocation, llvm::omp::Directive::OMPD_target); + // 5.8.1 Implicit Data-Mapping Attribute Rules // The following code follows the implicit data-mapping rules to map all the // symbols used inside the region that have not been explicitly mapped using @@ -1230,8 +1223,8 @@ genTeamsOp(Fortran::lower::AbstractConverter &converter, cp.processDefault(); cp.processNumTeams(stmtCtx, numTeamsClauseOperand); cp.processThreadLimit(stmtCtx, threadLimitClauseOperand); - cp.processTODO( - currentLocation, llvm::omp::Directive::OMPD_teams); + cp.processTODO(currentLocation, + llvm::omp::Directive::OMPD_teams); return genOpWithBody( OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval) @@ -1283,9 +1276,8 @@ static mlir::omp::DeclareTargetDeviceType getDeclareTargetInfo( cp.processEnter(symbolAndClause); cp.processLink(symbolAndClause); cp.processDeviceType(deviceType); - cp.processTODO( - converter.getCurrentLocation(), - llvm::omp::Directive::OMPD_declare_target); + cp.processTODO(converter.getCurrentLocation(), + llvm::omp::Directive::OMPD_declare_target); } return deviceType; @@ -1367,8 +1359,7 @@ genOmpSimpleStandalone(Fortran::lower::AbstractConverter &converter, break; case llvm::omp::Directive::OMPD_taskwait: ClauseProcessor(converter, semaCtx, opClauseList) - .processTODO( + .processTODO( currentLocation, llvm::omp::Directive::OMPD_taskwait); firOpBuilder.create(currentLocation); break; @@ -1550,11 +1541,8 @@ createSimdLoop(Fortran::lower::AbstractConverter &converter, cp.processIf(clause::If::DirectiveNameModifier::Simd, ifClauseOperand); cp.processSimdlen(simdlenClauseOperand); cp.processSafelen(safelenClauseOperand); - cp.processTODO(loc, ompDirective); + cp.processTODO(loc, ompDirective); mlir::TypeRange resultType; auto simdLoopOp = firOpBuilder.create( @@ -1607,8 +1595,7 @@ static void createWsloop(Fortran::lower::AbstractConverter &converter, cp.processScheduleChunk(stmtCtx, scheduleChunkClauseOperand); cp.processReduction(loc, reductionVars, reductionTypes, reductionDeclSymbols, &reductionSymbols); - cp.processTODO(loc, ompDirective); + cp.processTODO(loc, ompDirective); if (ReductionProcessor::doReductionByRef(reductionVars)) byrefOperand = firOpBuilder.getUnitAttr(); @@ -1670,11 +1657,9 @@ static void createSimdWsloop( const Fortran::parser::OmpClauseList &beginClauseList, const Fortran::parser::OmpClauseList *endClauseList, mlir::Location loc) { ClauseProcessor cp(converter, semaCtx, beginClauseList); - cp.processTODO< - Fortran::parser::OmpClause::Aligned, Fortran::parser::OmpClause::Allocate, - Fortran::parser::OmpClause::Linear, Fortran::parser::OmpClause::Safelen, - Fortran::parser::OmpClause::Simdlen, Fortran::parser::OmpClause::Order>( - loc, ompDirective); + cp.processTODO(loc, + ompDirective); // TODO: Add support for vectorization - add vectorization hints inside loop // body. // OpenMP standard does not specify the length of vector instructions. -- GitLab From 6b1cf0040059c407264d2609403c4fc090673167 Mon Sep 17 00:00:00 2001 From: Maksim Panchenko Date: Thu, 21 Mar 2024 14:05:21 -0700 Subject: [PATCH 195/296] [BOLT] Add support for Linux kernel static keys jump table (#86090) Runtime code modification used by static keys is the most ubiquitous self-modifying feature of the Linux kernel. The idea is to to eliminate the condition check and associated conditional jump on a hot path if that condition (based on a boolean value of a static key) does not change often. Whenever they condition changes, the kernel runtime modifies all code paths associated with that key flipping the code between nop and (unconditional) jump. --- bolt/include/bolt/Core/MCPlus.h | 1 + bolt/include/bolt/Core/MCPlusBuilder.h | 17 + bolt/lib/Core/BinaryContext.cpp | 8 +- bolt/lib/Core/BinaryFunction.cpp | 17 +- bolt/lib/Core/MCPlusBuilder.cpp | 22 ++ bolt/lib/Passes/BinaryPasses.cpp | 20 +- bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 385 +++++++++++++++++++++++ bolt/lib/Target/X86/X86MCPlusBuilder.cpp | 13 + bolt/test/X86/linux-static-keys.s | 67 ++++ 9 files changed, 547 insertions(+), 3 deletions(-) create mode 100644 bolt/test/X86/linux-static-keys.s diff --git a/bolt/include/bolt/Core/MCPlus.h b/bolt/include/bolt/Core/MCPlus.h index b6a9e73f2347..1d2360c18033 100644 --- a/bolt/include/bolt/Core/MCPlus.h +++ b/bolt/include/bolt/Core/MCPlus.h @@ -73,6 +73,7 @@ public: kOffset, /// Offset in the function. kLabel, /// MCSymbol pointing to this instruction. kSize, /// Size of the instruction. + kDynamicBranch, /// Jit instruction patched at runtime. kGeneric /// First generic annotation. }; diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h index 96b58f541623..198a8d8bf48f 100644 --- a/bolt/include/bolt/Core/MCPlusBuilder.h +++ b/bolt/include/bolt/Core/MCPlusBuilder.h @@ -1199,6 +1199,16 @@ public: /// Set instruction size. void setSize(MCInst &Inst, uint32_t Size) const; + /// Check if the branch instruction could be modified at runtime. + bool isDynamicBranch(const MCInst &Inst) const; + + /// Return ID for runtime-modifiable instruction. + std::optional getDynamicBranchID(const MCInst &Inst) const; + + /// Mark instruction as a dynamic branch, i.e. a branch that can be + /// overwritten at runtime. + void setDynamicBranch(MCInst &Inst, uint32_t ID) const; + /// Return MCSymbol that represents a target of this instruction at a given /// operand number \p OpNum. If there's no symbol associated with /// the operand - return nullptr. @@ -1688,6 +1698,13 @@ public: llvm_unreachable("not implemented"); } + /// Create long conditional branch with a target-specific conditional code + /// \p CC. + virtual void createLongCondBranch(MCInst &Inst, const MCSymbol *Target, + unsigned CC, MCContext *Ctx) const { + llvm_unreachable("not implemented"); + } + /// Reverses the branch condition in Inst and update its taken target to TBB. /// /// Returns true on success. diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp index b29ebbbfa18c..267f43f65e20 100644 --- a/bolt/lib/Core/BinaryContext.cpp +++ b/bolt/lib/Core/BinaryContext.cpp @@ -1939,7 +1939,13 @@ void BinaryContext::printInstruction(raw_ostream &OS, const MCInst &Instruction, OS << Endl; return; } - InstPrinter->printInst(&Instruction, 0, "", *STI, OS); + if (std::optional DynamicID = + MIB->getDynamicBranchID(Instruction)) { + OS << "\tjit\t" << MIB->getTargetSymbol(Instruction)->getName() + << " # ID: " << DynamicID; + } else { + InstPrinter->printInst(&Instruction, 0, "", *STI, OS); + } if (MIB->isCall(Instruction)) { if (MIB->isTailCall(Instruction)) OS << " # TAILCALL "; diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp index ce4dd29f542b..fdadef9dcd38 100644 --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -3350,6 +3350,16 @@ void BinaryFunction::fixBranches() { // Eliminate unnecessary conditional branch. if (TSuccessor == FSuccessor) { + // FIXME: at the moment, we cannot safely remove static key branches. + if (MIB->isDynamicBranch(*CondBranch)) { + if (opts::Verbosity) { + BC.outs() + << "BOLT-INFO: unable to remove redundant dynamic branch in " + << *this << '\n'; + } + continue; + } + BB->removeDuplicateConditionalSuccessor(CondBranch); if (TSuccessor != NextBB) BB->addBranchInstruction(TSuccessor); @@ -3358,8 +3368,13 @@ void BinaryFunction::fixBranches() { // Reverse branch condition and swap successors. auto swapSuccessors = [&]() { - if (MIB->isUnsupportedBranch(*CondBranch)) + if (MIB->isUnsupportedBranch(*CondBranch)) { + if (opts::Verbosity) { + BC.outs() << "BOLT-INFO: unable to swap successors in " << *this + << '\n'; + } return false; + } std::swap(TSuccessor, FSuccessor); BB->swapConditionalSuccessors(); auto L = BC.scopeLock(); diff --git a/bolt/lib/Core/MCPlusBuilder.cpp b/bolt/lib/Core/MCPlusBuilder.cpp index bd9bd0c45922..5b14ad5cdb88 100644 --- a/bolt/lib/Core/MCPlusBuilder.cpp +++ b/bolt/lib/Core/MCPlusBuilder.cpp @@ -303,6 +303,28 @@ void MCPlusBuilder::setSize(MCInst &Inst, uint32_t Size) const { setAnnotationOpValue(Inst, MCAnnotation::kSize, Size); } +bool MCPlusBuilder::isDynamicBranch(const MCInst &Inst) const { + if (!hasAnnotation(Inst, MCAnnotation::kDynamicBranch)) + return false; + assert(isBranch(Inst) && "Branch expected."); + return true; +} + +std::optional +MCPlusBuilder::getDynamicBranchID(const MCInst &Inst) const { + if (std::optional Value = + getAnnotationOpValue(Inst, MCAnnotation::kDynamicBranch)) { + assert(isBranch(Inst) && "Branch expected."); + return static_cast(*Value); + } + return std::nullopt; +} + +void MCPlusBuilder::setDynamicBranch(MCInst &Inst, uint32_t ID) const { + assert(isBranch(Inst) && "Branch expected."); + setAnnotationOpValue(Inst, MCAnnotation::kDynamicBranch, ID); +} + bool MCPlusBuilder::hasAnnotation(const MCInst &Inst, unsigned Index) const { return (bool)getAnnotationOpValue(Inst, Index); } diff --git a/bolt/lib/Passes/BinaryPasses.cpp b/bolt/lib/Passes/BinaryPasses.cpp index bf1c2ddd37dd..c0ba73108f57 100644 --- a/bolt/lib/Passes/BinaryPasses.cpp +++ b/bolt/lib/Passes/BinaryPasses.cpp @@ -107,6 +107,12 @@ static cl::opt cl::desc("print statistics about basic block ordering"), cl::init(0), cl::cat(BoltOptCategory)); +static cl::opt PrintLargeFunctions( + "print-large-functions", + cl::desc("print functions that could not be overwritten due to excessive " + "size"), + cl::init(false), cl::cat(BoltOptCategory)); + static cl::list PrintSortedBy("print-sorted-by", cl::CommaSeparated, cl::desc("print functions sorted by order of dyno stats"), @@ -570,8 +576,12 @@ Error CheckLargeFunctions::runOnFunctions(BinaryContext &BC) { uint64_t HotSize, ColdSize; std::tie(HotSize, ColdSize) = BC.calculateEmittedSize(BF, /*FixBranches=*/false); - if (HotSize > BF.getMaxSize()) + if (HotSize > BF.getMaxSize()) { + if (opts::PrintLargeFunctions) + BC.outs() << "BOLT-INFO: " << BF << " size exceeds allocated space by " + << (HotSize - BF.getMaxSize()) << " bytes\n"; BF.setSimple(false); + } }; ParallelUtilities::PredicateTy SkipFunc = [&](const BinaryFunction &BF) { @@ -852,6 +862,10 @@ uint64_t SimplifyConditionalTailCalls::fixTailCalls(BinaryFunction &BF) { assert(Result && "internal error analyzing conditional branch"); assert(CondBranch && "conditional branch expected"); + // Skip dynamic branches for now. + if (BF.getBinaryContext().MIB->isDynamicBranch(*CondBranch)) + continue; + // It's possible that PredBB is also a successor to BB that may have // been processed by a previous iteration of the SCTC loop, in which // case it may have been marked invalid. We should skip rewriting in @@ -1012,6 +1026,10 @@ uint64_t ShortenInstructions::shortenInstructions(BinaryFunction &Function) { const BinaryContext &BC = Function.getBinaryContext(); for (BinaryBasicBlock &BB : Function) { for (MCInst &Inst : BB) { + // Skip shortening instructions with Size annotation. + if (BC.MIB->getSize(Inst)) + continue; + MCInst OriginalInst; if (opts::Verbosity > 2) OriginalInst = Inst; diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp index a2bfd45a64e3..b028a455a6db 100644 --- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp +++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp @@ -14,7 +14,9 @@ #include "bolt/Rewrite/MetadataRewriter.h" #include "bolt/Rewrite/MetadataRewriters.h" #include "bolt/Utils/CommandLineOpts.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseSet.h" +#include "llvm/MC/MCDisassembler/MCDisassembler.h" #include "llvm/Support/BinaryStreamWriter.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" @@ -65,6 +67,16 @@ static cl::opt DumpStaticCalls("dump-static-calls", cl::init(false), cl::Hidden, cl::cat(BoltCategory)); +static cl::opt + DumpStaticKeys("dump-static-keys", + cl::desc("dump Linux kernel static keys jump table"), + cl::init(false), cl::Hidden, cl::cat(BoltCategory)); + +static cl::opt LongJumpLabels( + "long-jump-labels", + cl::desc("always use long jumps/nops for Linux kernel static keys"), + cl::init(false), cl::Hidden, cl::cat(BoltCategory)); + static cl::opt PrintORC("print-orc", cl::desc("print ORC unwind information for instructions"), @@ -151,6 +163,20 @@ class LinuxKernelRewriter final : public MetadataRewriter { /// Number of entries in the input file ORC sections. uint64_t NumORCEntries = 0; + /// Section containing static keys jump table. + ErrorOr StaticKeysJumpSection = std::errc::bad_address; + uint64_t StaticKeysJumpTableAddress = 0; + static constexpr size_t STATIC_KEYS_JUMP_ENTRY_SIZE = 8; + + struct JumpInfoEntry { + bool Likely; + bool InitValue; + }; + SmallVector JumpInfo; + + /// Static key entries that need nop conversion. + DenseSet NopIDs; + /// Section containing static call table. ErrorOr StaticCallSection = std::errc::bad_address; uint64_t StaticCallTableAddress = 0; @@ -235,6 +261,11 @@ class LinuxKernelRewriter final : public MetadataRewriter { /// Read .pci_fixup Error readPCIFixupTable(); + /// Handle static keys jump table. + Error readStaticKeysJumpTable(); + Error rewriteStaticKeysJumpTable(); + Error updateStaticKeysJumpTablePostEmit(); + /// Mark instructions referenced by kernel metadata. Error markInstructions(); @@ -268,6 +299,9 @@ public: if (Error E = readPCIFixupTable()) return E; + if (Error E = readStaticKeysJumpTable()) + return E; + return Error::success(); } @@ -290,12 +324,18 @@ public: if (Error E = rewriteStaticCalls()) return E; + if (Error E = rewriteStaticKeysJumpTable()) + return E; + return Error::success(); } Error postEmitFinalizer() override { updateLKMarkers(); + if (Error E = updateStaticKeysJumpTablePostEmit()) + return E; + return Error::success(); } }; @@ -1343,6 +1383,351 @@ Error LinuxKernelRewriter::readPCIFixupTable() { return Error::success(); } +/// Runtime code modification used by static keys is the most ubiquitous +/// self-modifying feature of the Linux kernel. The idea is to eliminate the +/// condition check and associated conditional jump on a hot path if that +/// condition (based on a boolean value of a static key) does not change often. +/// Whenever the condition changes, the kernel runtime modifies all code paths +/// associated with that key flipping the code between nop and (unconditional) +/// jump. The information about the code is stored in a static key jump table +/// and contains the list of entries of the following type from +/// include/linux/jump_label.h: +// +/// struct jump_entry { +/// s32 code; +/// s32 target; +/// long key; // key may be far away from the core kernel under KASLR +/// }; +/// +/// The list does not have to be stored in any sorted way, but it is sorted at +/// boot time (or module initialization time) first by "key" and then by "code". +/// jump_label_sort_entries() is responsible for sorting the table. +/// +/// The key in jump_entry structure uses lower two bits of the key address +/// (which itself is aligned) to store extra information. We are interested in +/// the lower bit which indicates if the key is likely to be set on the code +/// path associated with this jump_entry. +/// +/// static_key_{enable,disable}() functions modify the code based on key and +/// jump table entries. +/// +/// jump_label_update() updates all code entries for a given key. Batch mode is +/// used for x86. +/// +/// The actual patching happens in text_poke_bp_batch() that overrides the first +/// byte of the sequence with int3 before proceeding with actual code +/// replacement. +Error LinuxKernelRewriter::readStaticKeysJumpTable() { + const BinaryData *StaticKeysJumpTable = + BC.getBinaryDataByName("__start___jump_table"); + if (!StaticKeysJumpTable) + return Error::success(); + + StaticKeysJumpTableAddress = StaticKeysJumpTable->getAddress(); + + const BinaryData *Stop = BC.getBinaryDataByName("__stop___jump_table"); + if (!Stop) + return createStringError(errc::executable_format_error, + "missing __stop___jump_table symbol"); + + ErrorOr ErrorOrSection = + BC.getSectionForAddress(StaticKeysJumpTableAddress); + if (!ErrorOrSection) + return createStringError(errc::executable_format_error, + "no section matching __start___jump_table"); + + StaticKeysJumpSection = *ErrorOrSection; + if (!StaticKeysJumpSection->containsAddress(Stop->getAddress() - 1)) + return createStringError(errc::executable_format_error, + "__stop___jump_table not in the same section " + "as __start___jump_table"); + + if ((Stop->getAddress() - StaticKeysJumpTableAddress) % + STATIC_KEYS_JUMP_ENTRY_SIZE) + return createStringError(errc::executable_format_error, + "static keys jump table size error"); + + const uint64_t SectionAddress = StaticKeysJumpSection->getAddress(); + DataExtractor DE(StaticKeysJumpSection->getContents(), + BC.AsmInfo->isLittleEndian(), + BC.AsmInfo->getCodePointerSize()); + DataExtractor::Cursor Cursor(StaticKeysJumpTableAddress - SectionAddress); + uint32_t EntryID = 0; + while (Cursor && Cursor.tell() < Stop->getAddress() - SectionAddress) { + const uint64_t JumpAddress = + SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor); + const uint64_t TargetAddress = + SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor); + const uint64_t KeyAddress = + SectionAddress + Cursor.tell() + (int64_t)DE.getU64(Cursor); + + // Consume the status of the cursor. + if (!Cursor) + return createStringError( + errc::executable_format_error, + "out of bounds while reading static keys jump table: %s", + toString(Cursor.takeError()).c_str()); + + ++EntryID; + + JumpInfo.push_back(JumpInfoEntry()); + JumpInfoEntry &Info = JumpInfo.back(); + Info.Likely = KeyAddress & 1; + + if (opts::DumpStaticKeys) { + BC.outs() << "Static key jump entry: " << EntryID + << "\n\tJumpAddress: 0x" << Twine::utohexstr(JumpAddress) + << "\n\tTargetAddress: 0x" << Twine::utohexstr(TargetAddress) + << "\n\tKeyAddress: 0x" << Twine::utohexstr(KeyAddress) + << "\n\tIsLikely: " << Info.Likely << '\n'; + } + + BinaryFunction *BF = BC.getBinaryFunctionContainingAddress(JumpAddress); + if (!BF && opts::Verbosity) { + BC.outs() + << "BOLT-INFO: no function matches address 0x" + << Twine::utohexstr(JumpAddress) + << " of jump instruction referenced from static keys jump table\n"; + } + + if (!BF || !BC.shouldEmit(*BF)) + continue; + + MCInst *Inst = BF->getInstructionAtOffset(JumpAddress - BF->getAddress()); + if (!Inst) + return createStringError( + errc::executable_format_error, + "no instruction at static keys jump site address 0x%" PRIx64, + JumpAddress); + + if (!BF->containsAddress(TargetAddress)) + return createStringError( + errc::executable_format_error, + "invalid target of static keys jump at 0x%" PRIx64 " : 0x%" PRIx64, + JumpAddress, TargetAddress); + + const bool IsBranch = BC.MIB->isBranch(*Inst); + if (!IsBranch && !BC.MIB->isNoop(*Inst)) + return createStringError(errc::executable_format_error, + "jump or nop expected at address 0x%" PRIx64, + JumpAddress); + + const uint64_t Size = BC.computeInstructionSize(*Inst); + if (Size != 2 && Size != 5) { + return createStringError( + errc::executable_format_error, + "unexpected static keys jump size at address 0x%" PRIx64, + JumpAddress); + } + + MCSymbol *Target = BF->registerBranch(JumpAddress, TargetAddress); + MCInst StaticKeyBranch; + + // Create a conditional branch instruction. The actual conditional code type + // should not matter as long as it's a valid code. The instruction should be + // treated as a conditional branch for control-flow purposes. Before we emit + // the code, it will be converted to a different instruction in + // rewriteStaticKeysJumpTable(). + // + // NB: for older kernels, under LongJumpLabels option, we create long + // conditional branch to guarantee that code size estimation takes + // into account the extra bytes needed for long branch that will be used + // by the kernel patching code. Newer kernels can work with both short + // and long branches. The code for long conditional branch is larger + // than unconditional one, so we are pessimistic in our estimations. + if (opts::LongJumpLabels) + BC.MIB->createLongCondBranch(StaticKeyBranch, Target, 0, BC.Ctx.get()); + else + BC.MIB->createCondBranch(StaticKeyBranch, Target, 0, BC.Ctx.get()); + BC.MIB->moveAnnotations(std::move(*Inst), StaticKeyBranch); + BC.MIB->setDynamicBranch(StaticKeyBranch, EntryID); + *Inst = StaticKeyBranch; + + // IsBranch = InitialValue ^ LIKELY + // + // 0 0 0 + // 1 0 1 + // 1 1 0 + // 0 1 1 + // + // => InitialValue = IsBranch ^ LIKELY + Info.InitValue = IsBranch ^ Info.Likely; + + // Add annotations to facilitate manual code analysis. + BC.MIB->addAnnotation(*Inst, "Likely", Info.Likely); + BC.MIB->addAnnotation(*Inst, "InitValue", Info.InitValue); + if (!BC.MIB->getSize(*Inst)) + BC.MIB->setSize(*Inst, Size); + + if (opts::LongJumpLabels) + BC.MIB->setSize(*Inst, 5); + } + + BC.outs() << "BOLT-INFO: parsed " << EntryID << " static keys jump entries\n"; + + return Error::success(); +} + +// Pre-emit pass. Convert dynamic branch instructions into jumps that could be +// relaxed. In post-emit pass we will convert those jumps into nops when +// necessary. We do the unconditional conversion into jumps so that the jumps +// can be relaxed and the optimal size of jump/nop instruction is selected. +Error LinuxKernelRewriter::rewriteStaticKeysJumpTable() { + if (!StaticKeysJumpSection) + return Error::success(); + + uint64_t NumShort = 0; + uint64_t NumLong = 0; + for (BinaryFunction &BF : llvm::make_second_range(BC.getBinaryFunctions())) { + if (!BC.shouldEmit(BF)) + continue; + + for (BinaryBasicBlock &BB : BF) { + for (MCInst &Inst : BB) { + if (!BC.MIB->isDynamicBranch(Inst)) + continue; + + const uint32_t EntryID = *BC.MIB->getDynamicBranchID(Inst); + MCSymbol *Target = + const_cast(BC.MIB->getTargetSymbol(Inst)); + assert(Target && "Target symbol should be set."); + + const JumpInfoEntry &Info = JumpInfo[EntryID - 1]; + const bool IsBranch = Info.Likely ^ Info.InitValue; + + uint32_t Size = *BC.MIB->getSize(Inst); + if (Size == 2) + ++NumShort; + else if (Size == 5) + ++NumLong; + else + llvm_unreachable("Wrong size for static keys jump instruction."); + + MCInst NewInst; + // Replace the instruction with unconditional jump even if it needs to + // be nop in the binary. + if (opts::LongJumpLabels) { + BC.MIB->createLongUncondBranch(NewInst, Target, BC.Ctx.get()); + } else { + // Newer kernels can handle short and long jumps for static keys. + // Optimistically, emit short jump and check if it gets relaxed into + // a long one during post-emit. Only then convert the jump to a nop. + BC.MIB->createUncondBranch(NewInst, Target, BC.Ctx.get()); + } + + BC.MIB->moveAnnotations(std::move(Inst), NewInst); + Inst = NewInst; + + // Mark the instruction for nop conversion. + if (!IsBranch) + NopIDs.insert(EntryID); + + MCSymbol *Label = + BC.MIB->getOrCreateInstLabel(Inst, "__SK_", BC.Ctx.get()); + + // Create a relocation against the label. + const uint64_t EntryOffset = StaticKeysJumpTableAddress - + StaticKeysJumpSection->getAddress() + + (EntryID - 1) * 16; + StaticKeysJumpSection->addRelocation(EntryOffset, Label, + ELF::R_X86_64_PC32, + /*Addend*/ 0); + StaticKeysJumpSection->addRelocation(EntryOffset + 4, Target, + ELF::R_X86_64_PC32, /*Addend*/ 0); + } + } + } + + BC.outs() << "BOLT-INFO: the input contains " << NumShort << " short and " + << NumLong << " long static keys jumps in optimized functions\n"; + + return Error::success(); +} + +// Post-emit pass of static keys jump section. Convert jumps to nops. +Error LinuxKernelRewriter::updateStaticKeysJumpTablePostEmit() { + if (!StaticKeysJumpSection || !StaticKeysJumpSection->isFinalized()) + return Error::success(); + + const uint64_t SectionAddress = StaticKeysJumpSection->getAddress(); + DataExtractor DE(StaticKeysJumpSection->getOutputContents(), + BC.AsmInfo->isLittleEndian(), + BC.AsmInfo->getCodePointerSize()); + DataExtractor::Cursor Cursor(StaticKeysJumpTableAddress - SectionAddress); + const BinaryData *Stop = BC.getBinaryDataByName("__stop___jump_table"); + uint32_t EntryID = 0; + uint64_t NumShort = 0; + uint64_t NumLong = 0; + while (Cursor && Cursor.tell() < Stop->getAddress() - SectionAddress) { + const uint64_t JumpAddress = + SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor); + const uint64_t TargetAddress = + SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor); + const uint64_t KeyAddress = + SectionAddress + Cursor.tell() + (int64_t)DE.getU64(Cursor); + + // Consume the status of the cursor. + if (!Cursor) + return createStringError(errc::executable_format_error, + "out of bounds while updating static keys: %s", + toString(Cursor.takeError()).c_str()); + + ++EntryID; + + LLVM_DEBUG({ + dbgs() << "\n\tJumpAddress: 0x" << Twine::utohexstr(JumpAddress) + << "\n\tTargetAddress: 0x" << Twine::utohexstr(TargetAddress) + << "\n\tKeyAddress: 0x" << Twine::utohexstr(KeyAddress) << '\n'; + }); + + BinaryFunction *BF = + BC.getBinaryFunctionContainingAddress(JumpAddress, + /*CheckPastEnd*/ false, + /*UseMaxSize*/ true); + assert(BF && "Cannot get function for modified static key."); + + if (!BF->isEmitted()) + continue; + + // Disassemble instruction to collect stats even if nop-conversion is + // unnecessary. + MutableArrayRef Contents = MutableArrayRef( + reinterpret_cast(BF->getImageAddress()), BF->getImageSize()); + assert(Contents.size() && "Non-empty function image expected."); + + MCInst Inst; + uint64_t Size; + const uint64_t JumpOffset = JumpAddress - BF->getAddress(); + if (!BC.DisAsm->getInstruction(Inst, Size, Contents.slice(JumpOffset), 0, + nulls())) { + llvm_unreachable("Unable to disassemble jump instruction."); + } + assert(BC.MIB->isBranch(Inst) && "Branch instruction expected."); + + if (Size == 2) + ++NumShort; + else if (Size == 5) + ++NumLong; + else + llvm_unreachable("Unexpected size for static keys jump instruction."); + + // Check if we need to convert jump instruction into a nop. + if (!NopIDs.contains(EntryID)) + continue; + + SmallString<15> NopCode; + raw_svector_ostream VecOS(NopCode); + BC.MAB->writeNopData(VecOS, Size, BC.STI.get()); + for (uint64_t I = 0; I < Size; ++I) + Contents[JumpOffset + I] = NopCode[I]; + } + + BC.outs() << "BOLT-INFO: written " << NumShort << " short and " << NumLong + << " long static keys jumps in optimized functions\n"; + + return Error::success(); +} + } // namespace std::unique_ptr diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp index de55fbe51764..15f95f821777 100644 --- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp +++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp @@ -336,6 +336,9 @@ public: } bool isUnsupportedBranch(const MCInst &Inst) const override { + if (isDynamicBranch(Inst)) + return true; + switch (Inst.getOpcode()) { default: return false; @@ -2728,6 +2731,7 @@ public: void createUncondBranch(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx) const override { + Inst.clear(); Inst.setOpcode(X86::JMP_1); Inst.clear(); Inst.addOperand(MCOperand::createExpr( @@ -2776,6 +2780,15 @@ public: Inst.addOperand(MCOperand::createImm(CC)); } + void createLongCondBranch(MCInst &Inst, const MCSymbol *Target, unsigned CC, + MCContext *Ctx) const override { + Inst.setOpcode(X86::JCC_4); + Inst.clear(); + Inst.addOperand(MCOperand::createExpr( + MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx))); + Inst.addOperand(MCOperand::createImm(CC)); + } + bool reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx) const override { unsigned InvCC = getInvertedCondCode(getCondCode(Inst)); diff --git a/bolt/test/X86/linux-static-keys.s b/bolt/test/X86/linux-static-keys.s new file mode 100644 index 000000000000..08454bf97631 --- /dev/null +++ b/bolt/test/X86/linux-static-keys.s @@ -0,0 +1,67 @@ +# REQUIRES: system-linux + +## Check that BOLT correctly updates the Linux kernel static keys jump table. + +# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o +# RUN: %clang %cflags -nostdlib %t.o -o %t.exe \ +# RUN: -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr + +## Verify static keys jump bindings to instructions. + +# RUN: llvm-bolt %t.exe --print-normalized -o %t.out --keep-nops=0 \ +# RUN: --bolt-info=0 |& FileCheck %s + +## Verify the bindings again on the rewritten binary with nops removed. + +# RUN: llvm-bolt %t.out -o %t.out.1 --print-normalized |& FileCheck %s + +# CHECK: BOLT-INFO: Linux kernel binary detected +# CHECK: BOLT-INFO: parsed 2 static keys jump entries + + .text + .globl _start + .type _start, %function +_start: +# CHECK: Binary Function "_start" + nop +.L0: + jmp .L1 +# CHECK: jit +# CHECK-SAME: # ID: 1 {{.*}} # Likely: 0 # InitValue: 1 + nop +.L1: + .nops 5 +# CHECK: jit +# CHECK-SAME: # ID: 2 {{.*}} # Likely: 1 # InitValue: 1 +.L2: + nop + .size _start, .-_start + + .globl foo + .type foo, %function +foo: + ret + .size foo, .-foo + + +## Static keys jump table. + .rodata + .globl __start___jump_table + .type __start___jump_table, %object +__start___jump_table: + + .long .L0 - . # Jump address + .long .L1 - . # Target address + .quad 1 # Key address + + .long .L1 - . # Jump address + .long .L2 - . # Target address + .quad 0 # Key address + + .globl __stop___jump_table + .type __stop___jump_table, %object +__stop___jump_table: + +## Fake Linux Kernel sections. + .section __ksymtab,"a",@progbits + .section __ksymtab_gpl,"a",@progbits -- GitLab From 70a9c527b8c9857fb63a87b2d2025bf9defea7f2 Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Thu, 21 Mar 2024 22:28:44 +0100 Subject: [PATCH 196/296] [libc][bazel] Fix bazel build (#86190) Follow up on #86140 --- utils/bazel/llvm-project-overlay/libc/BUILD.bazel | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel index 2e8d475f196e..0d531a3dc12a 100644 --- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel @@ -97,6 +97,11 @@ libc_support_library( deps = [":llvm_libc_macros_float_macros"], ) +libc_support_library( + name = "llvm_libc_macros_fcntl_macros", + hdrs = ["include/llvm-libc-macros/linux/fcntl-macros.h"], +) + ############################### Support libraries ############################## libc_support_library( @@ -3271,6 +3276,7 @@ libc_function( ":__support_common", ":__support_osutil_syscall", ":errno", + ":llvm_libc_macros_fcntl_macros", ], ) -- GitLab From 628068113710d501e88b63a1506d66dd20ce7e94 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Thu, 21 Mar 2024 14:32:13 -0700 Subject: [PATCH 197/296] [BOLT] Output basic YAML profile in BAT mode Relax assumptions that YAML output is not supported in BAT mode. Set up basic infrastructure for emitting YAML for functions not covered by BAT, such as from `.bolt.org.text` section (code identical to input binary sans external refs), or non-rewritten functions in non-relocation mode (where the function stays in the same section but BAT mapping is not emitted). This diff only produces YAML profile for non-BAT functions (skipped, non-simple). YAML profile for BAT functions is added in follow-up diffs: - https://github.com/llvm/llvm-project/pull/76911 emits YAML profile with internal control flow information only (branch profile), - https://github.com/llvm/llvm-project/pull/76896 adds cross-function profile (calls profile). Test Plan: Added bolt/test/X86/bolt-address-translation-yaml.test Reviewers: ayermolo, dcci, maksfb, rafaelauler Reviewed By: rafaelauler Pull Request: https://github.com/llvm/llvm-project/pull/76910 --- bolt/docs/BAT.md | 5 +- .../bolt/Profile/BoltAddressTranslation.h | 3 + bolt/include/bolt/Profile/DataAggregator.h | 4 + bolt/lib/Profile/DataAggregator.cpp | 68 +- bolt/lib/Rewrite/RewriteInstance.cpp | 25 +- bolt/lib/Utils/CommandLineOpts.cpp | 4 + bolt/test/X86/Inputs/blarge_new.preagg.txt | 81 + bolt/test/X86/Inputs/blarge_new.yaml | 1648 +++++++++++++++++ .../test/X86/Inputs/blarge_new_bat.preagg.txt | 79 + .../X86/bolt-address-translation-yaml.test | 40 + 10 files changed, 1936 insertions(+), 21 deletions(-) create mode 100644 bolt/test/X86/Inputs/blarge_new.preagg.txt create mode 100644 bolt/test/X86/Inputs/blarge_new.yaml create mode 100644 bolt/test/X86/Inputs/blarge_new_bat.preagg.txt create mode 100644 bolt/test/X86/bolt-address-translation-yaml.test diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md index 060fc632f686..186b0e5ea89d 100644 --- a/bolt/docs/BAT.md +++ b/bolt/docs/BAT.md @@ -14,9 +14,8 @@ binary onto the original binary. # Usage `--enable-bat` flag controls the generation of BAT section. Sampled profile needs to be passed along with the optimized binary containing BAT section to -`perf2bolt` which reads BAT section and produces fdata profile for the original -binary. Note that YAML profile generation is not supported since BAT doesn't -contain the metadata for input functions. +`perf2bolt` which reads BAT section and produces profile for the original +binary. # Internals ## Section contents diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index 5f2f0959d93f..1f53f6d344ad 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -122,6 +122,9 @@ public: /// Returns BF hash by function output address (after BOLT). size_t getBFHash(uint64_t OutputAddress) const; + /// True if a given \p Address is a function with translation table entry. + bool isBATFunction(uint64_t Address) const { return Maps.count(Address); } + private: /// Helper to update \p Map by inserting one or more BAT entries reflecting /// \p BB for function located at \p FuncAddress. At least one entry will be diff --git a/bolt/include/bolt/Profile/DataAggregator.h b/bolt/include/bolt/Profile/DataAggregator.h index 5bb4d00024c5..f7840b49199f 100644 --- a/bolt/include/bolt/Profile/DataAggregator.h +++ b/bolt/include/bolt/Profile/DataAggregator.h @@ -463,6 +463,10 @@ private: /// Dump data structures into a file readable by llvm-bolt std::error_code writeAggregatedFile(StringRef OutputFilename) const; + /// Dump translated data structures into YAML + std::error_code writeBATYAML(BinaryContext &BC, + StringRef OutputFilename) const; + /// Filter out binaries based on PID void filterBinaryMMapInfo(); diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp index 6a64bcde911e..37c637a44a0e 100644 --- a/bolt/lib/Profile/DataAggregator.cpp +++ b/bolt/lib/Profile/DataAggregator.cpp @@ -16,6 +16,7 @@ #include "bolt/Core/BinaryFunction.h" #include "bolt/Profile/BoltAddressTranslation.h" #include "bolt/Profile/Heatmap.h" +#include "bolt/Profile/YAMLProfileWriter.h" #include "bolt/Utils/CommandLineOpts.h" #include "bolt/Utils/Utils.h" #include "llvm/ADT/STLExtras.h" @@ -85,6 +86,7 @@ MaxSamples("max-samples", cl::cat(AggregatorCategory)); extern cl::opt ProfileFormat; +extern cl::opt SaveProfile; cl::opt ReadPreAggregated( "pa", cl::desc("skip perf and read data from a pre-aggregated file format"), @@ -594,10 +596,21 @@ Error DataAggregator::readProfile(BinaryContext &BC) { convertBranchData(Function); } - if (opts::AggregateOnly && - opts::ProfileFormat == opts::ProfileFormatKind::PF_Fdata) { - if (std::error_code EC = writeAggregatedFile(opts::OutputFilename)) - report_error("cannot create output data file", EC); + if (opts::AggregateOnly) { + if (opts::ProfileFormat == opts::ProfileFormatKind::PF_Fdata) + if (std::error_code EC = writeAggregatedFile(opts::OutputFilename)) + report_error("cannot create output data file", EC); + + // BAT YAML is handled by DataAggregator since normal YAML output requires + // CFG which is not available in BAT mode. + if (usesBAT()) { + if (opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML) + if (std::error_code EC = writeBATYAML(BC, opts::OutputFilename)) + report_error("cannot create output data file", EC); + if (!opts::SaveProfile.empty()) + if (std::error_code EC = writeBATYAML(BC, opts::SaveProfile)) + report_error("cannot create output data file", EC); + } } return Error::success(); @@ -2258,6 +2271,53 @@ DataAggregator::writeAggregatedFile(StringRef OutputFilename) const { return std::error_code(); } +std::error_code DataAggregator::writeBATYAML(BinaryContext &BC, + StringRef OutputFilename) const { + std::error_code EC; + raw_fd_ostream OutFile(OutputFilename, EC, sys::fs::OpenFlags::OF_None); + if (EC) + return EC; + + yaml::bolt::BinaryProfile BP; + + // Fill out the header info. + BP.Header.Version = 1; + BP.Header.FileName = std::string(BC.getFilename()); + std::optional BuildID = BC.getFileBuildID(); + BP.Header.Id = BuildID ? std::string(*BuildID) : ""; + BP.Header.Origin = std::string(getReaderName()); + // Only the input binary layout order is supported. + BP.Header.IsDFSOrder = false; + // FIXME: Need to match hash function used to produce BAT hashes. + BP.Header.HashFunction = HashFunction::Default; + + ListSeparator LS(","); + raw_string_ostream EventNamesOS(BP.Header.EventNames); + for (const StringMapEntry &EventEntry : EventNames) + EventNamesOS << LS << EventEntry.first().str(); + + BP.Header.Flags = opts::BasicAggregation ? BinaryFunction::PF_SAMPLE + : BinaryFunction::PF_LBR; + + if (!opts::BasicAggregation) { + // Convert profile for functions not covered by BAT + for (auto &BFI : BC.getBinaryFunctions()) { + BinaryFunction &Function = BFI.second; + if (!Function.hasProfile()) + continue; + if (BAT->isBATFunction(Function.getAddress())) + continue; + BP.Functions.emplace_back( + YAMLProfileWriter::convert(Function, /*UseDFS=*/false)); + } + } + + // Write the profile. + yaml::Output Out(OutFile, nullptr, 0); + Out << BP; + return std::error_code(); +} + void DataAggregator::dump() const { DataReader::dump(); } void DataAggregator::dump(const LBREntry &LBR) const { diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp index cde195c17390..03f4298e817d 100644 --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -199,10 +199,7 @@ static cl::opt RelocationMode( "relocs", cl::desc("use relocations in the binary (default=autodetect)"), cl::cat(BoltCategory)); -static cl::opt -SaveProfile("w", - cl::desc("save recorded profile to a file"), - cl::cat(BoltOutputCategory)); +extern cl::opt SaveProfile; static cl::list SkipFunctionNames("skip-funcs", @@ -732,6 +729,13 @@ Error RewriteInstance::run() { // Skip disassembling if we have a translation table and we are running an // aggregation job. if (opts::AggregateOnly && BAT->enabledFor(InputFile)) { + // YAML profile in BAT mode requires CFG for .bolt.org.text functions + if (!opts::SaveProfile.empty() || + opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML) { + selectFunctionsToProcess(); + disassembleFunctions(); + buildFunctionsCFG(); + } processProfileData(); return Error::success(); } @@ -2027,14 +2031,6 @@ void RewriteInstance::adjustCommandLineOptions() { if (opts::Lite) BC->outs() << "BOLT-INFO: enabling lite mode\n"; - - if (!opts::SaveProfile.empty() && BAT->enabledFor(InputFile)) { - BC->errs() - << "BOLT-ERROR: unable to save profile in YAML format for input " - "file processed by BOLT. Please remove -w option and use branch " - "profile.\n"; - exit(1); - } } namespace { @@ -3126,12 +3122,13 @@ void RewriteInstance::processProfileData() { } } - if (!opts::SaveProfile.empty()) { + if (!opts::SaveProfile.empty() && !BAT->enabledFor(InputFile)) { YAMLProfileWriter PW(opts::SaveProfile); PW.writeProfile(*this); } if (opts::AggregateOnly && - opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML) { + opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML && + !BAT->enabledFor(InputFile)) { YAMLProfileWriter PW(opts::OutputFilename); PW.writeProfile(*this); } diff --git a/bolt/lib/Utils/CommandLineOpts.cpp b/bolt/lib/Utils/CommandLineOpts.cpp index e910fa4f8672..ba296c10c00a 100644 --- a/bolt/lib/Utils/CommandLineOpts.cpp +++ b/bolt/lib/Utils/CommandLineOpts.cpp @@ -162,6 +162,10 @@ cl::opt ProfileFormat( clEnumValN(PF_YAML, "yaml", "dense YAML representation")), cl::ZeroOrMore, cl::Hidden, cl::cat(BoltCategory)); +cl::opt SaveProfile("w", + cl::desc("save recorded profile to a file"), + cl::cat(BoltOutputCategory)); + cl::opt SplitEH("split-eh", cl::desc("split C++ exception handling code"), cl::Hidden, cl::cat(BoltOptCategory)); diff --git a/bolt/test/X86/Inputs/blarge_new.preagg.txt b/bolt/test/X86/Inputs/blarge_new.preagg.txt new file mode 100644 index 000000000000..e92f356d9188 --- /dev/null +++ b/bolt/test/X86/Inputs/blarge_new.preagg.txt @@ -0,0 +1,81 @@ +B 40164b 401608 109 0 +B 401611 4017e0 115 0 +B 4017f0 401616 117 0 +B 401ba2 4015da 6 0 +B 4015d5 401b60 1 0 +B 40159a 401b60 5 0 +B 401b9d 401b70 615 2 +B 401b90 401b99 344 37 +B 401ba2 40159f 8 0 +B 4015b0 401070 9 0 +B 401544 4014a0 6 0 +B 40188a 401928 5 0 +B 40152a 4014b0 21 0 +B 40169e 40165b 2 0 +B 4014dd 401070 12 1 +B 401509 4014ec 2 2 +B 401510 401030 673 0 +B 4019de 401080 1 0 +B 401500 401070 22 0 +B 401921 4014d6 9 0 +B 4019b3 401080 3 0 +B 40162d 401070 113 0 +B 4014d1 401800 27 0 +B 401a3f 401080 1 0 +B 4018d2 401050 17 0 +B 401664 4017c0 2 0 +B 401680 401070 2 0 +B 4017d0 401669 2 0 +B 4018f7 40190d 9 0 +B 4015bc 401592 6 0 +B 401964 401090 5 0 +B 4015f8 4015cd 1 0 +B 4015ec 401070 6 0 +F 40165b 401664 2 +F 4017c0 4017d0 2 +F 401669 401680 2 +F 40190d 401921 9 +F 4014d6 4014dd 9 +F 401800 4018d2 17 +F 4018d7 4018f7 9 +F 40159f 4015b0 8 +F 401515 401544 6 +F 401070 401500 1 +F 401070 401070 157 +F 4014a0 4014d1 6 +F 401616 40162d 112 +F 4019e3 401a3f 1 +F 4014e2 401500 19 +F 401090 401090 5 +F 401030 401030 673 +F 401505 401510 668 +F 401616 4017f0 2 +F 401070 4015b0 1 +F 4015da 4015ec 6 +F 401b60 401b90 6 +F 4019b8 4019de 1 +F 401969 4019b3 3 +F 401505 401509 2 +F 401515 40152a 21 +F 401592 40159a 4 +F 401050 401050 17 +F 4015cd 4015d5 1 +F 401070 4014dd 1 +F 401b99 401ba2 8 +F 401b70 401b90 326 +F 401b99 401b9d 324 +F 401592 4015bc 1 +F 401608 401611 109 +F 401b70 401b9d 268 +F 4015b5 4015bc 5 +F 401b99 401b90 1 +F 401b70 401ba2 5 +F 401632 40164b 108 +F 401080 401080 5 +F 4014b0 4014d1 21 +F 4017e0 4017f0 115 +F 4015f1 4015f8 1 +F 401685 40169e 2 +F 401928 401964 5 +F 401800 40188a 5 +F 4014ec 401500 2 diff --git a/bolt/test/X86/Inputs/blarge_new.yaml b/bolt/test/X86/Inputs/blarge_new.yaml new file mode 100644 index 000000000000..0380f5180e90 --- /dev/null +++ b/bolt/test/X86/Inputs/blarge_new.yaml @@ -0,0 +1,1648 @@ +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_EXEC + Machine: EM_X86_64 + Entry: 0x4016D0 +ProgramHeaders: + - Type: PT_PHDR + Flags: [ PF_R ] + VAddr: 0x400040 + Align: 0x8 + Offset: 0x40 + - Type: PT_INTERP + Flags: [ PF_R ] + FirstSec: .interp + LastSec: .interp + VAddr: 0x4002A8 + Offset: 0x2A8 + - Type: PT_LOAD + Flags: [ PF_R ] + FirstSec: .interp + LastSec: .rela.plt + VAddr: 0x400000 + Align: 0x1000 + Offset: 0x0 + - Type: PT_LOAD + Flags: [ PF_X, PF_R ] + FirstSec: .init + LastSec: .fini + VAddr: 0x401000 + Align: 0x1000 + Offset: 0x1000 + - Type: PT_LOAD + Flags: [ PF_R ] + FirstSec: .rodata + LastSec: .eh_frame + VAddr: 0x402000 + Align: 0x1000 + Offset: 0x2000 + - Type: PT_LOAD + Flags: [ PF_W, PF_R ] + FirstSec: .init_array + LastSec: .bss + VAddr: 0x403E00 + Align: 0x1000 + Offset: 0x2E00 + - Type: PT_DYNAMIC + Flags: [ PF_W, PF_R ] + FirstSec: .dynamic + LastSec: .dynamic + VAddr: 0x403E10 + Align: 0x8 + Offset: 0x2E10 + - Type: PT_NOTE + Flags: [ PF_R ] + FirstSec: .note.gnu.build-id + LastSec: .note.ABI-tag + VAddr: 0x4002C4 + Align: 0x4 + Offset: 0x2C4 + - Type: PT_GNU_EH_FRAME + Flags: [ PF_R ] + FirstSec: .eh_frame_hdr + LastSec: .eh_frame_hdr + VAddr: 0x402270 + Align: 0x4 + Offset: 0x2270 + - Type: PT_GNU_STACK + Flags: [ PF_W, PF_R ] + Align: 0x10 + Offset: 0x0 + - Type: PT_GNU_RELRO + Flags: [ PF_R ] + FirstSec: .init_array + LastSec: .got + VAddr: 0x403E00 + Offset: 0x2E00 +Sections: + - Name: .interp + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + Address: 0x4002A8 + AddressAlign: 0x1 + Content: 2F6C696236342F6C642D6C696E75782D7838362D36342E736F2E3200 + - Name: .note.gnu.build-id + Type: SHT_NOTE + Flags: [ SHF_ALLOC ] + Address: 0x4002C4 + AddressAlign: 0x4 + Notes: + - Name: GNU + Desc: 66CF856212C3B313EA98AD840984B20EA781118A + Type: NT_PRPSINFO + - Name: .note.ABI-tag + Type: SHT_NOTE + Flags: [ SHF_ALLOC ] + Address: 0x4002E8 + AddressAlign: 0x4 + Notes: + - Name: GNU + Desc: '00000000030000000200000000000000' + Type: NT_VERSION + - Name: .gnu.hash + Type: SHT_GNU_HASH + Flags: [ SHF_ALLOC ] + Address: 0x400308 + Link: .dynsym + AddressAlign: 0x8 + Header: + SymNdx: 0x1 + Shift2: 0x0 + BloomFilter: [ 0x0 ] + HashBuckets: [ 0x0 ] + HashValues: [ ] + - Name: .dynsym + Type: SHT_DYNSYM + Flags: [ SHF_ALLOC ] + Address: 0x400328 + Link: .dynstr + AddressAlign: 0x8 + - Name: .dynstr + Type: SHT_STRTAB + Flags: [ SHF_ALLOC ] + Address: 0x400430 + AddressAlign: 0x1 + - Name: .gnu.version + Type: SHT_GNU_versym + Flags: [ SHF_ALLOC ] + Address: 0x4004BA + Link: .dynsym + AddressAlign: 0x2 + Entries: [ 0, 2, 2, 3, 4, 2, 5, 5, 2, 0, 5 ] + - Name: .gnu.version_r + Type: SHT_GNU_verneed + Flags: [ SHF_ALLOC ] + Address: 0x4004D0 + Link: .dynstr + AddressAlign: 0x8 + Dependencies: + - Version: 1 + File: libm.so.6 + Entries: + - Name: GLIBC_2.2.5 + Hash: 157882997 + Flags: 0 + Other: 5 + - Name: GLIBC_2.29 + Hash: 110530953 + Flags: 0 + Other: 3 + - Version: 1 + File: libc.so.6 + Entries: + - Name: GLIBC_2.4 + Hash: 225011988 + Flags: 0 + Other: 4 + - Name: GLIBC_2.2.5 + Hash: 157882997 + Flags: 0 + Other: 2 + - Name: .rela.dyn + Type: SHT_RELA + Flags: [ SHF_ALLOC ] + Address: 0x400530 + Link: .dynsym + AddressAlign: 0x8 + Relocations: + - Offset: 0x403FF0 + Symbol: __libc_start_main + Type: R_X86_64_GLOB_DAT + - Offset: 0x403FF8 + Symbol: __gmon_start__ + Type: R_X86_64_GLOB_DAT + - Name: .rela.plt + Type: SHT_RELA + Flags: [ SHF_ALLOC, SHF_INFO_LINK ] + Address: 0x400560 + Link: .dynsym + AddressAlign: 0x8 + Info: .got.plt + Relocations: + - Offset: 0x404018 + Symbol: putchar + Type: R_X86_64_JUMP_SLOT + - Offset: 0x404020 + Symbol: puts + Type: R_X86_64_JUMP_SLOT + - Offset: 0x404028 + Symbol: pow + Type: R_X86_64_JUMP_SLOT + - Offset: 0x404030 + Symbol: __stack_chk_fail + Type: R_X86_64_JUMP_SLOT + - Offset: 0x404038 + Symbol: printf + Type: R_X86_64_JUMP_SLOT + - Offset: 0x404040 + Symbol: cos + Type: R_X86_64_JUMP_SLOT + - Offset: 0x404048 + Symbol: acos + Type: R_X86_64_JUMP_SLOT + - Offset: 0x404050 + Symbol: sqrt + Type: R_X86_64_JUMP_SLOT + - Name: .init + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC, SHF_EXECINSTR ] + Address: 0x401000 + AddressAlign: 0x4 + Offset: 0x1000 + Content: F30F1EFA4883EC08488B05E92F00004885C07402FFD04883C408C3 + - Name: .plt + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC, SHF_EXECINSTR ] + Address: 0x401020 + AddressAlign: 0x10 + EntSize: 0x10 + Content: FF35E22F0000FF25E42F00000F1F4000FF25E22F00006800000000E9E0FFFFFFFF25DA2F00006801000000E9D0FFFFFFFF25D22F00006802000000E9C0FFFFFFFF25CA2F00006803000000E9B0FFFFFFFF25C22F00006804000000E9A0FFFFFFFF25BA2F00006805000000E990FFFFFFFF25B22F00006806000000E980FFFFFFFF25AA2F00006807000000E970FFFFFF + - Name: .text + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC, SHF_EXECINSTR ] + Address: 0x4010B0 + AddressAlign: 0x10 + Content: 4156BF082040004155415455534883EC5064488B042528000000488944244831C0E86AFFFFFF488D742430488D7C2424488B0529100000F20F101529100000F20F100D2910000066480F6ED8488B050510000066480F6EC0E8F3060000BFBF20400031C0E857FFFFFF448B5C24244585DB7E2131DBF20F1044DC30BFCA204000B8010000004883C301E832FFFFFF395C24247FE1BF0A000000E8E2FEFFFF488D742430488D7C2424488B05B10F0000F20F1015C10F0000F20F100DC10F000066480F6ED8488B058D0F000066480F6EC0E87B060000BFBF20400031C0E8DFFEFFFF448B5424244585D27E2131DBF20F1044DC30BFCA204000B8010000004883C301E8BAFEFFFF395C24247FE1BF0A000000E86AFEFFFF488B053B0F0000F20F101D630F0000488D742430F20F10155E0F0000F20F100D5E0F0000488D7C242466480F6EC0E807060000BFBF20400031C0E86BFEFFFF448B4C24244585C97E2131DBF20F1044DC30BFCA204000B8010000004883C301E846FEFFFF395C24247FE1BF0A000000E8F6FDFFFF488D742430488D7C2424488B05BD0E0000F20F101DFD0E0000F20F100DFD0E000066480F6ED066480F6EC0E896050000BFBF20400031C0E8FAFDFFFF448B4424244585C07E2131DBF20F1044DC30BFCA204000B8010000004883C301E8D5FDFFFF395C24247FE1BF0A000000E885FDFFFF488B05460E0000F20F101DA60E0000488D742430F20F100DA10E0000F20F1005A10E0000488D7C242466480F6ED0E822050000BFBF20400031C0E886FDFFFF8B7C242485FF7E2131DBF20F1044DC30BFCA204000B8010000004883C301E863FDFFFF395C24247FE1BF0A000000E813FDFFFFF20F101D530E0000F20F1015530E0000488D742430F20F100D4E0E0000F20F10054E0E0000488D7C2424E8B4040000BFBF20400031C0E818FDFFFF8B74242485F67E2131DBF20F1044DC30BFCA204000B8010000004883C301E8F5FCFFFF395C24247FE1BF0A000000E8A5FCFFFFF20F101D050E0000F20F1015050E0000488D742430F20F100D000E0000F20F1005000E0000488D7C2424E846040000BFBF20400031C0E8AAFCFFFF8B4C242485C97E2131DBF20F1044DC30BFCA204000B8010000004883C301E887FCFFFF395C24247FE1BF0A000000E837FCFFFFF20F101DB70D0000F20F1015B70D0000488D742430F20F100DB20D0000F20F1005B20D0000488D7C2424E8D8030000BFBF20400031C0E83CFCFFFF8B54242485D27E2131DBF20F1044DC30BFCA204000B8010000004883C301E819FCFFFF395C24247FE1BF0A00000041BD09000000E8C3FBFFFF488B05940C00004889442410488B05800C000041BE280000004889442418488B05660C000041BC1100000048894424080F1F00488B05490C0000BD0900000048890424F20F101C24488D742430488D7C2424F20F10542408F20F104C2418F20F10442410E82A030000BFBF20400031C0E88EFBFFFF8B44242485C07E2131DBF20F1044DC30BFCA204000B8010000004883C301E86BFBFFFF395C24247FE1BF0A000000E81BFBFFFFF20F102424F20F5C25B60C0000F20F11242483ED017584F20F102DAC0C0000F20F586C2408F20F116C24084183EC010F8556FFFFFFF20F107C2418F20F5C3D900C0000F20F117C24184183EE010F8523FFFFFFF20F103D980B0000F20F587C2410F20F117C24104183ED010F85F3FEFFFFBF3020400031DBE8AEFAFFFF4889DF488D742428E8C10500008B54242889DEBFCE20400031C04883C302E8BBFAFFFF4881FBA086010075D4BF0A000000BB6901ED3FE863FAFFFF4889DF488D742428E8860500008B5424284889DE31C0BFDF2040004883C301E87FFAFFFF4881FB6941ED3F75D3BF58204000E83CFAFFFF660FEFD2660F28C2F20F111424E8CA010000F20F101424BF80204000B802000000660F28C8660F28C2E83EFAFFFFF20F101424F20F5815B10B0000F20F103DB10B0000660F2FFA73BBBFEE204000E8E9F9FFFF660FEFD2660F28C2F20F111424E857010000F20F101424BFA0204000B802000000660F28C8660F28C2E8EBF9FFFFF20F101424F20F58156E0B0000F20F103D6E0B0000660F2FFA73BB488B442448644833042528000000750F4883C45031C05B5D415C415D415EC3E89CF9FFFF662E0F1F8400000000006690F30F1EFA31ED4989D15E4889E24883E4F0505449C7C0201C400048C7C1B01B400048C7C7B0104000FF15F2280000F490F30F1EFAC3662E0F1F84000000000090B868404000483D684040007413B8000000004885C07409BF68404000FFE06690C30F1F440000662E0F1F840000000000BE684040004881EE684040004889F048C1EE3F48C1F8034801C648D1FE7411B8000000004885C07407BF68404000FFE0C30F1F440000662E0F1F840000000000803DE1280000007517554889E5E87EFFFFFFC605CF280000015DC30F1F440000C30F1F440000662E0F1F840000000000EB8E662E0F1F8400000000000F1F4000F20F5905480A0000F20F5E05480A0000C366662E0F1F8400000000000F1F4000F20F5905300A0000F20F5E05200A0000C3662E0F1F8400000000000F1F440000F20F5EC8534889F34883EC50F20F5ED0F20F110C24DD0424660FEFC9DB3C24DB2C24F20F5ED8F20F11542418DD442418D9C1D8CAD905E6090000D8CADEE9D905E0090000DCF9F20F115C2418D9C3D8C4D8CCD8CCD9CCDEC9DECAD9CADEE1D905C4090000DC4C2418DEC1D835BC090000D9C1D8CAD8CAD9C1D8CAD8E1DD5C2418F20F10442418660F2FC80F8398000000DDD8660F2EC8660F28D0C70701000000F20F51D20F87B6010000D9C9DB7C2430F20F100D90090000DD542418F20F10442418660F540586090000DB7C2420F20F58C2E879F7FFFFF20F11442418DD442418DB6C2430D8F1DEC1DD5C2418DB6C2420D9EEDFF1DDD87714F20F107C2418660F573D59090000F20F117C2418DB2C24D8350A090000DC6C2418DD1B4883C4505BC3660F1F440000DD5C2418F20F10442418C70703000000660F28F0660F2EC8F20F51F6F20F117424180F8736010000D9C9DB7C2420DC742418DD5C2418F20F10442418E827F7FFFFDB6C2420660FEFC9F20F11442418DD5C2420F20F10542420660F2ECA660F28DAF20F51DB0F870D010000F20F102DD5070000F20F591D8D080000F20F5EC5F20F116C2430F20F115C2420E8C8F6FFFFDB2C24F20F11442440F20F10442418D83553080000F20F580563080000F20F5E442430DB3C24E89DF6FFFFF20F104C2440F20F10642420DB2C24660F28D0F20F59CCF20F59D4D9C0F20F114C2440DC6C2440DD5C2440F20F10442440F20F11542440DC6C2440DD5C2440660F164424400F1103F20F10442418F20F580507080000F20F5E442430E83CF6FFFFF20F59442420DB2C24F20F11442418DC6C2418DD5B104883C4505BC3DB7C2430F20F11542418DB7C2420E82DF6FFFFF20F10542418DB6C2430DB6C2420E926FEFFFFDB7C2430DB7C2420E80DF6FFFFDB6C2430DB6C2420E9B2FEFFFF660F28C2F20F11542448F20F115C2420E8EBF5FFFFF20F103DB3060000F20F10442418F20F105C2420F20F591D5F070000F20F5EC7F20F117C2430F20F115C2420E89AF5FFFFDB2C24F20F10742420F20F10542448D83525070000F20F59F0660F28C2F20F11742440D9C0DB3C24DC6C2440DD1BE887F5FFFFF20F10442418F20F580511070000F20F5E442430E84EF5FFFFDB2C24F20F10542448F20F59442420F20F11442440DC6C2440660F28C2DD5B08E849F5FFFFE9CFFEFFFF0F1F400041B82000000031C031D2660F1F4400004889F948C1E70248C1E91E83E103488D1491488D0C85010000004801C04839CA72074829CA4883C0014183E80175D1488906C3662E0F1F8400000000000F1F00F30F1EFA41574C8D3D4322000041564989D641554989F541544189FC55488D2D34220000534C29FD4883EC08E81FF4FFFF48C1FD03741F31DB0F1F80000000004C89F24C89EE4489E741FF14DF4883C3014839DD75EA4883C4085B5D415C415D415E415FC366662E0F1F840000000000F30F1EFAC3 + - Name: .fini + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC, SHF_EXECINSTR ] + Address: 0x401C28 + AddressAlign: 0x4 + Content: F30F1EFA4883EC084883C408C3 + - Name: .rodata + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + Address: 0x402000 + AddressAlign: 0x10 + Offset: 0x2000 + Content: 01000200000000002A2A2A2A2A2A2A2A2A2043554249432046554E4354494F4E53202A2A2A2A2A2A2A2A2A2A2A0000002A2A2A2A2A2A2A2A2A20494E54454745522053515220524F4F5453202A2A2A2A2A2A2A2A2A2A2A002A2A2A2A2A2A2A2A2A20414E474C4520434F4E56455253494F4E202A2A2A2A2A2A2A2A2A2A2A000025332E30662064656772656573203D20252E3132662072616469616E730A0000252E3132662072616469616E73203D2025332E306620646567726565730A00536F6C7574696F6E733A0020256600737172742825336429203D202532640A007371727428256C5829203D2025580A0000000000000000F0BF00000000000014400000000000002440000000000000F03F0000000000003EC0000000000000404000000000000025C0000000000000314000000000000012C00000000000003FC000000000000036400000000000000CC000000000008041C06666666666662BC00000000000002840AE47E17A14AE284000000000000008409A999999999937C00000000000001840295C8FC2F5F850C000000000000020C000000000000041400000000000001E40D7A3703D0A572140000000000080464000000000000030403333333333331540333333333333FBBF00000000000028C077BE9F1A2FDDDC3F85EB51B81E85E33F000000000000D03FFCA9F1D24D62503F0000000000807640399D52A246DF413F9B0B6097FB2119400000000000806640182D4454FB21094000004040000010410000D8410000584200000000000000C0182D4454FB211940182D4454FB212940555555555555D53FFFFFFFFFFFFFFF7F000000000000000000000000000000800000000000000000 + - Name: .eh_frame_hdr + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + Address: 0x402270 + AddressAlign: 0x4 + Content: 011B033B5C0000000A000000B0EDFFFFA000000040EEFFFFC800000060F4FFFF7800000090F4FFFF8C00000050F5FFFF1001000070F5FFFF2401000090F5FFFF38010000F0F8FFFF6801000040F9FFFF80010000B0F9FFFFC8010000 + - Name: .eh_frame + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + Address: 0x4022D0 + AddressAlign: 0x8 + Content: 1400000000000000017A5200017810011B0C070890010000100000001C000000E0F3FFFF2F000000004407101000000030000000FCF3FFFF0500000000000000240000004400000008EDFFFF90000000000E10460E184A0F0B770880003F1A3B2A33242200000000440000006C00000070EDFFFF1406000000420E108E02470E188D03420E208C04410E288605410E308306440E800103F3050A0E30430E28410E20420E18420E10420E08410B00000010000000B400000038F4FFFF110000000000000010000000C800000044F4FFFF11000000000000002C000000DC00000050F4FFFF5C03000000450E108302470E600314010A0E10410E08470B0336010A0E10410E08410B00140000000C01000080F7FFFF4300000000000000000000004400000024010000B8F7FFFF6500000000460E108F02490E188E03450E208D04450E288C05440E308606480E388307470E406E0E38410E30410E28420E20420E18420E10420E0800100000006C010000E0F7FFFF050000000000000000000000 + - Name: .init_array + Type: SHT_INIT_ARRAY + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x403E00 + AddressAlign: 0x8 + EntSize: 0x8 + Offset: 0x2E00 + Content: B017400000000000 + - Name: .fini_array + Type: SHT_FINI_ARRAY + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x403E08 + AddressAlign: 0x8 + EntSize: 0x8 + Content: '8017400000000000' + - Name: .dynamic + Type: SHT_DYNAMIC + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x403E10 + Link: .dynstr + AddressAlign: 0x8 + Entries: + - Tag: DT_NEEDED + Value: 0x1 + - Tag: DT_NEEDED + Value: 0x28 + - Tag: DT_INIT + Value: 0x401000 + - Tag: DT_FINI + Value: 0x401C28 + - Tag: DT_INIT_ARRAY + Value: 0x403E00 + - Tag: DT_INIT_ARRAYSZ + Value: 0x8 + - Tag: DT_FINI_ARRAY + Value: 0x403E08 + - Tag: DT_FINI_ARRAYSZ + Value: 0x8 + - Tag: DT_GNU_HASH + Value: 0x400308 + - Tag: DT_STRTAB + Value: 0x400430 + - Tag: DT_SYMTAB + Value: 0x400328 + - Tag: DT_STRSZ + Value: 0x8A + - Tag: DT_SYMENT + Value: 0x18 + - Tag: DT_DEBUG + Value: 0x0 + - Tag: DT_PLTGOT + Value: 0x404000 + - Tag: DT_PLTRELSZ + Value: 0xC0 + - Tag: DT_PLTREL + Value: 0x7 + - Tag: DT_JMPREL + Value: 0x400560 + - Tag: DT_RELA + Value: 0x400530 + - Tag: DT_RELASZ + Value: 0x30 + - Tag: DT_RELAENT + Value: 0x18 + - Tag: DT_VERNEED + Value: 0x4004D0 + - Tag: DT_VERNEEDNUM + Value: 0x2 + - Tag: DT_VERSYM + Value: 0x4004BA + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 + - Name: .got + Type: SHT_PROGBITS + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x403FF0 + AddressAlign: 0x8 + EntSize: 0x8 + Content: '00000000000000000000000000000000' + - Name: .got.plt + Type: SHT_PROGBITS + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x404000 + AddressAlign: 0x8 + EntSize: 0x8 + Content: 103E400000000000000000000000000000000000000000003610400000000000461040000000000056104000000000006610400000000000761040000000000086104000000000009610400000000000A610400000000000 + - Name: .data + Type: SHT_PROGBITS + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x404058 + AddressAlign: 0x8 + Content: '00000000000000000000000000000000' + - Name: .tm_clone_table + Type: SHT_PROGBITS + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x404068 + AddressAlign: 0x8 + - Name: .bss + Type: SHT_NOBITS + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x404068 + AddressAlign: 0x1 + Size: 0x8 + - Name: .comment + Type: SHT_PROGBITS + Flags: [ SHF_MERGE, SHF_STRINGS ] + AddressAlign: 0x1 + EntSize: 0x1 + Content: 4743433A20285562756E747520392E342E302D317562756E7475317E31362E30342920392E342E3000 + - Name: .rela.init + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .init + Relocations: + - Offset: 0x40100B + Symbol: __gmon_start__ + Type: R_X86_64_REX_GOTPCRELX + Addend: -4 + - Name: .rela.text + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .text + Relocations: + - Offset: 0x4010B3 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 8 + - Offset: 0x4010D2 + Symbol: 'puts@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4010E3 + Symbol: .LC6 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4010EB + Symbol: .LC7 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4010F3 + Symbol: .LC8 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4010FF + Symbol: .LC4 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401109 + Symbol: SolveCubic + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40110E + Symbol: .rodata + Type: R_X86_64_32 + Addend: 191 + - Offset: 0x401115 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40112C + Symbol: .rodata + Type: R_X86_64_32 + Addend: 202 + - Offset: 0x40113A + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40114A + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40115B + Symbol: .LC6 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401163 + Symbol: .LC11 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40116B + Symbol: .LC12 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401177 + Symbol: .LC4 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401181 + Symbol: SolveCubic + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401186 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 191 + - Offset: 0x40118D + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4011A4 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 202 + - Offset: 0x4011B2 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4011C2 + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4011C9 + Symbol: .LC4 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4011D1 + Symbol: .LC13 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4011DE + Symbol: .LC14 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4011E6 + Symbol: .LC15 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4011F5 + Symbol: SolveCubic + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4011FA + Symbol: .rodata + Type: R_X86_64_32 + Addend: 191 + - Offset: 0x401201 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401218 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 202 + - Offset: 0x401226 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401236 + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401247 + Symbol: .LC4 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40124F + Symbol: .LC16 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401257 + Symbol: .LC17 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401266 + Symbol: SolveCubic + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40126B + Symbol: .rodata + Type: R_X86_64_32 + Addend: 191 + - Offset: 0x401272 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401289 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 202 + - Offset: 0x401297 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4012A7 + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4012AE + Symbol: .LC2 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4012B6 + Symbol: .LC18 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4012C3 + Symbol: .LC19 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4012CB + Symbol: .LC20 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4012DA + Symbol: SolveCubic + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4012DF + Symbol: .rodata + Type: R_X86_64_32 + Addend: 191 + - Offset: 0x4012E6 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4012FB + Symbol: .rodata + Type: R_X86_64_32 + Addend: 202 + - Offset: 0x401309 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401319 + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401321 + Symbol: .LC21 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401329 + Symbol: .LC22 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401336 + Symbol: .LC23 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40133E + Symbol: .LC24 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401348 + Symbol: SolveCubic + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40134D + Symbol: .rodata + Type: R_X86_64_32 + Addend: 191 + - Offset: 0x401354 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401369 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 202 + - Offset: 0x401377 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401387 + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40138F + Symbol: .LC25 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401397 + Symbol: .LC26 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4013A4 + Symbol: .LC27 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4013AC + Symbol: .LC28 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4013B6 + Symbol: SolveCubic + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4013BB + Symbol: .rodata + Type: R_X86_64_32 + Addend: 191 + - Offset: 0x4013C2 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4013D7 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 202 + - Offset: 0x4013E5 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4013F5 + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4013FD + Symbol: .LC29 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401405 + Symbol: .LC30 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401412 + Symbol: .LC31 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40141A + Symbol: .LC32 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401424 + Symbol: SolveCubic + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401429 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 191 + - Offset: 0x401430 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401445 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 202 + - Offset: 0x401453 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401469 + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401470 + Symbol: .LC4 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40147C + Symbol: .LC3 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40148E + Symbol: .LC2 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4014A3 + Symbol: .LC0 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4014D2 + Symbol: SolveCubic + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4014D7 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 191 + - Offset: 0x4014DE + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4014F3 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 202 + - Offset: 0x401501 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401511 + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40151E + Symbol: .LC33 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401530 + Symbol: .LC34 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401554 + Symbol: .LC35 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40156C + Symbol: .LC4 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401587 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 48 + - Offset: 0x40158E + Symbol: 'puts@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40159B + Symbol: usqrt + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4015A6 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 206 + - Offset: 0x4015B1 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4015C9 + Symbol: 'putchar@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4015D6 + Symbol: usqrt + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4015E4 + Symbol: .rodata + Type: R_X86_64_32 + Addend: 223 + - Offset: 0x4015ED + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4015FB + Symbol: .rodata + Type: R_X86_64_32 + Addend: 88 + - Offset: 0x401600 + Symbol: 'puts@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401612 + Symbol: deg2rad + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40161C + Symbol: .rodata + Type: R_X86_64_32 + Addend: 128 + - Offset: 0x40162E + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40163B + Symbol: .LC41 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401643 + Symbol: .LC42 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40164E + Symbol: .rodata + Type: R_X86_64_32 + Addend: 238 + - Offset: 0x401653 + Symbol: 'puts@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401665 + Symbol: rad2deg + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40166F + Symbol: .rodata + Type: R_X86_64_32 + Addend: 160 + - Offset: 0x401681 + Symbol: 'printf@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x40168E + Symbol: .LC45 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401696 + Symbol: .LC46 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4016C0 + Symbol: '__stack_chk_fail@@GLIBC_2.4' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4016E6 + Symbol: __libc_csu_fini + Type: R_X86_64_32S + - Offset: 0x4016ED + Symbol: __libc_csu_init + Type: R_X86_64_32S + - Offset: 0x4016F4 + Symbol: main + Type: R_X86_64_32S + - Offset: 0x4016FA + Symbol: '__libc_start_main@@GLIBC_2.2.5' + Type: R_X86_64_GOTPCRELX + Addend: -4 + - Offset: 0x401711 + Symbol: __TMC_END__ + Type: R_X86_64_32 + - Offset: 0x401717 + Symbol: .tm_clone_table + Type: R_X86_64_32S + - Offset: 0x40171E + Symbol: _ITM_deregisterTMCloneTable + Type: R_X86_64_32 + - Offset: 0x401728 + Symbol: .tm_clone_table + Type: R_X86_64_32 + - Offset: 0x401741 + Symbol: __TMC_END__ + Type: R_X86_64_32 + - Offset: 0x401748 + Symbol: .tm_clone_table + Type: R_X86_64_32S + - Offset: 0x401760 + Symbol: _ITM_registerTMCloneTable + Type: R_X86_64_32 + - Offset: 0x40176A + Symbol: .tm_clone_table + Type: R_X86_64_32 + - Offset: 0x401782 + Symbol: .bss + Type: R_X86_64_PC32 + Addend: -5 + - Offset: 0x401794 + Symbol: .bss + Type: R_X86_64_PC32 + Addend: -5 + - Offset: 0x4017C4 + Symbol: '.LC0 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4017CC + Symbol: .LC1 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4017E4 + Symbol: .LC1 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4017EC + Symbol: '.LC0 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401836 + Symbol: '.LC0 (2)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401840 + Symbol: '.LC1 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401860 + Symbol: '.LC2 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40186C + Symbol: '.LC3 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4018B4 + Symbol: .LC9 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4018C6 + Symbol: .LC10 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4018D3 + Symbol: 'pow@@GLIBC_2.29' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401903 + Symbol: '.LC12 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401912 + Symbol: '.LC0 (2)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401965 + Symbol: 'acos@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401997 + Symbol: '.LC6 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x40199F + Symbol: .LC5 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4019B4 + Symbol: 'cos@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x4019C9 + Symbol: '.LC0 (2)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4019D1 + Symbol: '.LC7 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x4019DF + Symbol: 'cos@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401A35 + Symbol: '.LC8 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401A40 + Symbol: 'cos@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401A6F + Symbol: 'sqrt@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401A8F + Symbol: 'sqrt@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401AB1 + Symbol: 'sqrt@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401AB9 + Symbol: '.LC6 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401ACD + Symbol: .LC5 + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401AE2 + Symbol: 'cos@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401AF7 + Symbol: '.LC0 (2)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401B15 + Symbol: 'sqrt@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401B23 + Symbol: '.LC7 (1)' + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401B2E + Symbol: 'cos@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401B53 + Symbol: 'sqrt@@GLIBC_2.2.5' + Type: R_X86_64_PLT32 + Addend: -4 + - Offset: 0x401BB9 + Symbol: __init_array_start + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401BD0 + Symbol: __init_array_end + Type: R_X86_64_PC32 + Addend: -4 + - Offset: 0x401BDD + Symbol: _init + Type: R_X86_64_PLT32 + Addend: -4 + - Name: .rela.eh_frame + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .eh_frame + Relocations: + - Offset: 0x4022F0 + Symbol: .text + Type: R_X86_64_PC32 + Addend: 1568 + - Offset: 0x402304 + Symbol: .text + Type: R_X86_64_PC32 + Addend: 1616 + - Offset: 0x402340 + Symbol: .text + Type: R_X86_64_PC32 + - Offset: 0x402388 + Symbol: .text + Type: R_X86_64_PC32 + Addend: 1808 + - Offset: 0x40239C + Symbol: .text + Type: R_X86_64_PC32 + Addend: 1840 + - Offset: 0x4023B0 + Symbol: .text + Type: R_X86_64_PC32 + Addend: 1872 + - Offset: 0x4023E0 + Symbol: .text + Type: R_X86_64_PC32 + Addend: 2736 + - Offset: 0x4023F8 + Symbol: .text + Type: R_X86_64_PC32 + Addend: 2816 + - Offset: 0x402440 + Symbol: .text + Type: R_X86_64_PC32 + Addend: 2928 + - Name: .rela.init_array + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .init_array + Relocations: + - Offset: 0x403E00 + Symbol: .text + Type: R_X86_64_64 + Addend: 1792 + - Name: .rela.fini_array + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .fini_array + Relocations: + - Offset: 0x403E08 + Symbol: .text + Type: R_X86_64_64 + Addend: 1744 + - Type: SectionHeaderTable + Sections: + - Name: .interp + - Name: .note.gnu.build-id + - Name: .note.ABI-tag + - Name: .gnu.hash + - Name: .dynsym + - Name: .dynstr + - Name: .gnu.version + - Name: .gnu.version_r + - Name: .rela.dyn + - Name: .rela.plt + - Name: .init + - Name: .rela.init + - Name: .plt + - Name: .text + - Name: .rela.text + - Name: .fini + - Name: .rodata + - Name: .eh_frame_hdr + - Name: .eh_frame + - Name: .rela.eh_frame + - Name: .init_array + - Name: .rela.init_array + - Name: .fini_array + - Name: .rela.fini_array + - Name: .dynamic + - Name: .got + - Name: .got.plt + - Name: .data + - Name: .tm_clone_table + - Name: .bss + - Name: .comment + - Name: .symtab + - Name: .strtab + - Name: .shstrtab +Symbols: + - Name: .interp + Type: STT_SECTION + Section: .interp + Value: 0x4002A8 + - Name: .note.gnu.build-id + Type: STT_SECTION + Section: .note.gnu.build-id + Value: 0x4002C4 + - Name: .note.ABI-tag + Type: STT_SECTION + Section: .note.ABI-tag + Value: 0x4002E8 + - Name: .gnu.hash + Type: STT_SECTION + Section: .gnu.hash + Value: 0x400308 + - Name: .dynsym + Type: STT_SECTION + Section: .dynsym + Value: 0x400328 + - Name: .dynstr + Type: STT_SECTION + Section: .dynstr + Value: 0x400430 + - Name: .gnu.version + Type: STT_SECTION + Section: .gnu.version + Value: 0x4004BA + - Name: .gnu.version_r + Type: STT_SECTION + Section: .gnu.version_r + Value: 0x4004D0 + - Name: .rela.dyn + Type: STT_SECTION + Section: .rela.dyn + Value: 0x400530 + - Name: .rela.plt + Type: STT_SECTION + Section: .rela.plt + Value: 0x400560 + - Name: .init + Type: STT_SECTION + Section: .init + Value: 0x401000 + - Name: .plt + Type: STT_SECTION + Section: .plt + Value: 0x401020 + - Name: .text + Type: STT_SECTION + Section: .text + Value: 0x4010B0 + - Name: .fini + Type: STT_SECTION + Section: .fini + Value: 0x401C28 + - Name: .rodata + Type: STT_SECTION + Section: .rodata + Value: 0x402000 + - Name: .eh_frame_hdr + Type: STT_SECTION + Section: .eh_frame_hdr + Value: 0x402270 + - Name: .eh_frame + Type: STT_SECTION + Section: .eh_frame + Value: 0x4022D0 + - Name: .init_array + Type: STT_SECTION + Section: .init_array + Value: 0x403E00 + - Name: .fini_array + Type: STT_SECTION + Section: .fini_array + Value: 0x403E08 + - Name: .dynamic + Type: STT_SECTION + Section: .dynamic + Value: 0x403E10 + - Name: .got + Type: STT_SECTION + Section: .got + Value: 0x403FF0 + - Name: .got.plt + Type: STT_SECTION + Section: .got.plt + Value: 0x404000 + - Name: .data + Type: STT_SECTION + Section: .data + Value: 0x404058 + - Name: .tm_clone_table + Type: STT_SECTION + Section: .tm_clone_table + Value: 0x404068 + - Name: .bss + Type: STT_SECTION + Section: .bss + Value: 0x404068 + - Name: .comment + Type: STT_SECTION + Section: .comment + - Name: basicmath_large.c + Type: STT_FILE + Index: SHN_ABS + - Name: .LC6 + Section: .rodata + Value: 0x402110 + - Name: .LC7 + Section: .rodata + Value: 0x402118 + - Name: .LC8 + Section: .rodata + Value: 0x402120 + - Name: .LC4 + Section: .rodata + Value: 0x402108 + - Name: .LC11 + Section: .rodata + Value: 0x402128 + - Name: .LC12 + Section: .rodata + Value: 0x402130 + - Name: .LC13 + Section: .rodata + Value: 0x402138 + - Name: .LC14 + Section: .rodata + Value: 0x402140 + - Name: .LC15 + Section: .rodata + Value: 0x402148 + - Name: .LC16 + Section: .rodata + Value: 0x402150 + - Name: .LC17 + Section: .rodata + Value: 0x402158 + - Name: .LC2 + Section: .rodata + Value: 0x4020F8 + - Name: .LC18 + Section: .rodata + Value: 0x402160 + - Name: .LC19 + Section: .rodata + Value: 0x402168 + - Name: .LC20 + Section: .rodata + Value: 0x402170 + - Name: .LC21 + Section: .rodata + Value: 0x402178 + - Name: .LC22 + Section: .rodata + Value: 0x402180 + - Name: .LC23 + Section: .rodata + Value: 0x402188 + - Name: .LC24 + Section: .rodata + Value: 0x402190 + - Name: .LC25 + Section: .rodata + Value: 0x402198 + - Name: .LC26 + Section: .rodata + Value: 0x4021A0 + - Name: .LC27 + Section: .rodata + Value: 0x4021A8 + - Name: .LC28 + Section: .rodata + Value: 0x4021B0 + - Name: .LC29 + Section: .rodata + Value: 0x4021B8 + - Name: .LC30 + Section: .rodata + Value: 0x4021C0 + - Name: .LC31 + Section: .rodata + Value: 0x4021C8 + - Name: .LC32 + Section: .rodata + Value: 0x4021D0 + - Name: .LC3 + Section: .rodata + Value: 0x402100 + - Name: .LC0 + Section: .rodata + Value: 0x4020F0 + - Name: .LC33 + Section: .rodata + Value: 0x4021D8 + - Name: .LC34 + Section: .rodata + Value: 0x4021E0 + - Name: .LC35 + Section: .rodata + Value: 0x4021E8 + - Name: .LC41 + Section: .rodata + Value: 0x4021F0 + - Name: .LC42 + Section: .rodata + Value: 0x4021F8 + - Name: .LC45 + Section: .rodata + Value: 0x402200 + - Name: .LC46 + Section: .rodata + Value: 0x402208 + - Name: crtstuff.c + Type: STT_FILE + Index: SHN_ABS + - Name: __TMC_LIST__ + Type: STT_OBJECT + Section: .tm_clone_table + Value: 0x404068 + - Name: deregister_tm_clones + Type: STT_FUNC + Section: .text + Value: 0x401710 + - Name: register_tm_clones + Type: STT_FUNC + Section: .text + Value: 0x401740 + - Name: __do_global_dtors_aux + Type: STT_FUNC + Section: .text + Value: 0x401780 + - Name: completed.8023 + Type: STT_OBJECT + Section: .bss + Value: 0x404068 + Size: 0x1 + - Name: __do_global_dtors_aux_fini_array_entry + Type: STT_OBJECT + Section: .fini_array + Value: 0x403E08 + - Name: frame_dummy + Type: STT_FUNC + Section: .text + Value: 0x4017B0 + - Name: __frame_dummy_init_array_entry + Type: STT_OBJECT + Section: .init_array + Value: 0x403E00 + - Name: rad2deg.c + Type: STT_FILE + Index: SHN_ABS + - Name: '.LC0 (1)' + Section: .rodata + Value: 0x402210 + - Name: .LC1 + Section: .rodata + Value: 0x402218 + - Name: cubic.c + Type: STT_FILE + Index: SHN_ABS + - Name: '.LC0 (2)' + Section: .rodata + Value: 0x402220 + - Name: '.LC1 (1)' + Section: .rodata + Value: 0x402224 + - Name: '.LC2 (1)' + Section: .rodata + Value: 0x402228 + - Name: '.LC3 (1)' + Section: .rodata + Value: 0x40222C + - Name: .LC9 + Section: .rodata + Value: 0x402248 + - Name: .LC10 + Section: .rodata + Value: 0x402250 + - Name: '.LC12 (1)' + Section: .rodata + Value: 0x402260 + - Name: '.LC6 (1)' + Section: .rodata + Value: 0x402170 + - Name: .LC5 + Section: .rodata + Value: 0x402230 + - Name: '.LC7 (1)' + Section: .rodata + Value: 0x402238 + - Name: '.LC8 (1)' + Section: .rodata + Value: 0x402240 + - Name: isqrt.c + Type: STT_FILE + Index: SHN_ABS + - Name: 'crtstuff.c (1)' + Type: STT_FILE + Index: SHN_ABS + - Name: __FRAME_END__ + Type: STT_OBJECT + Section: .eh_frame + Value: 0x40244C + - Type: STT_FILE + Index: SHN_ABS + - Name: __init_array_end + Section: .init_array + Value: 0x403E08 + - Name: _DYNAMIC + Type: STT_OBJECT + Section: .dynamic + Value: 0x403E10 + - Name: __init_array_start + Section: .init_array + Value: 0x403E00 + - Name: __GNU_EH_FRAME_HDR + Section: .eh_frame_hdr + Value: 0x402270 + - Name: _GLOBAL_OFFSET_TABLE_ + Type: STT_OBJECT + Section: .got.plt + Value: 0x404000 + - Name: __libc_csu_fini + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x401C20 + Size: 0x5 + - Name: 'putchar@@GLIBC_2.2.5' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: _ITM_deregisterTMCloneTable + Binding: STB_WEAK + - Name: data_start + Section: .data + Binding: STB_WEAK + Value: 0x404058 + - Name: 'puts@@GLIBC_2.2.5' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: usqrt + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x401B60 + Size: 0x43 + - Name: _edata + Section: .tm_clone_table + Binding: STB_GLOBAL + Value: 0x404068 + - Name: 'pow@@GLIBC_2.29' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: _fini + Type: STT_FUNC + Section: .fini + Binding: STB_GLOBAL + Value: 0x401C28 + Other: [ STV_HIDDEN ] + - Name: '__stack_chk_fail@@GLIBC_2.4' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: 'printf@@GLIBC_2.2.5' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: 'cos@@GLIBC_2.2.5' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: 'acos@@GLIBC_2.2.5' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: '__libc_start_main@@GLIBC_2.2.5' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: deg2rad + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x4017E0 + Size: 0x11 + - Name: __data_start + Section: .data + Binding: STB_GLOBAL + Value: 0x404058 + - Name: SolveCubic + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x401800 + Size: 0x35C + - Name: __gmon_start__ + Binding: STB_WEAK + - Name: __dso_handle + Type: STT_OBJECT + Section: .data + Binding: STB_GLOBAL + Value: 0x404060 + Other: [ STV_HIDDEN ] + - Name: _IO_stdin_used + Type: STT_OBJECT + Section: .rodata + Binding: STB_GLOBAL + Value: 0x402000 + Size: 0x4 + - Name: __libc_csu_init + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x401BB0 + Size: 0x65 + - Name: _end + Section: .bss + Binding: STB_GLOBAL + Value: 0x404070 + - Name: _dl_relocate_static_pie + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x401700 + Size: 0x5 + Other: [ STV_HIDDEN ] + - Name: _start + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x4016D0 + Size: 0x2F + - Name: rad2deg + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x4017C0 + Size: 0x11 + - Name: __bss_start + Section: .bss + Binding: STB_GLOBAL + Value: 0x404068 + - Name: main + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x4010B0 + Size: 0x614 + - Name: __TMC_END__ + Type: STT_OBJECT + Section: .tm_clone_table + Binding: STB_GLOBAL + Value: 0x404068 + Other: [ STV_HIDDEN ] + - Name: _ITM_registerTMCloneTable + Binding: STB_WEAK + - Name: 'sqrt@@GLIBC_2.2.5' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: _init + Type: STT_FUNC + Section: .init + Binding: STB_GLOBAL + Value: 0x401000 + Other: [ STV_HIDDEN ] +DynamicSymbols: + - Name: putchar + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: puts + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: pow + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: __stack_chk_fail + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: printf + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: cos + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: acos + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: __libc_start_main + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: __gmon_start__ + Binding: STB_WEAK + - Name: sqrt + Type: STT_FUNC + Binding: STB_GLOBAL +... diff --git a/bolt/test/X86/Inputs/blarge_new_bat.preagg.txt b/bolt/test/X86/Inputs/blarge_new_bat.preagg.txt new file mode 100644 index 000000000000..e9e4553aad95 --- /dev/null +++ b/bolt/test/X86/Inputs/blarge_new_bat.preagg.txt @@ -0,0 +1,79 @@ +B 40169e 40165b 7 0 +B 401664 800012 7 0 +B 401680 401070 7 0 +B 800022 401669 7 0 +B 401611 800000 121 0 +B 40162d 401070 119 0 +B 4015d5 800040 2 0 +B 800080 4015da 6 0 +B 40164b 401608 115 0 +B 800080 40159f 24 0 +B 4015ec 401070 6 0 +B 8001d0 401090 1 0 +B 4014d1 800082 25 0 +B 401510 401030 616 0 +B 8002ab 401080 1 0 +B 80007b 80004c 483 1 +B 800072 80004c 597 77 +B 80010c 800194 1 0 +B 401509 4014ec 1 0 +B 800010 401616 119 0 +B 80024a 401080 1 0 +B 800154 401050 20 0 +B 4014dd 401070 9 0 +B 80021f 401080 1 0 +B 800193 4014d6 8 0 +B 40159a 800040 19 0 +B 4015f8 4015cd 2 0 +B 40152a 4014b0 24 0 +B 401500 401070 15 0 +B 4015bc 401592 21 0 +B 401544 4014a0 1 0 +B 80004a 800052 24 0 +B 4015b0 401070 20 0 +B 800050 80007d 29 29 +F 401685 40169e 7 +F 4014a0 4014d1 1 +F 401090 401090 1 +F 401050 401050 20 +F 40159f 4015b0 20 +F 80007d 800080 27 +F 401515 401544 1 +F 4014e2 401500 13 +F 401592 40159a 19 +F 401505 401509 1 +F 4014b0 4014d1 24 +F 800194 8001d0 1 +F 8001d5 80021f 1 +F 401616 40162d 114 +F 80024f 8002ab 1 +F 800159 800193 7 +F 80004c 800050 26 +F 800224 80024a 1 +F 800082 80010c 1 +F 401080 401080 3 +F 401070 401070 168 +F 80004c 800072 555 +F 401616 800010 2 +F 401070 40162d 4 +F 800082 800154 20 +F 401669 401680 7 +F 40159f 800080 1 +F 4014ec 401500 1 +F 800012 800022 7 +F 401030 401030 616 +F 80004c 80007b 473 +F 800052 800072 24 +F 800040 80004a 21 +F 4015b5 4015bc 18 +F 4015cd 4015d5 2 +F 401592 4015bc 1 +F 4015da 4015ec 6 +F 4015f1 4015f8 2 +F 800000 800010 116 +F 401608 401611 115 +F 401632 40164b 114 +F 401515 40152a 24 +F 40165b 401664 7 +F 401505 401510 612 +F 4014d6 4014dd 8 diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test new file mode 100644 index 000000000000..25ff4e7fbfcc --- /dev/null +++ b/bolt/test/X86/bolt-address-translation-yaml.test @@ -0,0 +1,40 @@ +# Check new BAT format containing hashes for YAML profile. + +RUN: yaml2obj %p/Inputs/blarge_new.yaml &> %t.exe +RUN: llvm-bolt %t.exe -o %t.out --pa -p %p/Inputs/blarge_new.preagg.txt \ +RUN: --reorder-blocks=ext-tsp --split-functions --split-strategy=cdsplit \ +RUN: --reorder-functions=cdsort --enable-bat --dyno-stats --skip-funcs=main \ +RUN: 2>&1 | FileCheck --check-prefix WRITE-BAT-CHECK %s +RUN: perf2bolt %t.out --pa -p %p/Inputs/blarge_new_bat.preagg.txt -w %t.yaml -o %t.fdata \ +RUN: 2>&1 | FileCheck --check-prefix READ-BAT-CHECK %s +RUN: FileCheck --input-file %t.yaml --check-prefix YAML-BAT-CHECK %s +# Check that YAML converted from fdata matches YAML created directly with BAT. +RUN: llvm-bolt %t.exe -data %t.fdata -w %t.yaml-fdata -o /dev/null +RUN: FileCheck --input-file %t.yaml-fdata --check-prefix YAML-BAT-CHECK %s + +# Test resulting YAML profile with the original binary (no-stale mode) +RUN: llvm-bolt %t.exe -data %t.yaml -o %t.null -dyno-stats \ +RUN: | FileCheck --check-prefix CHECK-BOLT-YAML %s + +WRITE-BAT-CHECK: BOLT-INFO: Wrote 5 BAT maps +WRITE-BAT-CHECK: BOLT-INFO: Wrote 4 function and 22 basic block hashes +WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 344 + +READ-BAT-CHECK-NOT: BOLT-ERROR: unable to save profile in YAML format for input file processed by BOLT +READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries +READ-BAT-CHECK: PERF2BOLT: read 79 aggregated LBR entries + +YAML-BAT-CHECK: functions: +YAML-BAT-CHECK: - name: main +YAML-BAT-CHECK-NEXT: fid: 2 +YAML-BAT-CHECK-NEXT: hash: 0x9895746D48B2C876 +YAML-BAT-CHECK-NEXT: exec: 0 +YAML-BAT-CHECK-NEXT: nblocks: 46 +YAML-BAT-CHECK-NEXT: blocks: +YAML-BAT-CHECK-NEXT: - bid: 0 +YAML-BAT-CHECK-NEXT: insns: 26 +YAML-BAT-CHECK-NEXT: hash: 0xA900AE79CFD40000 +YAML-BAT-CHECK-NEXT: succ: [ { bid: 3, cnt: 0 }, { bid: 1, cnt: 0 } ] + +CHECK-BOLT-YAML: pre-processing profile using YAML profile reader +CHECK-BOLT-YAML-NEXT: 1 out of 16 functions in the binary (6.2%) have non-empty execution profile -- GitLab From cde54df39cab3a1d60a3e1862ab341609bee3cc3 Mon Sep 17 00:00:00 2001 From: Cooper Partin Date: Thu, 21 Mar 2024 14:43:15 -0700 Subject: [PATCH 198/296] Add support for PSV EntryFunctionName (#84409) This change introduces a version 3 of the PSV data that includes support for the name of the entry function as an offset into StringTable data to a null-terminated utf-8 string. Additional tests were added to ensure that the new value was properly serialized/deserialized from object data. Fixes #80175 --------- Co-authored-by: Cooper Partin --- llvm/include/llvm/BinaryFormat/DXContainer.h | 13 +++ llvm/include/llvm/MC/DXContainerPSVInfo.h | 25 ++-- llvm/include/llvm/MC/StringTableBuilder.h | 8 +- llvm/include/llvm/Object/DXContainer.h | 16 ++- .../include/llvm/ObjectYAML/DXContainerYAML.h | 5 +- llvm/lib/MC/DXContainerPSVInfo.cpp | 75 ++++++++---- llvm/lib/Object/DXContainer.cpp | 15 ++- llvm/lib/ObjectYAML/DXContainerEmitter.cpp | 3 +- llvm/lib/ObjectYAML/DXContainerYAML.cpp | 15 +++ .../DXContainer/PSVv3-amplification.yaml | 97 ++++++++++++++++ .../ObjectYAML/DXContainer/PSVv3-compute.yaml | 95 +++++++++++++++ .../ObjectYAML/DXContainer/PSVv3-domain.yaml | 105 +++++++++++++++++ .../DXContainer/PSVv3-geometry.yaml | 105 +++++++++++++++++ .../ObjectYAML/DXContainer/PSVv3-hull.yaml | 107 +++++++++++++++++ .../ObjectYAML/DXContainer/PSVv3-mesh.yaml | 109 ++++++++++++++++++ .../ObjectYAML/DXContainer/PSVv3-pixel.yaml | 99 ++++++++++++++++ .../ObjectYAML/DXContainer/PSVv3-vertex.yaml | 97 ++++++++++++++++ llvm/tools/obj2yaml/dxcontainer2yaml.cpp | 3 + 18 files changed, 940 insertions(+), 52 deletions(-) create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml create mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h index 532f9481766a..ba882c4a6f32 100644 --- a/llvm/include/llvm/BinaryFormat/DXContainer.h +++ b/llvm/include/llvm/BinaryFormat/DXContainer.h @@ -424,6 +424,19 @@ struct ResourceBindInfo : public v0::ResourceBindInfo { }; } // namespace v2 + +namespace v3 { +struct RuntimeInfo : public v2::RuntimeInfo { + uint32_t EntryNameOffset; + + void swapBytes() { sys::swapByteOrder(EntryNameOffset); } + + void swapBytes(Triple::EnvironmentType Stage) { + v2::RuntimeInfo::swapBytes(Stage); + } +}; + +} // namespace v3 } // namespace PSV #define COMPONENT_PRECISION(Val, Enum) Enum = Val, diff --git a/llvm/include/llvm/MC/DXContainerPSVInfo.h b/llvm/include/llvm/MC/DXContainerPSVInfo.h index 7d21c18d252f..bad2fe78eb8f 100644 --- a/llvm/include/llvm/MC/DXContainerPSVInfo.h +++ b/llvm/include/llvm/MC/DXContainerPSVInfo.h @@ -9,9 +9,11 @@ #ifndef LLVM_MC_DXCONTAINERPSVINFO_H #define LLVM_MC_DXCONTAINERPSVINFO_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/DXContainer.h" +#include "llvm/MC/StringTableBuilder.h" #include "llvm/TargetParser/Triple.h" #include @@ -45,8 +47,9 @@ struct PSVSignatureElement { // modifiable format, and can be used to serialize the data back into valid PSV // RuntimeInfo. struct PSVRuntimeInfo { + PSVRuntimeInfo() : DXConStrTabBuilder(StringTableBuilder::DXContainer) {} bool IsFinalized = false; - dxbc::PSV::v2::RuntimeInfo BaseData; + dxbc::PSV::v3::RuntimeInfo BaseData; SmallVector Resources; SmallVector InputElements; SmallVector OutputElements; @@ -64,6 +67,7 @@ struct PSVRuntimeInfo { std::array, 4> InputOutputMap; SmallVector InputPatchMap; SmallVector PatchOutputMap; + llvm::StringRef EntryName; // Serialize PSVInfo into the provided raw_ostream. The version field // specifies the data version to encode, the default value specifies encoding @@ -71,19 +75,12 @@ struct PSVRuntimeInfo { void write(raw_ostream &OS, uint32_t Version = std::numeric_limits::max()) const; - void finalize(Triple::EnvironmentType Stage) { - IsFinalized = true; - BaseData.SigInputElements = static_cast(InputElements.size()); - BaseData.SigOutputElements = static_cast(OutputElements.size()); - BaseData.SigPatchOrPrimElements = - static_cast(PatchOrPrimElements.size()); - if (!sys::IsBigEndianHost) - return; - BaseData.swapBytes(); - BaseData.swapBytes(Stage); - for (auto &Res : Resources) - Res.swapBytes(); - } + void finalize(Triple::EnvironmentType Stage); + +private: + SmallVector IndexBuffer; + SmallVector SignatureElements; + StringTableBuilder DXConStrTabBuilder; }; class Signature { diff --git a/llvm/include/llvm/MC/StringTableBuilder.h b/llvm/include/llvm/MC/StringTableBuilder.h index 4ee421e22c17..a738683548cf 100644 --- a/llvm/include/llvm/MC/StringTableBuilder.h +++ b/llvm/include/llvm/MC/StringTableBuilder.h @@ -74,12 +74,8 @@ public: /// Check if a string is contained in the string table. Since this class /// doesn't store the string values, this function can be used to check if /// storage needs to be done prior to adding the string. - bool contains(StringRef S) const { - return contains(CachedHashStringRef(S)); - } - bool contains(CachedHashStringRef S) const { - return StringIndexMap.count(S); - } + bool contains(StringRef S) const { return contains(CachedHashStringRef(S)); } + bool contains(CachedHashStringRef S) const { return StringIndexMap.count(S); } size_t getSize() const { return Size; } void clear(); diff --git a/llvm/include/llvm/Object/DXContainer.h b/llvm/include/llvm/Object/DXContainer.h index b6e3d321da24..19c83ba6c6e8 100644 --- a/llvm/include/llvm/Object/DXContainer.h +++ b/llvm/include/llvm/Object/DXContainer.h @@ -125,7 +125,8 @@ class PSVRuntimeInfo { uint32_t Size; using InfoStruct = std::variant; + dxbc::PSV::v1::RuntimeInfo, dxbc::PSV::v2::RuntimeInfo, + dxbc::PSV::v3::RuntimeInfo>; InfoStruct BasicInfo; ResourceArray Resources; StringRef StringTable; @@ -151,9 +152,11 @@ public: ResourceArray getResources() const { return Resources; } uint32_t getVersion() const { - return Size >= sizeof(dxbc::PSV::v2::RuntimeInfo) - ? 2 - : (Size >= sizeof(dxbc::PSV::v1::RuntimeInfo) ? 1 : 0); + return Size >= sizeof(dxbc::PSV::v3::RuntimeInfo) + ? 3 + : (Size >= sizeof(dxbc::PSV::v2::RuntimeInfo) ? 2 + : (Size >= sizeof(dxbc::PSV::v1::RuntimeInfo)) ? 1 + : 0); } uint32_t getResourceStride() const { return Resources.Stride; } @@ -161,6 +164,11 @@ public: const InfoStruct &getInfo() const { return BasicInfo; } template const T *getInfoAs() const { + if (const auto *P = std::get_if(&BasicInfo)) + return static_cast(P); + if (std::is_same::value) + return nullptr; + if (const auto *P = std::get_if(&BasicInfo)) return static_cast(P); if (std::is_same::value) diff --git a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h index f7f8d5e6bf47..9c4d9e19f11b 100644 --- a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h +++ b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h @@ -107,7 +107,7 @@ struct PSVInfo { // the format. uint32_t Version; - dxbc::PSV::v2::RuntimeInfo Info; + dxbc::PSV::v3::RuntimeInfo Info; uint32_t ResourceStride; SmallVector Resources; SmallVector SigInputElements; @@ -121,12 +121,15 @@ struct PSVInfo { MaskVector InputPatchMap; MaskVector PatchOutputMap; + StringRef EntryName; + void mapInfoForVersion(yaml::IO &IO); PSVInfo(); PSVInfo(const dxbc::PSV::v0::RuntimeInfo *P, uint16_t Stage); PSVInfo(const dxbc::PSV::v1::RuntimeInfo *P); PSVInfo(const dxbc::PSV::v2::RuntimeInfo *P); + PSVInfo(const dxbc::PSV::v3::RuntimeInfo *P, StringRef StringTable); }; struct SignatureParameter { diff --git a/llvm/lib/MC/DXContainerPSVInfo.cpp b/llvm/lib/MC/DXContainerPSVInfo.cpp index 48182fcd31df..aeff69380139 100644 --- a/llvm/lib/MC/DXContainerPSVInfo.cpp +++ b/llvm/lib/MC/DXContainerPSVInfo.cpp @@ -81,13 +81,18 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const { BindingSize = sizeof(dxbc::PSV::v0::ResourceBindInfo); break; case 2: - default: InfoSize = sizeof(dxbc::PSV::v2::RuntimeInfo); BindingSize = sizeof(dxbc::PSV::v2::ResourceBindInfo); + break; + case 3: + default: + InfoSize = sizeof(dxbc::PSV::v3::RuntimeInfo); + BindingSize = sizeof(dxbc::PSV::v2::ResourceBindInfo); } - // Write the size of the info. + // Write the size of the info. support::endian::write(OS, InfoSize, llvm::endianness::little); + // Write the info itself. OS.write(reinterpret_cast(&BaseData), InfoSize); @@ -104,32 +109,12 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const { if (Version == 0) return; - StringTableBuilder StrTabBuilder((StringTableBuilder::DXContainer)); - SmallVector IndexBuffer; - SmallVector SignatureElements; - SmallVector SemanticNames; - - ProcessElementList(StrTabBuilder, IndexBuffer, SignatureElements, - SemanticNames, InputElements); - ProcessElementList(StrTabBuilder, IndexBuffer, SignatureElements, - SemanticNames, OutputElements); - ProcessElementList(StrTabBuilder, IndexBuffer, SignatureElements, - SemanticNames, PatchOrPrimElements); - - StrTabBuilder.finalize(); - for (auto ElAndName : zip(SignatureElements, SemanticNames)) { - v0::SignatureElement &El = std::get<0>(ElAndName); - StringRef Name = std::get<1>(ElAndName); - El.NameOffset = static_cast(StrTabBuilder.getOffset(Name)); - if (sys::IsBigEndianHost) - El.swapBytes(); - } - - support::endian::write(OS, static_cast(StrTabBuilder.getSize()), + support::endian::write(OS, + static_cast(DXConStrTabBuilder.getSize()), llvm::endianness::little); // Write the string table. - StrTabBuilder.write(OS); + DXConStrTabBuilder.write(OS); // Write the index table size, then table. support::endian::write(OS, static_cast(IndexBuffer.size()), @@ -162,6 +147,46 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const { llvm::endianness::little); } +void PSVRuntimeInfo::finalize(Triple::EnvironmentType Stage) { + IsFinalized = true; + BaseData.SigInputElements = static_cast(InputElements.size()); + BaseData.SigOutputElements = static_cast(OutputElements.size()); + BaseData.SigPatchOrPrimElements = + static_cast(PatchOrPrimElements.size()); + + SmallVector SemanticNames; + + // Build a string table and set associated offsets to be written when + // write() is called + ProcessElementList(DXConStrTabBuilder, IndexBuffer, SignatureElements, + SemanticNames, InputElements); + ProcessElementList(DXConStrTabBuilder, IndexBuffer, SignatureElements, + SemanticNames, OutputElements); + ProcessElementList(DXConStrTabBuilder, IndexBuffer, SignatureElements, + SemanticNames, PatchOrPrimElements); + + DXConStrTabBuilder.add(EntryName); + + DXConStrTabBuilder.finalize(); + for (auto ElAndName : zip(SignatureElements, SemanticNames)) { + llvm::dxbc::PSV::v0::SignatureElement &El = std::get<0>(ElAndName); + StringRef Name = std::get<1>(ElAndName); + El.NameOffset = static_cast(DXConStrTabBuilder.getOffset(Name)); + if (sys::IsBigEndianHost) + El.swapBytes(); + } + + BaseData.EntryNameOffset = + static_cast(DXConStrTabBuilder.getOffset(EntryName)); + + if (!sys::IsBigEndianHost) + return; + BaseData.swapBytes(); + BaseData.swapBytes(Stage); + for (auto &Res : Resources) + Res.swapBytes(); +} + void Signature::write(raw_ostream &OS) { SmallVector SigParams; SigParams.reserve(Params.size()); diff --git a/llvm/lib/Object/DXContainer.cpp b/llvm/lib/Object/DXContainer.cpp index 935749afe338..3b1a6203a1f8 100644 --- a/llvm/lib/Object/DXContainer.cpp +++ b/llvm/lib/Object/DXContainer.cpp @@ -247,7 +247,14 @@ Error DirectX::PSVRuntimeInfo::parse(uint16_t ShaderKind) { const uint32_t PSVVersion = getVersion(); // Detect the PSVVersion by looking at the size field. - if (PSVVersion == 2) { + if (PSVVersion == 3) { + v3::RuntimeInfo Info; + if (Error Err = readStruct(PSVInfoData, Current, Info)) + return Err; + if (sys::IsBigEndianHost) + Info.swapBytes(ShaderStage); + BasicInfo = Info; + } else if (PSVVersion == 2) { v2::RuntimeInfo Info; if (Error Err = readStruct(PSVInfoData, Current, Info)) return Err; @@ -425,6 +432,8 @@ Error DirectX::PSVRuntimeInfo::parse(uint16_t ShaderKind) { } uint8_t DirectX::PSVRuntimeInfo::getSigInputCount() const { + if (const auto *P = std::get_if(&BasicInfo)) + return P->SigInputElements; if (const auto *P = std::get_if(&BasicInfo)) return P->SigInputElements; if (const auto *P = std::get_if(&BasicInfo)) @@ -433,6 +442,8 @@ uint8_t DirectX::PSVRuntimeInfo::getSigInputCount() const { } uint8_t DirectX::PSVRuntimeInfo::getSigOutputCount() const { + if (const auto *P = std::get_if(&BasicInfo)) + return P->SigOutputElements; if (const auto *P = std::get_if(&BasicInfo)) return P->SigOutputElements; if (const auto *P = std::get_if(&BasicInfo)) @@ -441,6 +452,8 @@ uint8_t DirectX::PSVRuntimeInfo::getSigOutputCount() const { } uint8_t DirectX::PSVRuntimeInfo::getSigPatchOrPrimCount() const { + if (const auto *P = std::get_if(&BasicInfo)) + return P->SigPatchOrPrimElements; if (const auto *P = std::get_if(&BasicInfo)) return P->SigPatchOrPrimElements; if (const auto *P = std::get_if(&BasicInfo)) diff --git a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp index 09a5e41c7123..f3a518df3175 100644 --- a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp +++ b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp @@ -198,8 +198,9 @@ void DXContainerWriter::writeParts(raw_ostream &OS) { if (!P.Info.has_value()) continue; mcdxbc::PSVRuntimeInfo PSV; - memcpy(&PSV.BaseData, &P.Info->Info, sizeof(dxbc::PSV::v2::RuntimeInfo)); + memcpy(&PSV.BaseData, &P.Info->Info, sizeof(dxbc::PSV::v3::RuntimeInfo)); PSV.Resources = P.Info->Resources; + PSV.EntryName = P.Info->EntryName; for (auto El : P.Info->SigInputElements) PSV.InputElements.push_back(mcdxbc::PSVSignatureElement{ diff --git a/llvm/lib/ObjectYAML/DXContainerYAML.cpp b/llvm/lib/ObjectYAML/DXContainerYAML.cpp index a6871e7855e4..38063670aee6 100644 --- a/llvm/lib/ObjectYAML/DXContainerYAML.cpp +++ b/llvm/lib/ObjectYAML/DXContainerYAML.cpp @@ -74,6 +74,16 @@ DXContainerYAML::PSVInfo::PSVInfo(const dxbc::PSV::v2::RuntimeInfo *P) memcpy(&Info, P, sizeof(dxbc::PSV::v2::RuntimeInfo)); } +DXContainerYAML::PSVInfo::PSVInfo(const dxbc::PSV::v3::RuntimeInfo *P, + StringRef StringTable) + : Version(3), + EntryName(StringTable.substr(P->EntryNameOffset, + StringTable.find('\0', P->EntryNameOffset) - + P->EntryNameOffset)) { + memset(&Info, 0, sizeof(Info)); + memcpy(&Info, P, sizeof(dxbc::PSV::v3::RuntimeInfo)); +} + namespace yaml { void MappingTraits::mapping( @@ -348,6 +358,11 @@ void DXContainerYAML::PSVInfo::mapInfoForVersion(yaml::IO &IO) { IO.mapRequired("NumThreadsX", Info.NumThreadsX); IO.mapRequired("NumThreadsY", Info.NumThreadsY); IO.mapRequired("NumThreadsZ", Info.NumThreadsZ); + + if (Version == 2) + return; + + IO.mapRequired("EntryName", EntryName); } } // namespace llvm diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml new file mode 100644 index 000000000000..09885bd529f0 --- /dev/null +++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml @@ -0,0 +1,97 @@ +# RUN: yaml2obj %s | obj2yaml | FileCheck %s + +--- !dxcontainer +Header: + Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] + Version: + Major: 1 + Minor: 0 + PartCount: 2 +Parts: + - Name: PSV0 + Size: 144 + PSVInfo: + Version: 3 + ShaderStage: 14 + PayloadSizeInBytes: 4092 + MinimumWaveLaneCount: 0 + MaximumWaveLaneCount: 4294967295 + UsesViewID: 0 + SigInputVectors: 0 + SigOutputVectors: [ 8, 16, 32, 64 ] + NumThreadsX: 512 + NumThreadsY: 1024 + NumThreadsZ: 2048 + EntryName: ASEntry + ResourceStride: 24 + Resources: + - Type: 1 + Space: 2 + LowerBound: 3 + UpperBound: 4 + Kind: 5 + Flags: 6 + - Type: 128 + Space: 32768 + LowerBound: 8388608 + UpperBound: 2147483648 + Kind: 65535 + Flags: 16776960 + SigInputElements: [] + SigOutputElements: [] + SigPatchOrPrimElements: [] + InputOutputMap: + - [ ] + - [ ] + - [ ] + - [ ] + - Name: DXIL + Size: 24 + Program: + MajorVersion: 6 + MinorVersion: 0 + ShaderKind: 14 + Size: 6 + DXILMajorVersion: 0 + DXILMinorVersion: 1 + DXILSize: 0 +... + +# CHECK: Name: PSV0 +# CHECK: PSVInfo: +# CHECK-NEXT: Version: 3 +# CHECK-NEXT: ShaderStage: 14 +# CHECK-NEXT: PayloadSizeInBytes: 4092 +# CHECK-NEXT: MinimumWaveLaneCount: 0 +# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 +# CHECK-NEXT: UsesViewID: 0 +# CHECK-NEXT: SigInputVectors: 0 +# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] +# CHECK-NEXT: NumThreadsX: 512 +# CHECK-NEXT: NumThreadsY: 1024 +# CHECK-NEXT: NumThreadsZ: 2048 +# CHECK-NEXT: EntryName: ASEntry +# CHECK-NEXT: ResourceStride: 24 +# CHECK-NEXT: Resources: +# CHECK-NEXT: - Type: 1 +# CHECK-NEXT: Space: 2 +# CHECK-NEXT: LowerBound: 3 +# CHECK-NEXT: UpperBound: 4 +# CHECK-NEXT: Kind: 5 +# CHECK-NEXT: Flags: 6 +# CHECK-NEXT: - Type: 128 +# CHECK-NEXT: Space: 32768 +# CHECK-NEXT: LowerBound: 8388608 +# CHECK-NEXT: UpperBound: 2147483648 +# CHECK-NEXT: Kind: 65535 +# CHECK-NEXT: Flags: 16776960 +# CHECK-NEXT: SigInputElements: [] +# CHECK-NEXT: SigOutputElements: [] +# CHECK-NEXT: SigPatchOrPrimElements: [] +# CHECK-NEXT: InputOutputMap: +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml new file mode 100644 index 000000000000..ee6fb112c772 --- /dev/null +++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml @@ -0,0 +1,95 @@ +# RUN: yaml2obj %s | obj2yaml | FileCheck %s + +--- !dxcontainer +Header: + Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] + Version: + Major: 1 + Minor: 0 + PartCount: 2 +Parts: + - Name: PSV0 + Size: 144 + PSVInfo: + Version: 3 + ShaderStage: 5 + MinimumWaveLaneCount: 0 + MaximumWaveLaneCount: 4294967295 + UsesViewID: 0 + SigInputVectors: 0 + SigOutputVectors: [ 8, 16, 32, 64 ] + NumThreadsX: 512 + NumThreadsY: 1024 + NumThreadsZ: 2048 + EntryName: CSEntry + ResourceStride: 24 + Resources: + - Type: 1 + Space: 2 + LowerBound: 3 + UpperBound: 4 + Kind: 5 + Flags: 6 + - Type: 128 + Space: 32768 + LowerBound: 8388608 + UpperBound: 2147483648 + Kind: 65535 + Flags: 16776960 + SigInputElements: [] + SigOutputElements: [] + SigPatchOrPrimElements: [] + InputOutputMap: + - [ ] + - [ ] + - [ ] + - [ ] + - Name: DXIL + Size: 24 + Program: + MajorVersion: 6 + MinorVersion: 0 + ShaderKind: 5 + Size: 6 + DXILMajorVersion: 0 + DXILMinorVersion: 1 + DXILSize: 0 +... + +# CHECK: Name: PSV0 +# CHECK: PSVInfo: +# CHECK-NEXT: Version: 3 +# CHECK-NEXT: ShaderStage: 5 +# CHECK-NEXT: MinimumWaveLaneCount: 0 +# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 +# CHECK-NEXT: UsesViewID: 0 +# CHECK-NEXT: SigInputVectors: 0 +# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] +# CHECK-NEXT: NumThreadsX: 512 +# CHECK-NEXT: NumThreadsY: 1024 +# CHECK-NEXT: NumThreadsZ: 2048 +# CHECK-NEXT: EntryName: CSEntry +# CHECK-NEXT: ResourceStride: 24 +# CHECK-NEXT: Resources: +# CHECK-NEXT: - Type: 1 +# CHECK-NEXT: Space: 2 +# CHECK-NEXT: LowerBound: 3 +# CHECK-NEXT: UpperBound: 4 +# CHECK-NEXT: Kind: 5 +# CHECK-NEXT: Flags: 6 +# CHECK-NEXT: - Type: 128 +# CHECK-NEXT: Space: 32768 +# CHECK-NEXT: LowerBound: 8388608 +# CHECK-NEXT: UpperBound: 2147483648 +# CHECK-NEXT: Kind: 65535 +# CHECK-NEXT: Flags: 16776960 +# CHECK-NEXT: SigInputElements: [] +# CHECK-NEXT: SigOutputElements: [] +# CHECK-NEXT: SigPatchOrPrimElements: [] +# CHECK-NEXT: InputOutputMap: +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml new file mode 100644 index 000000000000..dd367deae88e --- /dev/null +++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml @@ -0,0 +1,105 @@ +# RUN: yaml2obj %s | obj2yaml | FileCheck %s + +--- !dxcontainer +Header: + Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] + Version: + Major: 1 + Minor: 0 + PartCount: 2 +Parts: + - Name: PSV0 + Size: 144 + PSVInfo: + Version: 3 + ShaderStage: 4 + InputControlPointCount: 1024 + OutputPositionPresent: 1 + TessellatorDomain: 2056 + MinimumWaveLaneCount: 0 + MaximumWaveLaneCount: 4294967295 + UsesViewID: 0 + SigPatchConstOrPrimVectors: 0 + SigInputVectors: 0 + SigOutputVectors: [ 0, 16, 32, 64 ] + NumThreadsX: 512 + NumThreadsY: 1024 + NumThreadsZ: 2048 + EntryName: DSEntry + ResourceStride: 24 + Resources: + - Type: 1 + Space: 2 + LowerBound: 3 + UpperBound: 4 + Kind: 5 + Flags: 6 + - Type: 128 + Space: 32768 + LowerBound: 8388608 + UpperBound: 2147483648 + Kind: 65535 + Flags: 16776960 + SigInputElements: [] + SigOutputElements: [] + SigPatchOrPrimElements: [] + InputOutputMap: + - [ ] + - [ ] + - [ ] + - [ ] + PatchOutputMap: [] + - Name: DXIL + Size: 24 + Program: + MajorVersion: 6 + MinorVersion: 0 + ShaderKind: 4 + Size: 6 + DXILMajorVersion: 0 + DXILMinorVersion: 1 + DXILSize: 0 +... + +# CHECK: Name: PSV0 +# CHECK: PSVInfo: +# CHECK-NEXT: Version: 3 +# CHECK-NEXT: ShaderStage: 4 +# CHECK-NEXT: InputControlPointCount: 1024 +# CHECK-NEXT: OutputPositionPresent: 1 +# CHECK-NEXT: TessellatorDomain: 2056 +# CHECK-NEXT: MinimumWaveLaneCount: 0 +# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 +# CHECK-NEXT: UsesViewID: 0 +# CHECK-NEXT: SigPatchConstOrPrimVectors: 0 +# CHECK-NEXT: SigInputVectors: 0 +# CHECK-NEXT: SigOutputVectors: [ 0, 16, 32, 64 ] +# CHECK-NEXT: NumThreadsX: 512 +# CHECK-NEXT: NumThreadsY: 1024 +# CHECK-NEXT: NumThreadsZ: 2048 +# CHECK-NEXT: EntryName: DSEntry +# CHECK-NEXT: ResourceStride: 24 +# CHECK-NEXT: Resources: +# CHECK-NEXT: - Type: 1 +# CHECK-NEXT: Space: 2 +# CHECK-NEXT: LowerBound: 3 +# CHECK-NEXT: UpperBound: 4 +# CHECK-NEXT: Kind: 5 +# CHECK-NEXT: Flags: 6 +# CHECK-NEXT: - Type: 128 +# CHECK-NEXT: Space: 32768 +# CHECK-NEXT: LowerBound: 8388608 +# CHECK-NEXT: UpperBound: 2147483648 +# CHECK-NEXT: Kind: 65535 +# CHECK-NEXT: Flags: 16776960 +# CHECK-NEXT: SigInputElements: [] +# CHECK-NEXT: SigOutputElements: [] +# CHECK-NEXT: SigPatchOrPrimElements: [] +# CHECK-NEXT: InputOutputMap: +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: PatchOutputMap: [ ] +# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml new file mode 100644 index 000000000000..4c7680b63b02 --- /dev/null +++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml @@ -0,0 +1,105 @@ +# RUN: yaml2obj %s | obj2yaml | FileCheck %s + +--- !dxcontainer +Header: + Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] + Version: + Major: 1 + Minor: 0 + PartCount: 2 +Parts: + - Name: PSV0 + Size: 144 + PSVInfo: + Version: 3 + ShaderStage: 2 + InputPrimitive: 1024 + OutputTopology: 4096 + OutputStreamMask: 2056 + OutputPositionPresent: 1 + MinimumWaveLaneCount: 0 + MaximumWaveLaneCount: 4294967295 + UsesViewID: 0 + MaxVertexCount: 4096 + SigInputVectors: 0 + SigOutputVectors: [ 8, 16, 32, 64 ] + NumThreadsX: 512 + NumThreadsY: 1024 + NumThreadsZ: 2048 + EntryName: GSEntry + ResourceStride: 24 + Resources: + - Type: 1 + Space: 2 + LowerBound: 3 + UpperBound: 4 + Kind: 5 + Flags: 6 + - Type: 128 + Space: 32768 + LowerBound: 8388608 + UpperBound: 2147483648 + Kind: 65535 + Flags: 16776960 + SigInputElements: [] + SigOutputElements: [] + SigPatchOrPrimElements: [] + InputOutputMap: + - [ ] + - [ ] + - [ ] + - [ ] + - Name: DXIL + Size: 24 + Program: + MajorVersion: 6 + MinorVersion: 0 + ShaderKind: 2 + Size: 6 + DXILMajorVersion: 0 + DXILMinorVersion: 1 + DXILSize: 0 +... + +# CHECK: Name: PSV0 +# CHECK: PSVInfo: +# CHECK-NEXT: Version: 3 +# CHECK-NEXT: ShaderStage: 2 +# CHECK-NEXT: InputPrimitive: 1024 +# CHECK-NEXT: OutputTopology: 4096 +# CHECK-NEXT: OutputStreamMask: 2056 +# CHECK-NEXT: OutputPositionPresent: 1 +# CHECK-NEXT: MinimumWaveLaneCount: 0 +# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 +# CHECK-NEXT: UsesViewID: 0 +# CHECK-NEXT: MaxVertexCount: 4096 +# CHECK-NEXT: SigInputVectors: 0 +# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] +# CHECK-NEXT: NumThreadsX: 512 +# CHECK-NEXT: NumThreadsY: 1024 +# CHECK-NEXT: NumThreadsZ: 2048 +# CHECK-NEXT: EntryName: GSEntry +# CHECK-NEXT: ResourceStride: 24 +# CHECK-NEXT: Resources: +# CHECK-NEXT: - Type: 1 +# CHECK-NEXT: Space: 2 +# CHECK-NEXT: LowerBound: 3 +# CHECK-NEXT: UpperBound: 4 +# CHECK-NEXT: Kind: 5 +# CHECK-NEXT: Flags: 6 +# CHECK-NEXT: - Type: 128 +# CHECK-NEXT: Space: 32768 +# CHECK-NEXT: LowerBound: 8388608 +# CHECK-NEXT: UpperBound: 2147483648 +# CHECK-NEXT: Kind: 65535 +# CHECK-NEXT: Flags: 16776960 +# CHECK-NEXT: SigInputElements: [] +# CHECK-NEXT: SigOutputElements: [] +# CHECK-NEXT: SigPatchOrPrimElements: [] +# CHECK-NEXT: InputOutputMap: +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml new file mode 100644 index 000000000000..3bbad8a9b0ee --- /dev/null +++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml @@ -0,0 +1,107 @@ +# RUN: yaml2obj %s | obj2yaml | FileCheck %s + +--- !dxcontainer +Header: + Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] + Version: + Major: 1 + Minor: 0 + PartCount: 2 +Parts: + - Name: PSV0 + Size: 144 + PSVInfo: + Version: 3 + ShaderStage: 3 + InputControlPointCount: 1024 + OutputControlPointCount: 4096 + TessellatorDomain: 2056 + TessellatorOutputPrimitive: 8192 + MinimumWaveLaneCount: 0 + MaximumWaveLaneCount: 4294967295 + UsesViewID: 0 + SigPatchConstOrPrimVectors: 0 + SigInputVectors: 0 + SigOutputVectors: [ 0, 16, 32, 64 ] + NumThreadsX: 512 + NumThreadsY: 1024 + NumThreadsZ: 2048 + EntryName: HSEntry + ResourceStride: 24 + Resources: + - Type: 1 + Space: 2 + LowerBound: 3 + UpperBound: 4 + Kind: 5 + Flags: 6 + - Type: 128 + Space: 32768 + LowerBound: 8388608 + UpperBound: 2147483648 + Kind: 65535 + Flags: 16776960 + SigInputElements: [] + SigOutputElements: [] + SigPatchOrPrimElements: [] + InputOutputMap: + - [ ] + - [ ] + - [ ] + - [ ] + InputPatchMap: [] + - Name: DXIL + Size: 24 + Program: + MajorVersion: 6 + MinorVersion: 0 + ShaderKind: 3 + Size: 6 + DXILMajorVersion: 0 + DXILMinorVersion: 1 + DXILSize: 0 +... + +# CHECK: Name: PSV0 +# CHECK: PSVInfo: +# CHECK-NEXT: Version: 3 +# CHECK-NEXT: ShaderStage: 3 +# CHECK-NEXT: InputControlPointCount: 1024 +# CHECK-NEXT: OutputControlPointCount: 4096 +# CHECK-NEXT: TessellatorDomain: 2056 +# CHECK-NEXT: TessellatorOutputPrimitive: 8192 +# CHECK-NEXT: MinimumWaveLaneCount: 0 +# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 +# CHECK-NEXT: UsesViewID: 0 +# CHECK-NEXT: SigPatchConstOrPrimVectors: 0 +# CHECK-NEXT: SigInputVectors: 0 +# CHECK-NEXT: SigOutputVectors: [ 0, 16, 32, 64 ] +# CHECK-NEXT: NumThreadsX: 512 +# CHECK-NEXT: NumThreadsY: 1024 +# CHECK-NEXT: NumThreadsZ: 2048 +# CHECK-NEXT: EntryName: HSEntry +# CHECK-NEXT: ResourceStride: 24 +# CHECK-NEXT: Resources: +# CHECK-NEXT: - Type: 1 +# CHECK-NEXT: Space: 2 +# CHECK-NEXT: LowerBound: 3 +# CHECK-NEXT: UpperBound: 4 +# CHECK-NEXT: Kind: 5 +# CHECK-NEXT: Flags: 6 +# CHECK-NEXT: - Type: 128 +# CHECK-NEXT: Space: 32768 +# CHECK-NEXT: LowerBound: 8388608 +# CHECK-NEXT: UpperBound: 2147483648 +# CHECK-NEXT: Kind: 65535 +# CHECK-NEXT: Flags: 16776960 +# CHECK-NEXT: SigInputElements: [] +# CHECK-NEXT: SigOutputElements: [] +# CHECK-NEXT: SigPatchOrPrimElements: [] +# CHECK-NEXT: InputOutputMap: +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: InputPatchMap: [ ] +# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml new file mode 100644 index 000000000000..c5ea1fcf0780 --- /dev/null +++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml @@ -0,0 +1,109 @@ +# RUN: yaml2obj %s | obj2yaml | FileCheck %s + +--- !dxcontainer +Header: + Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] + Version: + Major: 1 + Minor: 0 + PartCount: 2 +Parts: + - Name: PSV0 + Size: 144 + PSVInfo: + Version: 3 + ShaderStage: 13 + GroupSharedBytesUsed: 1024 + GroupSharedBytesDependentOnViewID: 2056 + PayloadSizeInBytes: 4092 + MaxOutputVertices: 8196 + MaxOutputPrimitives: 4092 + MinimumWaveLaneCount: 0 + MaximumWaveLaneCount: 4294967295 + UsesViewID: 0 + SigPrimVectors: 128 + MeshOutputTopology: 16 + SigInputVectors: 0 + SigOutputVectors: [ 8, 16, 32, 64 ] + NumThreadsX: 512 + NumThreadsY: 1024 + NumThreadsZ: 2048 + EntryName: MSEntry + ResourceStride: 24 + Resources: + - Type: 1 + Space: 2 + LowerBound: 3 + UpperBound: 4 + Kind: 5 + Flags: 6 + - Type: 128 + Space: 32768 + LowerBound: 8388608 + UpperBound: 2147483648 + Kind: 65535 + Flags: 16776960 + SigInputElements: [] + SigOutputElements: [] + SigPatchOrPrimElements: [] + InputOutputMap: + - [ ] + - [ ] + - [ ] + - [ ] + - Name: DXIL + Size: 24 + Program: + MajorVersion: 6 + MinorVersion: 0 + ShaderKind: 13 + Size: 6 + DXILMajorVersion: 0 + DXILMinorVersion: 1 + DXILSize: 0 +... + +# CHECK: Name: PSV0 +# CHECK: PSVInfo: +# CHECK-NEXT: Version: 3 +# CHECK-NEXT: ShaderStage: 13 +# CHECK-NEXT: GroupSharedBytesUsed: 1024 +# CHECK-NEXT: GroupSharedBytesDependentOnViewID: 2056 +# CHECK-NEXT: PayloadSizeInBytes: 4092 +# CHECK-NEXT: MaxOutputVertices: 8196 +# CHECK-NEXT: MaxOutputPrimitives: 4092 +# CHECK-NEXT: MinimumWaveLaneCount: 0 +# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 +# CHECK-NEXT: UsesViewID: 0 +# CHECK-NEXT: SigPrimVectors: 128 +# CHECK-NEXT: MeshOutputTopology: 16 +# CHECK-NEXT: SigInputVectors: 0 +# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] +# CHECK-NEXT: NumThreadsX: 512 +# CHECK-NEXT: NumThreadsY: 1024 +# CHECK-NEXT: NumThreadsZ: 2048 +# CHECK-NEXT: EntryName: MSEntry +# CHECK-NEXT: ResourceStride: 24 +# CHECK-NEXT: Resources: +# CHECK-NEXT: - Type: 1 +# CHECK-NEXT: Space: 2 +# CHECK-NEXT: LowerBound: 3 +# CHECK-NEXT: UpperBound: 4 +# CHECK-NEXT: Kind: 5 +# CHECK-NEXT: Flags: 6 +# CHECK-NEXT: - Type: 128 +# CHECK-NEXT: Space: 32768 +# CHECK-NEXT: LowerBound: 8388608 +# CHECK-NEXT: UpperBound: 2147483648 +# CHECK-NEXT: Kind: 65535 +# CHECK-NEXT: Flags: 16776960 +# CHECK-NEXT: SigInputElements: [] +# CHECK-NEXT: SigOutputElements: [] +# CHECK-NEXT: SigPatchOrPrimElements: [] +# CHECK-NEXT: InputOutputMap: +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml new file mode 100644 index 000000000000..b28d5ec8074d --- /dev/null +++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml @@ -0,0 +1,99 @@ +# RUN: yaml2obj %s | obj2yaml | FileCheck %s + +--- !dxcontainer +Header: + Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] + Version: + Major: 1 + Minor: 0 + PartCount: 2 +Parts: + - Name: PSV0 + Size: 144 + PSVInfo: + Version: 3 + ShaderStage: 0 + DepthOutput: 7 + SampleFrequency: 96 + MinimumWaveLaneCount: 0 + MaximumWaveLaneCount: 4294967295 + UsesViewID: 0 + SigInputVectors: 0 + SigOutputVectors: [ 8, 16, 32, 64 ] + NumThreadsX: 512 + NumThreadsY: 1024 + NumThreadsZ: 2048 + EntryName: PSEntry + ResourceStride: 24 + Resources: + - Type: 1 + Space: 2 + LowerBound: 3 + UpperBound: 4 + Kind: 5 + Flags: 6 + - Type: 128 + Space: 32768 + LowerBound: 8388608 + UpperBound: 2147483648 + Kind: 65535 + Flags: 16776960 + SigInputElements: [] + SigOutputElements: [] + SigPatchOrPrimElements: [] + InputOutputMap: + - [ ] + - [ ] + - [ ] + - [ ] + - Name: DXIL + Size: 24 + Program: + MajorVersion: 6 + MinorVersion: 0 + ShaderKind: 0 + Size: 6 + DXILMajorVersion: 0 + DXILMinorVersion: 1 + DXILSize: 0 +... + +# CHECK: Name: PSV0 +# CHECK: PSVInfo: +# CHECK-NEXT: Version: 3 +# CHECK-NEXT: ShaderStage: 0 +# CHECK-NEXT: DepthOutput: 7 +# CHECK-NEXT: SampleFrequency: 96 +# CHECK-NEXT: MinimumWaveLaneCount: 0 +# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 +# CHECK-NEXT: UsesViewID: 0 +# CHECK-NEXT: SigInputVectors: 0 +# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] +# CHECK-NEXT: NumThreadsX: 512 +# CHECK-NEXT: NumThreadsY: 1024 +# CHECK-NEXT: NumThreadsZ: 2048 +# CHECK-NEXT: EntryName: PSEntry +# CHECK-NEXT: ResourceStride: 24 +# CHECK-NEXT: Resources: +# CHECK-NEXT: - Type: 1 +# CHECK-NEXT: Space: 2 +# CHECK-NEXT: LowerBound: 3 +# CHECK-NEXT: UpperBound: 4 +# CHECK-NEXT: Kind: 5 +# CHECK-NEXT: Flags: 6 +# CHECK-NEXT: - Type: 128 +# CHECK-NEXT: Space: 32768 +# CHECK-NEXT: LowerBound: 8388608 +# CHECK-NEXT: UpperBound: 2147483648 +# CHECK-NEXT: Kind: 65535 +# CHECK-NEXT: Flags: 16776960 +# CHECK-NEXT: SigInputElements: [] +# CHECK-NEXT: SigOutputElements: [] +# CHECK-NEXT: SigPatchOrPrimElements: [] +# CHECK-NEXT: InputOutputMap: +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml new file mode 100644 index 000000000000..d1fb55839931 --- /dev/null +++ b/llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml @@ -0,0 +1,97 @@ +# RUN: yaml2obj %s | obj2yaml | FileCheck %s + +--- !dxcontainer +Header: + Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] + Version: + Major: 1 + Minor: 0 + PartCount: 2 +Parts: + - Name: PSV0 + Size: 144 + PSVInfo: + Version: 3 + ShaderStage: 1 + OutputPositionPresent: 1 + MinimumWaveLaneCount: 0 + MaximumWaveLaneCount: 4294967295 + UsesViewID: 0 + SigInputVectors: 0 + SigOutputVectors: [ 8, 16, 32, 64 ] + NumThreadsX: 512 + NumThreadsY: 1024 + NumThreadsZ: 2048 + EntryName: VSEntry + ResourceStride: 24 + Resources: + - Type: 1 + Space: 2 + LowerBound: 3 + UpperBound: 4 + Kind: 5 + Flags: 6 + - Type: 128 + Space: 32768 + LowerBound: 8388608 + UpperBound: 2147483648 + Kind: 65535 + Flags: 16776960 + SigInputElements: [] + SigOutputElements: [] + SigPatchOrPrimElements: [] + InputOutputMap: + - [ ] + - [ ] + - [ ] + - [ ] + - Name: DXIL + Size: 24 + Program: + MajorVersion: 6 + MinorVersion: 0 + ShaderKind: 1 + Size: 6 + DXILMajorVersion: 0 + DXILMinorVersion: 1 + DXILSize: 0 +... + +# CHECK: Name: PSV0 +# CHECK: PSVInfo: +# CHECK-NEXT: Version: 3 +# CHECK-NEXT: ShaderStage: 1 +# CHECK-NEXT: OutputPositionPresent: 1 +# CHECK-NEXT: MinimumWaveLaneCount: 0 +# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 +# CHECK-NEXT: UsesViewID: 0 +# CHECK-NEXT: SigInputVectors: 0 +# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] +# CHECK-NEXT: NumThreadsX: 512 +# CHECK-NEXT: NumThreadsY: 1024 +# CHECK-NEXT: NumThreadsZ: 2048 +# CHECK-NEXT: EntryName: VSEntry +# CHECK-NEXT: ResourceStride: 24 +# CHECK-NEXT: Resources: +# CHECK-NEXT: - Type: 1 +# CHECK-NEXT: Space: 2 +# CHECK-NEXT: LowerBound: 3 +# CHECK-NEXT: UpperBound: 4 +# CHECK-NEXT: Kind: 5 +# CHECK-NEXT: Flags: 6 +# CHECK-NEXT: - Type: 128 +# CHECK-NEXT: Space: 32768 +# CHECK-NEXT: LowerBound: 8388608 +# CHECK-NEXT: UpperBound: 2147483648 +# CHECK-NEXT: Kind: 65535 +# CHECK-NEXT: Flags: 16776960 +# CHECK-NEXT: SigInputElements: [] +# CHECK-NEXT: SigOutputElements: [] +# CHECK-NEXT: SigPatchOrPrimElements: [] +# CHECK-NEXT: InputOutputMap: +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: - [ ] +# CHECK-NEXT: Name diff --git a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp index 69d9b9a2f784..ec4f5c74498f 100644 --- a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp +++ b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp @@ -99,6 +99,9 @@ dumpDXContainer(MemoryBufferRef Source) { else if (const auto *P = std::get_if(&PSVInfo->getInfo())) NewPart.Info = DXContainerYAML::PSVInfo(P); + else if (const auto *P = + std::get_if(&PSVInfo->getInfo())) + NewPart.Info = DXContainerYAML::PSVInfo(P, PSVInfo->getStringTable()); NewPart.Info->ResourceStride = PSVInfo->getResourceStride(); for (auto Res : PSVInfo->getResources()) NewPart.Info->Resources.push_back(Res); -- GitLab From b19bf3e888f95c35bf641cd0eee18a8da702f6fe Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Thu, 21 Mar 2024 15:48:35 -0600 Subject: [PATCH 199/296] [clang][SPIRV] Don't warn on -mcmodel (#86039) The code model doesn't affect the sub-compilation, so don't check it. Followup to #70740. --- clang/lib/Driver/ToolChains/Clang.cpp | 4 ++-- clang/test/Driver/unsupported-option-gpu.c | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index bc9cc8ce6cf5..86a287db72a4 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -5863,8 +5863,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, } else if (Triple.getArch() == llvm::Triple::x86_64) { Ok = llvm::is_contained({"small", "kernel", "medium", "large", "tiny"}, CM); - } else if (Triple.isNVPTX() || Triple.isAMDGPU()) { - // NVPTX/AMDGPU does not care about the code model and will accept + } else if (Triple.isNVPTX() || Triple.isAMDGPU() || Triple.isSPIRV()) { + // NVPTX/AMDGPU/SPIRV does not care about the code model and will accept // whatever works for the host. Ok = true; } else if (Triple.isSPARC64()) { diff --git a/clang/test/Driver/unsupported-option-gpu.c b/clang/test/Driver/unsupported-option-gpu.c index f23cb71ebfb0..5618b2cba72e 100644 --- a/clang/test/Driver/unsupported-option-gpu.c +++ b/clang/test/Driver/unsupported-option-gpu.c @@ -2,4 +2,5 @@ // DEFINE: %{check} = %clang -### --target=x86_64-linux-gnu -c -mcmodel=medium // RUN: %{check} -x cuda %s --cuda-path=%S/Inputs/CUDA/usr/local/cuda --offload-arch=sm_60 --no-cuda-version-check -fbasic-block-sections=all +// RUN: %{check} -x hip %s --offload=spirv64 -nogpulib -nogpuinc // RUN: %{check} -x hip %s --rocm-path=%S/Inputs/rocm -nogpulib -nogpuinc -- GitLab From b609a4d7ea8b716f5f0ec83d10945362f42e730d Mon Sep 17 00:00:00 2001 From: alx32 <103613512+alx32@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:50:44 -0700 Subject: [PATCH 200/296] [lld-macho][NFC] Refactor insertions into inputSections (#85692) Before this change, after `InputSection` objects are created, they need to be added to the appropriate container for tracking. The logic for selecting the appropriate container lives in `Driver.cpp` / `gatherInputSections`, where the `InputSection` is added to the matching container depending on the input config and the type of `InputSection`. Also, multiple other locations also insert directly into `inputSections` array - assuming that that is the appropriate container for the `InputSection`'s they create. Currently this is the correct assumption, however an upcoming feature will change this. For an upcoming feature (relative method lists), we need to route `InputSection`'s either to `inputSections` array or to a synthetic section, depending on weather the relative method list optimization is enabled or not. We can achieve the above either by duplicating some of the logic or refactoring the routing and `InputSection`'s and reusing that. The refactoring & code sharing approach seems the correct way to go - as such this diff performs the refactoring while not introducing any functional changes. Later on we can just call `addInputSection` and not have to worry about routing logic. --------- --- lld/MachO/Driver.cpp | 42 ++++----------------------------- lld/MachO/InputSection.cpp | 38 +++++++++++++++++++++++++++++ lld/MachO/InputSection.h | 3 +++ lld/MachO/ObjC.cpp | 10 ++++---- lld/MachO/SyntheticSections.cpp | 4 ++-- 5 files changed, 52 insertions(+), 45 deletions(-) diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp index 36248925d65a..919a14b8bcf0 100644 --- a/lld/MachO/Driver.cpp +++ b/lld/MachO/Driver.cpp @@ -612,7 +612,7 @@ static void replaceCommonSymbols() { if (!osec) osec = ConcatOutputSection::getOrCreateForInput(isec); isec->parent = osec; - inputSections.push_back(isec); + addInputSection(isec); // FIXME: CommonSymbol should store isReferencedDynamically, noDeadStrip // and pass them on here. @@ -1220,53 +1220,18 @@ static void createFiles(const InputArgList &args) { static void gatherInputSections() { TimeTraceScope timeScope("Gathering input sections"); - int inputOrder = 0; for (const InputFile *file : inputFiles) { for (const Section *section : file->sections) { // Compact unwind entries require special handling elsewhere. (In // contrast, EH frames are handled like regular ConcatInputSections.) if (section->name == section_names::compactUnwind) continue; - ConcatOutputSection *osec = nullptr; - for (const Subsection &subsection : section->subsections) { - if (auto *isec = dyn_cast(subsection.isec)) { - if (isec->isCoalescedWeak()) - continue; - if (config->emitInitOffsets && - sectionType(isec->getFlags()) == S_MOD_INIT_FUNC_POINTERS) { - in.initOffsets->addInput(isec); - continue; - } - isec->outSecOff = inputOrder++; - if (!osec) - osec = ConcatOutputSection::getOrCreateForInput(isec); - isec->parent = osec; - inputSections.push_back(isec); - } else if (auto *isec = - dyn_cast(subsection.isec)) { - if (isec->getName() == section_names::objcMethname) { - if (in.objcMethnameSection->inputOrder == UnspecifiedInputOrder) - in.objcMethnameSection->inputOrder = inputOrder++; - in.objcMethnameSection->addInput(isec); - } else { - if (in.cStringSection->inputOrder == UnspecifiedInputOrder) - in.cStringSection->inputOrder = inputOrder++; - in.cStringSection->addInput(isec); - } - } else if (auto *isec = - dyn_cast(subsection.isec)) { - if (in.wordLiteralSection->inputOrder == UnspecifiedInputOrder) - in.wordLiteralSection->inputOrder = inputOrder++; - in.wordLiteralSection->addInput(isec); - } else { - llvm_unreachable("unexpected input section kind"); - } - } + for (const Subsection &subsection : section->subsections) + addInputSection(subsection.isec); } if (!file->objCImageInfo.empty()) in.objCImageInfo->addFile(file); } - assert(inputOrder <= UnspecifiedInputOrder); } static void foldIdenticalLiterals() { @@ -1422,6 +1387,7 @@ bool link(ArrayRef argsArr, llvm::raw_ostream &stdoutOS, concatOutputSections.clear(); inputFiles.clear(); inputSections.clear(); + inputSectionsOrder = 0; loadedArchives.clear(); loadedObjectFrameworks.clear(); missingAutolinkWarnings.clear(); diff --git a/lld/MachO/InputSection.cpp b/lld/MachO/InputSection.cpp index 8f5affb1dc21..22930d52dd1d 100644 --- a/lld/MachO/InputSection.cpp +++ b/lld/MachO/InputSection.cpp @@ -37,6 +37,44 @@ static_assert(sizeof(void *) != 8 || "instances of it"); std::vector macho::inputSections; +int macho::inputSectionsOrder = 0; + +// Call this function to add a new InputSection and have it routed to the +// appropriate container. Depending on its type and current config, it will +// either be added to 'inputSections' vector or to a synthetic section. +void lld::macho::addInputSection(InputSection *inputSection) { + if (auto *isec = dyn_cast(inputSection)) { + if (isec->isCoalescedWeak()) + return; + if (config->emitInitOffsets && + sectionType(isec->getFlags()) == S_MOD_INIT_FUNC_POINTERS) { + in.initOffsets->addInput(isec); + return; + } + isec->outSecOff = inputSectionsOrder++; + auto *osec = ConcatOutputSection::getOrCreateForInput(isec); + isec->parent = osec; + inputSections.push_back(isec); + } else if (auto *isec = dyn_cast(inputSection)) { + if (isec->getName() == section_names::objcMethname) { + if (in.objcMethnameSection->inputOrder == UnspecifiedInputOrder) + in.objcMethnameSection->inputOrder = inputSectionsOrder++; + in.objcMethnameSection->addInput(isec); + } else { + if (in.cStringSection->inputOrder == UnspecifiedInputOrder) + in.cStringSection->inputOrder = inputSectionsOrder++; + in.cStringSection->addInput(isec); + } + } else if (auto *isec = dyn_cast(inputSection)) { + if (in.wordLiteralSection->inputOrder == UnspecifiedInputOrder) + in.wordLiteralSection->inputOrder = inputSectionsOrder++; + in.wordLiteralSection->addInput(isec); + } else { + llvm_unreachable("unexpected input section kind"); + } + + assert(inputSectionsOrder <= UnspecifiedInputOrder); +} uint64_t InputSection::getFileSize() const { return isZeroFill(getFlags()) ? 0 : getSize(); diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h index b25f0638f4c6..694bdf734907 100644 --- a/lld/MachO/InputSection.h +++ b/lld/MachO/InputSection.h @@ -302,6 +302,8 @@ bool isEhFrameSection(const InputSection *); bool isGccExceptTabSection(const InputSection *); extern std::vector inputSections; +// This is used as a counter for specyfing input order for input sections +extern int inputSectionsOrder; namespace section_names { @@ -369,6 +371,7 @@ constexpr const char addrSig[] = "__llvm_addrsig"; } // namespace section_names +void addInputSection(InputSection *inputSection); } // namespace macho std::string toString(const macho::InputSection *); diff --git a/lld/MachO/ObjC.cpp b/lld/MachO/ObjC.cpp index 40df2243b26f..66959cfb665f 100644 --- a/lld/MachO/ObjC.cpp +++ b/lld/MachO/ObjC.cpp @@ -790,7 +790,7 @@ void ObjcCategoryMerger::emitAndLinkProtocolList( infoCategoryWriter.catPtrListInfo.align); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; listSec->live = true; - allInputSections.push_back(listSec); + addInputSection(listSec); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; @@ -848,7 +848,7 @@ void ObjcCategoryMerger::emitAndLinkPointerList( infoCategoryWriter.catPtrListInfo.align); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; listSec->live = true; - allInputSections.push_back(listSec); + addInputSection(listSec); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; @@ -889,7 +889,7 @@ ObjcCategoryMerger::emitCatListEntrySec(const std::string &forCateogryName, bodyData, infoCategoryWriter.catListInfo.align); newCatList->parent = infoCategoryWriter.catListInfo.outputSection; newCatList->live = true; - allInputSections.push_back(newCatList); + addInputSection(newCatList); newCatList->parent = infoCategoryWriter.catListInfo.outputSection; @@ -927,7 +927,7 @@ Defined *ObjcCategoryMerger::emitCategoryBody(const std::string &name, bodyData, infoCategoryWriter.catBodyInfo.align); newBodySec->parent = infoCategoryWriter.catBodyInfo.outputSection; newBodySec->live = true; - allInputSections.push_back(newBodySec); + addInputSection(newBodySec); std::string symName = objc::symbol_names::category + baseClassName + "_$_(" + name + ")"; @@ -1132,7 +1132,7 @@ void ObjcCategoryMerger::generateCatListForNonErasedCategories( infoCategoryWriter.catListInfo.align); listSec->parent = infoCategoryWriter.catListInfo.outputSection; listSec->live = true; - allInputSections.push_back(listSec); + addInputSection(listSec); std::string slotSymName = "<__objc_catlist slot for category "; slotSymName += nonErasedCatBody->getName(); diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp index 7ee3261ce307..1b3694528de1 100644 --- a/lld/MachO/SyntheticSections.cpp +++ b/lld/MachO/SyntheticSections.cpp @@ -793,7 +793,7 @@ void StubHelperSection::setUp() { in.imageLoaderCache->parent = ConcatOutputSection::getOrCreateForInput(in.imageLoaderCache); - inputSections.push_back(in.imageLoaderCache); + addInputSection(in.imageLoaderCache); // Since this isn't in the symbol table or in any input file, the noDeadStrip // argument doesn't matter. dyldPrivate = @@ -855,7 +855,7 @@ ConcatInputSection *ObjCSelRefsSection::makeSelRef(StringRef methname) { /*addend=*/static_cast(methnameOffset), /*referent=*/in.objcMethnameSection->isec}); objcSelref->parent = ConcatOutputSection::getOrCreateForInput(objcSelref); - inputSections.push_back(objcSelref); + addInputSection(objcSelref); objcSelref->isFinal = true; methnameToSelref[CachedHashStringRef(methname)] = objcSelref; return objcSelref; -- GitLab From e4a672ef85f76c3402b81640e1e83e5d3069d1b9 Mon Sep 17 00:00:00 2001 From: alx32 <103613512+alx32@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:53:09 -0700 Subject: [PATCH 201/296] [lld][macho] Fix gcc category merging warning (#86091) Fixing gcc warning regarding creating non-null-terminated string: ``` ../../lld/MachO/ObjC.cpp:1226:10: warning: 'char* strncpy(char*, const char*, size_t)' output truncated before terminating nul copying as many bytes from a string as its length [-Wstringop-truncation] 1226 | strncpy(strData, str, len); | ~~~~~~~^~~~~~~~~~~~~~~~~~~ ../../lld/MachO/ObjC.cpp: In member function 'void {anonymous}::ObjcCategoryMerger::emitAndLinkPointerList(lld::macho::Defined*, uint32_t, const {anonymous}::ObjcCategoryMerger::ClassExtensionInfo&, const {anonymous}::ObjcCategoryMerger::PointerListInfo&)': ../../lld/MachO/ObjC.cpp:1223:24: note: length computed here 1223 | uint32_t len = strlen(str); | ~~~~~~^~~~~ ``` This is not actually a bug, as `newSectionData` returns a zero-initialized memory region, so the null terminator will be there. --- lld/MachO/ObjC.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lld/MachO/ObjC.cpp b/lld/MachO/ObjC.cpp index 66959cfb665f..5902b82d30f5 100644 --- a/lld/MachO/ObjC.cpp +++ b/lld/MachO/ObjC.cpp @@ -1221,9 +1221,11 @@ void ObjcCategoryMerger::doCleanup() { generatedSectionData.clear(); } StringRef ObjcCategoryMerger::newStringData(const char *str) { uint32_t len = strlen(str); - auto &data = newSectionData(len + 1); + uint32_t bufSize = len + 1; + auto &data = newSectionData(bufSize); char *strData = reinterpret_cast(data.data()); - strncpy(strData, str, len); + // Copy the string chars and null-terminator + memcpy(strData, str, bufSize); return StringRef(strData, len); } -- GitLab From e470ca89ba77b2f200ff3a8ad65c74028f42c5f7 Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Thu, 21 Mar 2024 15:03:34 -0700 Subject: [PATCH 202/296] [InstallAPI] Report exports discovered in binary but not in interface (#86025) This patch completes the classes of errors installapi can detect. --- .../clang/Basic/DiagnosticInstallAPIKinds.td | 1 + .../include/clang/InstallAPI/DylibVerifier.h | 13 +- clang/lib/InstallAPI/DylibVerifier.cpp | 171 ++++++- clang/test/InstallAPI/diagnostics-cpp.test | 2 + clang/test/InstallAPI/linker-symbols.test | 440 ++++++++++++++++++ .../mismatching-objc-class-symbols.test | 269 +++++++++++ clang/test/InstallAPI/symbol-flags.test | 290 ++++++++++++ .../clang-installapi/ClangInstallAPI.cpp | 2 +- llvm/lib/TextAPI/BinaryReader/DylibReader.cpp | 7 +- 9 files changed, 1174 insertions(+), 21 deletions(-) create mode 100644 clang/test/InstallAPI/linker-symbols.test create mode 100644 clang/test/InstallAPI/mismatching-objc-class-symbols.test create mode 100644 clang/test/InstallAPI/symbol-flags.test diff --git a/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td b/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td index f99a5fca64cb..a4c6e630ac5f 100644 --- a/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td +++ b/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td @@ -26,6 +26,7 @@ def warn_library_hidden_symbol : Warning<"declaration has external linkage, but def warn_header_hidden_symbol : Warning<"symbol exported in dynamic library, but marked hidden in declaration '%0'">, InGroup; def err_header_hidden_symbol : Error<"symbol exported in dynamic library, but marked hidden in declaration '%0'">; def err_header_symbol_missing : Error<"no declaration found for exported symbol '%0' in dynamic library">; +def warn_header_symbol_missing : Warning<"no declaration was found for exported symbol '%0' in dynamic library">, InGroup; def warn_header_availability_mismatch : Warning<"declaration '%0' is marked %select{available|unavailable}1," " but symbol is %select{not |}2exported in dynamic library">, InGroup; def err_header_availability_mismatch : Error<"declaration '%0' is marked %select{available|unavailable}1," diff --git a/clang/include/clang/InstallAPI/DylibVerifier.h b/clang/include/clang/InstallAPI/DylibVerifier.h index bbfa8711313e..49de24763f1f 100644 --- a/clang/include/clang/InstallAPI/DylibVerifier.h +++ b/clang/include/clang/InstallAPI/DylibVerifier.h @@ -28,7 +28,7 @@ enum class VerificationMode { /// lifetime of InstallAPI. /// As declarations are collected during AST traversal, they are /// compared as symbols against what is available in the binary dylib. -class DylibVerifier { +class DylibVerifier : llvm::MachO::RecordVisitor { private: struct SymbolContext; @@ -72,6 +72,9 @@ public: Result verify(ObjCIVarRecord *R, const FrontendAttrs *FA, const StringRef SuperClass); + // Scan through dylib slices and report any remaining missing exports. + Result verifyRemainingSymbols(); + /// Initialize target for verification. void setTarget(const Target &T); @@ -128,6 +131,14 @@ private: /// Find matching dylib slice for target triple that is being parsed. void assignSlice(const Target &T); + /// Shared implementation for verifying exported symbols in dylib. + void visitSymbolInDylib(const Record &R, SymbolContext &SymCtx); + + void visitGlobal(const GlobalRecord &R) override; + void visitObjCInterface(const ObjCInterfaceRecord &R) override; + void visitObjCCategory(const ObjCCategoryRecord &R) override; + void visitObjCIVar(const ObjCIVarRecord &R, const StringRef Super); + /// Gather annotations for symbol for error reporting. std::string getAnnotatedName(const Record *R, SymbolContext &SymCtx, bool ValidSourceLoc = true); diff --git a/clang/lib/InstallAPI/DylibVerifier.cpp b/clang/lib/InstallAPI/DylibVerifier.cpp index 24e0d0addf2f..94b8e9cd3233 100644 --- a/clang/lib/InstallAPI/DylibVerifier.cpp +++ b/clang/lib/InstallAPI/DylibVerifier.cpp @@ -66,17 +66,15 @@ std::string DylibVerifier::getAnnotatedName(const Record *R, Annotation += "(tlv) "; // Check if symbol represents only part of a @interface declaration. - const bool IsAnnotatedObjCClass = - ((SymCtx.ObjCIFKind != ObjCIFSymbolKind::None) && - (SymCtx.ObjCIFKind <= ObjCIFSymbolKind::EHType)); - - if (IsAnnotatedObjCClass) { - if (SymCtx.ObjCIFKind == ObjCIFSymbolKind::EHType) - Annotation += "Exception Type of "; - if (SymCtx.ObjCIFKind == ObjCIFSymbolKind::MetaClass) - Annotation += "Metaclass of "; - if (SymCtx.ObjCIFKind == ObjCIFSymbolKind::Class) - Annotation += "Class of "; + switch (SymCtx.ObjCIFKind) { + default: + break; + case ObjCIFSymbolKind::EHType: + return Annotation + "Exception Type of " + PrettyName; + case ObjCIFSymbolKind::MetaClass: + return Annotation + "Metaclass of " + PrettyName; + case ObjCIFSymbolKind::Class: + return Annotation + "Class of " + PrettyName; } // Only print symbol type prefix or leading "_" if there is no source location @@ -90,9 +88,6 @@ std::string DylibVerifier::getAnnotatedName(const Record *R, return Annotation + PrettyName; } - if (IsAnnotatedObjCClass) - return Annotation + PrettyName; - switch (SymCtx.Kind) { case EncodeKind::GlobalSymbol: return Annotation + PrettyName; @@ -332,9 +327,9 @@ bool DylibVerifier::compareSymbolFlags(const Record *R, SymbolContext &SymCtx, } if (!DR->isThreadLocalValue() && R->isThreadLocalValue()) { Ctx.emitDiag([&]() { - SymCtx.FA->D->getLocation(), - Ctx.Diag->Report(diag::err_header_symbol_flags_mismatch) - << getAnnotatedName(DR, SymCtx) << R->isThreadLocalValue(); + Ctx.Diag->Report(SymCtx.FA->D->getLocation(), + diag::err_header_symbol_flags_mismatch) + << getAnnotatedName(R, SymCtx) << R->isThreadLocalValue(); }); return false; } @@ -520,5 +515,147 @@ void DylibVerifier::VerifierContext::emitDiag( Report(); } +// The existence of weak-defined RTTI can not always be inferred from the +// header files because they can be generated as part of an implementation +// file. +// InstallAPI doesn't warn about weak-defined RTTI, because this doesn't affect +// static linking and so can be ignored for text-api files. +static bool shouldIgnoreCpp(StringRef Name, bool IsWeakDef) { + return (IsWeakDef && + (Name.starts_with("__ZTI") || Name.starts_with("__ZTS"))); +} +void DylibVerifier::visitSymbolInDylib(const Record &R, SymbolContext &SymCtx) { + // Undefined symbols should not be in InstallAPI generated text-api files. + if (R.isUndefined()) { + updateState(Result::Valid); + return; + } + + // Internal symbols should not be in InstallAPI generated text-api files. + if (R.isInternal()) { + updateState(Result::Valid); + return; + } + + // Allow zippered symbols with potentially mismatching availability + // between macOS and macCatalyst in the final text-api file. + const StringRef SymbolName(SymCtx.SymbolName); + if (const Symbol *Sym = Exports->findSymbol(SymCtx.Kind, SymCtx.SymbolName, + SymCtx.ObjCIFKind)) { + if (Sym->hasArchitecture(Ctx.Target.Arch)) { + updateState(Result::Ignore); + return; + } + } + + if (shouldIgnoreCpp(SymbolName, R.isWeakDefined())) { + updateState(Result::Valid); + return; + } + + // All checks at this point classify as some kind of violation that should be + // reported. + + // Regardless of verification mode, error out on mismatched special linker + // symbols. + if (SymbolName.starts_with("$ld$")) { + Ctx.emitDiag([&]() { + Ctx.Diag->Report(diag::err_header_symbol_missing) + << getAnnotatedName(&R, SymCtx, /*ValidSourceLoc=*/false); + }); + updateState(Result::Invalid); + return; + } + + // Missing declarations for exported symbols are hard errors on Pedantic mode. + if (Mode == VerificationMode::Pedantic) { + Ctx.emitDiag([&]() { + Ctx.Diag->Report(diag::err_header_symbol_missing) + << getAnnotatedName(&R, SymCtx, /*ValidSourceLoc=*/false); + }); + updateState(Result::Invalid); + return; + } + + // Missing declarations for exported symbols are warnings on ErrorsAndWarnings + // mode. + if (Mode == VerificationMode::ErrorsAndWarnings) { + Ctx.emitDiag([&]() { + Ctx.Diag->Report(diag::warn_header_symbol_missing) + << getAnnotatedName(&R, SymCtx, /*ValidSourceLoc=*/false); + }); + updateState(Result::Ignore); + return; + } + + // Missing declarations are dropped for ErrorsOnly mode. It is the last + // remaining mode. + updateState(Result::Ignore); + return; +} + +void DylibVerifier::visitGlobal(const GlobalRecord &R) { + if (R.isVerified()) + return; + SymbolContext SymCtx; + SimpleSymbol Sym = parseSymbol(R.getName()); + SymCtx.SymbolName = Sym.Name; + SymCtx.Kind = Sym.Kind; + visitSymbolInDylib(R, SymCtx); +} + +void DylibVerifier::visitObjCIVar(const ObjCIVarRecord &R, + const StringRef Super) { + if (R.isVerified()) + return; + SymbolContext SymCtx; + SymCtx.SymbolName = ObjCIVarRecord::createScopedName(Super, R.getName()); + SymCtx.Kind = EncodeKind::ObjectiveCInstanceVariable; + visitSymbolInDylib(R, SymCtx); +} + +void DylibVerifier::visitObjCInterface(const ObjCInterfaceRecord &R) { + if (R.isVerified()) + return; + SymbolContext SymCtx; + SymCtx.SymbolName = R.getName(); + SymCtx.ObjCIFKind = assignObjCIFSymbolKind(&R); + if (SymCtx.ObjCIFKind > ObjCIFSymbolKind::EHType) { + if (R.hasExceptionAttribute()) { + SymCtx.Kind = EncodeKind::ObjectiveCClassEHType; + visitSymbolInDylib(R, SymCtx); + } + SymCtx.Kind = EncodeKind::ObjectiveCClass; + visitSymbolInDylib(R, SymCtx); + } else { + SymCtx.Kind = R.hasExceptionAttribute() ? EncodeKind::ObjectiveCClassEHType + : EncodeKind::ObjectiveCClass; + visitSymbolInDylib(R, SymCtx); + } + + for (const ObjCIVarRecord *IV : R.getObjCIVars()) + visitObjCIVar(*IV, R.getName()); +} + +void DylibVerifier::visitObjCCategory(const ObjCCategoryRecord &R) { + for (const ObjCIVarRecord *IV : R.getObjCIVars()) + visitObjCIVar(*IV, R.getSuperClassName()); +} + +DylibVerifier::Result DylibVerifier::verifyRemainingSymbols() { + if (getState() == Result::NoVerify) + return Result::NoVerify; + assert(!Dylib.empty() && "No binary to verify against"); + + Ctx.DiscoveredFirstError = false; + Ctx.PrintArch = true; + for (std::shared_ptr Slice : Dylib) { + Ctx.Target = Slice->getTarget(); + Ctx.DylibSlice = Slice.get(); + Slice->visit(*this); + } + return getState(); +} + } // namespace installapi } // namespace clang diff --git a/clang/test/InstallAPI/diagnostics-cpp.test b/clang/test/InstallAPI/diagnostics-cpp.test index 658886537507..51cca129ea0a 100644 --- a/clang/test/InstallAPI/diagnostics-cpp.test +++ b/clang/test/InstallAPI/diagnostics-cpp.test @@ -21,6 +21,8 @@ CHECK-NEXT: CPP.h:5:7: error: declaration has external linkage, but symbol has i CHECK-NEXT: CPP.h:6:7: error: dynamic library symbol '(weak-def) Bar::init()' is weak defined, but its declaration is not CHECK-NEXT: int init(); CHECK-NEXT: ^ +CHECK-NEXT: warning: violations found for arm64 +CHECK-NEXT: error: no declaration found for exported symbol 'int foo(unsigned int)' in dynamic library //--- inputs.json.in { diff --git a/clang/test/InstallAPI/linker-symbols.test b/clang/test/InstallAPI/linker-symbols.test new file mode 100644 index 000000000000..1e4ddf9c45d5 --- /dev/null +++ b/clang/test/InstallAPI/linker-symbols.test @@ -0,0 +1,440 @@ +; RUN: rm -rf %t +; RUN: split-file %s %t +; RUN: sed -e "s|DSTROOT|%/t|g" %t/inputs.json.in > %t/inputs.json + +; RUN: yaml2obj %t/MagicSymbols.yaml -o %t/MagicSymbols + +; RUN: not clang-installapi -target x86_64-apple-macosx13 \ +; RUN: -install_name \ +; RUN: /System/Library/Frameworks/SpecialLinkerSymbols.framework/Versions/A/SpecialLinkerSymbols \ +; RUN: -current_version 1 -compatibility_version 1 \ +; RUN: %t/inputs.json -o %t/output.tbd \ +; RUN: --verify-mode=ErrorsOnly \ +; RUN: --verify-against=%t/MagicSymbols 2>&1 | FileCheck %s + +CHECK: warning: violations found for x86_64 +CHECK: error: no declaration found for exported symbol '$ld$add$os10.4$_symbol2' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$add$os10.5$_symbol2' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$hide$os10.6$_symbol1' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$hide$os10.7$_symbol1' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$weak$os10.5$_symbol3' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$weak$os10.4$_symbol3' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$install_name$os10.4$/System/Library/Frameworks/A.framework/Versions/A/A' in dynamic library +CHECK: error: no declaration found for exported symbol '$ld$install_name$os10.5$/System/Library/Frameworks/B.framework/Versions/A/B' in dynamic library + +;--- MagicSymbols.h +#ifndef SPECIAL_LINKER_SYMBOLS_H +#define SPECIAL_LINKER_SYMBOLS_H + +extern const int SpecialLinkerSymbolsVersion; + +extern int symbol1; +extern int symbol3; + +#endif // SPECIAL_LINKER_SYMBOLS_H + +;--- inputs.json.in +{ + "headers": [ { + "path" : "DSTROOT/MagicSymbols.h", + "type" : "project" + } + ], + "version": "3" +} + +;--- MagicSymbols.yaml +--- !mach-o +FileHeader: + magic: 0xFEEDFACF + cputype: 0x1000007 + cpusubtype: 0x3 + filetype: 0x6 + ncmds: 12 + sizeofcmds: 952 + flags: 0x100085 + reserved: 0x0 +LoadCommands: + - cmd: LC_SEGMENT_64 + cmdsize: 232 + segname: __TEXT + vmaddr: 0 + vmsize: 4096 + fileoff: 0 + filesize: 4096 + maxprot: 5 + initprot: 5 + nsects: 2 + flags: 0 + Sections: + - sectname: __text + segname: __TEXT + addr: 0xBD8 + size: 0 + offset: 0xBD8 + align: 0 + reloff: 0x0 + nreloc: 0 + flags: 0x80000000 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: '' + - sectname: __const + segname: __TEXT + addr: 0xBD8 + size: 4 + offset: 0xBD8 + align: 2 + reloff: 0x0 + nreloc: 0 + flags: 0x0 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: '07000000' + - cmd: LC_SEGMENT_64 + cmdsize: 232 + segname: __DATA + vmaddr: 4096 + vmsize: 4096 + fileoff: 4096 + filesize: 4096 + maxprot: 3 + initprot: 3 + nsects: 2 + flags: 0 + Sections: + - sectname: __data + segname: __DATA + addr: 0x1000 + size: 8 + offset: 0x1000 + align: 2 + reloff: 0x0 + nreloc: 0 + flags: 0x0 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: 4D00000009030000 + - sectname: __common + segname: __DATA + addr: 0x1008 + size: 8 + offset: 0x0 + align: 0 + reloff: 0x0 + nreloc: 0 + flags: 0x1 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + - cmd: LC_SEGMENT_64 + cmdsize: 72 + segname: __LINKEDIT + vmaddr: 8192 + vmsize: 944 + fileoff: 8192 + filesize: 944 + maxprot: 1 + initprot: 1 + nsects: 0 + flags: 0 + - cmd: LC_DYLD_INFO_ONLY + cmdsize: 48 + rebase_off: 0 + rebase_size: 0 + bind_off: 0 + bind_size: 0 + weak_bind_off: 0 + weak_bind_size: 0 + lazy_bind_off: 0 + lazy_bind_size: 0 + export_off: 8192 + export_size: 376 + - cmd: LC_SYMTAB + cmdsize: 24 + symoff: 8576 + nsyms: 12 + stroff: 8768 + strsize: 368 + - cmd: LC_DYSYMTAB + cmdsize: 80 + ilocalsym: 0 + nlocalsym: 0 + iextdefsym: 0 + nextdefsym: 11 + iundefsym: 11 + nundefsym: 1 + tocoff: 0 + ntoc: 0 + modtaboff: 0 + nmodtab: 0 + extrefsymoff: 0 + nextrefsyms: 0 + indirectsymoff: 0 + nindirectsyms: 0 + extreloff: 0 + nextrel: 0 + locreloff: 0 + nlocrel: 0 + - cmd: LC_ID_DYLIB + cmdsize: 120 + dylib: + name: 24 + timestamp: 0 + current_version: 65536 + compatibility_version: 65536 + Content: '/System/Library/Frameworks/SpecialLinkerSymbols.framework/Versions/A/SpecialLinkerSymbols' + ZeroPadBytes: 7 + - cmd: LC_UUID + cmdsize: 24 + uuid: 4C4C4478-5555-3144-A106-356C3C9DACA3 + - cmd: LC_BUILD_VERSION + cmdsize: 32 + platform: 1 + minos: 851968 + sdk: 983040 + ntools: 1 + Tools: + - tool: 4 + version: 1245184 + - cmd: LC_LOAD_DYLIB + cmdsize: 56 + dylib: + name: 24 + timestamp: 0 + current_version: 88539136 + compatibility_version: 65536 + Content: '/usr/lib/libSystem.B.dylib' + ZeroPadBytes: 6 + - cmd: LC_FUNCTION_STARTS + cmdsize: 16 + dataoff: 8568 + datasize: 8 + - cmd: LC_DATA_IN_CODE + cmdsize: 16 + dataoff: 8576 + datasize: 0 +LinkEditData: + ExportTrie: + TerminalSize: 0 + NodeOffset: 0 + Name: '' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 0 + NodeOffset: 11 + Name: _ + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 50 + Name: SpecialLinkerSymbolsVersion + Flags: 0x0 + Address: 0xBD8 + Other: 0x0 + ImportName: '' + - TerminalSize: 0 + NodeOffset: 55 + Name: symbol + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 63 + Name: '3' + Flags: 0x0 + Address: 0x1004 + Other: 0x0 + ImportName: '' + - TerminalSize: 3 + NodeOffset: 68 + Name: '1' + Flags: 0x0 + Address: 0x1000 + Other: 0x0 + ImportName: '' + - TerminalSize: 0 + NodeOffset: 73 + Name: '$ld$' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 0 + NodeOffset: 134 + Name: 'add$os10.' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 162 + Name: '4$_symbol2' + Flags: 0x0 + Address: 0x1008 + Other: 0x0 + ImportName: '' + - TerminalSize: 3 + NodeOffset: 167 + Name: '5$_symbol2' + Flags: 0x0 + Address: 0x1009 + Other: 0x0 + ImportName: '' + - TerminalSize: 0 + NodeOffset: 172 + Name: 'hide$os10.' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 200 + Name: '6$_symbol1' + Flags: 0x0 + Address: 0x100A + Other: 0x0 + ImportName: '' + - TerminalSize: 3 + NodeOffset: 205 + Name: '7$_symbol1' + Flags: 0x0 + Address: 0x100B + Other: 0x0 + ImportName: '' + - TerminalSize: 0 + NodeOffset: 210 + Name: 'weak$os10.' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 238 + Name: '5$_symbol3' + Flags: 0x0 + Address: 0x100F + Other: 0x0 + ImportName: '' + - TerminalSize: 3 + NodeOffset: 243 + Name: '4$_symbol3' + Flags: 0x0 + Address: 0x100E + Other: 0x0 + ImportName: '' + - TerminalSize: 0 + NodeOffset: 248 + Name: 'install_name$os10.' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 362 + Name: '4$/System/Library/Frameworks/A.framework/Versions/A/A' + Flags: 0x0 + Address: 0x100C + Other: 0x0 + ImportName: '' + - TerminalSize: 3 + NodeOffset: 367 + Name: '5$/System/Library/Frameworks/B.framework/Versions/A/B' + Flags: 0x0 + Address: 0x100D + Other: 0x0 + ImportName: '' + NameList: + - n_strx: 2 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4104 + - n_strx: 26 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4105 + - n_strx: 50 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4106 + - n_strx: 75 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4107 + - n_strx: 100 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4108 + - n_strx: 176 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4109 + - n_strx: 252 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4110 + - n_strx: 277 + n_type: 0xF + n_sect: 4 + n_desc: 0 + n_value: 4111 + - n_strx: 302 + n_type: 0xF + n_sect: 2 + n_desc: 0 + n_value: 3032 + - n_strx: 331 + n_type: 0xF + n_sect: 3 + n_desc: 0 + n_value: 4096 + - n_strx: 340 + n_type: 0xF + n_sect: 3 + n_desc: 0 + n_value: 4100 + - n_strx: 349 + n_type: 0x1 + n_sect: 0 + n_desc: 256 + n_value: 0 + StringTable: + - ' ' + - '$ld$add$os10.4$_symbol2' + - '$ld$add$os10.5$_symbol2' + - '$ld$hide$os10.6$_symbol1' + - '$ld$hide$os10.7$_symbol1' + - '$ld$install_name$os10.4$/System/Library/Frameworks/A.framework/Versions/A/A' + - '$ld$install_name$os10.5$/System/Library/Frameworks/B.framework/Versions/A/B' + - '$ld$weak$os10.4$_symbol3' + - '$ld$weak$os10.5$_symbol3' + - _SpecialLinkerSymbolsVersion + - _symbol1 + - _symbol3 + - dyld_stub_binder + - '' + - '' +... diff --git a/clang/test/InstallAPI/mismatching-objc-class-symbols.test b/clang/test/InstallAPI/mismatching-objc-class-symbols.test new file mode 100644 index 000000000000..3b4acf1035ac --- /dev/null +++ b/clang/test/InstallAPI/mismatching-objc-class-symbols.test @@ -0,0 +1,269 @@ +; RUN: rm -rf %t +; RUN: split-file %s %t +; RUN: sed -e "s|DSTROOT|%/t|g" %t/inputs.json.in > %t/inputs.json +; RUN: yaml2obj %t/swift-objc-class.yaml -o %t/libswift-objc.dylib + +// Try out dylib that only has 1 symbol for a ObjCClass, with no declarations in header. +; RUN: clang-installapi -target arm64-apple-macos14 -dynamiclib \ +; RUN: -install_name tmp.dylib --verify-against=%t/libswift-objc.dylib \ +; RUN: -I%t/usr/include %t/inputs.json -o %t/missing.tbd \ +; RUN: --verify-mode=ErrorsAndWarnings 2>&1 | FileCheck --check-prefix MISSING_DECL %s +; RUN: llvm-readtapi --compare %t/missing.tbd %t/missing-expected.tbd + +// Try out a dylib that only has 1 symbol for a ObjCClass, +// but a complete ObjCClass decl in header. +; RUN: clang-installapi -target arm64-apple-macos14 -dynamiclib \ +; RUN: -install_name tmp.dylib --verify-against=%t/libswift-objc.dylib \ +; RUN: -I%t/usr/include %t/inputs.json -o %t/mismatching.tbd \ +; RUN: --verify-mode=Pedantic -DFULL_DECL 2>&1 | FileCheck --check-prefix MISMATCH_DECL %s +; RUN: llvm-readtapi -compare %t/mismatching.tbd %t/mismatching-expected.tbd + +// Try out a dylib that only has 1 symbol for a ObjCClass, but is represented in header. +; RUN: clang-installapi -target arm64-apple-macos14 \ +; RUN: -install_name tmp.dylib --verify-against=%t/libswift-objc.dylib \ +; RUN: -I%t/usr/include %t/inputs.json -o %t/matching.tbd \ +; RUN: --verify-mode=Pedantic \ +; RUN: -DHAS_META_DECL 2>&1 | FileCheck --allow-empty %s + +; MISSING_DECL: violations found for arm64 +; MISSING_DECL-NEXT: warning: no declaration was found for exported symbol 'Metaclass of Suggestion' in dynamic library + +; MISMATCH_DECL: violations found for arm64-apple-macos14 +; MISMATCH_DECL: warning: declaration has external linkage, but dynamic library doesn't have symbol 'Class of Suggestion' + +; CHECK-NOT: error +; CHECK-NOT: warning + + +;--- usr/include/mismatch.h +#if HAS_META_DECL +int metaclass __asm("_OBJC_METACLASS_$_Suggestion"); +#endif + +#if FULL_DECL +@interface Suggestion +@end +#endif + +;--- inputs.json.in +{ + "headers": [ { + "path" : "DSTROOT/usr/include/mismatch.h", + "type" : "public" + } + ], + "version": "3" +} + +;--- missing-expected.tbd +--- !tapi-tbd +tbd-version: 4 +targets: [ arm64-macos ] +flags: [ not_app_extension_safe ] +install-name: tmp.dylib +current-version: 0 +compatibility-version: 0 +... + +;--- mismatching-expected.tbd +--- !tapi-tbd +tbd-version: 4 +targets: [ arm64-macos ] +flags: [ not_app_extension_safe ] +install-name: tmp.dylib +current-version: 0 +compatibility-version: 0 +exports: + - targets: [ arm64-macos ] + objc-classes: [ Suggestion ] +... + +;--- swift-objc-class.yaml +--- !mach-o +FileHeader: + magic: 0xFEEDFACF + cputype: 0x100000C + cpusubtype: 0x0 + filetype: 0x6 + ncmds: 13 + sizeofcmds: 752 + flags: 0x100085 + reserved: 0x0 +LoadCommands: + - cmd: LC_SEGMENT_64 + cmdsize: 232 + segname: __TEXT + vmaddr: 0 + vmsize: 16384 + fileoff: 0 + filesize: 16384 + maxprot: 5 + initprot: 5 + nsects: 2 + flags: 0 + Sections: + - sectname: __text + segname: __TEXT + addr: 0x330 + size: 0 + offset: 0x330 + align: 0 + reloff: 0x0 + nreloc: 0 + flags: 0x80000000 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: '' + - sectname: __const + segname: __TEXT + addr: 0x330 + size: 1 + offset: 0x330 + align: 0 + reloff: 0x0 + nreloc: 0 + flags: 0x0 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: '61' + - cmd: LC_SEGMENT_64 + cmdsize: 72 + segname: __LINKEDIT + vmaddr: 16384 + vmsize: 416 + fileoff: 16384 + filesize: 416 + maxprot: 1 + initprot: 1 + nsects: 0 + flags: 0 + - cmd: LC_DYLD_INFO_ONLY + cmdsize: 48 + rebase_off: 0 + rebase_size: 0 + bind_off: 0 + bind_size: 0 + weak_bind_off: 0 + weak_bind_size: 0 + lazy_bind_off: 0 + lazy_bind_size: 0 + export_off: 16384 + export_size: 40 + - cmd: LC_SYMTAB + cmdsize: 24 + symoff: 16432 + nsyms: 2 + stroff: 16464 + strsize: 48 + - cmd: LC_DYSYMTAB + cmdsize: 80 + ilocalsym: 0 + nlocalsym: 0 + iextdefsym: 0 + nextdefsym: 1 + iundefsym: 1 + nundefsym: 1 + tocoff: 0 + ntoc: 0 + modtaboff: 0 + nmodtab: 0 + extrefsymoff: 0 + nextrefsyms: 0 + indirectsymoff: 0 + nindirectsyms: 0 + extreloff: 0 + nextrel: 0 + locreloff: 0 + nlocrel: 0 + - cmd: LC_ID_DYLIB + cmdsize: 40 + dylib: + name: 24 + timestamp: 0 + current_version: 0 + compatibility_version: 0 + Content: tmp.dylib + ZeroPadBytes: 7 + - cmd: LC_UUID + cmdsize: 24 + uuid: 4C4C4443-5555-3144-A142-97179769CBE0 + - cmd: LC_BUILD_VERSION + cmdsize: 32 + platform: 1 + minos: 917504 + sdk: 983040 + ntools: 1 + Tools: + - tool: 4 + version: 1245184 + - cmd: LC_LOAD_DYLIB + cmdsize: 96 + dylib: + name: 24 + timestamp: 0 + current_version: 197656576 + compatibility_version: 19660800 + Content: '/System/Library/Frameworks/Foundation.framework/Versions/C/Foundation' + ZeroPadBytes: 3 + - cmd: LC_LOAD_DYLIB + cmdsize: 56 + dylib: + name: 24 + timestamp: 0 + current_version: 88473600 + compatibility_version: 65536 + Content: '/usr/lib/libSystem.B.dylib' + ZeroPadBytes: 6 + - cmd: LC_FUNCTION_STARTS + cmdsize: 16 + dataoff: 16424 + datasize: 8 + - cmd: LC_DATA_IN_CODE + cmdsize: 16 + dataoff: 16432 + datasize: 0 + - cmd: LC_CODE_SIGNATURE + cmdsize: 16 + dataoff: 16512 + datasize: 288 +LinkEditData: + ExportTrie: + TerminalSize: 0 + NodeOffset: 0 + Name: '' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 32 + Name: '_OBJC_METACLASS_$_Suggestion' + Flags: 0x0 + Address: 0x330 + Other: 0x0 + ImportName: '' + NameList: + - n_strx: 2 + n_type: 0xF + n_sect: 2 + n_desc: 0 + n_value: 816 + - n_strx: 31 + n_type: 0x1 + n_sect: 0 + n_desc: 512 + n_value: 0 + StringTable: + - ' ' + - '_OBJC_METACLASS_$_Suggestion' + - dyld_stub_binder + FunctionStarts: [ 0x330 ] +... +// Generated from: +// xcrun -sdk macosx clang tmp.c -dynamiclib -install_name tmp.dylib +// tmp.c: +// __attribute__((visibility("default"))) +// const char Meta __asm("_OBJC_METACLASS_$_Suggestion") = 'a'; diff --git a/clang/test/InstallAPI/symbol-flags.test b/clang/test/InstallAPI/symbol-flags.test new file mode 100644 index 000000000000..3f68afd17e3b --- /dev/null +++ b/clang/test/InstallAPI/symbol-flags.test @@ -0,0 +1,290 @@ +; RUN: rm -rf %t +; RUN: split-file %s %t +; RUN: sed -e "s|DSTROOT|%/t|g" %t/inputs.json.in > %t/inputs.json + +; RUN: yaml2obj %t/flags.yaml -o %t/SymbolFlags + +; RUN: not clang-installapi -x c++ --target=arm64-apple-macos13 \ +; RUN: -install_name /System/Library/Frameworks/SymbolFlags.framework/Versions/A/SymbolFlags \ +; RUN: -current_version 1 -compatibility_version 1 \ +; RUN: %t/inputs.json -o output.tbd \ +; RUN: --verify-against=%t/SymbolFlags \ +; RUN: --verify-mode=ErrorsOnly 2>&1 | FileCheck %s + +; CHECK: project.h:2:21: error: declaration '(tlv) val' is thread local, but symbol is not in dynamic library +; CHECK-NEXT: extern __thread int val; +; CHECK: project.h:3:13: error: dynamic library symbol '(weak-def) __Z12my_weak_funcv' is weak defined, but its declaration is not +; CHECK-NEXT: extern void my_weak_func(); + +;--- project.h +extern void my_func(); +extern __thread int val; +extern void my_weak_func(); + +;--- inputs.json.in +{ + "headers": [ { + "path" : "DSTROOT/project.h", + "type" : "project" + } + ], + "version": "3" +} + +;--- flags.yaml +--- !mach-o +FileHeader: + magic: 0xFEEDFACF + cputype: 0x100000C + cpusubtype: 0x0 + filetype: 0x6 + ncmds: 14 + sizeofcmds: 912 + flags: 0x118085 + reserved: 0x0 +LoadCommands: + - cmd: LC_SEGMENT_64 + cmdsize: 232 + segname: __TEXT + vmaddr: 0 + vmsize: 16384 + fileoff: 0 + filesize: 16384 + maxprot: 5 + initprot: 5 + nsects: 2 + flags: 0 + Sections: + - sectname: __text + segname: __TEXT + addr: 0xFB0 + size: 8 + offset: 0xFB0 + align: 2 + reloff: 0x0 + nreloc: 0 + flags: 0x80000400 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: C0035FD6C0035FD6 + - sectname: __unwind_info + segname: __TEXT + addr: 0xFB8 + size: 4152 + offset: 0xFB8 + align: 2 + reloff: 0x0 + nreloc: 0 + flags: 0x0 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + content: 010000001C000000010000002000000000000000200000000200000000000002B00F00003800000038000000B80F00000000000038000000030000000C0001001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + - cmd: LC_SEGMENT_64 + cmdsize: 152 + segname: __DATA + vmaddr: 16384 + vmsize: 16384 + fileoff: 16384 + filesize: 0 + maxprot: 3 + initprot: 3 + nsects: 1 + flags: 0 + Sections: + - sectname: __common + segname: __DATA + addr: 0x4000 + size: 4 + offset: 0x0 + align: 2 + reloff: 0x0 + nreloc: 0 + flags: 0x1 + reserved1: 0x0 + reserved2: 0x0 + reserved3: 0x0 + - cmd: LC_SEGMENT_64 + cmdsize: 72 + segname: __LINKEDIT + vmaddr: 32768 + vmsize: 480 + fileoff: 16384 + filesize: 480 + maxprot: 1 + initprot: 1 + nsects: 0 + flags: 0 + - cmd: LC_DYLD_INFO_ONLY + cmdsize: 48 + rebase_off: 0 + rebase_size: 0 + bind_off: 0 + bind_size: 0 + weak_bind_off: 0 + weak_bind_size: 0 + lazy_bind_off: 0 + lazy_bind_size: 0 + export_off: 16384 + export_size: 64 + - cmd: LC_SYMTAB + cmdsize: 24 + symoff: 16456 + nsyms: 4 + stroff: 16520 + strsize: 56 + - cmd: LC_DYSYMTAB + cmdsize: 80 + ilocalsym: 0 + nlocalsym: 0 + iextdefsym: 0 + nextdefsym: 3 + iundefsym: 3 + nundefsym: 1 + tocoff: 0 + ntoc: 0 + modtaboff: 0 + nmodtab: 0 + extrefsymoff: 0 + nextrefsyms: 0 + indirectsymoff: 0 + nindirectsyms: 0 + extreloff: 0 + nextrel: 0 + locreloff: 0 + nlocrel: 0 + - cmd: LC_ID_DYLIB + cmdsize: 96 + dylib: + name: 24 + timestamp: 0 + current_version: 65536 + compatibility_version: 65536 + Content: '/System/Library/Frameworks/SymbolFlags.framework/Versions/A/SymbolFlags' + ZeroPadBytes: 1 + - cmd: LC_UUID + cmdsize: 24 + uuid: 4C4C4436-5555-3144-A1AF-5D3063ACFC99 + - cmd: LC_BUILD_VERSION + cmdsize: 32 + platform: 1 + minos: 851968 + sdk: 983040 + ntools: 1 + Tools: + - tool: 4 + version: 1245184 + - cmd: LC_LOAD_DYLIB + cmdsize: 48 + dylib: + name: 24 + timestamp: 0 + current_version: 117985024 + compatibility_version: 65536 + Content: '/usr/lib/libc++.1.dylib' + ZeroPadBytes: 1 + - cmd: LC_LOAD_DYLIB + cmdsize: 56 + dylib: + name: 24 + timestamp: 0 + current_version: 88473600 + compatibility_version: 65536 + Content: '/usr/lib/libSystem.B.dylib' + ZeroPadBytes: 6 + - cmd: LC_FUNCTION_STARTS + cmdsize: 16 + dataoff: 16448 + datasize: 8 + - cmd: LC_DATA_IN_CODE + cmdsize: 16 + dataoff: 16456 + datasize: 0 + - cmd: LC_CODE_SIGNATURE + cmdsize: 16 + dataoff: 16576 + datasize: 288 +LinkEditData: + ExportTrie: + TerminalSize: 0 + NodeOffset: 0 + Name: '' + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 0 + NodeOffset: 5 + Name: _ + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 4 + NodeOffset: 16 + Name: val + Flags: 0x0 + Address: 0x4000 + Other: 0x0 + ImportName: '' + - TerminalSize: 0 + NodeOffset: 22 + Name: _Z + Flags: 0x0 + Address: 0x0 + Other: 0x0 + ImportName: '' + Children: + - TerminalSize: 3 + NodeOffset: 52 + Name: 7my_funcv + Flags: 0x0 + Address: 0xFB0 + Other: 0x0 + ImportName: '' + - TerminalSize: 3 + NodeOffset: 57 + Name: 12my_weak_funcv + Flags: 0x4 + Address: 0xFB4 + Other: 0x0 + ImportName: '' + NameList: + - n_strx: 2 + n_type: 0xF + n_sect: 1 + n_desc: 0 + n_value: 4016 + - n_strx: 15 + n_type: 0xF + n_sect: 1 + n_desc: 128 + n_value: 4020 + - n_strx: 34 + n_type: 0xF + n_sect: 3 + n_desc: 0 + n_value: 16384 + - n_strx: 39 + n_type: 0x1 + n_sect: 0 + n_desc: 512 + n_value: 0 + StringTable: + - ' ' + - __Z7my_funcv + - __Z12my_weak_funcv + - _val + - dyld_stub_binder + FunctionStarts: [ 0xFB0, 0xFB4 ] +... + +/// Generated from: +// clang++ -mtargetos=macosx13 -arch arm64 flags.cpp +// flags.cpp: +// __attribute__((visibility("default"))) void my_func() {} +// __attribute__((weak)) void my_weak_func() {} +// int val = 0; diff --git a/clang/tools/clang-installapi/ClangInstallAPI.cpp b/clang/tools/clang-installapi/ClangInstallAPI.cpp index 54e82d78d4d2..13061cfa36ee 100644 --- a/clang/tools/clang-installapi/ClangInstallAPI.cpp +++ b/clang/tools/clang-installapi/ClangInstallAPI.cpp @@ -123,7 +123,7 @@ static bool run(ArrayRef Args, const char *ProgName) { } } - if (Ctx.Verifier->getState() == DylibVerifier::Result::Invalid) + if (Ctx.Verifier->verifyRemainingSymbols() == DylibVerifier::Result::Invalid) return EXIT_FAILURE; // After symbols have been collected, prepare to write output. diff --git a/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp b/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp index 0694d8f28df6..2e36d4a8b98c 100644 --- a/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp +++ b/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp @@ -293,8 +293,11 @@ static Error readSymbols(MachOObjectFile *Obj, RecordsSlice &Slice, RecordLinkage Linkage = RecordLinkage::Unknown; SymbolFlags RecordFlags = SymbolFlags::None; - if (Opt.Undefineds && (Flags & SymbolRef::SF_Undefined)) { - Linkage = RecordLinkage::Undefined; + if (Flags & SymbolRef::SF_Undefined) { + if (Opt.Undefineds) + Linkage = RecordLinkage::Undefined; + else + continue; if (Flags & SymbolRef::SF_Weak) RecordFlags |= SymbolFlags::WeakReferenced; } else if (Flags & SymbolRef::SF_Exported) { -- GitLab From b8e53630f899ddb8a2ec0d37bcb86608d58c4960 Mon Sep 17 00:00:00 2001 From: Alexander Richardson Date: Thu, 21 Mar 2024 15:04:19 -0700 Subject: [PATCH 203/296] [compiler-rt] Avoid pulling in __cxa_pure_virtual When building optimized versions of the runtime libraries the compiler is generally able to elide these references, but when building them for maximum debug info (with -O0), these references remain which causes the test suite to fail for tests that do not pull in the C++ standard library. Reviewed By: vitalybuka Pull Request: https://github.com/llvm/llvm-project/pull/84613 --- .../sanitizer_stacktrace_printer.h | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h index 10361a320344..e39cb891575e 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h @@ -30,10 +30,15 @@ class StackTracePrinter { virtual void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, uptr address, const AddressInfo *info, - bool vs_style, - const char *strip_path_prefix = "") = 0; + bool vs_style, const char *strip_path_prefix = "") { + // Should be pure virtual, but we can't depend on __cxa_pure_virtual. + UNIMPLEMENTED(); + } - virtual bool RenderNeedsSymbolization(const char *format) = 0; + virtual bool RenderNeedsSymbolization(const char *format) { + // Should be pure virtual, but we can't depend on __cxa_pure_virtual. + UNIMPLEMENTED(); + } void RenderSourceLocation(InternalScopedString *buffer, const char *file, int line, int column, bool vs_style, @@ -44,7 +49,10 @@ class StackTracePrinter { const char *strip_path_prefix); virtual void RenderData(InternalScopedString *buffer, const char *format, const DataInfo *DI, - const char *strip_path_prefix = "") = 0; + const char *strip_path_prefix = "") { + // Should be pure virtual, but we can't depend on __cxa_pure_virtual. + UNIMPLEMENTED(); + } private: // To be called from StackTracePrinter::GetOrInit -- GitLab From c56211b2430cf63ba3a469a4ae89cf2e829e9332 Mon Sep 17 00:00:00 2001 From: Roland McGrath Date: Thu, 21 Mar 2024 15:11:31 -0700 Subject: [PATCH 204/296] [libc] Make math-macros.h C++-friendly (#86206) The isfinite, isnan, and isinf "functions" are specified by C99..C23 to be macros that act as type-generic functions. Defining them as their __builtin_* counterparts works fine for this. However, in C++ the identifiers need to be usable in different contexts, such as being declared inside a C++ namespace. So define inline constexpr template functions for them under `#ifdef __cplusplus`. --- libc/include/llvm-libc-macros/math-macros.h | 32 ++++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/libc/include/llvm-libc-macros/math-macros.h b/libc/include/llvm-libc-macros/math-macros.h index db8a4ea65bd6..2605535b927d 100644 --- a/libc/include/llvm-libc-macros/math-macros.h +++ b/libc/include/llvm-libc-macros/math-macros.h @@ -30,10 +30,6 @@ #define FP_LLOGB0 (-LONG_MAX - 1) #define FP_LLOGBNAN LONG_MAX -#define isfinite(x) __builtin_isfinite(x) -#define isinf(x) __builtin_isinf(x) -#define isnan(x) __builtin_isnan(x) - #ifdef __FAST_MATH__ #define math_errhandling 0 #elif defined(__NO_MATH_ERRNO__) @@ -44,4 +40,32 @@ #define math_errhandling (MATH_ERRNO | MATH_ERREXCEPT) #endif +// These must be type-generic functions. The C standard specifies them as +// being macros rather than functions, in fact. However, in C++ it's important +// that there be function declarations that don't interfere with other uses of +// the identifier, even in places with parentheses where a function-like macro +// will be expanded (such as a function declaration in a C++ namespace). + +#ifdef __cplusplus + +template inline constexpr bool isfinite(T x) { + return __builtin_isfinite(x); +} + +template inline constexpr bool isinf(T x) { + return __builtin_isinf(x); +} + +template inline constexpr bool isnan(T x) { + return __builtin_isnan(x); +} + +#else + +#define isfinite(x) __builtin_isfinite(x) +#define isinf(x) __builtin_isinf(x) +#define isnan(x) __builtin_isnan(x) + +#endif + #endif // LLVM_LIBC_MACROS_MATH_MACROS_H -- GitLab From 00f3454bbe04ae8cf0eeda981c439e7f97390bd4 Mon Sep 17 00:00:00 2001 From: Slava Zakharin Date: Thu, 21 Mar 2024 15:12:31 -0700 Subject: [PATCH 205/296] [flang][runtime] Added pseudo file unit for simplified PRINT. (#86134) A file unit is emulated via a temporary buffer that accumulates the output, which is printed out via std::printf at the end of the IO statement. This implementation will be used for the offload devices. --- flang/runtime/CMakeLists.txt | 2 + flang/runtime/external-unit.cpp | 333 ++++++++++++++++++++++++++++++++ flang/runtime/io-stmt.cpp | 10 + flang/runtime/lock.h | 17 +- flang/runtime/pseudo-unit.cpp | 167 ++++++++++++++++ flang/runtime/tools.h | 21 ++ flang/runtime/unit.cpp | 319 +----------------------------- flang/runtime/unit.h | 61 +++++- 8 files changed, 615 insertions(+), 315 deletions(-) create mode 100644 flang/runtime/external-unit.cpp create mode 100644 flang/runtime/pseudo-unit.cpp diff --git a/flang/runtime/CMakeLists.txt b/flang/runtime/CMakeLists.txt index 7dd60b5edcd5..021474871154 100644 --- a/flang/runtime/CMakeLists.txt +++ b/flang/runtime/CMakeLists.txt @@ -129,6 +129,7 @@ set(sources exceptions.cpp execute.cpp extensions.cpp + external-unit.cpp extrema.cpp file.cpp findloc.cpp @@ -149,6 +150,7 @@ set(sources numeric.cpp pointer.cpp product.cpp + pseudo-unit.cpp ragged.cpp random.cpp reduction.cpp diff --git a/flang/runtime/external-unit.cpp b/flang/runtime/external-unit.cpp new file mode 100644 index 000000000000..9d650ceca4a8 --- /dev/null +++ b/flang/runtime/external-unit.cpp @@ -0,0 +1,333 @@ +//===-- runtime/external-unit.cpp -----------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implemenation of ExternalFileUnit for RT_USE_PSEUDO_FILE_UNIT=0. +// +//===----------------------------------------------------------------------===// + +#include "tools.h" + +#if !defined(RT_USE_PSEUDO_FILE_UNIT) + +#include "io-error.h" +#include "lock.h" +#include "unit-map.h" +#include "unit.h" +#include +#include + +namespace Fortran::runtime::io { + +// The per-unit data structures are created on demand so that Fortran I/O +// should work without a Fortran main program. +static Lock unitMapLock; +static Lock createOpenLock; +static UnitMap *unitMap{nullptr}; + +void FlushOutputOnCrash(const Terminator &terminator) { + if (!defaultOutput && !errorOutput) { + return; + } + IoErrorHandler handler{terminator}; + handler.HasIoStat(); // prevent nested crash if flush has error + CriticalSection critical{unitMapLock}; + if (defaultOutput) { + defaultOutput->FlushOutput(handler); + } + if (errorOutput) { + errorOutput->FlushOutput(handler); + } +} + +ExternalFileUnit *ExternalFileUnit::LookUp(int unit) { + return GetUnitMap().LookUp(unit); +} + +ExternalFileUnit *ExternalFileUnit::LookUpOrCreate( + int unit, const Terminator &terminator, bool &wasExtant) { + return GetUnitMap().LookUpOrCreate(unit, terminator, wasExtant); +} + +ExternalFileUnit *ExternalFileUnit::LookUpOrCreateAnonymous(int unit, + Direction dir, Fortran::common::optional isUnformatted, + const Terminator &terminator) { + // Make sure that the returned anonymous unit has been opened + // not just created in the unitMap. + CriticalSection critical{createOpenLock}; + bool exists{false}; + ExternalFileUnit *result{ + GetUnitMap().LookUpOrCreate(unit, terminator, exists)}; + if (result && !exists) { + IoErrorHandler handler{terminator}; + result->OpenAnonymousUnit( + dir == Direction::Input ? OpenStatus::Unknown : OpenStatus::Replace, + Action::ReadWrite, Position::Rewind, Convert::Unknown, handler); + result->isUnformatted = isUnformatted; + } + return result; +} + +ExternalFileUnit *ExternalFileUnit::LookUp( + const char *path, std::size_t pathLen) { + return GetUnitMap().LookUp(path, pathLen); +} + +ExternalFileUnit &ExternalFileUnit::CreateNew( + int unit, const Terminator &terminator) { + bool wasExtant{false}; + ExternalFileUnit *result{ + GetUnitMap().LookUpOrCreate(unit, terminator, wasExtant)}; + RUNTIME_CHECK(terminator, result && !wasExtant); + return *result; +} + +ExternalFileUnit *ExternalFileUnit::LookUpForClose(int unit) { + return GetUnitMap().LookUpForClose(unit); +} + +ExternalFileUnit &ExternalFileUnit::NewUnit( + const Terminator &terminator, bool forChildIo) { + ExternalFileUnit &unit{GetUnitMap().NewUnit(terminator)}; + unit.createdForInternalChildIo_ = forChildIo; + return unit; +} + +bool ExternalFileUnit::OpenUnit(Fortran::common::optional status, + Fortran::common::optional action, Position position, + OwningPtr &&newPath, std::size_t newPathLength, Convert convert, + IoErrorHandler &handler) { + if (convert == Convert::Unknown) { + convert = executionEnvironment.conversion; + } + swapEndianness_ = convert == Convert::Swap || + (convert == Convert::LittleEndian && !isHostLittleEndian) || + (convert == Convert::BigEndian && isHostLittleEndian); + bool impliedClose{false}; + if (IsConnected()) { + bool isSamePath{newPath.get() && path() && pathLength() == newPathLength && + std::memcmp(path(), newPath.get(), newPathLength) == 0}; + if (status && *status != OpenStatus::Old && isSamePath) { + handler.SignalError("OPEN statement for connected unit may not have " + "explicit STATUS= other than 'OLD'"); + return impliedClose; + } + if (!newPath.get() || isSamePath) { + // OPEN of existing unit, STATUS='OLD' or unspecified, not new FILE= + newPath.reset(); + return impliedClose; + } + // Otherwise, OPEN on open unit with new FILE= implies CLOSE + DoImpliedEndfile(handler); + FlushOutput(handler); + TruncateFrame(0, handler); + Close(CloseStatus::Keep, handler); + impliedClose = true; + } + if (newPath.get() && newPathLength > 0) { + if (const auto *already{ + GetUnitMap().LookUp(newPath.get(), newPathLength)}) { + handler.SignalError(IostatOpenAlreadyConnected, + "OPEN(UNIT=%d,FILE='%.*s'): file is already connected to unit %d", + unitNumber_, static_cast(newPathLength), newPath.get(), + already->unitNumber_); + return impliedClose; + } + } + set_path(std::move(newPath), newPathLength); + Open(status.value_or(OpenStatus::Unknown), action, position, handler); + auto totalBytes{knownSize()}; + if (access == Access::Direct) { + if (!openRecl) { + handler.SignalError(IostatOpenBadRecl, + "OPEN(UNIT=%d,ACCESS='DIRECT'): record length is not known", + unitNumber()); + } else if (*openRecl <= 0) { + handler.SignalError(IostatOpenBadRecl, + "OPEN(UNIT=%d,ACCESS='DIRECT',RECL=%jd): record length is invalid", + unitNumber(), static_cast(*openRecl)); + } else if (totalBytes && (*totalBytes % *openRecl != 0)) { + handler.SignalError(IostatOpenBadRecl, + "OPEN(UNIT=%d,ACCESS='DIRECT',RECL=%jd): record length is not an " + "even divisor of the file size %jd", + unitNumber(), static_cast(*openRecl), + static_cast(*totalBytes)); + } + recordLength = openRecl; + } + endfileRecordNumber.reset(); + currentRecordNumber = 1; + if (totalBytes && access == Access::Direct && openRecl.value_or(0) > 0) { + endfileRecordNumber = 1 + (*totalBytes / *openRecl); + } + if (position == Position::Append) { + if (totalBytes) { + frameOffsetInFile_ = *totalBytes; + } + if (access != Access::Stream) { + if (!endfileRecordNumber) { + // Fake it so that we can backspace relative from the end + endfileRecordNumber = std::numeric_limits::max() - 2; + } + currentRecordNumber = *endfileRecordNumber; + } + } + return impliedClose; +} + +void ExternalFileUnit::OpenAnonymousUnit( + Fortran::common::optional status, + Fortran::common::optional action, Position position, + Convert convert, IoErrorHandler &handler) { + // I/O to an unconnected unit reads/creates a local file, e.g. fort.7 + std::size_t pathMaxLen{32}; + auto path{SizedNew{handler}(pathMaxLen)}; + std::snprintf(path.get(), pathMaxLen, "fort.%d", unitNumber_); + OpenUnit(status, action, position, std::move(path), std::strlen(path.get()), + convert, handler); +} + +void ExternalFileUnit::CloseUnit(CloseStatus status, IoErrorHandler &handler) { + DoImpliedEndfile(handler); + FlushOutput(handler); + Close(status, handler); +} + +void ExternalFileUnit::DestroyClosed() { + GetUnitMap().DestroyClosed(*this); // destroys *this +} + +Iostat ExternalFileUnit::SetDirection(Direction direction) { + if (direction == Direction::Input) { + if (mayRead()) { + direction_ = Direction::Input; + return IostatOk; + } else { + return IostatReadFromWriteOnly; + } + } else { + if (mayWrite()) { + direction_ = Direction::Output; + return IostatOk; + } else { + return IostatWriteToReadOnly; + } + } +} + +UnitMap &ExternalFileUnit::CreateUnitMap() { + Terminator terminator{__FILE__, __LINE__}; + IoErrorHandler handler{terminator}; + UnitMap &newUnitMap{*New{terminator}().release()}; + + bool wasExtant{false}; + ExternalFileUnit &out{*newUnitMap.LookUpOrCreate( + FORTRAN_DEFAULT_OUTPUT_UNIT, terminator, wasExtant)}; + RUNTIME_CHECK(terminator, !wasExtant); + out.Predefine(1); + handler.SignalError(out.SetDirection(Direction::Output)); + out.isUnformatted = false; + defaultOutput = &out; + + ExternalFileUnit &in{*newUnitMap.LookUpOrCreate( + FORTRAN_DEFAULT_INPUT_UNIT, terminator, wasExtant)}; + RUNTIME_CHECK(terminator, !wasExtant); + in.Predefine(0); + handler.SignalError(in.SetDirection(Direction::Input)); + in.isUnformatted = false; + defaultInput = ∈ + + ExternalFileUnit &error{ + *newUnitMap.LookUpOrCreate(FORTRAN_ERROR_UNIT, terminator, wasExtant)}; + RUNTIME_CHECK(terminator, !wasExtant); + error.Predefine(2); + handler.SignalError(error.SetDirection(Direction::Output)); + error.isUnformatted = false; + errorOutput = &error; + + return newUnitMap; +} + +// A back-up atexit() handler for programs that don't terminate with a main +// program END or a STOP statement or other Fortran-initiated program shutdown, +// such as programs with a C main() that terminate normally. It flushes all +// external I/O units. It is registered once the first time that any external +// I/O is attempted. +static void CloseAllExternalUnits() { + IoErrorHandler handler{"Fortran program termination"}; + ExternalFileUnit::CloseAll(handler); +} + +UnitMap &ExternalFileUnit::GetUnitMap() { + if (unitMap) { + return *unitMap; + } + { + CriticalSection critical{unitMapLock}; + if (unitMap) { + return *unitMap; + } + unitMap = &CreateUnitMap(); + } + std::atexit(CloseAllExternalUnits); + return *unitMap; +} + +void ExternalFileUnit::CloseAll(IoErrorHandler &handler) { + CriticalSection critical{unitMapLock}; + if (unitMap) { + unitMap->CloseAll(handler); + FreeMemoryAndNullify(unitMap); + } + defaultOutput = nullptr; + defaultInput = nullptr; + errorOutput = nullptr; +} + +void ExternalFileUnit::FlushAll(IoErrorHandler &handler) { + CriticalSection critical{unitMapLock}; + if (unitMap) { + unitMap->FlushAll(handler); + } +} + +int ExternalFileUnit::GetAsynchronousId(IoErrorHandler &handler) { + if (!mayAsynchronous()) { + handler.SignalError(IostatBadAsynchronous); + return -1; + } else { + for (int j{0}; 64 * j < maxAsyncIds; ++j) { + if (auto least{asyncIdAvailable_[j].LeastElement()}) { + asyncIdAvailable_[j].reset(*least); + return 64 * j + static_cast(*least); + } + } + handler.SignalError(IostatTooManyAsyncOps); + return -1; + } +} + +bool ExternalFileUnit::Wait(int id) { + if (static_cast(id) >= maxAsyncIds || + asyncIdAvailable_[id / 64].test(id % 64)) { + return false; + } else { + if (id == 0) { // means "all IDs" + for (int j{0}; 64 * j < maxAsyncIds; ++j) { + asyncIdAvailable_[j].set(); + } + asyncIdAvailable_[0].reset(0); + } else { + asyncIdAvailable_[id / 64].set(id % 64); + } + return true; + } +} + +} // namespace Fortran::runtime::io + +#endif // !defined(RT_USE_PSEUDO_FILE_UNIT) diff --git a/flang/runtime/io-stmt.cpp b/flang/runtime/io-stmt.cpp index 075d7b5ae518..e3f1214324d8 100644 --- a/flang/runtime/io-stmt.cpp +++ b/flang/runtime/io-stmt.cpp @@ -227,7 +227,17 @@ ConnectionState &ExternalIoStatementBase::GetConnectionState() { return unit_; } int ExternalIoStatementBase::EndIoStatement() { CompleteOperation(); auto result{IoStatementBase::EndIoStatement()}; +#if !defined(RT_USE_PSEUDO_FILE_UNIT) unit_.EndIoStatement(); // annihilates *this in unit_.u_ +#else + // Fetch the unit pointer before *this disappears. + ExternalFileUnit *unitPtr{&unit_}; + // The pseudo file units are dynamically allocated + // and are not tracked in the unit map. + // They have to be destructed and deallocated here. + unitPtr->~ExternalFileUnit(); + FreeMemory(unitPtr); +#endif return result; } diff --git a/flang/runtime/lock.h b/flang/runtime/lock.h index 5fdcf4745c21..61b06a62ff7c 100644 --- a/flang/runtime/lock.h +++ b/flang/runtime/lock.h @@ -12,6 +12,7 @@ #define FORTRAN_RUNTIME_LOCK_H_ #include "terminator.h" +#include "tools.h" // Avoid if possible to avoid introduction of C++ runtime // library dependence. @@ -35,7 +36,17 @@ namespace Fortran::runtime { class Lock { public: -#if USE_PTHREADS +#if RT_USE_PSEUDO_LOCK + // No lock implementation, e.g. for using together + // with RT_USE_PSEUDO_FILE_UNIT. + // The users of Lock class may use it under + // USE_PTHREADS and otherwise, so it has to provide + // all the interfaces. + void Take() {} + bool Try() { return true; } + void Drop() {} + bool TakeIfNoDeadlock() { return true; } +#elif USE_PTHREADS Lock() { pthread_mutex_init(&mutex_, nullptr); } ~Lock() { pthread_mutex_destroy(&mutex_); } void Take() { @@ -79,7 +90,9 @@ public: } private: -#if USE_PTHREADS +#if RT_USE_PSEUDO_FILE_UNIT + // No state. +#elif USE_PTHREADS pthread_mutex_t mutex_{}; volatile bool isBusy_{false}; volatile pthread_t holder_; diff --git a/flang/runtime/pseudo-unit.cpp b/flang/runtime/pseudo-unit.cpp new file mode 100644 index 000000000000..8b5f36e2233a --- /dev/null +++ b/flang/runtime/pseudo-unit.cpp @@ -0,0 +1,167 @@ +//===-- runtime/pseudo-unit.cpp -------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implemenation of ExternalFileUnit and PseudoOpenFile for +// RT_USE_PSEUDO_FILE_UNIT=1. +// +//===----------------------------------------------------------------------===// + +#include "tools.h" + +#if defined(RT_USE_PSEUDO_FILE_UNIT) + +#include "io-error.h" +#include "unit.h" +#include + +namespace Fortran::runtime::io { + +void FlushOutputOnCrash(const Terminator &) {} + +ExternalFileUnit *ExternalFileUnit::LookUp(int) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +ExternalFileUnit *ExternalFileUnit::LookUpOrCreate( + int, const Terminator &, bool &) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +ExternalFileUnit *ExternalFileUnit::LookUpOrCreateAnonymous(int unit, + Direction direction, Fortran::common::optional, + const Terminator &terminator) { + if (direction != Direction::Output) { + terminator.Crash("ExternalFileUnit only supports output IO"); + } + return New{terminator}(unit).release(); +} + +ExternalFileUnit *ExternalFileUnit::LookUp(const char *, std::size_t) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +ExternalFileUnit &ExternalFileUnit::CreateNew(int, const Terminator &) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +ExternalFileUnit *ExternalFileUnit::LookUpForClose(int) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +ExternalFileUnit &ExternalFileUnit::NewUnit(const Terminator &, bool) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +bool ExternalFileUnit::OpenUnit(Fortran::common::optional status, + Fortran::common::optional, Position, OwningPtr &&, + std::size_t, Convert, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +void ExternalFileUnit::OpenAnonymousUnit(Fortran::common::optional, + Fortran::common::optional, Position, Convert convert, + IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +void ExternalFileUnit::CloseUnit(CloseStatus, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +void ExternalFileUnit::DestroyClosed() { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +Iostat ExternalFileUnit::SetDirection(Direction direction) { + if (direction != Direction::Output) { + return IostatReadFromWriteOnly; + } + direction_ = direction; + return IostatOk; +} + +void ExternalFileUnit::CloseAll(IoErrorHandler &) {} + +void ExternalFileUnit::FlushAll(IoErrorHandler &) {} + +int ExternalFileUnit::GetAsynchronousId(IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +bool ExternalFileUnit::Wait(int) { + Terminator{__FILE__, __LINE__}.Crash("unsupported"); +} + +void PseudoOpenFile::set_mayAsynchronous(bool yes) { + if (yes) { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); + } +} + +Fortran::common::optional +PseudoOpenFile::knownSize() const { + Terminator{__FILE__, __LINE__}.Crash("unsupported"); +} + +void PseudoOpenFile::Open(OpenStatus, Fortran::common::optional, + Position, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +void PseudoOpenFile::Close(CloseStatus, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +std::size_t PseudoOpenFile::Read( + FileOffset, char *, std::size_t, std::size_t, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +std::size_t PseudoOpenFile::Write(FileOffset at, const char *buffer, + std::size_t bytes, IoErrorHandler &handler) { + if (at) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); + } + // TODO: use persistent string buffer that can be reallocated + // as needed, and only freed at destruction of *this. + auto string{SizedNew{handler}(bytes + 1)}; + std::memcpy(string.get(), buffer, bytes); + string.get()[bytes] = '\0'; + std::printf("%s", string.get()); + return bytes; +} + +void PseudoOpenFile::Truncate(FileOffset, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +int PseudoOpenFile::ReadAsynchronously( + FileOffset, char *, std::size_t, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +int PseudoOpenFile::WriteAsynchronously( + FileOffset, const char *, std::size_t, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +void PseudoOpenFile::Wait(int, IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +void PseudoOpenFile::WaitAll(IoErrorHandler &handler) { + handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +Position PseudoOpenFile::InquirePosition() const { + Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION); +} + +} // namespace Fortran::runtime::io + +#endif // defined(RT_USE_PSEUDO_FILE_UNIT) diff --git a/flang/runtime/tools.h b/flang/runtime/tools.h index df25eb888233..c70a1b438e33 100644 --- a/flang/runtime/tools.h +++ b/flang/runtime/tools.h @@ -21,6 +21,27 @@ #include #include +/// \macro RT_PRETTY_FUNCTION +/// Gets a user-friendly looking function signature for the current scope +/// using the best available method on each platform. The exact format of the +/// resulting string is implementation specific and non-portable, so this should +/// only be used, for example, for logging or diagnostics. +/// Copy of LLVM_PRETTY_FUNCTION +#if defined(_MSC_VER) +#define RT_PRETTY_FUNCTION __FUNCSIG__ +#elif defined(__GNUC__) || defined(__clang__) +#define RT_PRETTY_FUNCTION __PRETTY_FUNCTION__ +#else +#define RT_PRETTY_FUNCTION __func__ +#endif + +#if defined(RT_DEVICE_COMPILATION) +// Use the pseudo lock and pseudo file unit implementations +// for the device. +#define RT_USE_PSEUDO_LOCK 1 +#define RT_USE_PSEUDO_FILE_UNIT 1 +#endif + namespace Fortran::runtime { class Terminator; diff --git a/flang/runtime/unit.cpp b/flang/runtime/unit.cpp index 82f0e68cc20a..67f4775ae0a9 100644 --- a/flang/runtime/unit.cpp +++ b/flang/runtime/unit.cpp @@ -5,293 +5,23 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - +// +// Implementation of ExternalFileUnit common for both +// RT_USE_PSEUDO_FILE_UNIT=0 and RT_USE_PSEUDO_FILE_UNIT=1. +// +//===----------------------------------------------------------------------===// #include "unit.h" #include "io-error.h" #include "lock.h" #include "tools.h" -#include "unit-map.h" -#include "flang/Runtime/magic-numbers.h" -#include #include #include namespace Fortran::runtime::io { -// The per-unit data structures are created on demand so that Fortran I/O -// should work without a Fortran main program. -static Lock unitMapLock; -static Lock createOpenLock; -static UnitMap *unitMap{nullptr}; -static ExternalFileUnit *defaultInput{nullptr}; // unit 5 -static ExternalFileUnit *defaultOutput{nullptr}; // unit 6 -static ExternalFileUnit *errorOutput{nullptr}; // unit 0 extension - -void FlushOutputOnCrash(const Terminator &terminator) { - if (!defaultOutput && !errorOutput) { - return; - } - IoErrorHandler handler{terminator}; - handler.HasIoStat(); // prevent nested crash if flush has error - CriticalSection critical{unitMapLock}; - if (defaultOutput) { - defaultOutput->FlushOutput(handler); - } - if (errorOutput) { - errorOutput->FlushOutput(handler); - } -} - -ExternalFileUnit *ExternalFileUnit::LookUp(int unit) { - return GetUnitMap().LookUp(unit); -} - -ExternalFileUnit *ExternalFileUnit::LookUpOrCreate( - int unit, const Terminator &terminator, bool &wasExtant) { - return GetUnitMap().LookUpOrCreate(unit, terminator, wasExtant); -} - -ExternalFileUnit *ExternalFileUnit::LookUpOrCreateAnonymous(int unit, - Direction dir, Fortran::common::optional isUnformatted, - const Terminator &terminator) { - // Make sure that the returned anonymous unit has been opened - // not just created in the unitMap. - CriticalSection critical{createOpenLock}; - bool exists{false}; - ExternalFileUnit *result{ - GetUnitMap().LookUpOrCreate(unit, terminator, exists)}; - if (result && !exists) { - IoErrorHandler handler{terminator}; - result->OpenAnonymousUnit( - dir == Direction::Input ? OpenStatus::Unknown : OpenStatus::Replace, - Action::ReadWrite, Position::Rewind, Convert::Unknown, handler); - result->isUnformatted = isUnformatted; - } - return result; -} - -ExternalFileUnit *ExternalFileUnit::LookUp( - const char *path, std::size_t pathLen) { - return GetUnitMap().LookUp(path, pathLen); -} - -ExternalFileUnit &ExternalFileUnit::CreateNew( - int unit, const Terminator &terminator) { - bool wasExtant{false}; - ExternalFileUnit *result{ - GetUnitMap().LookUpOrCreate(unit, terminator, wasExtant)}; - RUNTIME_CHECK(terminator, result && !wasExtant); - return *result; -} - -ExternalFileUnit *ExternalFileUnit::LookUpForClose(int unit) { - return GetUnitMap().LookUpForClose(unit); -} - -ExternalFileUnit &ExternalFileUnit::NewUnit( - const Terminator &terminator, bool forChildIo) { - ExternalFileUnit &unit{GetUnitMap().NewUnit(terminator)}; - unit.createdForInternalChildIo_ = forChildIo; - return unit; -} - -bool ExternalFileUnit::OpenUnit(Fortran::common::optional status, - Fortran::common::optional action, Position position, - OwningPtr &&newPath, std::size_t newPathLength, Convert convert, - IoErrorHandler &handler) { - if (convert == Convert::Unknown) { - convert = executionEnvironment.conversion; - } - swapEndianness_ = convert == Convert::Swap || - (convert == Convert::LittleEndian && !isHostLittleEndian) || - (convert == Convert::BigEndian && isHostLittleEndian); - bool impliedClose{false}; - if (IsConnected()) { - bool isSamePath{newPath.get() && path() && pathLength() == newPathLength && - std::memcmp(path(), newPath.get(), newPathLength) == 0}; - if (status && *status != OpenStatus::Old && isSamePath) { - handler.SignalError("OPEN statement for connected unit may not have " - "explicit STATUS= other than 'OLD'"); - return impliedClose; - } - if (!newPath.get() || isSamePath) { - // OPEN of existing unit, STATUS='OLD' or unspecified, not new FILE= - newPath.reset(); - return impliedClose; - } - // Otherwise, OPEN on open unit with new FILE= implies CLOSE - DoImpliedEndfile(handler); - FlushOutput(handler); - TruncateFrame(0, handler); - Close(CloseStatus::Keep, handler); - impliedClose = true; - } - if (newPath.get() && newPathLength > 0) { - if (const auto *already{ - GetUnitMap().LookUp(newPath.get(), newPathLength)}) { - handler.SignalError(IostatOpenAlreadyConnected, - "OPEN(UNIT=%d,FILE='%.*s'): file is already connected to unit %d", - unitNumber_, static_cast(newPathLength), newPath.get(), - already->unitNumber_); - return impliedClose; - } - } - set_path(std::move(newPath), newPathLength); - Open(status.value_or(OpenStatus::Unknown), action, position, handler); - auto totalBytes{knownSize()}; - if (access == Access::Direct) { - if (!openRecl) { - handler.SignalError(IostatOpenBadRecl, - "OPEN(UNIT=%d,ACCESS='DIRECT'): record length is not known", - unitNumber()); - } else if (*openRecl <= 0) { - handler.SignalError(IostatOpenBadRecl, - "OPEN(UNIT=%d,ACCESS='DIRECT',RECL=%jd): record length is invalid", - unitNumber(), static_cast(*openRecl)); - } else if (totalBytes && (*totalBytes % *openRecl != 0)) { - handler.SignalError(IostatOpenBadRecl, - "OPEN(UNIT=%d,ACCESS='DIRECT',RECL=%jd): record length is not an " - "even divisor of the file size %jd", - unitNumber(), static_cast(*openRecl), - static_cast(*totalBytes)); - } - recordLength = openRecl; - } - endfileRecordNumber.reset(); - currentRecordNumber = 1; - if (totalBytes && access == Access::Direct && openRecl.value_or(0) > 0) { - endfileRecordNumber = 1 + (*totalBytes / *openRecl); - } - if (position == Position::Append) { - if (totalBytes) { - frameOffsetInFile_ = *totalBytes; - } - if (access != Access::Stream) { - if (!endfileRecordNumber) { - // Fake it so that we can backspace relative from the end - endfileRecordNumber = std::numeric_limits::max() - 2; - } - currentRecordNumber = *endfileRecordNumber; - } - } - return impliedClose; -} - -void ExternalFileUnit::OpenAnonymousUnit( - Fortran::common::optional status, - Fortran::common::optional action, Position position, - Convert convert, IoErrorHandler &handler) { - // I/O to an unconnected unit reads/creates a local file, e.g. fort.7 - std::size_t pathMaxLen{32}; - auto path{SizedNew{handler}(pathMaxLen)}; - std::snprintf(path.get(), pathMaxLen, "fort.%d", unitNumber_); - OpenUnit(status, action, position, std::move(path), std::strlen(path.get()), - convert, handler); -} - -void ExternalFileUnit::CloseUnit(CloseStatus status, IoErrorHandler &handler) { - DoImpliedEndfile(handler); - FlushOutput(handler); - Close(status, handler); -} - -void ExternalFileUnit::DestroyClosed() { - GetUnitMap().DestroyClosed(*this); // destroys *this -} - -Iostat ExternalFileUnit::SetDirection(Direction direction) { - if (direction == Direction::Input) { - if (mayRead()) { - direction_ = Direction::Input; - return IostatOk; - } else { - return IostatReadFromWriteOnly; - } - } else { - if (mayWrite()) { - direction_ = Direction::Output; - return IostatOk; - } else { - return IostatWriteToReadOnly; - } - } -} - -UnitMap &ExternalFileUnit::CreateUnitMap() { - Terminator terminator{__FILE__, __LINE__}; - IoErrorHandler handler{terminator}; - UnitMap &newUnitMap{*New{terminator}().release()}; - - bool wasExtant{false}; - ExternalFileUnit &out{*newUnitMap.LookUpOrCreate( - FORTRAN_DEFAULT_OUTPUT_UNIT, terminator, wasExtant)}; - RUNTIME_CHECK(terminator, !wasExtant); - out.Predefine(1); - handler.SignalError(out.SetDirection(Direction::Output)); - out.isUnformatted = false; - defaultOutput = &out; - - ExternalFileUnit &in{*newUnitMap.LookUpOrCreate( - FORTRAN_DEFAULT_INPUT_UNIT, terminator, wasExtant)}; - RUNTIME_CHECK(terminator, !wasExtant); - in.Predefine(0); - handler.SignalError(in.SetDirection(Direction::Input)); - in.isUnformatted = false; - defaultInput = ∈ - - ExternalFileUnit &error{ - *newUnitMap.LookUpOrCreate(FORTRAN_ERROR_UNIT, terminator, wasExtant)}; - RUNTIME_CHECK(terminator, !wasExtant); - error.Predefine(2); - handler.SignalError(error.SetDirection(Direction::Output)); - error.isUnformatted = false; - errorOutput = &error; - - return newUnitMap; -} - -// A back-up atexit() handler for programs that don't terminate with a main -// program END or a STOP statement or other Fortran-initiated program shutdown, -// such as programs with a C main() that terminate normally. It flushes all -// external I/O units. It is registered once the first time that any external -// I/O is attempted. -static void CloseAllExternalUnits() { - IoErrorHandler handler{"Fortran program termination"}; - ExternalFileUnit::CloseAll(handler); -} - -UnitMap &ExternalFileUnit::GetUnitMap() { - if (unitMap) { - return *unitMap; - } - { - CriticalSection critical{unitMapLock}; - if (unitMap) { - return *unitMap; - } - unitMap = &CreateUnitMap(); - } - std::atexit(CloseAllExternalUnits); - return *unitMap; -} - -void ExternalFileUnit::CloseAll(IoErrorHandler &handler) { - CriticalSection critical{unitMapLock}; - if (unitMap) { - unitMap->CloseAll(handler); - FreeMemoryAndNullify(unitMap); - } - defaultOutput = nullptr; - defaultInput = nullptr; - errorOutput = nullptr; -} - -void ExternalFileUnit::FlushAll(IoErrorHandler &handler) { - CriticalSection critical{unitMapLock}; - if (unitMap) { - unitMap->FlushAll(handler); - } -} +ExternalFileUnit *defaultInput{nullptr}; // unit 5 +ExternalFileUnit *defaultOutput{nullptr}; // unit 6 +ExternalFileUnit *errorOutput{nullptr}; // unit 0 extension static inline void SwapEndianness( char *data, std::size_t bytes, std::size_t elementBytes) { @@ -999,39 +729,6 @@ void ExternalFileUnit::PopChildIo(ChildIo &child) { child_.reset(child.AcquirePrevious().release()); // deletes top child } -int ExternalFileUnit::GetAsynchronousId(IoErrorHandler &handler) { - if (!mayAsynchronous()) { - handler.SignalError(IostatBadAsynchronous); - return -1; - } else { - for (int j{0}; 64 * j < maxAsyncIds; ++j) { - if (auto least{asyncIdAvailable_[j].LeastElement()}) { - asyncIdAvailable_[j].reset(*least); - return 64 * j + static_cast(*least); - } - } - handler.SignalError(IostatTooManyAsyncOps); - return -1; - } -} - -bool ExternalFileUnit::Wait(int id) { - if (static_cast(id) >= maxAsyncIds || - asyncIdAvailable_[id / 64].test(id % 64)) { - return false; - } else { - if (id == 0) { // means "all IDs" - for (int j{0}; 64 * j < maxAsyncIds; ++j) { - asyncIdAvailable_[j].set(); - } - asyncIdAvailable_[0].reset(0); - } else { - asyncIdAvailable_[id / 64].set(id % 64); - } - return true; - } -} - std::int32_t ExternalFileUnit::ReadHeaderOrFooter(std::int64_t frameOffset) { std::int32_t word; char *wordPtr{reinterpret_cast(&word)}; diff --git a/flang/runtime/unit.h b/flang/runtime/unit.h index fc5bead7e1d9..5f854abd42f6 100644 --- a/flang/runtime/unit.h +++ b/flang/runtime/unit.h @@ -31,10 +31,67 @@ namespace Fortran::runtime::io { class UnitMap; class ChildIo; +class ExternalFileUnit; + +// Predefined file units. +extern ExternalFileUnit *defaultInput; // unit 5 +extern ExternalFileUnit *defaultOutput; // unit 6 +extern ExternalFileUnit *errorOutput; // unit 0 extension + +#if defined(RT_USE_PSEUDO_FILE_UNIT) +// A flavor of OpenFile class that pretends to be a terminal, +// and only provides basic buffering of the output +// in an internal buffer, and Write's the output +// using std::printf(). Since it does not rely on file system +// APIs, it can be used to implement external output +// for offload devices. +class PseudoOpenFile { +public: + using FileOffset = std::int64_t; + + const char *path() const { return nullptr; } + std::size_t pathLength() const { return 0; } + void set_path(OwningPtr &&, std::size_t bytes) {} + bool mayRead() const { return false; } + bool mayWrite() const { return true; } + bool mayPosition() const { return false; } + bool mayAsynchronous() const { return false; } + void set_mayAsynchronous(bool yes); + // Pretend to be a terminal to force the output + // at the end of IO statement. + bool isTerminal() const { return true; } + bool isWindowsTextFile() const { return false; } + Fortran::common::optional knownSize() const; + bool IsConnected() const { return false; } + void Open(OpenStatus, Fortran::common::optional, Position, + IoErrorHandler &); + void Predefine(int fd) {} + void Close(CloseStatus, IoErrorHandler &); + std::size_t Read(FileOffset, char *, std::size_t minBytes, + std::size_t maxBytes, IoErrorHandler &); + std::size_t Write(FileOffset, const char *, std::size_t, IoErrorHandler &); + void Truncate(FileOffset, IoErrorHandler &); + int ReadAsynchronously(FileOffset, char *, std::size_t, IoErrorHandler &); + int WriteAsynchronously( + FileOffset, const char *, std::size_t, IoErrorHandler &); + void Wait(int id, IoErrorHandler &); + void WaitAll(IoErrorHandler &); + Position InquirePosition() const; +}; +#endif // defined(RT_USE_PSEUDO_FILE_UNIT) + +#if !defined(RT_USE_PSEUDO_FILE_UNIT) +using OpenFileClass = OpenFile; +using FileFrameClass = FileFrame; +#else // defined(RT_USE_PSEUDO_FILE_UNIT) +using OpenFileClass = PseudoOpenFile; +// Use not so big buffer for the pseudo file unit frame. +using FileFrameClass = FileFrame; +#endif // defined(RT_USE_PSEUDO_FILE_UNIT) class ExternalFileUnit : public ConnectionState, - public OpenFile, - public FileFrame { + public OpenFileClass, + public FileFrameClass { public: static constexpr int maxAsyncIds{64 * 16}; -- GitLab From 6f9297fc4da9df776aef7ee9a18ac426053aaed4 Mon Sep 17 00:00:00 2001 From: Alexander Richardson Date: Thu, 21 Mar 2024 15:22:06 -0700 Subject: [PATCH 206/296] [compiler-rt] Fix build race with COMPILER_RT_TEST_STANDALONE_BUILD_LIBS Since this standalone build configuration uses the runtime libraries that are being built just now, we need to ensure that e.g. the TSan unit tests depend on the tsan runtime library. Also fix TSAN_DEPS being overridden to not include the tsan runtime (commit .....). This change fixes a build race seen in the CI checks for TsanRtlTest-x86_64-Test in https://github.com/llvm/llvm-project/pull/83088. Reviewed By: vitalybuka Pull Request: https://github.com/llvm/llvm-project/pull/83650 --- compiler-rt/cmake/Modules/CompilerRTCompile.cmake | 2 +- compiler-rt/lib/msan/tests/CMakeLists.txt | 6 +++--- compiler-rt/lib/tsan/tests/CMakeLists.txt | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compiler-rt/cmake/Modules/CompilerRTCompile.cmake b/compiler-rt/cmake/Modules/CompilerRTCompile.cmake index d5b2e7970f11..1629db18f1c2 100644 --- a/compiler-rt/cmake/Modules/CompilerRTCompile.cmake +++ b/compiler-rt/cmake/Modules/CompilerRTCompile.cmake @@ -46,7 +46,7 @@ function(sanitizer_test_compile obj_list source arch) # Write out architecture-specific flags into TARGET_CFLAGS variable. get_target_flags_for_arch(${arch} TARGET_CFLAGS) set(COMPILE_DEPS ${TEST_COMPILE_DEPS}) - if(NOT COMPILER_RT_STANDALONE_BUILD) + if(NOT COMPILER_RT_STANDALONE_BUILD OR COMPILER_RT_TEST_STANDALONE_BUILD_LIBS) list(APPEND COMPILE_DEPS ${TEST_DEPS}) endif() clang_compile(${output_obj} ${source} diff --git a/compiler-rt/lib/msan/tests/CMakeLists.txt b/compiler-rt/lib/msan/tests/CMakeLists.txt index 4f09f1e6a691..e0771dd5a1a7 100644 --- a/compiler-rt/lib/msan/tests/CMakeLists.txt +++ b/compiler-rt/lib/msan/tests/CMakeLists.txt @@ -69,9 +69,9 @@ macro(msan_compile obj_list source arch kind cflags) sanitizer_test_compile( ${obj_list} ${source} ${arch} KIND ${kind} - COMPILE_DEPS ${MSAN_UNITTEST_HEADERS} + COMPILE_DEPS ${MSAN_UNITTEST_HEADERS} libcxx_msan_${arch}-build DEPS msan - CFLAGS -isystem ${CMAKE_CURRENT_BINARY_DIR}/../libcxx_msan_${arch}/include/c++/v1 + CFLAGS -isystem ${MSAN_LIBCXX_DIR}/../include/c++/v1 ${MSAN_UNITTEST_INSTRUMENTED_CFLAGS} ${cflags} ) endmacro() @@ -120,7 +120,7 @@ macro(add_msan_tests_for_arch arch kind cflags) set(MSAN_TEST_DEPS ${MSAN_TEST_OBJECTS} libcxx_msan_${arch}-build ${MSAN_LOADABLE_SO} "${MSAN_LIBCXX_DIR}/libc++.a" "${MSAN_LIBCXX_DIR}/libc++abi.a") - list(APPEND MSAN_TEST_DEPS msan) + list(APPEND MSAN_TEST_DEPS msan libcxx_msan_${arch}-build) get_target_flags_for_arch(${arch} TARGET_LINK_FLAGS) add_compiler_rt_test(MsanUnitTests "Msan-${arch}${kind}-Test" ${arch} OBJECTS ${MSAN_TEST_OBJECTS} "${MSAN_LIBCXX_DIR}/libc++.a" "${MSAN_LIBCXX_DIR}/libc++abi.a" diff --git a/compiler-rt/lib/tsan/tests/CMakeLists.txt b/compiler-rt/lib/tsan/tests/CMakeLists.txt index ad8cc9b0eb05..1bc08bbf7450 100644 --- a/compiler-rt/lib/tsan/tests/CMakeLists.txt +++ b/compiler-rt/lib/tsan/tests/CMakeLists.txt @@ -67,7 +67,7 @@ endforeach() set(TSAN_DEPS tsan) # TSan uses C++ standard library headers. if (TARGET cxx-headers OR HAVE_LIBCXX) - set(TSAN_DEPS cxx-headers) + list(APPEND TSAN_DEPS cxx-headers) endif() # add_tsan_unittest( -- GitLab From 8d1affb87181b9636b87e04a245bcde06f8a7d47 Mon Sep 17 00:00:00 2001 From: Diego Caballero Date: Thu, 21 Mar 2024 15:28:36 -0700 Subject: [PATCH 207/296] Update @dcaballe in CODEOWNERS (#86177) It fixes a few rules that don't seem to be working and adding myself to a few paths where I've been contributing and can offer my review. Also minor sorting changes. --- .github/CODEOWNERS | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c246c42b0904..9cf6f5d79d41 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -59,8 +59,8 @@ clang/test/AST/Interp/ @tbaederr /mlir/Dialect/*/Transforms/Bufferize.cpp @matthias-springer # Linalg Dialect in MLIR. -/mlir/include/mlir/Dialect/Linalg @dcaballe @nicolasvasilache -/mlir/lib/Dialect/Linalg @dcaballe @nicolasvasilache +/mlir/include/mlir/Dialect/Linalg/* @dcaballe @nicolasvasilache +/mlir/lib/Dialect/Linalg/* @dcaballe @nicolasvasilache /mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp @MaheshRavishankar @nicolasvasilache /mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp @MaheshRavishankar @nicolasvasilache /mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp @MaheshRavishankar @nicolasvasilache @@ -77,14 +77,14 @@ clang/test/AST/Interp/ @tbaederr /mlir/**/*SME* @banach-space @dcaballe @nicolasvasilache /mlir/**/*SVE* @banach-space @dcaballe @nicolasvasilache /mlir/**/*VectorInterfaces* @dcaballe @nicolasvasilache -/mlir/**/*VectorToSCF* @banach-space @dcaballe @nicolasvasilache @matthias-springer +/mlir/**/*VectorToSCF* @banach-space @dcaballe @matthias-springer @nicolasvasilache /mlir/**/*VectorToLLVM* @banach-space @dcaballe @nicolasvasilache /mlir/**/*X86Vector* @aartbik @dcaballe @nicolasvasilache -/mlir/include/mlir/Dialect/Vector @dcaballe @nicolasvasilache -/mlir/lib/Dialect/Vector @dcaballe @nicolasvasilache -/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp @MaheshRavishankar @nicolasvasilache -/mlir/**/*EmulateNarrowType* @hanhanW +/mlir/include/mlir/Dialect/Vector/* @dcaballe @nicolasvasilache +/mlir/lib/Dialect/Vector/* @dcaballe @nicolasvasilache /mlir/lib/Dialect/Vector/Transforms/* @hanhanW @nicolasvasilache +/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp @MaheshRavishankar @nicolasvasilache +/mlir/**/*EmulateNarrowType* @dcaballe @hanhanW # Presburger library in MLIR /mlir/**/*Presburger* @Groverkss @Superty -- GitLab From 1538b82fd395a2fba90842b8a8010b8dcc919499 Mon Sep 17 00:00:00 2001 From: Cooper Partin Date: Thu, 21 Mar 2024 15:40:29 -0700 Subject: [PATCH 208/296] Revert "Add support for PSV EntryFunctionName (#84409)" (#86211) This reverts commit cde54df39cab3a1d60a3e1862ab341609bee3cc3. Co-authored-by: Cooper Partin --- llvm/include/llvm/BinaryFormat/DXContainer.h | 13 --- llvm/include/llvm/MC/DXContainerPSVInfo.h | 25 ++-- llvm/include/llvm/MC/StringTableBuilder.h | 8 +- llvm/include/llvm/Object/DXContainer.h | 16 +-- .../include/llvm/ObjectYAML/DXContainerYAML.h | 5 +- llvm/lib/MC/DXContainerPSVInfo.cpp | 75 ++++-------- llvm/lib/Object/DXContainer.cpp | 15 +-- llvm/lib/ObjectYAML/DXContainerEmitter.cpp | 3 +- llvm/lib/ObjectYAML/DXContainerYAML.cpp | 15 --- .../DXContainer/PSVv3-amplification.yaml | 97 ---------------- .../ObjectYAML/DXContainer/PSVv3-compute.yaml | 95 --------------- .../ObjectYAML/DXContainer/PSVv3-domain.yaml | 105 ----------------- .../DXContainer/PSVv3-geometry.yaml | 105 ----------------- .../ObjectYAML/DXContainer/PSVv3-hull.yaml | 107 ----------------- .../ObjectYAML/DXContainer/PSVv3-mesh.yaml | 109 ------------------ .../ObjectYAML/DXContainer/PSVv3-pixel.yaml | 99 ---------------- .../ObjectYAML/DXContainer/PSVv3-vertex.yaml | 97 ---------------- llvm/tools/obj2yaml/dxcontainer2yaml.cpp | 3 - 18 files changed, 52 insertions(+), 940 deletions(-) delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml delete mode 100644 llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h index ba882c4a6f32..532f9481766a 100644 --- a/llvm/include/llvm/BinaryFormat/DXContainer.h +++ b/llvm/include/llvm/BinaryFormat/DXContainer.h @@ -424,19 +424,6 @@ struct ResourceBindInfo : public v0::ResourceBindInfo { }; } // namespace v2 - -namespace v3 { -struct RuntimeInfo : public v2::RuntimeInfo { - uint32_t EntryNameOffset; - - void swapBytes() { sys::swapByteOrder(EntryNameOffset); } - - void swapBytes(Triple::EnvironmentType Stage) { - v2::RuntimeInfo::swapBytes(Stage); - } -}; - -} // namespace v3 } // namespace PSV #define COMPONENT_PRECISION(Val, Enum) Enum = Val, diff --git a/llvm/include/llvm/MC/DXContainerPSVInfo.h b/llvm/include/llvm/MC/DXContainerPSVInfo.h index bad2fe78eb8f..7d21c18d252f 100644 --- a/llvm/include/llvm/MC/DXContainerPSVInfo.h +++ b/llvm/include/llvm/MC/DXContainerPSVInfo.h @@ -9,11 +9,9 @@ #ifndef LLVM_MC_DXCONTAINERPSVINFO_H #define LLVM_MC_DXCONTAINERPSVINFO_H -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/DXContainer.h" -#include "llvm/MC/StringTableBuilder.h" #include "llvm/TargetParser/Triple.h" #include @@ -47,9 +45,8 @@ struct PSVSignatureElement { // modifiable format, and can be used to serialize the data back into valid PSV // RuntimeInfo. struct PSVRuntimeInfo { - PSVRuntimeInfo() : DXConStrTabBuilder(StringTableBuilder::DXContainer) {} bool IsFinalized = false; - dxbc::PSV::v3::RuntimeInfo BaseData; + dxbc::PSV::v2::RuntimeInfo BaseData; SmallVector Resources; SmallVector InputElements; SmallVector OutputElements; @@ -67,7 +64,6 @@ struct PSVRuntimeInfo { std::array, 4> InputOutputMap; SmallVector InputPatchMap; SmallVector PatchOutputMap; - llvm::StringRef EntryName; // Serialize PSVInfo into the provided raw_ostream. The version field // specifies the data version to encode, the default value specifies encoding @@ -75,12 +71,19 @@ struct PSVRuntimeInfo { void write(raw_ostream &OS, uint32_t Version = std::numeric_limits::max()) const; - void finalize(Triple::EnvironmentType Stage); - -private: - SmallVector IndexBuffer; - SmallVector SignatureElements; - StringTableBuilder DXConStrTabBuilder; + void finalize(Triple::EnvironmentType Stage) { + IsFinalized = true; + BaseData.SigInputElements = static_cast(InputElements.size()); + BaseData.SigOutputElements = static_cast(OutputElements.size()); + BaseData.SigPatchOrPrimElements = + static_cast(PatchOrPrimElements.size()); + if (!sys::IsBigEndianHost) + return; + BaseData.swapBytes(); + BaseData.swapBytes(Stage); + for (auto &Res : Resources) + Res.swapBytes(); + } }; class Signature { diff --git a/llvm/include/llvm/MC/StringTableBuilder.h b/llvm/include/llvm/MC/StringTableBuilder.h index a738683548cf..4ee421e22c17 100644 --- a/llvm/include/llvm/MC/StringTableBuilder.h +++ b/llvm/include/llvm/MC/StringTableBuilder.h @@ -74,8 +74,12 @@ public: /// Check if a string is contained in the string table. Since this class /// doesn't store the string values, this function can be used to check if /// storage needs to be done prior to adding the string. - bool contains(StringRef S) const { return contains(CachedHashStringRef(S)); } - bool contains(CachedHashStringRef S) const { return StringIndexMap.count(S); } + bool contains(StringRef S) const { + return contains(CachedHashStringRef(S)); + } + bool contains(CachedHashStringRef S) const { + return StringIndexMap.count(S); + } size_t getSize() const { return Size; } void clear(); diff --git a/llvm/include/llvm/Object/DXContainer.h b/llvm/include/llvm/Object/DXContainer.h index 19c83ba6c6e8..b6e3d321da24 100644 --- a/llvm/include/llvm/Object/DXContainer.h +++ b/llvm/include/llvm/Object/DXContainer.h @@ -125,8 +125,7 @@ class PSVRuntimeInfo { uint32_t Size; using InfoStruct = std::variant; + dxbc::PSV::v1::RuntimeInfo, dxbc::PSV::v2::RuntimeInfo>; InfoStruct BasicInfo; ResourceArray Resources; StringRef StringTable; @@ -152,11 +151,9 @@ public: ResourceArray getResources() const { return Resources; } uint32_t getVersion() const { - return Size >= sizeof(dxbc::PSV::v3::RuntimeInfo) - ? 3 - : (Size >= sizeof(dxbc::PSV::v2::RuntimeInfo) ? 2 - : (Size >= sizeof(dxbc::PSV::v1::RuntimeInfo)) ? 1 - : 0); + return Size >= sizeof(dxbc::PSV::v2::RuntimeInfo) + ? 2 + : (Size >= sizeof(dxbc::PSV::v1::RuntimeInfo) ? 1 : 0); } uint32_t getResourceStride() const { return Resources.Stride; } @@ -164,11 +161,6 @@ public: const InfoStruct &getInfo() const { return BasicInfo; } template const T *getInfoAs() const { - if (const auto *P = std::get_if(&BasicInfo)) - return static_cast(P); - if (std::is_same::value) - return nullptr; - if (const auto *P = std::get_if(&BasicInfo)) return static_cast(P); if (std::is_same::value) diff --git a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h index 9c4d9e19f11b..f7f8d5e6bf47 100644 --- a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h +++ b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h @@ -107,7 +107,7 @@ struct PSVInfo { // the format. uint32_t Version; - dxbc::PSV::v3::RuntimeInfo Info; + dxbc::PSV::v2::RuntimeInfo Info; uint32_t ResourceStride; SmallVector Resources; SmallVector SigInputElements; @@ -121,15 +121,12 @@ struct PSVInfo { MaskVector InputPatchMap; MaskVector PatchOutputMap; - StringRef EntryName; - void mapInfoForVersion(yaml::IO &IO); PSVInfo(); PSVInfo(const dxbc::PSV::v0::RuntimeInfo *P, uint16_t Stage); PSVInfo(const dxbc::PSV::v1::RuntimeInfo *P); PSVInfo(const dxbc::PSV::v2::RuntimeInfo *P); - PSVInfo(const dxbc::PSV::v3::RuntimeInfo *P, StringRef StringTable); }; struct SignatureParameter { diff --git a/llvm/lib/MC/DXContainerPSVInfo.cpp b/llvm/lib/MC/DXContainerPSVInfo.cpp index aeff69380139..48182fcd31df 100644 --- a/llvm/lib/MC/DXContainerPSVInfo.cpp +++ b/llvm/lib/MC/DXContainerPSVInfo.cpp @@ -81,18 +81,13 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const { BindingSize = sizeof(dxbc::PSV::v0::ResourceBindInfo); break; case 2: - InfoSize = sizeof(dxbc::PSV::v2::RuntimeInfo); - BindingSize = sizeof(dxbc::PSV::v2::ResourceBindInfo); - break; - case 3: default: - InfoSize = sizeof(dxbc::PSV::v3::RuntimeInfo); + InfoSize = sizeof(dxbc::PSV::v2::RuntimeInfo); BindingSize = sizeof(dxbc::PSV::v2::ResourceBindInfo); } - // Write the size of the info. - support::endian::write(OS, InfoSize, llvm::endianness::little); + support::endian::write(OS, InfoSize, llvm::endianness::little); // Write the info itself. OS.write(reinterpret_cast(&BaseData), InfoSize); @@ -109,12 +104,32 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const { if (Version == 0) return; - support::endian::write(OS, - static_cast(DXConStrTabBuilder.getSize()), + StringTableBuilder StrTabBuilder((StringTableBuilder::DXContainer)); + SmallVector IndexBuffer; + SmallVector SignatureElements; + SmallVector SemanticNames; + + ProcessElementList(StrTabBuilder, IndexBuffer, SignatureElements, + SemanticNames, InputElements); + ProcessElementList(StrTabBuilder, IndexBuffer, SignatureElements, + SemanticNames, OutputElements); + ProcessElementList(StrTabBuilder, IndexBuffer, SignatureElements, + SemanticNames, PatchOrPrimElements); + + StrTabBuilder.finalize(); + for (auto ElAndName : zip(SignatureElements, SemanticNames)) { + v0::SignatureElement &El = std::get<0>(ElAndName); + StringRef Name = std::get<1>(ElAndName); + El.NameOffset = static_cast(StrTabBuilder.getOffset(Name)); + if (sys::IsBigEndianHost) + El.swapBytes(); + } + + support::endian::write(OS, static_cast(StrTabBuilder.getSize()), llvm::endianness::little); // Write the string table. - DXConStrTabBuilder.write(OS); + StrTabBuilder.write(OS); // Write the index table size, then table. support::endian::write(OS, static_cast(IndexBuffer.size()), @@ -147,46 +162,6 @@ void PSVRuntimeInfo::write(raw_ostream &OS, uint32_t Version) const { llvm::endianness::little); } -void PSVRuntimeInfo::finalize(Triple::EnvironmentType Stage) { - IsFinalized = true; - BaseData.SigInputElements = static_cast(InputElements.size()); - BaseData.SigOutputElements = static_cast(OutputElements.size()); - BaseData.SigPatchOrPrimElements = - static_cast(PatchOrPrimElements.size()); - - SmallVector SemanticNames; - - // Build a string table and set associated offsets to be written when - // write() is called - ProcessElementList(DXConStrTabBuilder, IndexBuffer, SignatureElements, - SemanticNames, InputElements); - ProcessElementList(DXConStrTabBuilder, IndexBuffer, SignatureElements, - SemanticNames, OutputElements); - ProcessElementList(DXConStrTabBuilder, IndexBuffer, SignatureElements, - SemanticNames, PatchOrPrimElements); - - DXConStrTabBuilder.add(EntryName); - - DXConStrTabBuilder.finalize(); - for (auto ElAndName : zip(SignatureElements, SemanticNames)) { - llvm::dxbc::PSV::v0::SignatureElement &El = std::get<0>(ElAndName); - StringRef Name = std::get<1>(ElAndName); - El.NameOffset = static_cast(DXConStrTabBuilder.getOffset(Name)); - if (sys::IsBigEndianHost) - El.swapBytes(); - } - - BaseData.EntryNameOffset = - static_cast(DXConStrTabBuilder.getOffset(EntryName)); - - if (!sys::IsBigEndianHost) - return; - BaseData.swapBytes(); - BaseData.swapBytes(Stage); - for (auto &Res : Resources) - Res.swapBytes(); -} - void Signature::write(raw_ostream &OS) { SmallVector SigParams; SigParams.reserve(Params.size()); diff --git a/llvm/lib/Object/DXContainer.cpp b/llvm/lib/Object/DXContainer.cpp index 3b1a6203a1f8..935749afe338 100644 --- a/llvm/lib/Object/DXContainer.cpp +++ b/llvm/lib/Object/DXContainer.cpp @@ -247,14 +247,7 @@ Error DirectX::PSVRuntimeInfo::parse(uint16_t ShaderKind) { const uint32_t PSVVersion = getVersion(); // Detect the PSVVersion by looking at the size field. - if (PSVVersion == 3) { - v3::RuntimeInfo Info; - if (Error Err = readStruct(PSVInfoData, Current, Info)) - return Err; - if (sys::IsBigEndianHost) - Info.swapBytes(ShaderStage); - BasicInfo = Info; - } else if (PSVVersion == 2) { + if (PSVVersion == 2) { v2::RuntimeInfo Info; if (Error Err = readStruct(PSVInfoData, Current, Info)) return Err; @@ -432,8 +425,6 @@ Error DirectX::PSVRuntimeInfo::parse(uint16_t ShaderKind) { } uint8_t DirectX::PSVRuntimeInfo::getSigInputCount() const { - if (const auto *P = std::get_if(&BasicInfo)) - return P->SigInputElements; if (const auto *P = std::get_if(&BasicInfo)) return P->SigInputElements; if (const auto *P = std::get_if(&BasicInfo)) @@ -442,8 +433,6 @@ uint8_t DirectX::PSVRuntimeInfo::getSigInputCount() const { } uint8_t DirectX::PSVRuntimeInfo::getSigOutputCount() const { - if (const auto *P = std::get_if(&BasicInfo)) - return P->SigOutputElements; if (const auto *P = std::get_if(&BasicInfo)) return P->SigOutputElements; if (const auto *P = std::get_if(&BasicInfo)) @@ -452,8 +441,6 @@ uint8_t DirectX::PSVRuntimeInfo::getSigOutputCount() const { } uint8_t DirectX::PSVRuntimeInfo::getSigPatchOrPrimCount() const { - if (const auto *P = std::get_if(&BasicInfo)) - return P->SigPatchOrPrimElements; if (const auto *P = std::get_if(&BasicInfo)) return P->SigPatchOrPrimElements; if (const auto *P = std::get_if(&BasicInfo)) diff --git a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp index f3a518df3175..09a5e41c7123 100644 --- a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp +++ b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp @@ -198,9 +198,8 @@ void DXContainerWriter::writeParts(raw_ostream &OS) { if (!P.Info.has_value()) continue; mcdxbc::PSVRuntimeInfo PSV; - memcpy(&PSV.BaseData, &P.Info->Info, sizeof(dxbc::PSV::v3::RuntimeInfo)); + memcpy(&PSV.BaseData, &P.Info->Info, sizeof(dxbc::PSV::v2::RuntimeInfo)); PSV.Resources = P.Info->Resources; - PSV.EntryName = P.Info->EntryName; for (auto El : P.Info->SigInputElements) PSV.InputElements.push_back(mcdxbc::PSVSignatureElement{ diff --git a/llvm/lib/ObjectYAML/DXContainerYAML.cpp b/llvm/lib/ObjectYAML/DXContainerYAML.cpp index 38063670aee6..a6871e7855e4 100644 --- a/llvm/lib/ObjectYAML/DXContainerYAML.cpp +++ b/llvm/lib/ObjectYAML/DXContainerYAML.cpp @@ -74,16 +74,6 @@ DXContainerYAML::PSVInfo::PSVInfo(const dxbc::PSV::v2::RuntimeInfo *P) memcpy(&Info, P, sizeof(dxbc::PSV::v2::RuntimeInfo)); } -DXContainerYAML::PSVInfo::PSVInfo(const dxbc::PSV::v3::RuntimeInfo *P, - StringRef StringTable) - : Version(3), - EntryName(StringTable.substr(P->EntryNameOffset, - StringTable.find('\0', P->EntryNameOffset) - - P->EntryNameOffset)) { - memset(&Info, 0, sizeof(Info)); - memcpy(&Info, P, sizeof(dxbc::PSV::v3::RuntimeInfo)); -} - namespace yaml { void MappingTraits::mapping( @@ -358,11 +348,6 @@ void DXContainerYAML::PSVInfo::mapInfoForVersion(yaml::IO &IO) { IO.mapRequired("NumThreadsX", Info.NumThreadsX); IO.mapRequired("NumThreadsY", Info.NumThreadsY); IO.mapRequired("NumThreadsZ", Info.NumThreadsZ); - - if (Version == 2) - return; - - IO.mapRequired("EntryName", EntryName); } } // namespace llvm diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml deleted file mode 100644 index 09885bd529f0..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-amplification.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 14 - PayloadSizeInBytes: 4092 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigInputVectors: 0 - SigOutputVectors: [ 8, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: ASEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 14 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 14 -# CHECK-NEXT: PayloadSizeInBytes: 4092 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: ASEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml deleted file mode 100644 index ee6fb112c772..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-compute.yaml +++ /dev/null @@ -1,95 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 5 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigInputVectors: 0 - SigOutputVectors: [ 8, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: CSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 5 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 5 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: CSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml deleted file mode 100644 index dd367deae88e..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-domain.yaml +++ /dev/null @@ -1,105 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 4 - InputControlPointCount: 1024 - OutputPositionPresent: 1 - TessellatorDomain: 2056 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigPatchConstOrPrimVectors: 0 - SigInputVectors: 0 - SigOutputVectors: [ 0, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: DSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - PatchOutputMap: [] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 4 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 4 -# CHECK-NEXT: InputControlPointCount: 1024 -# CHECK-NEXT: OutputPositionPresent: 1 -# CHECK-NEXT: TessellatorDomain: 2056 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigPatchConstOrPrimVectors: 0 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 0, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: DSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: PatchOutputMap: [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml deleted file mode 100644 index 4c7680b63b02..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-geometry.yaml +++ /dev/null @@ -1,105 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 2 - InputPrimitive: 1024 - OutputTopology: 4096 - OutputStreamMask: 2056 - OutputPositionPresent: 1 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - MaxVertexCount: 4096 - SigInputVectors: 0 - SigOutputVectors: [ 8, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: GSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 2 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 2 -# CHECK-NEXT: InputPrimitive: 1024 -# CHECK-NEXT: OutputTopology: 4096 -# CHECK-NEXT: OutputStreamMask: 2056 -# CHECK-NEXT: OutputPositionPresent: 1 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: MaxVertexCount: 4096 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: GSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml deleted file mode 100644 index 3bbad8a9b0ee..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-hull.yaml +++ /dev/null @@ -1,107 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 3 - InputControlPointCount: 1024 - OutputControlPointCount: 4096 - TessellatorDomain: 2056 - TessellatorOutputPrimitive: 8192 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigPatchConstOrPrimVectors: 0 - SigInputVectors: 0 - SigOutputVectors: [ 0, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: HSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - InputPatchMap: [] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 3 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 3 -# CHECK-NEXT: InputControlPointCount: 1024 -# CHECK-NEXT: OutputControlPointCount: 4096 -# CHECK-NEXT: TessellatorDomain: 2056 -# CHECK-NEXT: TessellatorOutputPrimitive: 8192 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigPatchConstOrPrimVectors: 0 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 0, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: HSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: InputPatchMap: [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml deleted file mode 100644 index c5ea1fcf0780..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-mesh.yaml +++ /dev/null @@ -1,109 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 13 - GroupSharedBytesUsed: 1024 - GroupSharedBytesDependentOnViewID: 2056 - PayloadSizeInBytes: 4092 - MaxOutputVertices: 8196 - MaxOutputPrimitives: 4092 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigPrimVectors: 128 - MeshOutputTopology: 16 - SigInputVectors: 0 - SigOutputVectors: [ 8, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: MSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 13 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 13 -# CHECK-NEXT: GroupSharedBytesUsed: 1024 -# CHECK-NEXT: GroupSharedBytesDependentOnViewID: 2056 -# CHECK-NEXT: PayloadSizeInBytes: 4092 -# CHECK-NEXT: MaxOutputVertices: 8196 -# CHECK-NEXT: MaxOutputPrimitives: 4092 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigPrimVectors: 128 -# CHECK-NEXT: MeshOutputTopology: 16 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: MSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml deleted file mode 100644 index b28d5ec8074d..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-pixel.yaml +++ /dev/null @@ -1,99 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 0 - DepthOutput: 7 - SampleFrequency: 96 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigInputVectors: 0 - SigOutputVectors: [ 8, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: PSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 0 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 0 -# CHECK-NEXT: DepthOutput: 7 -# CHECK-NEXT: SampleFrequency: 96 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: PSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: Name diff --git a/llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml b/llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml deleted file mode 100644 index d1fb55839931..000000000000 --- a/llvm/test/ObjectYAML/DXContainer/PSVv3-vertex.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# RUN: yaml2obj %s | obj2yaml | FileCheck %s - ---- !dxcontainer -Header: - Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] - Version: - Major: 1 - Minor: 0 - PartCount: 2 -Parts: - - Name: PSV0 - Size: 144 - PSVInfo: - Version: 3 - ShaderStage: 1 - OutputPositionPresent: 1 - MinimumWaveLaneCount: 0 - MaximumWaveLaneCount: 4294967295 - UsesViewID: 0 - SigInputVectors: 0 - SigOutputVectors: [ 8, 16, 32, 64 ] - NumThreadsX: 512 - NumThreadsY: 1024 - NumThreadsZ: 2048 - EntryName: VSEntry - ResourceStride: 24 - Resources: - - Type: 1 - Space: 2 - LowerBound: 3 - UpperBound: 4 - Kind: 5 - Flags: 6 - - Type: 128 - Space: 32768 - LowerBound: 8388608 - UpperBound: 2147483648 - Kind: 65535 - Flags: 16776960 - SigInputElements: [] - SigOutputElements: [] - SigPatchOrPrimElements: [] - InputOutputMap: - - [ ] - - [ ] - - [ ] - - [ ] - - Name: DXIL - Size: 24 - Program: - MajorVersion: 6 - MinorVersion: 0 - ShaderKind: 1 - Size: 6 - DXILMajorVersion: 0 - DXILMinorVersion: 1 - DXILSize: 0 -... - -# CHECK: Name: PSV0 -# CHECK: PSVInfo: -# CHECK-NEXT: Version: 3 -# CHECK-NEXT: ShaderStage: 1 -# CHECK-NEXT: OutputPositionPresent: 1 -# CHECK-NEXT: MinimumWaveLaneCount: 0 -# CHECK-NEXT: MaximumWaveLaneCount: 4294967295 -# CHECK-NEXT: UsesViewID: 0 -# CHECK-NEXT: SigInputVectors: 0 -# CHECK-NEXT: SigOutputVectors: [ 8, 16, 32, 64 ] -# CHECK-NEXT: NumThreadsX: 512 -# CHECK-NEXT: NumThreadsY: 1024 -# CHECK-NEXT: NumThreadsZ: 2048 -# CHECK-NEXT: EntryName: VSEntry -# CHECK-NEXT: ResourceStride: 24 -# CHECK-NEXT: Resources: -# CHECK-NEXT: - Type: 1 -# CHECK-NEXT: Space: 2 -# CHECK-NEXT: LowerBound: 3 -# CHECK-NEXT: UpperBound: 4 -# CHECK-NEXT: Kind: 5 -# CHECK-NEXT: Flags: 6 -# CHECK-NEXT: - Type: 128 -# CHECK-NEXT: Space: 32768 -# CHECK-NEXT: LowerBound: 8388608 -# CHECK-NEXT: UpperBound: 2147483648 -# CHECK-NEXT: Kind: 65535 -# CHECK-NEXT: Flags: 16776960 -# CHECK-NEXT: SigInputElements: [] -# CHECK-NEXT: SigOutputElements: [] -# CHECK-NEXT: SigPatchOrPrimElements: [] -# CHECK-NEXT: InputOutputMap: -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: - [ ] -# CHECK-NEXT: Name diff --git a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp index ec4f5c74498f..69d9b9a2f784 100644 --- a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp +++ b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp @@ -99,9 +99,6 @@ dumpDXContainer(MemoryBufferRef Source) { else if (const auto *P = std::get_if(&PSVInfo->getInfo())) NewPart.Info = DXContainerYAML::PSVInfo(P); - else if (const auto *P = - std::get_if(&PSVInfo->getInfo())) - NewPart.Info = DXContainerYAML::PSVInfo(P, PSVInfo->getStringTable()); NewPart.Info->ResourceStride = PSVInfo->getResourceStride(); for (auto Res : PSVInfo->getResources()) NewPart.Info->Resources.push_back(Res); -- GitLab From dc74bf7a5412df82223f7062d9a6b814abbfca45 Mon Sep 17 00:00:00 2001 From: Cyndy Ishida Date: Thu, 21 Mar 2024 15:41:49 -0700 Subject: [PATCH 209/296] Add myself as codeowner for InstallAPI & TextAPI --- .github/CODEOWNERS | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9cf6f5d79d41..fea132c8fe78 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -119,3 +119,8 @@ clang/test/AST/Interp/ @tbaederr # Bazel build system. /utils/bazel/ @rupprecht + +# InstallAPI and TextAPI +/llvm/**/TextAPI/ @cyndyishida +/clang/**/InstallAPI/ @cyndyishida +/clang/tools/clang-installapi/ @cyndyishida -- GitLab From 06d245242e3e24cd4558f545fb5ceba0582c4f03 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 22 Mar 2024 07:08:51 +0800 Subject: [PATCH 210/296] [RISCV] Recursively split concat_vector into smaller LMULs when lowering (#85825) This is a reimplementation of the combine added in #83035 but as a lowering instead of a combine, so we don't regress the test case added in e59f120e3a14ccdc55fcb7be996efaa768daabe0 by interfering with the strided load combine Previously the combine had to concatenate the split vectors with insert_subvector instead of concat_vectors to prevent an infinite combine loop. And the reasoning behind keeping it as a combine was because if we emitted the insert_subvector during lowering then we didn't fold away inserts of undef subvectors. However it turns out we can avoid this if we just do this in lowering and select a concat_vector directly, since we get the undef folding for free with `DAG.getNode(ISD::CONCAT_VECTOR, ...)` via foldCONCAT_VECTORS. --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 24 + .../CodeGen/RISCV/rvv/active_lane_mask.ll | 91 +- .../RISCV/rvv/combine-store-extract-crash.ll | 4 +- .../CodeGen/RISCV/rvv/extract-subvector.ll | 3 +- .../RISCV/rvv/fixed-vectors-shuffle-concat.ll | 254 ++-- .../rvv/fixed-vectors-strided-load-combine.ll | 105 +- .../CodeGen/RISCV/rvv/fpclamptosat_vec.ll | 1066 ++++++++++------- llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll | 13 +- llvm/test/CodeGen/RISCV/rvv/pr63596.ll | 37 +- 9 files changed, 844 insertions(+), 753 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index a3ebfb34ad7a..71059b5bdc0f 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -6611,6 +6611,30 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op, // better than going through the stack, as the default expansion does. SDLoc DL(Op); MVT VT = Op.getSimpleValueType(); + MVT ContainerVT = VT; + if (VT.isFixedLengthVector()) + ContainerVT = ::getContainerForFixedLengthVector(DAG, VT, Subtarget); + + // Recursively split concat_vectors with more than 2 operands: + // + // concat_vector op1, op2, op3, op4 + // -> + // concat_vector (concat_vector op1, op2), (concat_vector op3, op4) + // + // This reduces the length of the chain of vslideups and allows us to + // perform the vslideups at a smaller LMUL, limited to MF2. + if (Op.getNumOperands() > 2 && + ContainerVT.bitsGE(getLMUL1VT(ContainerVT))) { + MVT HalfVT = VT.getHalfNumVectorElementsVT(); + assert(isPowerOf2_32(Op.getNumOperands())); + size_t HalfNumOps = Op.getNumOperands() / 2; + SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, + Op->ops().take_front(HalfNumOps)); + SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, + Op->ops().drop_front(HalfNumOps)); + return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); + } + unsigned NumOpElts = Op.getOperand(0).getSimpleValueType().getVectorMinNumElements(); SDValue Vec = DAG.getUNDEF(VT); diff --git a/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll b/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll index 87d95d7596d4..139579b3d2a3 100644 --- a/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll +++ b/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll @@ -161,72 +161,71 @@ define <64 x i1> @fv64(ptr %p, i64 %index, i64 %tc) { define <128 x i1> @fv128(ptr %p, i64 %index, i64 %tc) { ; CHECK-LABEL: fv128: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma ; CHECK-NEXT: lui a0, %hi(.LCPI10_0) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_0) +; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma ; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vid.v v16 -; CHECK-NEXT: vsaddu.vx v16, v16, a1 -; CHECK-NEXT: vmsltu.vx v0, v16, a2 -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 4, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 2 ; CHECK-NEXT: lui a0, %hi(.LCPI10_1) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_1) -; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) +; CHECK-NEXT: vle8.v v9, (a0) ; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 6, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 4 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v10, v16, a2 +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v8, v16, a2 +; CHECK-NEXT: vsetivli zero, 4, e8, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v8, v10, 2 ; CHECK-NEXT: lui a0, %hi(.LCPI10_2) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_2) ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 8, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 6 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 6, e8, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v8, v9, 4 ; CHECK-NEXT: lui a0, %hi(.LCPI10_3) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_3) ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 10, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 8 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 6 +; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma ; CHECK-NEXT: lui a0, %hi(.LCPI10_4) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_4) -; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 12, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 10 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vid.v v16 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v0, v16, a2 +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 4, e8, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v0, v9, 2 ; CHECK-NEXT: lui a0, %hi(.LCPI10_5) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_5) ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetivli zero, 14, e8, m1, tu, ma -; CHECK-NEXT: vslideup.vi v0, v16, 12 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 6, e8, mf2, tu, ma +; CHECK-NEXT: vslideup.vi v0, v9, 4 ; CHECK-NEXT: lui a0, %hi(.LCPI10_6) ; CHECK-NEXT: addi a0, a0, %lo(.LCPI10_6) ; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma -; CHECK-NEXT: vle8.v v8, (a0) -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsaddu.vx v8, v16, a1 -; CHECK-NEXT: vmsltu.vx v16, v8, a2 -; CHECK-NEXT: vsetvli zero, zero, e8, m1, ta, ma -; CHECK-NEXT: vslideup.vi v0, v16, 14 +; CHECK-NEXT: vle8.v v9, (a0) +; CHECK-NEXT: vsext.vf8 v16, v9 +; CHECK-NEXT: vsaddu.vx v16, v16, a1 +; CHECK-NEXT: vmsltu.vx v9, v16, a2 +; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v0, v9, 6 +; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; CHECK-NEXT: vslideup.vi v0, v8, 8 ; CHECK-NEXT: ret %mask = call <128 x i1> @llvm.get.active.lane.mask.v128i1.i64(i64 %index, i64 %tc) ret <128 x i1> %mask diff --git a/llvm/test/CodeGen/RISCV/rvv/combine-store-extract-crash.ll b/llvm/test/CodeGen/RISCV/rvv/combine-store-extract-crash.ll index c64216180c2a..ed434deea1a8 100644 --- a/llvm/test/CodeGen/RISCV/rvv/combine-store-extract-crash.ll +++ b/llvm/test/CodeGen/RISCV/rvv/combine-store-extract-crash.ll @@ -19,7 +19,7 @@ define void @test(ptr %ref_array, ptr %sad_array) { ; RV32-NEXT: th.swia a0, (a1), 4, 0 ; RV32-NEXT: vsetivli zero, 4, e8, mf4, ta, ma ; RV32-NEXT: vle8.v v10, (a3) -; RV32-NEXT: vsetivli zero, 8, e8, m1, tu, ma +; RV32-NEXT: vsetivli zero, 8, e8, mf2, ta, ma ; RV32-NEXT: vslideup.vi v10, v9, 4 ; RV32-NEXT: vsetivli zero, 16, e32, m4, ta, ma ; RV32-NEXT: vzext.vf4 v12, v10 @@ -42,7 +42,7 @@ define void @test(ptr %ref_array, ptr %sad_array) { ; RV64-NEXT: th.swia a0, (a1), 4, 0 ; RV64-NEXT: vsetivli zero, 4, e8, mf4, ta, ma ; RV64-NEXT: vle8.v v10, (a3) -; RV64-NEXT: vsetivli zero, 8, e8, m1, tu, ma +; RV64-NEXT: vsetivli zero, 8, e8, mf2, ta, ma ; RV64-NEXT: vslideup.vi v10, v9, 4 ; RV64-NEXT: vsetivli zero, 16, e32, m4, ta, ma ; RV64-NEXT: vzext.vf4 v12, v10 diff --git a/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll b/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll index 76aa2b913c65..e15e6452163b 100644 --- a/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll +++ b/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll @@ -469,9 +469,8 @@ define @extract_nxv6f16_nxv12f16_6( %in) ; CHECK: # %bb.0: ; CHECK-NEXT: csrr a0, vlenb ; CHECK-NEXT: srli a0, a0, 2 -; CHECK-NEXT: vsetvli zero, a0, e16, m1, ta, ma -; CHECK-NEXT: vslidedown.vx v13, v10, a0 ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma +; CHECK-NEXT: vslidedown.vx v13, v10, a0 ; CHECK-NEXT: vslidedown.vx v12, v9, a0 ; CHECK-NEXT: add a1, a0, a0 ; CHECK-NEXT: vsetvli zero, a1, e16, m1, ta, ma diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll index e5bef20fd9e2..98e6b8f2dd76 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-concat.ll @@ -19,14 +19,11 @@ define <8 x i32> @concat_2xv4i32(<4 x i32> %a, <4 x i32> %b) { define <8 x i32> @concat_4xv2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x i32> %d) { ; CHECK-LABEL: concat_4xv2i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vmv1r.v v12, v11 -; CHECK-NEXT: vmv1r.v v14, v9 -; CHECK-NEXT: vsetivli zero, 4, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v14, 2 -; CHECK-NEXT: vsetivli zero, 6, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v10, 4 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v10, v11, 2 +; CHECK-NEXT: vslideup.vi v8, v9, 2 ; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-NEXT: vslideup.vi v8, v12, 6 +; CHECK-NEXT: vslideup.vi v8, v10, 4 ; CHECK-NEXT: ret %ab = shufflevector <2 x i32> %a, <2 x i32> %b, <4 x i32> %cd = shufflevector <2 x i32> %c, <2 x i32> %d, <4 x i32> @@ -37,24 +34,18 @@ define <8 x i32> @concat_4xv2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x define <8 x i32> @concat_8xv1i32(<1 x i32> %a, <1 x i32> %b, <1 x i32> %c, <1 x i32> %d, <1 x i32> %e, <1 x i32> %f, <1 x i32> %g, <1 x i32> %h) { ; CHECK-LABEL: concat_8xv1i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vmv1r.v v16, v15 -; CHECK-NEXT: vmv1r.v v18, v13 -; CHECK-NEXT: vmv1r.v v20, v11 -; CHECK-NEXT: vmv1r.v v22, v9 -; CHECK-NEXT: vsetivli zero, 2, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v22, 1 -; CHECK-NEXT: vsetivli zero, 3, e32, m2, tu, ma +; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v14, v15, 1 +; CHECK-NEXT: vslideup.vi v12, v13, 1 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v12, v14, 2 +; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v10, v11, 1 +; CHECK-NEXT: vslideup.vi v8, v9, 1 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma ; CHECK-NEXT: vslideup.vi v8, v10, 2 -; CHECK-NEXT: vsetivli zero, 4, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v20, 3 -; CHECK-NEXT: vsetivli zero, 5, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v12, 4 -; CHECK-NEXT: vsetivli zero, 6, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v18, 5 -; CHECK-NEXT: vsetivli zero, 7, e32, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v14, 6 ; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-NEXT: vslideup.vi v8, v16, 7 +; CHECK-NEXT: vslideup.vi v8, v12, 4 ; CHECK-NEXT: ret %ab = shufflevector <1 x i32> %a, <1 x i32> %b, <2 x i32> %cd = shufflevector <1 x i32> %c, <1 x i32> %d, <2 x i32> @@ -80,15 +71,14 @@ define <16 x i32> @concat_2xv8i32(<8 x i32> %a, <8 x i32> %b) { define <16 x i32> @concat_4xv4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d) { ; CHECK-LABEL: concat_4xv4i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vmv1r.v v12, v11 -; CHECK-NEXT: vmv1r.v v16, v10 -; CHECK-NEXT: vmv1r.v v20, v9 -; CHECK-NEXT: vsetivli zero, 8, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v20, 4 -; CHECK-NEXT: vsetivli zero, 12, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v16, 8 +; CHECK-NEXT: vmv1r.v v14, v11 +; CHECK-NEXT: vmv1r.v v12, v10 +; CHECK-NEXT: vmv1r.v v10, v9 +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vslideup.vi v12, v14, 4 +; CHECK-NEXT: vslideup.vi v8, v10, 4 ; CHECK-NEXT: vsetivli zero, 16, e32, m4, ta, ma -; CHECK-NEXT: vslideup.vi v8, v12, 12 +; CHECK-NEXT: vslideup.vi v8, v12, 8 ; CHECK-NEXT: ret %ab = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> %cd = shufflevector <4 x i32> %c, <4 x i32> %d, <8 x i32> @@ -99,26 +89,18 @@ define <16 x i32> @concat_4xv4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x define <16 x i32> @concat_8xv2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c, <2 x i32> %d, <2 x i32> %e, <2 x i32> %f, <2 x i32> %g, <2 x i32> %h) { ; CHECK-LABEL: concat_8xv2i32: ; CHECK: # %bb.0: -; CHECK-NEXT: vmv1r.v v16, v15 -; CHECK-NEXT: vmv1r.v v20, v14 -; CHECK-NEXT: vmv1r.v v24, v13 -; CHECK-NEXT: vmv1r.v v28, v11 -; CHECK-NEXT: vmv1r.v v4, v10 -; CHECK-NEXT: vmv1r.v v0, v9 -; CHECK-NEXT: vsetivli zero, 4, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v0, 2 -; CHECK-NEXT: vsetivli zero, 6, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v4, 4 -; CHECK-NEXT: vsetivli zero, 8, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v28, 6 -; CHECK-NEXT: vsetivli zero, 10, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v12, 8 -; CHECK-NEXT: vsetivli zero, 12, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v24, 10 -; CHECK-NEXT: vsetivli zero, 14, e32, m4, tu, ma -; CHECK-NEXT: vslideup.vi v8, v20, 12 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v14, v15, 2 +; CHECK-NEXT: vslideup.vi v12, v13, 2 +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vslideup.vi v12, v14, 4 +; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-NEXT: vslideup.vi v10, v11, 2 +; CHECK-NEXT: vslideup.vi v8, v9, 2 +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v10, 4 ; CHECK-NEXT: vsetivli zero, 16, e32, m4, ta, ma -; CHECK-NEXT: vslideup.vi v8, v16, 14 +; CHECK-NEXT: vslideup.vi v8, v12, 8 ; CHECK-NEXT: ret %ab = shufflevector <2 x i32> %a, <2 x i32> %b, <4 x i32> %cd = shufflevector <2 x i32> %c, <2 x i32> %d, <4 x i32> @@ -152,29 +134,27 @@ define <32 x i32> @concat_2xv16i32(<16 x i32> %a, <16 x i32> %b) { define <32 x i32> @concat_4xv8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) { ; VLA-LABEL: concat_4xv8i32: ; VLA: # %bb.0: -; VLA-NEXT: vmv2r.v v16, v14 -; VLA-NEXT: vmv2r.v v24, v12 -; VLA-NEXT: vmv2r.v v0, v10 -; VLA-NEXT: vsetivli zero, 16, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v0, 8 -; VLA-NEXT: vsetivli zero, 24, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v24, 16 +; VLA-NEXT: vmv2r.v v20, v14 +; VLA-NEXT: vmv2r.v v16, v12 +; VLA-NEXT: vmv2r.v v12, v10 +; VLA-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLA-NEXT: vslideup.vi v16, v20, 8 +; VLA-NEXT: vslideup.vi v8, v12, 8 ; VLA-NEXT: li a0, 32 ; VLA-NEXT: vsetvli zero, a0, e32, m8, ta, ma -; VLA-NEXT: vslideup.vi v8, v16, 24 +; VLA-NEXT: vslideup.vi v8, v16, 16 ; VLA-NEXT: ret ; ; VLS-LABEL: concat_4xv8i32: ; VLS: # %bb.0: -; VLS-NEXT: vmv2r.v v16, v14 -; VLS-NEXT: vmv2r.v v24, v12 -; VLS-NEXT: vmv2r.v v0, v10 -; VLS-NEXT: vsetivli zero, 16, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v0, 8 -; VLS-NEXT: vsetivli zero, 24, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v24, 16 +; VLS-NEXT: vmv2r.v v20, v14 +; VLS-NEXT: vmv2r.v v16, v12 +; VLS-NEXT: vmv2r.v v12, v10 +; VLS-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLS-NEXT: vslideup.vi v16, v20, 8 +; VLS-NEXT: vslideup.vi v8, v12, 8 ; VLS-NEXT: vsetvli a0, zero, e32, m8, ta, ma -; VLS-NEXT: vslideup.vi v8, v16, 24 +; VLS-NEXT: vslideup.vi v8, v16, 16 ; VLS-NEXT: ret %ab = shufflevector <8 x i32> %a, <8 x i32> %b, <16 x i32> %cd = shufflevector <8 x i32> %c, <8 x i32> %d, <16 x i32> @@ -185,123 +165,49 @@ define <32 x i32> @concat_4xv8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x define <32 x i32> @concat_8xv4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d, <4 x i32> %e, <4 x i32> %f, <4 x i32> %g, <4 x i32> %h) { ; VLA-LABEL: concat_8xv4i32: ; VLA: # %bb.0: -; VLA-NEXT: addi sp, sp, -16 -; VLA-NEXT: .cfi_def_cfa_offset 16 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 5 -; VLA-NEXT: sub sp, sp, a0 -; VLA-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb -; VLA-NEXT: vmv1r.v v16, v15 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 3 -; VLA-NEXT: mv a1, a0 -; VLA-NEXT: slli a0, a0, 1 -; VLA-NEXT: add a0, a0, a1 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLA-NEXT: vmv1r.v v16, v14 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 4 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLA-NEXT: vmv1r.v v16, v13 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 3 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill +; VLA-NEXT: vmv1r.v v18, v15 +; VLA-NEXT: vmv1r.v v20, v14 +; VLA-NEXT: vmv1r.v v22, v13 ; VLA-NEXT: vmv1r.v v16, v12 -; VLA-NEXT: addi a0, sp, 16 -; VLA-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLA-NEXT: vmv1r.v v0, v11 -; VLA-NEXT: vmv1r.v v24, v10 -; VLA-NEXT: vmv1r.v v16, v9 -; VLA-NEXT: vsetivli zero, 8, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v16, 4 -; VLA-NEXT: vsetivli zero, 12, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v24, 8 -; VLA-NEXT: vsetivli zero, 16, e32, m8, tu, ma -; VLA-NEXT: vslideup.vi v8, v0, 12 -; VLA-NEXT: vsetivli zero, 20, e32, m8, tu, ma -; VLA-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLA-NEXT: vslideup.vi v8, v16, 16 -; VLA-NEXT: vsetivli zero, 24, e32, m8, tu, ma -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 3 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLA-NEXT: vslideup.vi v8, v16, 20 -; VLA-NEXT: vsetivli zero, 28, e32, m8, tu, ma -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 4 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLA-NEXT: vslideup.vi v8, v16, 24 +; VLA-NEXT: vmv1r.v v14, v11 +; VLA-NEXT: vmv1r.v v12, v10 +; VLA-NEXT: vmv1r.v v10, v9 +; VLA-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; VLA-NEXT: vslideup.vi v20, v18, 4 +; VLA-NEXT: vslideup.vi v16, v22, 4 +; VLA-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLA-NEXT: vslideup.vi v16, v20, 8 +; VLA-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; VLA-NEXT: vslideup.vi v12, v14, 4 +; VLA-NEXT: vslideup.vi v8, v10, 4 +; VLA-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLA-NEXT: vslideup.vi v8, v12, 8 ; VLA-NEXT: li a0, 32 ; VLA-NEXT: vsetvli zero, a0, e32, m8, ta, ma -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 3 -; VLA-NEXT: mv a1, a0 -; VLA-NEXT: slli a0, a0, 1 -; VLA-NEXT: add a0, a0, a1 -; VLA-NEXT: add a0, sp, a0 -; VLA-NEXT: addi a0, a0, 16 -; VLA-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLA-NEXT: vslideup.vi v8, v16, 28 -; VLA-NEXT: csrr a0, vlenb -; VLA-NEXT: slli a0, a0, 5 -; VLA-NEXT: add sp, sp, a0 -; VLA-NEXT: addi sp, sp, 16 +; VLA-NEXT: vslideup.vi v8, v16, 16 ; VLA-NEXT: ret ; ; VLS-LABEL: concat_8xv4i32: ; VLS: # %bb.0: -; VLS-NEXT: addi sp, sp, -16 -; VLS-NEXT: .cfi_def_cfa_offset 16 -; VLS-NEXT: addi sp, sp, -512 -; VLS-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb -; VLS-NEXT: vmv1r.v v16, v15 -; VLS-NEXT: addi a0, sp, 400 -; VLS-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLS-NEXT: vmv1r.v v16, v14 -; VLS-NEXT: addi a0, sp, 272 -; VLS-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLS-NEXT: vmv1r.v v16, v13 -; VLS-NEXT: addi a0, sp, 144 -; VLS-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill +; VLS-NEXT: vmv1r.v v18, v15 +; VLS-NEXT: vmv1r.v v20, v14 +; VLS-NEXT: vmv1r.v v22, v13 ; VLS-NEXT: vmv1r.v v16, v12 -; VLS-NEXT: addi a0, sp, 16 -; VLS-NEXT: vs8r.v v16, (a0) # Unknown-size Folded Spill -; VLS-NEXT: vmv1r.v v0, v11 -; VLS-NEXT: vmv1r.v v24, v10 -; VLS-NEXT: vmv1r.v v16, v9 -; VLS-NEXT: vsetivli zero, 8, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v16, 4 -; VLS-NEXT: vsetivli zero, 12, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v24, 8 -; VLS-NEXT: vsetivli zero, 16, e32, m8, tu, ma -; VLS-NEXT: vslideup.vi v8, v0, 12 -; VLS-NEXT: vsetivli zero, 20, e32, m8, tu, ma -; VLS-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLS-NEXT: vslideup.vi v8, v16, 16 -; VLS-NEXT: vsetivli zero, 24, e32, m8, tu, ma -; VLS-NEXT: addi a0, sp, 144 -; VLS-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLS-NEXT: vslideup.vi v8, v16, 20 -; VLS-NEXT: vsetivli zero, 28, e32, m8, tu, ma -; VLS-NEXT: addi a0, sp, 272 -; VLS-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLS-NEXT: vslideup.vi v8, v16, 24 +; VLS-NEXT: vmv1r.v v14, v11 +; VLS-NEXT: vmv1r.v v12, v10 +; VLS-NEXT: vmv1r.v v10, v9 +; VLS-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; VLS-NEXT: vslideup.vi v20, v18, 4 +; VLS-NEXT: vslideup.vi v16, v22, 4 +; VLS-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLS-NEXT: vslideup.vi v16, v20, 8 +; VLS-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; VLS-NEXT: vslideup.vi v12, v14, 4 +; VLS-NEXT: vslideup.vi v8, v10, 4 +; VLS-NEXT: vsetivli zero, 16, e32, m4, ta, ma +; VLS-NEXT: vslideup.vi v8, v12, 8 ; VLS-NEXT: vsetvli a0, zero, e32, m8, ta, ma -; VLS-NEXT: addi a0, sp, 400 -; VLS-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload -; VLS-NEXT: vslideup.vi v8, v16, 28 -; VLS-NEXT: addi sp, sp, 512 -; VLS-NEXT: addi sp, sp, 16 +; VLS-NEXT: vslideup.vi v8, v16, 16 ; VLS-NEXT: ret %ab = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> %cd = shufflevector <4 x i32> %c, <4 x i32> %d, <8 x i32> diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll index 4ec2e59672ad..657d52354aa3 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-combine.ll @@ -27,13 +27,14 @@ define void @widen_3xv4i16(ptr %x, ptr %z) { ; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma ; CHECK-NEXT: vle16.v v8, (a0) ; CHECK-NEXT: addi a2, a0, 8 -; CHECK-NEXT: vle16.v v10, (a2) +; CHECK-NEXT: vle16.v v9, (a2) ; CHECK-NEXT: addi a0, a0, 16 -; CHECK-NEXT: vle16.v v12, (a0) -; CHECK-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v10, 4 -; CHECK-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v12, 8 +; CHECK-NEXT: vle16.v v10, (a0) +; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; CHECK-NEXT: vslideup.vi v8, v9, 4 +; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v10, 8 +; CHECK-NEXT: vsetivli zero, 12, e16, m2, ta, ma ; CHECK-NEXT: vse16.v v8, (a1) ; CHECK-NEXT: ret %a = load <4 x i16>, ptr %x @@ -72,20 +73,18 @@ define void @widen_4xv4i16(ptr %x, ptr %z) { define void @widen_4xv4i16_unaligned(ptr %x, ptr %z) { ; CHECK-NO-MISALIGN-LABEL: widen_4xv4i16_unaligned: ; CHECK-NO-MISALIGN: # %bb.0: -; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 8, e8, mf2, ta, ma +; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 8, e16, m1, ta, ma ; CHECK-NO-MISALIGN-NEXT: vle8.v v8, (a0) -; CHECK-NO-MISALIGN-NEXT: addi a2, a0, 8 -; CHECK-NO-MISALIGN-NEXT: vle8.v v10, (a2) ; CHECK-NO-MISALIGN-NEXT: addi a2, a0, 16 -; CHECK-NO-MISALIGN-NEXT: vle8.v v12, (a2) +; CHECK-NO-MISALIGN-NEXT: vle8.v v10, (a2) +; CHECK-NO-MISALIGN-NEXT: addi a2, a0, 8 ; CHECK-NO-MISALIGN-NEXT: addi a0, a0, 24 -; CHECK-NO-MISALIGN-NEXT: vle8.v v14, (a0) -; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v10, 4 -; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v12, 8 +; CHECK-NO-MISALIGN-NEXT: vle8.v v9, (a0) +; CHECK-NO-MISALIGN-NEXT: vle8.v v11, (a2) +; CHECK-NO-MISALIGN-NEXT: vslideup.vi v10, v9, 4 +; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v11, 4 ; CHECK-NO-MISALIGN-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v14, 12 +; CHECK-NO-MISALIGN-NEXT: vslideup.vi v8, v10, 8 ; CHECK-NO-MISALIGN-NEXT: vse16.v v8, (a1) ; CHECK-NO-MISALIGN-NEXT: ret ; @@ -187,18 +186,17 @@ define void @strided_constant_mismatch_4xv4i16(ptr %x, ptr %z) { ; CHECK: # %bb.0: ; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma ; CHECK-NEXT: vle16.v v8, (a0) -; CHECK-NEXT: addi a2, a0, 2 -; CHECK-NEXT: vle16.v v10, (a2) ; CHECK-NEXT: addi a2, a0, 6 -; CHECK-NEXT: vle16.v v12, (a2) +; CHECK-NEXT: vle16.v v10, (a2) +; CHECK-NEXT: addi a2, a0, 2 ; CHECK-NEXT: addi a0, a0, 8 -; CHECK-NEXT: vle16.v v14, (a0) -; CHECK-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v10, 4 -; CHECK-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; CHECK-NEXT: vslideup.vi v8, v12, 8 +; CHECK-NEXT: vle16.v v9, (a0) +; CHECK-NEXT: vle16.v v11, (a2) +; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; CHECK-NEXT: vslideup.vi v10, v9, 4 +; CHECK-NEXT: vslideup.vi v8, v11, 4 ; CHECK-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; CHECK-NEXT: vslideup.vi v8, v14, 12 +; CHECK-NEXT: vslideup.vi v8, v10, 8 ; CHECK-NEXT: vse16.v v8, (a1) ; CHECK-NEXT: ret %a = load <4 x i16>, ptr %x @@ -258,17 +256,16 @@ define void @strided_runtime_mismatch_4xv4i16(ptr %x, ptr %z, i64 %s, i64 %t) { ; RV32-NEXT: vsetivli zero, 4, e16, mf2, ta, ma ; RV32-NEXT: vle16.v v8, (a0) ; RV32-NEXT: add a0, a0, a2 -; RV32-NEXT: vle16.v v10, (a0) -; RV32-NEXT: add a0, a0, a4 -; RV32-NEXT: vle16.v v12, (a0) -; RV32-NEXT: add a0, a0, a2 -; RV32-NEXT: vle16.v v14, (a0) -; RV32-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; RV32-NEXT: vslideup.vi v8, v10, 4 -; RV32-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; RV32-NEXT: vslideup.vi v8, v12, 8 +; RV32-NEXT: add a4, a0, a4 +; RV32-NEXT: vle16.v v10, (a4) +; RV32-NEXT: add a2, a4, a2 +; RV32-NEXT: vle16.v v9, (a2) +; RV32-NEXT: vle16.v v11, (a0) +; RV32-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; RV32-NEXT: vslideup.vi v10, v9, 4 +; RV32-NEXT: vslideup.vi v8, v11, 4 ; RV32-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; RV32-NEXT: vslideup.vi v8, v14, 12 +; RV32-NEXT: vslideup.vi v8, v10, 8 ; RV32-NEXT: vse16.v v8, (a1) ; RV32-NEXT: ret ; @@ -277,17 +274,16 @@ define void @strided_runtime_mismatch_4xv4i16(ptr %x, ptr %z, i64 %s, i64 %t) { ; RV64-NEXT: vsetivli zero, 4, e16, mf2, ta, ma ; RV64-NEXT: vle16.v v8, (a0) ; RV64-NEXT: add a0, a0, a2 -; RV64-NEXT: vle16.v v10, (a0) -; RV64-NEXT: add a0, a0, a3 -; RV64-NEXT: vle16.v v12, (a0) -; RV64-NEXT: add a0, a0, a2 -; RV64-NEXT: vle16.v v14, (a0) -; RV64-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; RV64-NEXT: vslideup.vi v8, v10, 4 -; RV64-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; RV64-NEXT: vslideup.vi v8, v12, 8 +; RV64-NEXT: add a3, a0, a3 +; RV64-NEXT: vle16.v v10, (a3) +; RV64-NEXT: add a2, a3, a2 +; RV64-NEXT: vle16.v v9, (a2) +; RV64-NEXT: vle16.v v11, (a0) +; RV64-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; RV64-NEXT: vslideup.vi v10, v9, 4 +; RV64-NEXT: vslideup.vi v8, v11, 4 ; RV64-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; RV64-NEXT: vslideup.vi v8, v14, 12 +; RV64-NEXT: vslideup.vi v8, v10, 8 ; RV64-NEXT: vse16.v v8, (a1) ; RV64-NEXT: ret ; @@ -296,17 +292,16 @@ define void @strided_runtime_mismatch_4xv4i16(ptr %x, ptr %z, i64 %s, i64 %t) { ; ZVE64F-NEXT: vsetivli zero, 4, e16, mf2, ta, ma ; ZVE64F-NEXT: vle16.v v8, (a0) ; ZVE64F-NEXT: add a0, a0, a2 -; ZVE64F-NEXT: vle16.v v10, (a0) -; ZVE64F-NEXT: add a0, a0, a3 -; ZVE64F-NEXT: vle16.v v12, (a0) -; ZVE64F-NEXT: add a0, a0, a2 -; ZVE64F-NEXT: vle16.v v14, (a0) -; ZVE64F-NEXT: vsetivli zero, 8, e16, m2, tu, ma -; ZVE64F-NEXT: vslideup.vi v8, v10, 4 -; ZVE64F-NEXT: vsetivli zero, 12, e16, m2, tu, ma -; ZVE64F-NEXT: vslideup.vi v8, v12, 8 +; ZVE64F-NEXT: add a3, a0, a3 +; ZVE64F-NEXT: vle16.v v10, (a3) +; ZVE64F-NEXT: add a2, a3, a2 +; ZVE64F-NEXT: vle16.v v9, (a2) +; ZVE64F-NEXT: vle16.v v11, (a0) +; ZVE64F-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; ZVE64F-NEXT: vslideup.vi v10, v9, 4 +; ZVE64F-NEXT: vslideup.vi v8, v11, 4 ; ZVE64F-NEXT: vsetivli zero, 16, e16, m2, ta, ma -; ZVE64F-NEXT: vslideup.vi v8, v14, 12 +; ZVE64F-NEXT: vslideup.vi v8, v10, 8 ; ZVE64F-NEXT: vse16.v v8, (a1) ; ZVE64F-NEXT: ret %a = load <4 x i16>, ptr %x diff --git a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll index eb7894ede046..b3bda5973eb8 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll @@ -441,57 +441,50 @@ define <4 x i32> @stest_f16i32(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 ; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnclip.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb @@ -609,57 +602,50 @@ define <4 x i32> @utesth_f16i32(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 ; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb @@ -787,60 +773,53 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vslideup.vi v8, v10, 2 ; CHECK-V-NEXT: li a0, -1 ; CHECK-V-NEXT: srli a0, a0, 32 -; CHECK-V-NEXT: vmin.vx v8, v10, a0 +; CHECK-V-NEXT: vmin.vx v8, v8, a0 ; CHECK-V-NEXT: vmax.vx v10, v8, zero ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnsrl.wi v8, v10, 0 @@ -1404,90 +1383,125 @@ define <8 x i16> @stest_f16i16(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 4 ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnclip.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -1682,90 +1696,125 @@ define <8 x i16> @utesth_f16i16(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 -; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma -; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma ; CHECK-V-NEXT: csrr a0, vlenb ; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 4 +; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma +; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -1982,94 +2031,129 @@ define <8 x i16> @ustest_f16i16(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vslideup.vi v8, v10, 4 ; CHECK-V-NEXT: lui a0, 16 ; CHECK-V-NEXT: addi a0, a0, -1 -; CHECK-V-NEXT: vmin.vx v8, v10, a0 +; CHECK-V-NEXT: vmin.vx v8, v8, a0 ; CHECK-V-NEXT: vmax.vx v10, v8, zero ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnsrl.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -3723,57 +3807,50 @@ define <4 x i32> @stest_f16i32_mm(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 ; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnclip.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb @@ -3889,57 +3966,50 @@ define <4 x i32> @utesth_f16i32_mm(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 ; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb @@ -4066,60 +4136,53 @@ define <4 x i32> @ustest_f16i32_mm(<4 x half> %x) { ; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 ; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 4 * vlenb -; CHECK-V-NEXT: lhu s0, 24(a0) -; CHECK-V-NEXT: lhu s1, 16(a0) -; CHECK-V-NEXT: lhu s2, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu a0, 24(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e64, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e64, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 -; CHECK-V-NEXT: add a0, sp, a0 -; CHECK-V-NEXT: addi a0, a0, 16 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e64, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; CHECK-V-NEXT: vsetivli zero, 2, e64, m1, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e64, m2, ta, ma ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 ; CHECK-V-NEXT: add a0, sp, a0 ; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 +; CHECK-V-NEXT: vslideup.vi v8, v10, 2 ; CHECK-V-NEXT: li a0, -1 ; CHECK-V-NEXT: srli a0, a0, 32 -; CHECK-V-NEXT: vmin.vx v8, v10, a0 +; CHECK-V-NEXT: vmin.vx v8, v8, a0 ; CHECK-V-NEXT: vmax.vx v10, v8, zero ; CHECK-V-NEXT: vsetvli zero, zero, e32, m1, ta, ma ; CHECK-V-NEXT: vnsrl.wi v8, v10, 0 @@ -4671,90 +4734,125 @@ define <8 x i16> @stest_f16i16_mm(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 4 ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnclip.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -4947,90 +5045,125 @@ define <8 x i16> @utesth_f16i16_mm(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.lu.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-V-NEXT: vmv.s.x v10, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl2r.v v8, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v10, v8, 4 ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnclipu.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload @@ -5246,94 +5379,129 @@ define <8 x i16> @ustest_f16i16_mm(<8 x half> %x) { ; CHECK-V-NEXT: .cfi_offset s5, -56 ; CHECK-V-NEXT: .cfi_offset s6, -64 ; CHECK-V-NEXT: csrr a1, vlenb -; CHECK-V-NEXT: slli a1, a1, 1 +; CHECK-V-NEXT: slli a1, a1, 2 ; CHECK-V-NEXT: sub sp, sp, a1 -; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 2 * vlenb -; CHECK-V-NEXT: lhu s0, 56(a0) -; CHECK-V-NEXT: lhu s1, 48(a0) -; CHECK-V-NEXT: lhu s2, 40(a0) -; CHECK-V-NEXT: lhu s3, 32(a0) -; CHECK-V-NEXT: lhu s4, 24(a0) -; CHECK-V-NEXT: lhu s5, 16(a0) -; CHECK-V-NEXT: lhu s6, 0(a0) -; CHECK-V-NEXT: lhu a0, 8(a0) +; CHECK-V-NEXT: .cfi_escape 0x0f, 0x0e, 0x72, 0x00, 0x11, 0xd0, 0x00, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 80 + 4 * vlenb +; CHECK-V-NEXT: lhu s0, 0(a0) +; CHECK-V-NEXT: lhu s1, 8(a0) +; CHECK-V-NEXT: lhu s2, 16(a0) +; CHECK-V-NEXT: lhu s3, 24(a0) +; CHECK-V-NEXT: lhu s4, 32(a0) +; CHECK-V-NEXT: lhu s5, 40(a0) +; CHECK-V-NEXT: lhu s6, 48(a0) +; CHECK-V-NEXT: lhu a0, 56(a0) ; CHECK-V-NEXT: fmv.w.x fa0, a0 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz ; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s6 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v8, v10, 1 -; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s5 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 3, e32, m2, tu, ma -; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 2 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma ; CHECK-V-NEXT: fmv.w.x fa0, s4 +; CHECK-V-NEXT: vmv.s.x v8, a0 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 4, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 -; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 3 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs2r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s3 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 5, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s2 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 4 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s2 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 6, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 5 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: fmv.w.x fa0, s1 ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 7, e32, m2, tu, ma +; CHECK-V-NEXT: vsetivli zero, 1, e32, m1, ta, ma +; CHECK-V-NEXT: fmv.w.x fa0, s0 ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 -; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 6 -; CHECK-V-NEXT: vs2r.v v10, (a0) # Unknown-size Folded Spill -; CHECK-V-NEXT: fmv.w.x fa0, s0 +; CHECK-V-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill ; CHECK-V-NEXT: call __extendhfsf2 ; CHECK-V-NEXT: fcvt.l.s a0, fa0, rtz -; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma ; CHECK-V-NEXT: vmv.s.x v8, a0 ; CHECK-V-NEXT: addi a0, sp, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 1 +; CHECK-V-NEXT: vsetivli zero, 4, e32, m1, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 +; CHECK-V-NEXT: vl1r.v v9, (a0) # Unknown-size Folded Reload +; CHECK-V-NEXT: vslideup.vi v8, v9, 2 +; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma +; CHECK-V-NEXT: csrr a0, vlenb +; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: add a0, sp, a0 +; CHECK-V-NEXT: addi a0, a0, 16 ; CHECK-V-NEXT: vl2r.v v10, (a0) # Unknown-size Folded Reload -; CHECK-V-NEXT: vslideup.vi v10, v8, 7 +; CHECK-V-NEXT: vslideup.vi v8, v10, 4 ; CHECK-V-NEXT: lui a0, 16 ; CHECK-V-NEXT: addi a0, a0, -1 -; CHECK-V-NEXT: vmin.vx v8, v10, a0 +; CHECK-V-NEXT: vmin.vx v8, v8, a0 ; CHECK-V-NEXT: vmax.vx v10, v8, zero ; CHECK-V-NEXT: vsetvli zero, zero, e16, m1, ta, ma ; CHECK-V-NEXT: vnsrl.wi v8, v10, 0 ; CHECK-V-NEXT: csrr a0, vlenb -; CHECK-V-NEXT: slli a0, a0, 1 +; CHECK-V-NEXT: slli a0, a0, 2 ; CHECK-V-NEXT: add sp, sp, a0 ; CHECK-V-NEXT: ld ra, 72(sp) # 8-byte Folded Reload ; CHECK-V-NEXT: ld s0, 64(sp) # 8-byte Folded Reload diff --git a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll index f3ae03af7c78..0b236f6d3ff3 100644 --- a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll @@ -2136,17 +2136,18 @@ define @mgather_baseidx_nxv32i8(ptr %base, ; RV64-NEXT: vluxei64.v v13, (a0), v24, v0.t ; RV64-NEXT: srli a1, a1, 2 ; RV64-NEXT: vsetvli a3, zero, e8, mf2, ta, ma -; RV64-NEXT: vslidedown.vx v0, v16, a1 -; RV64-NEXT: vsetvli a1, zero, e64, m8, ta, ma -; RV64-NEXT: vsext.vf8 v16, v10 -; RV64-NEXT: vsetvli zero, zero, e8, m1, ta, mu -; RV64-NEXT: vluxei64.v v14, (a0), v16, v0.t +; RV64-NEXT: vslidedown.vx v8, v16, a1 ; RV64-NEXT: vsetvli a1, zero, e8, mf4, ta, ma -; RV64-NEXT: vslidedown.vx v0, v0, a2 +; RV64-NEXT: vslidedown.vx v0, v8, a2 ; RV64-NEXT: vsetvli a1, zero, e64, m8, ta, ma ; RV64-NEXT: vsext.vf8 v16, v11 ; RV64-NEXT: vsetvli zero, zero, e8, m1, ta, mu ; RV64-NEXT: vluxei64.v v15, (a0), v16, v0.t +; RV64-NEXT: vsetvli zero, zero, e64, m8, ta, ma +; RV64-NEXT: vsext.vf8 v16, v10 +; RV64-NEXT: vsetvli zero, zero, e8, m1, ta, mu +; RV64-NEXT: vmv1r.v v0, v8 +; RV64-NEXT: vluxei64.v v14, (a0), v16, v0.t ; RV64-NEXT: vmv4r.v v8, v12 ; RV64-NEXT: ret %ptrs = getelementptr inbounds i8, ptr %base, %idxs diff --git a/llvm/test/CodeGen/RISCV/rvv/pr63596.ll b/llvm/test/CodeGen/RISCV/rvv/pr63596.ll index c27488b18a01..d13d67fd0a88 100644 --- a/llvm/test/CodeGen/RISCV/rvv/pr63596.ll +++ b/llvm/test/CodeGen/RISCV/rvv/pr63596.ll @@ -9,39 +9,38 @@ define <4 x float> @foo(ptr %0) nounwind { ; CHECK-NEXT: sd s0, 32(sp) # 8-byte Folded Spill ; CHECK-NEXT: sd s1, 24(sp) # 8-byte Folded Spill ; CHECK-NEXT: sd s2, 16(sp) # 8-byte Folded Spill -; CHECK-NEXT: lhu s0, 6(a0) -; CHECK-NEXT: lhu s1, 4(a0) -; CHECK-NEXT: lhu s2, 0(a0) -; CHECK-NEXT: lhu a0, 2(a0) +; CHECK-NEXT: lhu s0, 0(a0) +; CHECK-NEXT: lhu s1, 2(a0) +; CHECK-NEXT: lhu s2, 4(a0) +; CHECK-NEXT: lhu a0, 6(a0) ; CHECK-NEXT: fmv.w.x fa0, a0 ; CHECK-NEXT: call __extendhfsf2 -; CHECK-NEXT: fsw fa0, 8(sp) +; CHECK-NEXT: fsw fa0, 4(sp) ; CHECK-NEXT: fmv.w.x fa0, s2 ; CHECK-NEXT: call __extendhfsf2 -; CHECK-NEXT: fsw fa0, 0(sp) +; CHECK-NEXT: fsw fa0, 12(sp) ; CHECK-NEXT: fmv.w.x fa0, s1 ; CHECK-NEXT: call __extendhfsf2 -; CHECK-NEXT: fsw fa0, 12(sp) +; CHECK-NEXT: fsw fa0, 8(sp) ; CHECK-NEXT: fmv.w.x fa0, s0 ; CHECK-NEXT: call __extendhfsf2 -; CHECK-NEXT: fsw fa0, 4(sp) -; CHECK-NEXT: addi a0, sp, 8 +; CHECK-NEXT: fsw fa0, 0(sp) +; CHECK-NEXT: addi a0, sp, 4 ; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; CHECK-NEXT: vle32.v v9, (a0) -; CHECK-NEXT: mv a0, sp ; CHECK-NEXT: vle32.v v8, (a0) -; CHECK-NEXT: vsetivli zero, 2, e32, m1, tu, ma -; CHECK-NEXT: vslideup.vi v8, v9, 1 ; CHECK-NEXT: addi a0, sp, 12 -; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma ; CHECK-NEXT: vle32.v v9, (a0) -; CHECK-NEXT: vsetivli zero, 3, e32, m1, tu, ma -; CHECK-NEXT: vslideup.vi v8, v9, 2 -; CHECK-NEXT: addi a0, sp, 4 +; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v9, v8, 1 +; CHECK-NEXT: addi a0, sp, 8 ; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma -; CHECK-NEXT: vle32.v v9, (a0) +; CHECK-NEXT: vle32.v v10, (a0) +; CHECK-NEXT: mv a0, sp +; CHECK-NEXT: vle32.v v8, (a0) +; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma +; CHECK-NEXT: vslideup.vi v8, v10, 1 ; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma -; CHECK-NEXT: vslideup.vi v8, v9, 3 +; CHECK-NEXT: vslideup.vi v8, v9, 2 ; CHECK-NEXT: ld ra, 40(sp) # 8-byte Folded Reload ; CHECK-NEXT: ld s0, 32(sp) # 8-byte Folded Reload ; CHECK-NEXT: ld s1, 24(sp) # 8-byte Folded Reload -- GitLab From 51d5b6581912c8495360a09a0e6be978e0374d90 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 22 Mar 2024 07:26:29 +0800 Subject: [PATCH 211/296] [RISCV] Handle scalable ops with < EEW / 2 narrow types in combineBinOp_VLToVWBinOp_VL (#84158) We can remove the restriction that the narrow type needs to be exactly EEW / 2 for scalable ISD::{ADD,SUB,MUL} nodes. This allows us to perform the combine even if we can't fully fold the extend into the widening op. VP intrinsics already do this, since they are lowered to _VL nodes which don't have this restriction. The "exactly EEW / 2" narrow type restriction prevented us from emitting V{S,Z}EXT_VL nodes with i1 element types which crash when we try to select them, since no other legal type is double the size of i1, see the test case added in this PR `i1_zext`. So to preserve this, this adds a check for i1 narrow types instead. --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 8 +- .../RISCV/rvv/vscale-vw-web-simplification.ll | 38 +- llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll | 483 ++++++++++-------- llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll | 384 +++++++------- llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll | 320 ++++++------ 5 files changed, 631 insertions(+), 602 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 71059b5bdc0f..5a2fb0239e0a 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -13715,12 +13715,8 @@ struct NodeExtensionHelper { SDValue NarrowElt = OrigOperand.getOperand(0); MVT NarrowVT = NarrowElt.getSimpleValueType(); - - unsigned ScalarBits = VT.getScalarSizeInBits(); - unsigned NarrowScalarBits = NarrowVT.getScalarSizeInBits(); - - // Ensure the extension's semantic is equivalent to rvv vzext or vsext. - if (ScalarBits != NarrowScalarBits * 2) + // i1 types are legal but we can't select V{S,Z}EXT_VLs with them. + if (NarrowVT.getVectorElementType() == MVT::i1) break; SupportsZExt = Opc == ISD::ZERO_EXTEND; diff --git a/llvm/test/CodeGen/RISCV/rvv/vscale-vw-web-simplification.ll b/llvm/test/CodeGen/RISCV/rvv/vscale-vw-web-simplification.ll index 972fa66917a5..e56dca0732bb 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vscale-vw-web-simplification.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vscale-vw-web-simplification.ll @@ -283,18 +283,19 @@ define @vwop_vscale_sext_i8i32_multiple_users(ptr %x, ptr %y, ; ; FOLDING-LABEL: vwop_vscale_sext_i8i32_multiple_users: ; FOLDING: # %bb.0: -; FOLDING-NEXT: vsetvli a3, zero, e32, m1, ta, ma +; FOLDING-NEXT: vsetvli a3, zero, e16, mf2, ta, ma ; FOLDING-NEXT: vle8.v v8, (a0) ; FOLDING-NEXT: vle8.v v9, (a1) ; FOLDING-NEXT: vle8.v v10, (a2) -; FOLDING-NEXT: vsext.vf4 v11, v8 -; FOLDING-NEXT: vsext.vf4 v8, v9 -; FOLDING-NEXT: vsext.vf4 v9, v10 -; FOLDING-NEXT: vmul.vv v8, v11, v8 -; FOLDING-NEXT: vadd.vv v10, v11, v9 -; FOLDING-NEXT: vsub.vv v9, v11, v9 -; FOLDING-NEXT: vor.vv v8, v8, v10 -; FOLDING-NEXT: vor.vv v8, v8, v9 +; FOLDING-NEXT: vsext.vf2 v11, v8 +; FOLDING-NEXT: vsext.vf2 v8, v9 +; FOLDING-NEXT: vsext.vf2 v9, v10 +; FOLDING-NEXT: vwmul.vv v10, v11, v8 +; FOLDING-NEXT: vwadd.vv v8, v11, v9 +; FOLDING-NEXT: vwsub.vv v12, v11, v9 +; FOLDING-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; FOLDING-NEXT: vor.vv v8, v10, v8 +; FOLDING-NEXT: vor.vv v8, v8, v12 ; FOLDING-NEXT: ret %a = load , ptr %x %b = load , ptr %y @@ -563,18 +564,19 @@ define @vwop_vscale_zext_i8i32_multiple_users(ptr %x, ptr %y, ; ; FOLDING-LABEL: vwop_vscale_zext_i8i32_multiple_users: ; FOLDING: # %bb.0: -; FOLDING-NEXT: vsetvli a3, zero, e32, m1, ta, ma +; FOLDING-NEXT: vsetvli a3, zero, e16, mf2, ta, ma ; FOLDING-NEXT: vle8.v v8, (a0) ; FOLDING-NEXT: vle8.v v9, (a1) ; FOLDING-NEXT: vle8.v v10, (a2) -; FOLDING-NEXT: vzext.vf4 v11, v8 -; FOLDING-NEXT: vzext.vf4 v8, v9 -; FOLDING-NEXT: vzext.vf4 v9, v10 -; FOLDING-NEXT: vmul.vv v8, v11, v8 -; FOLDING-NEXT: vadd.vv v10, v11, v9 -; FOLDING-NEXT: vsub.vv v9, v11, v9 -; FOLDING-NEXT: vor.vv v8, v8, v10 -; FOLDING-NEXT: vor.vv v8, v8, v9 +; FOLDING-NEXT: vzext.vf2 v11, v8 +; FOLDING-NEXT: vzext.vf2 v8, v9 +; FOLDING-NEXT: vzext.vf2 v9, v10 +; FOLDING-NEXT: vwmulu.vv v10, v11, v8 +; FOLDING-NEXT: vwaddu.vv v8, v11, v9 +; FOLDING-NEXT: vwsubu.vv v12, v11, v9 +; FOLDING-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; FOLDING-NEXT: vor.vv v8, v10, v8 +; FOLDING-NEXT: vor.vv v8, v8, v12 ; FOLDING-NEXT: ret %a = load , ptr %x %b = load , ptr %y diff --git a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll index a559fbf2bc8a..66a7eea18be5 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV32 %s +; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck -check-prefixes=CHECK,RV64 %s define @vwadd_vv_nxv1i64_nxv1i32( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv1i64_nxv1i32: @@ -421,10 +421,10 @@ define @vwaddu_wx_nxv8i64_nxv8i32( %va, i32 define @vwadd_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -435,10 +435,10 @@ define @vwadd_vv_nxv1i64_nxv1i16( %va, @vwaddu_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -451,10 +451,10 @@ define @vwadd_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -469,10 +469,10 @@ define @vwaddu_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -485,9 +485,9 @@ define @vwaddu_vx_nxv1i64_nxv1i16( %va, i16 define @vwadd_wv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v9 +; CHECK-NEXT: vwadd.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -497,9 +497,9 @@ define @vwadd_wv_nxv1i64_nxv1i16( %va, @vwaddu_wv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v9 +; CHECK-NEXT: vwaddu.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -511,9 +511,9 @@ define @vwadd_wx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v9 +; CHECK-NEXT: vwadd.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -527,9 +527,9 @@ define @vwaddu_wx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v9 +; CHECK-NEXT: vwaddu.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -541,10 +541,10 @@ define @vwaddu_wx_nxv1i64_nxv1i16( %va, i16 define @vwadd_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -555,10 +555,10 @@ define @vwadd_vv_nxv2i64_nxv2i16( %va, @vwaddu_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -571,10 +571,10 @@ define @vwadd_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -589,10 +589,10 @@ define @vwaddu_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -605,9 +605,9 @@ define @vwaddu_vx_nxv2i64_nxv2i16( %va, i16 define @vwadd_wv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v11, v10 +; CHECK-NEXT: vwadd.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -617,9 +617,9 @@ define @vwadd_wv_nxv2i64_nxv2i16( %va, @vwaddu_wv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v11, v10 +; CHECK-NEXT: vwaddu.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -631,9 +631,9 @@ define @vwadd_wx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v11, v10 +; CHECK-NEXT: vwadd.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -647,9 +647,9 @@ define @vwaddu_wx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v11, v10 +; CHECK-NEXT: vwaddu.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -661,10 +661,10 @@ define @vwaddu_wx_nxv2i64_nxv2i16( %va, i16 define @vwadd_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vsext.vf4 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vsext.vf2 v14, v9 +; CHECK-NEXT: vwadd.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -675,10 +675,10 @@ define @vwadd_vv_nxv4i64_nxv4i16( %va, @vwaddu_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v12, v8 +; CHECK-NEXT: vzext.vf2 v14, v9 +; CHECK-NEXT: vwaddu.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -691,10 +691,10 @@ define @vwadd_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vsext.vf4 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vsext.vf2 v14, v9 +; CHECK-NEXT: vwadd.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -709,10 +709,10 @@ define @vwaddu_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v12, v8 +; CHECK-NEXT: vzext.vf2 v14, v9 +; CHECK-NEXT: vwaddu.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -725,9 +725,9 @@ define @vwaddu_vx_nxv4i64_nxv4i16( %va, i16 define @vwadd_wv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v14, v12 +; CHECK-NEXT: vwadd.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -737,9 +737,9 @@ define @vwadd_wv_nxv4i64_nxv4i16( %va, @vwaddu_wv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v14, v12 +; CHECK-NEXT: vwaddu.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -751,9 +751,9 @@ define @vwadd_wx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v14, v12 +; CHECK-NEXT: vwadd.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -767,9 +767,9 @@ define @vwaddu_wx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v14, v12 +; CHECK-NEXT: vwaddu.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -781,10 +781,10 @@ define @vwaddu_wx_nxv4i64_nxv4i16( %va, i16 define @vwadd_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vsext.vf4 v24, v10 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vsext.vf2 v20, v10 +; CHECK-NEXT: vwadd.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -795,10 +795,10 @@ define @vwadd_vv_nxv8i64_nxv8i16( %va, @vwaddu_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v24, v10 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v16, v8 +; CHECK-NEXT: vzext.vf2 v20, v10 +; CHECK-NEXT: vwaddu.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -811,10 +811,10 @@ define @vwadd_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vsext.vf4 v24, v10 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vsext.vf2 v20, v10 +; CHECK-NEXT: vwadd.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -829,10 +829,10 @@ define @vwaddu_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v24, v10 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v16, v8 +; CHECK-NEXT: vzext.vf2 v20, v10 +; CHECK-NEXT: vwaddu.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -845,9 +845,9 @@ define @vwaddu_vx_nxv8i64_nxv8i16( %va, i16 define @vwadd_wv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v20, v16 +; CHECK-NEXT: vwadd.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -857,9 +857,9 @@ define @vwadd_wv_nxv8i64_nxv8i16( %va, @vwaddu_wv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v20, v16 +; CHECK-NEXT: vwaddu.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -871,9 +871,9 @@ define @vwadd_wx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v20, v16 +; CHECK-NEXT: vwadd.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -887,9 +887,9 @@ define @vwaddu_wx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v20, v16 +; CHECK-NEXT: vwaddu.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -901,10 +901,10 @@ define @vwaddu_wx_nxv8i64_nxv8i16( %va, i16 define @vwadd_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -915,10 +915,10 @@ define @vwadd_vv_nxv1i64_nxv1i8( %va, @vwaddu_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -931,10 +931,10 @@ define @vwadd_vx_nxv1i64_nxv1i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -949,10 +949,10 @@ define @vwaddu_vx_nxv1i64_nxv1i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v8, v9 -; CHECK-NEXT: vadd.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -965,9 +965,9 @@ define @vwaddu_vx_nxv1i64_nxv1i8( %va, i8 %b define @vwadd_wv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v9 +; CHECK-NEXT: vwadd.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -977,9 +977,9 @@ define @vwadd_wv_nxv1i64_nxv1i8( %va, @vwaddu_wv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v9 +; CHECK-NEXT: vwaddu.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -991,9 +991,9 @@ define @vwadd_wx_nxv1i64_nxv1i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v9 +; CHECK-NEXT: vwadd.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1007,9 +1007,9 @@ define @vwaddu_wx_nxv1i64_nxv1i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v9 -; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v9 +; CHECK-NEXT: vwaddu.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1021,10 +1021,10 @@ define @vwaddu_wx_nxv1i64_nxv1i8( %va, i8 % define @vwadd_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1035,10 +1035,10 @@ define @vwadd_vv_nxv2i64_nxv2i8( %va, @vwaddu_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1051,10 +1051,10 @@ define @vwadd_vx_nxv2i64_nxv2i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwadd.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1069,10 +1069,10 @@ define @vwaddu_vx_nxv2i64_nxv2i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v12, v9 -; CHECK-NEXT: vadd.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwaddu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1085,9 +1085,9 @@ define @vwaddu_vx_nxv2i64_nxv2i8( %va, i8 %b define @vwadd_wv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v11, v10 +; CHECK-NEXT: vwadd.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -1097,9 +1097,9 @@ define @vwadd_wv_nxv2i64_nxv2i8( %va, @vwaddu_wv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v11, v10 +; CHECK-NEXT: vwaddu.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -1111,9 +1111,9 @@ define @vwadd_wx_nxv2i64_nxv2i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v11, v10 +; CHECK-NEXT: vwadd.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1127,9 +1127,9 @@ define @vwaddu_wx_nxv2i64_nxv2i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v10 -; CHECK-NEXT: vadd.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v11, v10 +; CHECK-NEXT: vwaddu.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1141,10 +1141,10 @@ define @vwaddu_wx_nxv2i64_nxv2i8( %va, i8 % define @vwadd_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vsext.vf8 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v14, v9 +; CHECK-NEXT: vwadd.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1155,10 +1155,10 @@ define @vwadd_vv_nxv4i64_nxv4i8( %va, @vwaddu_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v8 -; CHECK-NEXT: vzext.vf8 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v14, v9 +; CHECK-NEXT: vwaddu.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1171,10 +1171,10 @@ define @vwadd_vx_nxv4i64_nxv4i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vsext.vf8 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v14, v9 +; CHECK-NEXT: vwadd.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1189,10 +1189,10 @@ define @vwaddu_vx_nxv4i64_nxv4i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v8 -; CHECK-NEXT: vzext.vf8 v16, v9 -; CHECK-NEXT: vadd.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v14, v9 +; CHECK-NEXT: vwaddu.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1205,9 +1205,9 @@ define @vwaddu_vx_nxv4i64_nxv4i8( %va, i8 %b define @vwadd_wv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v14, v12 +; CHECK-NEXT: vwadd.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -1217,9 +1217,9 @@ define @vwadd_wv_nxv4i64_nxv4i8( %va, @vwaddu_wv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v14, v12 +; CHECK-NEXT: vwaddu.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -1231,9 +1231,9 @@ define @vwadd_wx_nxv4i64_nxv4i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v14, v12 +; CHECK-NEXT: vwadd.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1247,9 +1247,9 @@ define @vwaddu_wx_nxv4i64_nxv4i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v12 -; CHECK-NEXT: vadd.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v14, v12 +; CHECK-NEXT: vwaddu.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1261,10 +1261,10 @@ define @vwaddu_wx_nxv4i64_nxv4i8( %va, i8 % define @vwadd_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwadd_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsext.vf8 v24, v9 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v20, v9 +; CHECK-NEXT: vwadd.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1275,10 +1275,10 @@ define @vwadd_vv_nxv8i64_nxv8i8( %va, @vwaddu_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwaddu_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v8 -; CHECK-NEXT: vzext.vf8 v24, v9 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v20, v9 +; CHECK-NEXT: vwaddu.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1291,10 +1291,10 @@ define @vwadd_vx_nxv8i64_nxv8i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsext.vf8 v24, v9 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v20, v9 +; CHECK-NEXT: vwadd.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1309,10 +1309,10 @@ define @vwaddu_vx_nxv8i64_nxv8i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v8 -; CHECK-NEXT: vzext.vf8 v24, v9 -; CHECK-NEXT: vadd.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v20, v9 +; CHECK-NEXT: vwaddu.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1325,9 +1325,9 @@ define @vwaddu_vx_nxv8i64_nxv8i8( %va, i8 %b define @vwadd_wv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwadd_wv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v20, v16 +; CHECK-NEXT: vwadd.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = sext %vb to %vd = add %va, %vc @@ -1337,9 +1337,9 @@ define @vwadd_wv_nxv8i64_nxv8i8( %va, @vwaddu_wv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwaddu_wv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v20, v16 +; CHECK-NEXT: vwaddu.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = zext %vb to %vd = add %va, %vc @@ -1351,9 +1351,9 @@ define @vwadd_wx_nxv8i64_nxv8i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v20, v16 +; CHECK-NEXT: vwadd.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1367,9 +1367,9 @@ define @vwaddu_wx_nxv8i64_nxv8i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v24, v16 -; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v20, v16 +; CHECK-NEXT: vwaddu.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1377,3 +1377,34 @@ define @vwaddu_wx_nxv8i64_nxv8i8( %va, i8 % %vc = add %va, %vb ret %vc } + +; Make sure that we don't introduce any V{S,Z}EXT_VL nodes with i1 types from +; combineBinOp_VLToVWBinOp_VL, since they can't be selected. +define @i1_zext( %va, %vb, ptr %p) { +; RV32-LABEL: i1_zext: +; RV32: # %bb.0: +; RV32-NEXT: vsetvli a1, zero, e64, m1, ta, ma +; RV32-NEXT: vmv.v.i v9, 0 +; RV32-NEXT: vmerge.vim v9, v9, 1, v0 +; RV32-NEXT: vadd.vv v8, v9, v8 +; RV32-NEXT: li a1, 42 +; RV32-NEXT: sh a1, 0(a0) +; RV32-NEXT: ret +; +; RV64-LABEL: i1_zext: +; RV64: # %bb.0: +; RV64-NEXT: vsetvli a1, zero, e64, m1, ta, mu +; RV64-NEXT: vadd.vi v8, v8, 1, v0.t +; RV64-NEXT: li a1, 42 +; RV64-NEXT: sh a1, 0(a0) +; RV64-NEXT: ret + %vc = zext %va to + %vd = add %vc, %vb + +; Introduce an illegal type so that the DAG changes after legalizing +; types. Otherwise the legalize vector ops phase will be run immediately after +; the legalize types phase, and the zext will already be in non-i1 form by the +; time combineBinOp_VLToVWBinOp_VL is called. + store i9 42, ptr %p + ret %vd +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll index 3634162eefd6..539a4bdb27ad 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwmul-sdnode.ll @@ -341,10 +341,10 @@ define @vwmulsu_vx_nxv8i64_nxv8i32( %va, i3 define @vwmul_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -355,10 +355,10 @@ define @vwmul_vv_nxv1i64_nxv1i16( %va, @vwmulu_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -369,10 +369,10 @@ define @vwmulu_vv_nxv1i64_nxv1i16( %va, @vwmulsu_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -385,10 +385,10 @@ define @vwmul_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -403,10 +403,10 @@ define @vwmulu_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -421,10 +421,10 @@ define @vwmulsu_vx_nxv1i64_nxv1i16( %va, i1 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -437,10 +437,10 @@ define @vwmulsu_vx_nxv1i64_nxv1i16( %va, i1 define @vwmul_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -451,10 +451,10 @@ define @vwmul_vv_nxv2i64_nxv2i16( %va, @vwmulu_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -465,10 +465,10 @@ define @vwmulu_vv_nxv2i64_nxv2i16( %va, @vwmulsu_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -481,10 +481,10 @@ define @vwmul_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -499,10 +499,10 @@ define @vwmulu_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -517,10 +517,10 @@ define @vwmulsu_vx_nxv2i64_nxv2i16( %va, i1 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vzext.vf4 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vzext.vf2 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -533,10 +533,10 @@ define @vwmulsu_vx_nxv2i64_nxv2i16( %va, i1 define @vwmul_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vsext.vf4 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vsext.vf2 v14, v9 +; CHECK-NEXT: vwmul.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -547,10 +547,10 @@ define @vwmul_vv_nxv4i64_nxv4i16( %va, @vwmulu_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v12, v8 +; CHECK-NEXT: vzext.vf2 v14, v9 +; CHECK-NEXT: vwmulu.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -561,10 +561,10 @@ define @vwmulu_vv_nxv4i64_nxv4i16( %va, @vwmulsu_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vzext.vf2 v14, v9 +; CHECK-NEXT: vwmulsu.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -577,10 +577,10 @@ define @vwmul_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vsext.vf4 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vsext.vf2 v14, v9 +; CHECK-NEXT: vwmul.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -595,10 +595,10 @@ define @vwmulu_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v12, v8 +; CHECK-NEXT: vzext.vf2 v14, v9 +; CHECK-NEXT: vwmulu.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -613,10 +613,10 @@ define @vwmulsu_vx_nxv4i64_nxv4i16( %va, i1 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vzext.vf4 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vzext.vf2 v14, v9 +; CHECK-NEXT: vwmulsu.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -629,10 +629,10 @@ define @vwmulsu_vx_nxv4i64_nxv4i16( %va, i1 define @vwmul_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vsext.vf4 v24, v10 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vsext.vf2 v20, v10 +; CHECK-NEXT: vwmul.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -643,10 +643,10 @@ define @vwmul_vv_nxv8i64_nxv8i16( %va, @vwmulu_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v24, v10 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v16, v8 +; CHECK-NEXT: vzext.vf2 v20, v10 +; CHECK-NEXT: vwmulu.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -657,10 +657,10 @@ define @vwmulu_vv_nxv8i64_nxv8i16( %va, @vwmulsu_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v24, v10 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vzext.vf2 v20, v10 +; CHECK-NEXT: vwmulsu.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -673,10 +673,10 @@ define @vwmul_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vsext.vf4 v24, v10 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vsext.vf2 v20, v10 +; CHECK-NEXT: vwmul.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -691,10 +691,10 @@ define @vwmulu_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v24, v10 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v16, v8 +; CHECK-NEXT: vzext.vf2 v20, v10 +; CHECK-NEXT: vwmulu.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -709,10 +709,10 @@ define @vwmulsu_vx_nxv8i64_nxv8i16( %va, i1 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vzext.vf4 v24, v10 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vzext.vf2 v20, v10 +; CHECK-NEXT: vwmulsu.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement undef, i16 %b, i16 0 %splat = shufflevector %head, undef, zeroinitializer @@ -725,10 +725,10 @@ define @vwmulsu_vx_nxv8i64_nxv8i16( %va, i1 define @vwmul_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -739,10 +739,10 @@ define @vwmul_vv_nxv1i64_nxv1i8( %va, @vwmulu_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -753,10 +753,10 @@ define @vwmulu_vv_nxv1i64_nxv1i8( %va, @vwmulsu_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -769,10 +769,10 @@ define @vwmul_vx_nxv1i64_nxv1i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -787,10 +787,10 @@ define @vwmulu_vx_nxv1i64_nxv1i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -805,10 +805,10 @@ define @vwmulsu_vx_nxv1i64_nxv1i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v8, v9 -; CHECK-NEXT: vmul.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -821,10 +821,10 @@ define @vwmulsu_vx_nxv1i64_nxv1i8( %va, i8 % define @vwmul_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -835,10 +835,10 @@ define @vwmul_vv_nxv2i64_nxv2i8( %va, @vwmulu_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -849,10 +849,10 @@ define @vwmulu_vv_nxv2i64_nxv2i8( %va, @vwmulsu_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -865,10 +865,10 @@ define @vwmul_vx_nxv2i64_nxv2i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwmul.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -883,10 +883,10 @@ define @vwmulu_vx_nxv2i64_nxv2i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -901,10 +901,10 @@ define @vwmulsu_vx_nxv2i64_nxv2i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vzext.vf8 v12, v9 -; CHECK-NEXT: vmul.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vzext.vf4 v11, v9 +; CHECK-NEXT: vwmulsu.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -917,10 +917,10 @@ define @vwmulsu_vx_nxv2i64_nxv2i8( %va, i8 % define @vwmul_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vsext.vf8 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v14, v9 +; CHECK-NEXT: vwmul.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -931,10 +931,10 @@ define @vwmul_vv_nxv4i64_nxv4i8( %va, @vwmulu_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v8 -; CHECK-NEXT: vzext.vf8 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v14, v9 +; CHECK-NEXT: vwmulu.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -945,10 +945,10 @@ define @vwmulu_vv_nxv4i64_nxv4i8( %va, @vwmulsu_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vzext.vf8 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v14, v9 +; CHECK-NEXT: vwmulsu.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -961,10 +961,10 @@ define @vwmul_vx_nxv4i64_nxv4i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vsext.vf8 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v14, v9 +; CHECK-NEXT: vwmul.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -979,10 +979,10 @@ define @vwmulu_vx_nxv4i64_nxv4i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v8 -; CHECK-NEXT: vzext.vf8 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v14, v9 +; CHECK-NEXT: vwmulu.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -997,10 +997,10 @@ define @vwmulsu_vx_nxv4i64_nxv4i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vzext.vf8 v16, v9 -; CHECK-NEXT: vmul.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vzext.vf4 v14, v9 +; CHECK-NEXT: vwmulsu.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -1013,10 +1013,10 @@ define @vwmulsu_vx_nxv4i64_nxv4i8( %va, i8 % define @vwmul_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwmul_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsext.vf8 v24, v9 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v20, v9 +; CHECK-NEXT: vwmul.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1027,10 +1027,10 @@ define @vwmul_vv_nxv8i64_nxv8i8( %va, @vwmulu_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwmulu_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v8 -; CHECK-NEXT: vzext.vf8 v24, v9 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v20, v9 +; CHECK-NEXT: vwmulu.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = zext %va to %vd = zext %vb to @@ -1041,10 +1041,10 @@ define @vwmulu_vv_nxv8i64_nxv8i8( %va, @vwmulsu_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwmulsu_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vzext.vf8 v24, v9 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v20, v9 +; CHECK-NEXT: vwmulsu.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = zext %vb to @@ -1057,10 +1057,10 @@ define @vwmul_vx_nxv8i64_nxv8i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsext.vf8 v24, v9 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v20, v9 +; CHECK-NEXT: vwmul.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -1075,10 +1075,10 @@ define @vwmulu_vx_nxv8i64_nxv8i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v8 -; CHECK-NEXT: vzext.vf8 v24, v9 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v20, v9 +; CHECK-NEXT: vwmulu.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer @@ -1093,10 +1093,10 @@ define @vwmulsu_vx_nxv8i64_nxv8i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vzext.vf8 v24, v9 -; CHECK-NEXT: vmul.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vzext.vf4 v20, v9 +; CHECK-NEXT: vwmulsu.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement undef, i8 %b, i8 0 %splat = shufflevector %head, undef, zeroinitializer diff --git a/llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll index 123469ade0ed..852814d648bf 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwsub-sdnode.ll @@ -421,10 +421,10 @@ define @vwsubu_wx_nxv8i64_nxv8i32( %va, i32 define @vwsub_vv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v8, v9 -; CHECK-NEXT: vsub.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -451,10 +451,10 @@ define @vwsub_vx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v8, v9 -; CHECK-NEXT: vsub.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -483,9 +483,9 @@ define @vwsubu_vx_nxv1i64_nxv1i16( %va, i16 define @vwsub_wv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v9 +; CHECK-NEXT: vwsub.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -495,9 +495,9 @@ define @vwsub_wv_nxv1i64_nxv1i16( %va, @vwsubu_wv_nxv1i64_nxv1i16( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv1i64_nxv1i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v9 +; CHECK-NEXT: vwsubu.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -509,9 +509,9 @@ define @vwsub_wx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v9 +; CHECK-NEXT: vwsub.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -525,9 +525,9 @@ define @vwsubu_wx_nxv1i64_nxv1i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf4 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf2 v10, v9 +; CHECK-NEXT: vwsubu.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -539,10 +539,10 @@ define @vwsubu_wx_nxv1i64_nxv1i16( %va, i16 define @vwsub_vv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v12, v9 -; CHECK-NEXT: vsub.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -569,10 +569,10 @@ define @vwsub_vx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v10, v8 -; CHECK-NEXT: vsext.vf4 v12, v9 -; CHECK-NEXT: vsub.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v10, v8 +; CHECK-NEXT: vsext.vf2 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -601,9 +601,9 @@ define @vwsubu_vx_nxv2i64_nxv2i16( %va, i16 define @vwsub_wv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v11, v10 +; CHECK-NEXT: vwsub.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -613,9 +613,9 @@ define @vwsub_wv_nxv2i64_nxv2i16( %va, @vwsubu_wv_nxv2i64_nxv2i16( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv2i64_nxv2i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v11, v10 +; CHECK-NEXT: vwsubu.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -627,9 +627,9 @@ define @vwsub_wx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf2 v11, v10 +; CHECK-NEXT: vwsub.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -643,9 +643,9 @@ define @vwsubu_wx_nxv2i64_nxv2i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf4 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf2 v11, v10 +; CHECK-NEXT: vwsubu.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -657,10 +657,10 @@ define @vwsubu_wx_nxv2i64_nxv2i16( %va, i16 define @vwsub_vv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vsext.vf4 v16, v9 -; CHECK-NEXT: vsub.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vsext.vf2 v14, v9 +; CHECK-NEXT: vwsub.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -687,10 +687,10 @@ define @vwsub_vx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v12, v8 -; CHECK-NEXT: vsext.vf4 v16, v9 -; CHECK-NEXT: vsub.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v12, v8 +; CHECK-NEXT: vsext.vf2 v14, v9 +; CHECK-NEXT: vwsub.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -719,9 +719,9 @@ define @vwsubu_vx_nxv4i64_nxv4i16( %va, i16 define @vwsub_wv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v14, v12 +; CHECK-NEXT: vwsub.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -731,9 +731,9 @@ define @vwsub_wv_nxv4i64_nxv4i16( %va, @vwsubu_wv_nxv4i64_nxv4i16( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv4i64_nxv4i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v14, v12 +; CHECK-NEXT: vwsubu.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -745,9 +745,9 @@ define @vwsub_wx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf2 v14, v12 +; CHECK-NEXT: vwsub.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -761,9 +761,9 @@ define @vwsubu_wx_nxv4i64_nxv4i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m1, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf4 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf2 v14, v12 +; CHECK-NEXT: vwsubu.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -775,10 +775,10 @@ define @vwsubu_wx_nxv4i64_nxv4i16( %va, i16 define @vwsub_vv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vsext.vf4 v24, v10 -; CHECK-NEXT: vsub.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vsext.vf2 v20, v10 +; CHECK-NEXT: vwsub.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -805,10 +805,10 @@ define @vwsub_vx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v16, v8 -; CHECK-NEXT: vsext.vf4 v24, v10 -; CHECK-NEXT: vsub.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v16, v8 +; CHECK-NEXT: vsext.vf2 v20, v10 +; CHECK-NEXT: vwsub.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -837,9 +837,9 @@ define @vwsubu_vx_nxv8i64_nxv8i16( %va, i16 define @vwsub_wv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v20, v16 +; CHECK-NEXT: vwsub.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -849,9 +849,9 @@ define @vwsub_wv_nxv8i64_nxv8i16( %va, @vwsubu_wv_nxv8i64_nxv8i16( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv8i64_nxv8i16: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v20, v16 +; CHECK-NEXT: vwsubu.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -863,9 +863,9 @@ define @vwsub_wx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf4 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf2 v20, v16 +; CHECK-NEXT: vwsub.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -879,9 +879,9 @@ define @vwsubu_wx_nxv8i64_nxv8i16( %va, i16 ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e16, m2, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf4 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf2 v20, v16 +; CHECK-NEXT: vwsubu.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i16 %b, i16 0 %splat = shufflevector %head, poison, zeroinitializer @@ -893,10 +893,10 @@ define @vwsubu_wx_nxv8i64_nxv8i16( %va, i16 define @vwsub_vv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v8, v9 -; CHECK-NEXT: vsub.vv v8, v10, v8 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -923,10 +923,10 @@ define @vwsub_vx_nxv1i64_nxv1i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v8, v9 -; CHECK-NEXT: vsub.vv v8, v10, v8 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -955,9 +955,9 @@ define @vwsubu_vx_nxv1i64_nxv1i8( %va, i8 %b define @vwsub_wv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v9 +; CHECK-NEXT: vwsub.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -967,9 +967,9 @@ define @vwsub_wv_nxv1i64_nxv1i8( %va, @vwsubu_wv_nxv1i64_nxv1i8( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv1i64_nxv1i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli a0, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v9 +; CHECK-NEXT: vwsubu.wv v8, v8, v10 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -981,9 +981,9 @@ define @vwsub_wx_nxv1i64_nxv1i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v9 +; CHECK-NEXT: vwsub.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -997,9 +997,9 @@ define @vwsubu_wx_nxv1i64_nxv1i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf8, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vzext.vf8 v10, v9 -; CHECK-NEXT: vsub.vv v8, v8, v10 +; CHECK-NEXT: vsetvli zero, zero, e32, mf2, ta, ma +; CHECK-NEXT: vzext.vf4 v10, v9 +; CHECK-NEXT: vwsubu.wv v8, v8, v10 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1011,10 +1011,10 @@ define @vwsubu_wx_nxv1i64_nxv1i8( %va, i8 % define @vwsub_vv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v12, v9 -; CHECK-NEXT: vsub.vv v8, v10, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1041,10 +1041,10 @@ define @vwsub_vx_nxv2i64_nxv2i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v10, v8 -; CHECK-NEXT: vsext.vf8 v12, v9 -; CHECK-NEXT: vsub.vv v8, v10, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v10, v8 +; CHECK-NEXT: vsext.vf4 v11, v9 +; CHECK-NEXT: vwsub.vv v8, v10, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1073,9 +1073,9 @@ define @vwsubu_vx_nxv2i64_nxv2i8( %va, i8 %b define @vwsub_wv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v11, v10 +; CHECK-NEXT: vwsub.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -1085,9 +1085,9 @@ define @vwsub_wv_nxv2i64_nxv2i8( %va, @vwsubu_wv_nxv2i64_nxv2i8( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv2i64_nxv2i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli a0, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v11, v10 +; CHECK-NEXT: vwsubu.wv v8, v8, v11 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -1099,9 +1099,9 @@ define @vwsub_wx_nxv2i64_nxv2i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vsext.vf4 v11, v10 +; CHECK-NEXT: vwsub.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1115,9 +1115,9 @@ define @vwsubu_wx_nxv2i64_nxv2i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv.v.x v10, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m2, ta, ma -; CHECK-NEXT: vzext.vf8 v12, v10 -; CHECK-NEXT: vsub.vv v8, v8, v12 +; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma +; CHECK-NEXT: vzext.vf4 v11, v10 +; CHECK-NEXT: vwsubu.wv v8, v8, v11 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1129,10 +1129,10 @@ define @vwsubu_wx_nxv2i64_nxv2i8( %va, i8 % define @vwsub_vv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vsext.vf8 v16, v9 -; CHECK-NEXT: vsub.vv v8, v12, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v14, v9 +; CHECK-NEXT: vwsub.vv v8, v12, v14 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1159,10 +1159,10 @@ define @vwsub_vx_nxv4i64_nxv4i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v12, v8 -; CHECK-NEXT: vsext.vf8 v16, v9 -; CHECK-NEXT: vsub.vv v8, v12, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v12, v8 +; CHECK-NEXT: vsext.vf4 v14, v9 +; CHECK-NEXT: vwsub.vv v8, v12, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1191,9 +1191,9 @@ define @vwsubu_vx_nxv4i64_nxv4i8( %va, i8 %b define @vwsub_wv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v14, v12 +; CHECK-NEXT: vwsub.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -1203,9 +1203,9 @@ define @vwsub_wv_nxv4i64_nxv4i8( %va, @vwsubu_wv_nxv4i64_nxv4i8( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv4i64_nxv4i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v14, v12 +; CHECK-NEXT: vwsubu.wv v8, v8, v14 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -1217,9 +1217,9 @@ define @vwsub_wx_nxv4i64_nxv4i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vsext.vf4 v14, v12 +; CHECK-NEXT: vwsub.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1233,9 +1233,9 @@ define @vwsubu_wx_nxv4i64_nxv4i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, mf2, ta, ma ; CHECK-NEXT: vmv.v.x v12, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m4, ta, ma -; CHECK-NEXT: vzext.vf8 v16, v12 -; CHECK-NEXT: vsub.vv v8, v8, v16 +; CHECK-NEXT: vsetvli zero, zero, e32, m2, ta, ma +; CHECK-NEXT: vzext.vf4 v14, v12 +; CHECK-NEXT: vwsubu.wv v8, v8, v14 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1247,10 +1247,10 @@ define @vwsubu_wx_nxv4i64_nxv4i8( %va, i8 % define @vwsub_vv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwsub_vv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsext.vf8 v24, v9 -; CHECK-NEXT: vsub.vv v8, v16, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v20, v9 +; CHECK-NEXT: vwsub.vv v8, v16, v20 ; CHECK-NEXT: ret %vc = sext %va to %vd = sext %vb to @@ -1277,10 +1277,10 @@ define @vwsub_vx_nxv8i64_nxv8i8( %va, i8 %b) ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v9, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v16, v8 -; CHECK-NEXT: vsext.vf8 v24, v9 -; CHECK-NEXT: vsub.vv v8, v16, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v16, v8 +; CHECK-NEXT: vsext.vf4 v20, v9 +; CHECK-NEXT: vwsub.vv v8, v16, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1309,9 +1309,9 @@ define @vwsubu_vx_nxv8i64_nxv8i8( %va, i8 %b define @vwsub_wv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwsub_wv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v20, v16 +; CHECK-NEXT: vwsub.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = sext %vb to %vd = sub %va, %vc @@ -1321,9 +1321,9 @@ define @vwsub_wv_nxv8i64_nxv8i8( %va, @vwsubu_wv_nxv8i64_nxv8i8( %va, %vb) { ; CHECK-LABEL: vwsubu_wv_nxv8i64_nxv8i8: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli a0, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli a0, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v20, v16 +; CHECK-NEXT: vwsubu.wv v8, v8, v20 ; CHECK-NEXT: ret %vc = zext %vb to %vd = sub %va, %vc @@ -1335,9 +1335,9 @@ define @vwsub_wx_nxv8i64_nxv8i8( %va, i8 %b ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vsext.vf8 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vsext.vf4 v20, v16 +; CHECK-NEXT: vwsub.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer @@ -1351,9 +1351,9 @@ define @vwsubu_wx_nxv8i64_nxv8i8( %va, i8 % ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a1, zero, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.x v16, a0 -; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vzext.vf8 v24, v16 -; CHECK-NEXT: vsub.vv v8, v8, v24 +; CHECK-NEXT: vsetvli zero, zero, e32, m4, ta, ma +; CHECK-NEXT: vzext.vf4 v20, v16 +; CHECK-NEXT: vwsubu.wv v8, v8, v20 ; CHECK-NEXT: ret %head = insertelement poison, i8 %b, i8 0 %splat = shufflevector %head, poison, zeroinitializer -- GitLab From d22cf4365ca58ccf1db21341d63ac49921f1c47a Mon Sep 17 00:00:00 2001 From: Nathan Lanza Date: Thu, 21 Mar 2024 19:52:19 -0400 Subject: [PATCH 212/296] [cmake] Place clang behind mlir in the liist of external projects In preparation for the initial ClangIR upstreaming process, move clang behind MLIR in the list of external projects. Otherwise, cmake will attempt to build clang before MLIR. reland of https://github.com/llvm/llvm-project/pull/86050 Reviewers: Pull Request: https://github.com/llvm/llvm-project/pull/86210 --- llvm/tools/CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/llvm/tools/CMakeLists.txt b/llvm/tools/CMakeLists.txt index c6116ac81d12..cde57367934e 100644 --- a/llvm/tools/CMakeLists.txt +++ b/llvm/tools/CMakeLists.txt @@ -37,12 +37,13 @@ add_llvm_tool_subdirectory(llvm-profdata) # Projects supported via LLVM_EXTERNAL_*_SOURCE_DIR need to be explicitly # specified. -add_llvm_external_project(clang) add_llvm_external_project(lld) -add_llvm_external_project(lldb) add_llvm_external_project(mlir) -# Flang depends on mlir, so place it afterward +# ClangIR and Flang depend on mlir, lldb and Flang depend on clang, sort them +# accordingly so place them afterwards +add_llvm_external_project(clang) add_llvm_external_project(flang) +add_llvm_external_project(lldb) add_llvm_external_project(bolt) # Automatically add remaining sub-directories containing a 'CMakeLists.txt' -- GitLab From e66b670f3bf9312f696e66c31152ae535207d6bb Mon Sep 17 00:00:00 2001 From: Nathan Lanza Date: Thu, 21 Mar 2024 19:53:48 -0400 Subject: [PATCH 213/296] [CIR][Basic][NFC] Add the CIR language to the Language enum Add the CIR language to the Language enum and the standard usages of it. commit-id:fd12b2c2 Reviewers: bcardosolopes, AaronBallman, erichkeane Reviewed By: AaronBallman, bcardosolopes Pull Request: https://github.com/llvm/llvm-project/pull/86072 --- clang/include/clang/Basic/LangStandard.h | 5 +++-- clang/include/clang/Driver/Types.def | 1 + clang/lib/Basic/LangStandards.cpp | 3 +++ .../Serialization/SymbolGraphSerializer.cpp | 1 + clang/lib/Frontend/CompilerInvocation.cpp | 13 +++++++++++-- clang/lib/Frontend/FrontendActions.cpp | 1 + clang/lib/Frontend/FrontendOptions.cpp | 1 + 7 files changed, 21 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/Basic/LangStandard.h b/clang/include/clang/Basic/LangStandard.h index 199e24c67316..8e25afc83366 100644 --- a/clang/include/clang/Basic/LangStandard.h +++ b/clang/include/clang/Basic/LangStandard.h @@ -26,8 +26,9 @@ enum class Language : uint8_t { /// Assembly: we accept this only so that we can preprocess it. Asm, - /// LLVM IR: we accept this so that we can run the optimizer on it, - /// and compile it to assembly or object code. + /// LLVM IR & CIR: we accept these so that we can run the optimizer on them, + /// and compile them to assembly or object code (or LLVM for CIR). + CIR, LLVM_IR, ///@{ Languages that the frontend can parse and compile. diff --git a/clang/include/clang/Driver/Types.def b/clang/include/clang/Driver/Types.def index f72c27e1ee70..0e0cae5fb706 100644 --- a/clang/include/clang/Driver/Types.def +++ b/clang/include/clang/Driver/Types.def @@ -90,6 +90,7 @@ TYPE("ir", LLVM_BC, INVALID, "bc", phases TYPE("lto-ir", LTO_IR, INVALID, "s", phases::Compile, phases::Backend, phases::Assemble, phases::Link) TYPE("lto-bc", LTO_BC, INVALID, "o", phases::Compile, phases::Backend, phases::Assemble, phases::Link) +TYPE("cir", CIR, INVALID, "cir", phases::Compile, phases::Backend, phases::Assemble, phases::Link) // Misc. TYPE("ast", AST, INVALID, "ast", phases::Compile, phases::Backend, phases::Assemble, phases::Link) TYPE("ifs", IFS, INVALID, "ifs", phases::IfsMerge) diff --git a/clang/lib/Basic/LangStandards.cpp b/clang/lib/Basic/LangStandards.cpp index cb2c07723499..c8c9292abcb2 100644 --- a/clang/lib/Basic/LangStandards.cpp +++ b/clang/lib/Basic/LangStandards.cpp @@ -21,6 +21,8 @@ StringRef clang::languageToString(Language L) { return "Asm"; case Language::LLVM_IR: return "LLVM IR"; + case Language::CIR: + return "ClangIR"; case Language::C: return "C"; case Language::CXX: @@ -92,6 +94,7 @@ LangStandard::Kind clang::getDefaultLanguageStandard(clang::Language Lang, switch (Lang) { case Language::Unknown: case Language::LLVM_IR: + case Language::CIR: llvm_unreachable("Invalid input kind!"); case Language::OpenCL: return LangStandard::lang_opencl12; diff --git a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp index 349b93e2a232..545860acb7db 100644 --- a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp +++ b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp @@ -208,6 +208,7 @@ StringRef getLanguageName(Language Lang) { case Language::Unknown: case Language::Asm: case Language::LLVM_IR: + case Language::CIR: llvm_unreachable("Unsupported language kind"); } diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 0df6a82ccd89..7bd91d4791ec 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -2757,6 +2757,9 @@ static void GenerateFrontendArgs(const FrontendOptions &Opts, case Language::HLSL: Lang = "hlsl"; break; + case Language::CIR: + Lang = "cir"; + break; } GenerateArg(Consumer, OPT_x, @@ -2958,6 +2961,7 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args, .Cases("ast", "pcm", "precompiled-header", InputKind(Language::Unknown, InputKind::Precompiled)) .Case("ir", Language::LLVM_IR) + .Case("cir", Language::CIR) .Default(Language::Unknown); if (DashX.isUnknown()) @@ -3323,6 +3327,7 @@ static bool IsInputCompatibleWithStandard(InputKind IK, switch (IK.getLanguage()) { case Language::Unknown: case Language::LLVM_IR: + case Language::CIR: llvm_unreachable("should not parse language flags for this input"); case Language::C: @@ -3388,6 +3393,8 @@ static StringRef GetInputKindName(InputKind IK) { return "Asm"; case Language::LLVM_IR: return "LLVM IR"; + case Language::CIR: + return "Clang IR"; case Language::HLSL: return "HLSL"; @@ -3403,7 +3410,8 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts, const llvm::Triple &T, InputKind IK) { if (IK.getFormat() == InputKind::Precompiled || - IK.getLanguage() == Language::LLVM_IR) { + IK.getLanguage() == Language::LLVM_IR || + IK.getLanguage() == Language::CIR) { if (Opts.ObjCAutoRefCount) GenerateArg(Consumer, OPT_fobjc_arc); if (Opts.PICLevel != 0) @@ -3689,7 +3697,8 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args, unsigned NumErrorsBefore = Diags.getNumErrors(); if (IK.getFormat() == InputKind::Precompiled || - IK.getLanguage() == Language::LLVM_IR) { + IK.getLanguage() == Language::LLVM_IR || + IK.getLanguage() == Language::CIR) { // ObjCAAutoRefCount and Sanitize LangOpts are used to setup the // PassManager in BackendUtil.cpp. They need to be initialized no matter // what the input type is. diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index 81fcd8d5ae9b..3fd1cdd3b479 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -1083,6 +1083,7 @@ void PrintPreambleAction::ExecuteAction() { case Language::CUDA: case Language::HIP: case Language::HLSL: + case Language::CIR: break; case Language::Unknown: diff --git a/clang/lib/Frontend/FrontendOptions.cpp b/clang/lib/Frontend/FrontendOptions.cpp index bf83b27c1367..32ed99571e85 100644 --- a/clang/lib/Frontend/FrontendOptions.cpp +++ b/clang/lib/Frontend/FrontendOptions.cpp @@ -34,5 +34,6 @@ InputKind FrontendOptions::getInputKindForExtension(StringRef Extension) { .Case("hip", Language::HIP) .Cases("ll", "bc", Language::LLVM_IR) .Case("hlsl", Language::HLSL) + .Case("cir", Language::CIR) .Default(Language::Unknown); } -- GitLab From 3942bd2fb56380aa050977dc6aede011e191d9b0 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Thu, 21 Mar 2024 17:05:50 -0700 Subject: [PATCH 214/296] [SLP]Fix a crash if the argument of call was affected by minbitwidt analysis. Need to support proper type conversion for function arguments to avoid compiler crash. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 9 +- .../X86/call-arg-reduced-by-minbitwidth.ll | 82 +++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/call-arg-reduced-by-minbitwidth.ll diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 36b446962c4a..7295ae0ba90b 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -12473,12 +12473,12 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { if (UseIntrinsic && isVectorIntrinsicWithOverloadTypeAtArg(ID, -1)) TysForDecl.push_back( FixedVectorType::get(CI->getType(), E->Scalars.size())); + auto *CEI = cast(VL0); for (unsigned I : seq(0, CI->arg_size())) { ValueList OpVL; // Some intrinsics have scalar arguments. This argument should not be // vectorized. if (UseIntrinsic && isVectorIntrinsicWithScalarOpAtArg(ID, I)) { - CallInst *CEI = cast(VL0); ScalarArg = CEI->getArgOperand(I); OpVecs.push_back(CEI->getArgOperand(I)); if (isVectorIntrinsicWithOverloadTypeAtArg(ID, I)) @@ -12491,6 +12491,13 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n"); return E->VectorizedValue; } + ScalarArg = CEI->getArgOperand(I); + if (cast(OpVec->getType())->getElementType() != + ScalarArg->getType()) { + auto *CastTy = FixedVectorType::get(ScalarArg->getType(), + VecTy->getNumElements()); + OpVec = Builder.CreateIntCast(OpVec, CastTy, GetOperandSignedness(I)); + } LLVM_DEBUG(dbgs() << "SLP: OpVec[" << I << "]: " << *OpVec << "\n"); OpVecs.push_back(OpVec); if (UseIntrinsic && isVectorIntrinsicWithOverloadTypeAtArg(ID, I)) diff --git a/llvm/test/Transforms/SLPVectorizer/X86/call-arg-reduced-by-minbitwidth.ll b/llvm/test/Transforms/SLPVectorizer/X86/call-arg-reduced-by-minbitwidth.ll new file mode 100644 index 000000000000..27c9655f94d3 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/X86/call-arg-reduced-by-minbitwidth.ll @@ -0,0 +1,82 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-pc-windows-msvc19.34.0 < %s | FileCheck %s + +define void @test(ptr %0, i8 %1, i1 %cmp12.i) { +; CHECK-LABEL: define void @test( +; CHECK-SAME: ptr [[TMP0:%.*]], i8 [[TMP1:%.*]], i1 [[CMP12_I:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP2:%.*]] = insertelement <8 x i1> poison, i1 [[CMP12_I]], i32 0 +; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> poison, <8 x i32> zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x i8> poison, i8 [[TMP1]], i32 0 +; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x i8> [[TMP4]], <8 x i8> poison, <8 x i32> zeroinitializer +; CHECK-NEXT: br label [[PRE:%.*]] +; CHECK: pre: +; CHECK-NEXT: [[TMP6:%.*]] = zext <8 x i8> [[TMP5]] to <8 x i32> +; CHECK-NEXT: [[TMP7:%.*]] = call <8 x i32> @llvm.umax.v8i32(<8 x i32> [[TMP6]], <8 x i32> ) +; CHECK-NEXT: [[TMP8:%.*]] = trunc <8 x i32> [[TMP7]] to <8 x i8> +; CHECK-NEXT: [[TMP9:%.*]] = add <8 x i8> [[TMP8]], +; CHECK-NEXT: [[TMP10:%.*]] = select <8 x i1> [[TMP3]], <8 x i8> [[TMP9]], <8 x i8> [[TMP5]] +; CHECK-NEXT: store <8 x i8> [[TMP10]], ptr [[TMP0]], align 1 +; CHECK-NEXT: br label [[PRE]] +; +entry: + %idx11 = getelementptr i8, ptr %0, i64 1 + %idx22 = getelementptr i8, ptr %0, i64 2 + %idx33 = getelementptr i8, ptr %0, i64 3 + %idx44 = getelementptr i8, ptr %0, i64 4 + %idx55 = getelementptr i8, ptr %0, i64 5 + %idx66 = getelementptr i8, ptr %0, i64 6 + %idx77 = getelementptr i8, ptr %0, i64 7 + br label %pre + +pre: + %conv.i = zext i8 %1 to i32 + %2 = tail call i32 @llvm.umax.i32(i32 %conv.i, i32 1) + %.sroa.speculated.i = add i32 %2, 1 + %intensity.0.i = select i1 %cmp12.i, i32 %.sroa.speculated.i, i32 %conv.i + %conv14.i = trunc i32 %intensity.0.i to i8 + store i8 %conv14.i, ptr %0, align 1 + %conv.i.1 = zext i8 %1 to i32 + %3 = tail call i32 @llvm.umax.i32(i32 %conv.i.1, i32 1) + %ss1 = add i32 %3, 1 + %ii1 = select i1 %cmp12.i, i32 %ss1, i32 %conv.i.1 + %conv14.i.1 = trunc i32 %ii1 to i8 + store i8 %conv14.i.1, ptr %idx11, align 1 + %conv.i.2 = zext i8 %1 to i32 + %4 = tail call i32 @llvm.umax.i32(i32 %conv.i.2, i32 1) + %ss2 = add i32 %4, 1 + %ii2 = select i1 %cmp12.i, i32 %ss2, i32 %conv.i.2 + %conv14.i.2 = trunc i32 %ii2 to i8 + store i8 %conv14.i.2, ptr %idx22, align 1 + %conv.i.3 = zext i8 %1 to i32 + %5 = tail call i32 @llvm.umax.i32(i32 %conv.i.3, i32 1) + %ss3 = add i32 %5, 1 + %ii3 = select i1 %cmp12.i, i32 %ss3, i32 %conv.i.3 + %conv14.i.3 = trunc i32 %ii3 to i8 + store i8 %conv14.i.3, ptr %idx33, align 1 + %conv.i.4 = zext i8 %1 to i32 + %6 = tail call i32 @llvm.umax.i32(i32 %conv.i.4, i32 1) + %ss4 = add i32 %6, 1 + %ii4 = select i1 %cmp12.i, i32 %ss4, i32 %conv.i.4 + %conv14.i.4 = trunc i32 %ii4 to i8 + store i8 %conv14.i.4, ptr %idx44, align 1 + %conv.i.5 = zext i8 %1 to i32 + %7 = tail call i32 @llvm.umax.i32(i32 %conv.i.5, i32 1) + %ss5 = add i32 %7, 1 + %ii5 = select i1 %cmp12.i, i32 %ss5, i32 %conv.i.5 + %conv14.i.5 = trunc i32 %ii5 to i8 + store i8 %conv14.i.5, ptr %idx55, align 1 + %conv.i.6 = zext i8 %1 to i32 + %8 = tail call i32 @llvm.umax.i32(i32 %conv.i.6, i32 1) + %ss6 = add i32 %8, 1 + %ii6 = select i1 %cmp12.i, i32 %ss6, i32 %conv.i.6 + %conv14.i.6 = trunc i32 %ii6 to i8 + store i8 %conv14.i.6, ptr %idx66, align 1 + %conv.i.7 = zext i8 %1 to i32 + %9 = tail call i32 @llvm.umax.i32(i32 %conv.i.7, i32 1) + %ss7 = add i32 %9, 1 + %ii7 = select i1 %cmp12.i, i32 %ss7, i32 %conv.i.7 + %conv14.i.7 = trunc i32 %ii7 to i8 + store i8 %conv14.i.7, ptr %idx77, align 1 + br label %pre +} -- GitLab From 7564566779eb07e9daf41a351b09cf7607871845 Mon Sep 17 00:00:00 2001 From: Jonas Paulsson Date: Wed, 20 Mar 2024 11:58:46 -0400 Subject: [PATCH 215/296] Reapply "Move assertion for AdjustsStack from PEI to MachineVerifier (#85698)" - The check is now actually done in both PEI and the MachineVerifier. - More .mir tests trivially updated with "adjustsStack: true" as needed. --- llvm/lib/CodeGen/MachineVerifier.cpp | 6 +++++ .../clear-dead-implicit-def-impdef.mir | 2 ++ ...plicit-def-remat-requires-impdef-check.mir | 2 ++ ...implicit-def-with-impdef-greedy-assert.mir | 2 ++ .../CodeGen/AMDGPU/fold-restore-undef-use.mir | 2 ++ .../greedy-alloc-fail-sgpr1024-spill.mir | 1 + .../ran-out-of-sgprs-allocation-failure.mir | 1 + .../CodeGen/AMDGPU/sched-crash-dbg-value.mir | 2 ++ .../AMDGPU/sgpr-spill-wrong-stack-id.mir | 1 + .../AMDGPU/snippet-copy-bundle-regression.mir | 1 + .../virtregrewrite-undef-identity-copy.mir | 1 + ...no-register-coalescing-in-returnsTwice.mir | 2 ++ .../CodeGen/Hexagon/regalloc-bad-undef.mir | 2 +- .../SystemZ/RAbasic-invalid-LR-update.mir | 2 ++ .../SystemZ/clear-liverange-spillreg.mir | 1 + llvm/test/CodeGen/SystemZ/int-cmp-56.mir | 4 +++ .../SystemZ/regcoal-subranges-update.mir | 2 ++ llvm/test/CodeGen/X86/callbr-asm-kill.mir | 1 + llvm/test/CodeGen/X86/late-remat-update.mir | 1 + llvm/test/CodeGen/X86/limit-split-cost.mir | 1 + llvm/test/CodeGen/X86/regalloc-copy-hints.mir | 1 + .../CodeGen/X86/statepoint-fastregalloc.mir | 4 +++ .../X86/statepoint-invoke-ra-enter-at-end.mir | 2 +- .../X86/statepoint-invoke-ra-hoist-copies.mir | 2 +- .../statepoint-invoke-ra-inline-spiller.mir | 2 +- ...tatepoint-invoke-ra-remove-back-copies.mir | 2 +- .../test/CodeGen/X86/statepoint-invoke-ra.mir | 2 +- .../CodeGen/X86/statepoint-vreg-folding.mir | 2 +- .../memory-operand-folding-tieddef.mir | 1 + .../InstrRef/memory-operand-load-folding.mir | 1 + .../MIR/InstrRef/phi-coalesce-subreg.mir | 1 + .../DebugInfo/MIR/InstrRef/phi-coalescing.mir | 1 + .../MIR/InstrRef/phi-on-stack-coalesced.mir | 1 + .../MIR/InstrRef/phi-on-stack-coalesced2.mir | 1 + .../MIR/InstrRef/phi-regallocd-to-stack.mir | 1 + .../MIR/InstrRef/phi-through-regalloc.mir | 1 + .../MIR/InstrRef/stack-coloring-dbg-phi.mir | 1 + .../MIR/InstrRef/survives-livedebugvars.mir | 1 + .../Mips/livedebugvars-stop-trimming-loc.mir | 2 ++ .../live-debug-vars-unused-arg-debugonly.mir | 2 +- .../MIR/X86/live-debug-vars-unused-arg.mir | 2 +- .../X86/livedebugvars-crossbb-interval.mir | 1 + .../X86/live-debug-vars-intervals.mir | 4 +++ .../MachineVerifier/test_adjustsstack.mir | 26 +++++++++++++++++++ 44 files changed, 92 insertions(+), 9 deletions(-) create mode 100644 llvm/test/MachineVerifier/test_adjustsstack.mir diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index c69d36fc7fdd..005efe48ac0c 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -3697,6 +3697,9 @@ void MachineVerifier::verifyStackFrame() { if (I.getOpcode() == FrameSetupOpcode) { if (BBState.ExitIsSetup) report("FrameSetup is after another FrameSetup", &I); + if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack()) + report("AdjustsStack not set in presence of a frame pseudo " + "instruction.", &I); BBState.ExitValue -= TII->getFrameTotalSize(I); BBState.ExitIsSetup = true; } @@ -3712,6 +3715,9 @@ void MachineVerifier::verifyStackFrame() { errs() << "FrameDestroy <" << Size << "> is after FrameSetup <" << AbsSPAdj << ">.\n"; } + if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack()) + report("AdjustsStack not set in presence of a frame pseudo " + "instruction.", &I); BBState.ExitValue += Size; BBState.ExitIsSetup = false; } diff --git a/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir b/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir index 9040937d027d..1592c86e267f 100644 --- a/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir +++ b/llvm/test/CodeGen/AArch64/clear-dead-implicit-def-impdef.mir @@ -3,6 +3,8 @@ --- name: func tracksRegLiveness: true +frameInfo: + adjustsStack: true body: | bb.0: liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6 diff --git a/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir b/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir index aa94a03786f5..47aa34e3c011 100644 --- a/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir +++ b/llvm/test/CodeGen/AArch64/implicit-def-remat-requires-impdef-check.mir @@ -22,6 +22,7 @@ name: inst_stores_to_dead_spill_implicit_def_impdef tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true body: | bb.0: @@ -59,6 +60,7 @@ body: | name: inst_stores_to_dead_spill_movimm_impdef tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true body: | bb.0: diff --git a/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir b/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir index e5395b20afd4..a5d74ef75f0a 100644 --- a/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir +++ b/llvm/test/CodeGen/AArch64/implicit-def-with-impdef-greedy-assert.mir @@ -4,6 +4,8 @@ --- name: widget tracksRegLiveness: true +frameInfo: + adjustsStack: true jumpTable: kind: label-difference32 entries: diff --git a/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir b/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir index 3616d617f84a..5ef8a94eeaa7 100644 --- a/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir +++ b/llvm/test/CodeGen/AMDGPU/fold-restore-undef-use.mir @@ -8,6 +8,8 @@ --- name: restore_undef_copy_use tracksRegLiveness: true +frameInfo: + adjustsStack: true machineFunctionInfo: maxKernArgAlign: 1 isEntryFunction: true diff --git a/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir b/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir index bdd89a907790..dde84af57ed2 100644 --- a/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir +++ b/llvm/test/CodeGen/AMDGPU/greedy-alloc-fail-sgpr1024-spill.mir @@ -13,6 +13,7 @@ name: greedy_fail_alloc_sgpr1024_spill tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true machineFunctionInfo: explicitKernArgSize: 16 diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir index 2ccc24152a9f..fdfc9b043cc9 100644 --- a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir +++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir @@ -24,6 +24,7 @@ registers: - { id: 10, class: sreg_64_xexec, preferred-register: '$vcc' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: maxKernArgAlign: 1 diff --git a/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir b/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir index c0d199920bd9..09037709d51d 100644 --- a/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir +++ b/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir @@ -181,6 +181,8 @@ legalized: false regBankSelected: false selected: false tracksRegLiveness: true +frameInfo: + adjustsStack: true liveins: - { reg: '$vgpr0', virtual-reg: '%0' } - { reg: '$vgpr1', virtual-reg: '%1' } diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir b/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir index efbdbca9da6b..c6ccbd99bf89 100644 --- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir +++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-wrong-stack-id.mir @@ -78,6 +78,7 @@ name: sgpr_spill_wrong_stack_id tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true machineFunctionInfo: scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3 diff --git a/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir b/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir index 355829825146..f8ec6bb5d943 100644 --- a/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir +++ b/llvm/test/CodeGen/AMDGPU/snippet-copy-bundle-regression.mir @@ -21,6 +21,7 @@ name: kernel tracksRegLiveness: true frameInfo: + adjustsStack: true hasCalls: true machineFunctionInfo: isEntryFunction: true diff --git a/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir b/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir index 3d9db687ffa1..6659e9532376 100644 --- a/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir +++ b/llvm/test/CodeGen/AMDGPU/virtregrewrite-undef-identity-copy.mir @@ -20,6 +20,7 @@ name: undef_identity_copy tracksRegLiveness: true frameInfo: maxAlignment: 4 + adjustsStack: true hasCalls: true machineFunctionInfo: isEntryFunction: true diff --git a/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir b/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir index 5c59566247d8..b4bbb9be8ae4 100644 --- a/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir +++ b/llvm/test/CodeGen/ARM/no-register-coalescing-in-returnsTwice.mir @@ -86,6 +86,8 @@ --- name: main exposesReturnsTwice: true +frameInfo: + adjustsStack: true stack: - { id: 0, name: P0, size: 80, alignment: 8, local-offset: -80 } - { id: 1, name: jb1, size: 160, alignment: 8, local-offset: -240 } diff --git a/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir b/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir index 67f4dd72ea0b..9468b18bf8e4 100644 --- a/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir +++ b/llvm/test/CodeGen/Hexagon/regalloc-bad-undef.mir @@ -135,7 +135,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 0 - adjustsStack: false + adjustsStack: true hasCalls: true maxCallFrameSize: 0 hasOpaqueSPAdjustment: false diff --git a/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir b/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir index 3b308ce3d0d2..adeec15b1755 100644 --- a/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir +++ b/llvm/test/CodeGen/SystemZ/RAbasic-invalid-LR-update.mir @@ -25,6 +25,8 @@ name: autogen_SD21418 alignment: 4 tracksRegLiveness: true +frameInfo: + adjustsStack: true registers: - { id: 0, class: vr128bit } - { id: 1, class: vr128bit } diff --git a/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir b/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir index 7ff7d9b8b709..197c3d8551fc 100644 --- a/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir +++ b/llvm/test/CodeGen/SystemZ/clear-liverange-spillreg.mir @@ -157,6 +157,7 @@ registers: - { id: 129, class: grx32bit } - { id: 130, class: fp64bit } frameInfo: + adjustsStack: true hasCalls: true body: | bb.0: diff --git a/llvm/test/CodeGen/SystemZ/int-cmp-56.mir b/llvm/test/CodeGen/SystemZ/int-cmp-56.mir index e52fd44ae47d..3e00b6065eb9 100644 --- a/llvm/test/CodeGen/SystemZ/int-cmp-56.mir +++ b/llvm/test/CodeGen/SystemZ/int-cmp-56.mir @@ -48,6 +48,7 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | @@ -125,6 +126,7 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | @@ -202,6 +204,7 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | @@ -279,6 +282,7 @@ liveins: - { reg: '$r2d', virtual-reg: '%0' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir b/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir index f709b70ff1b7..bf5855010bf9 100644 --- a/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir +++ b/llvm/test/CodeGen/SystemZ/regcoal-subranges-update.mir @@ -49,6 +49,8 @@ body: | --- name: segfault tracksRegLiveness: true +frameInfo: + adjustsStack: true liveins: [] body: | ; CHECK-LABEL: name: segfault diff --git a/llvm/test/CodeGen/X86/callbr-asm-kill.mir b/llvm/test/CodeGen/X86/callbr-asm-kill.mir index 86c58c4715ed..0dded37c97af 100644 --- a/llvm/test/CodeGen/X86/callbr-asm-kill.mir +++ b/llvm/test/CodeGen/X86/callbr-asm-kill.mir @@ -45,6 +45,7 @@ liveins: - { reg: '$rsi', virtual-reg: '%3' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/CodeGen/X86/late-remat-update.mir b/llvm/test/CodeGen/X86/late-remat-update.mir index 84a78f84728c..dd4e99c6df14 100644 --- a/llvm/test/CodeGen/X86/late-remat-update.mir +++ b/llvm/test/CodeGen/X86/late-remat-update.mir @@ -66,6 +66,7 @@ registers: liveins: - { reg: '$edi', virtual-reg: '%0' } frameInfo: + adjustsStack: true hasCalls: true body: | bb.0.entry: diff --git a/llvm/test/CodeGen/X86/limit-split-cost.mir b/llvm/test/CodeGen/X86/limit-split-cost.mir index 6f5329e5b332..7ec0404e0f73 100644 --- a/llvm/test/CodeGen/X86/limit-split-cost.mir +++ b/llvm/test/CodeGen/X86/limit-split-cost.mir @@ -86,6 +86,7 @@ registers: liveins: - { reg: '$edi', virtual-reg: '%0' } frameInfo: + adjustsStack: true hasCalls: true body: | bb.0.entry: diff --git a/llvm/test/CodeGen/X86/regalloc-copy-hints.mir b/llvm/test/CodeGen/X86/regalloc-copy-hints.mir index 13b5a541fa22..d09bcd6a6b40 100644 --- a/llvm/test/CodeGen/X86/regalloc-copy-hints.mir +++ b/llvm/test/CodeGen/X86/regalloc-copy-hints.mir @@ -103,6 +103,7 @@ registers: - { id: 82, class: gr32 } frameInfo: maxAlignment: 4 + adjustsStack: true hasCalls: true fixedStack: - { id: 0, size: 4, alignment: 4, stack-id: default, isImmutable: true } diff --git a/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir b/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir index 02c931067300..8bac14018a7d 100644 --- a/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir +++ b/llvm/test/CodeGen/X86/statepoint-fastregalloc.mir @@ -6,6 +6,8 @@ --- name: test_relocate tracksRegLiveness: true +frameInfo: + adjustsStack: true body: | bb.0.entry: liveins: $rdi @@ -25,6 +27,8 @@ body: | --- name: test_relocate_multi_regmasks tracksRegLiveness: true +frameInfo: + adjustsStack: true body: | bb.0.entry: liveins: $rdi diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir index 11968f17c70a..5f05270729fd 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir @@ -231,7 +231,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 1 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir index aae2f3870138..cf9128260f19 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir @@ -398,7 +398,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 1 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir index 87f5f0f96c50..fcebc69d9b2e 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir @@ -175,7 +175,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 4 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir index 49253968fcca..8bb39a03f7e3 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir @@ -226,7 +226,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 4 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir index 858ff3f1888b..da651039ce21 100644 --- a/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir +++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir @@ -172,7 +172,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 4 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir b/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir index e24d5e8af1f5..d40a9a06d162 100644 --- a/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir +++ b/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir @@ -114,7 +114,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 8 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-folding-tieddef.mir b/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-folding-tieddef.mir index cece656d0897..5ebd1a89ae92 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-folding-tieddef.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-folding-tieddef.mir @@ -100,6 +100,7 @@ registers: - { id: 38, class: gr8 } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-load-folding.mir b/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-load-folding.mir index f0af93804048..b0bff30c4c82 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-load-folding.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/memory-operand-load-folding.mir @@ -87,6 +87,7 @@ liveins: - { reg: '$edi', virtual-reg: '%0' } - { reg: '$xmm0', virtual-reg: '%1' } frameInfo: + adjustsStack: true hasCalls: true body: | bb.0.if.then: diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-coalesce-subreg.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-coalesce-subreg.mir index 51d3f7e1a6a4..d73d8a375906 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-coalesce-subreg.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-coalesce-subreg.mir @@ -97,6 +97,7 @@ liveins: - { reg: '$esi', virtual-reg: '%4' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir index bc1c7ebac6ce..6460263c6025 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-coalescing.mir @@ -106,6 +106,7 @@ liveins: - { reg: '$rsi', virtual-reg: '%5' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced.mir index d59333e73fbc..68c9bf6c89dd 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced.mir @@ -70,6 +70,7 @@ liveins: - { reg: '$esi', virtual-reg: '%2' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced2.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced2.mir index ab2647d3b45a..cf17af4ba430 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced2.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-on-stack-coalesced2.mir @@ -71,6 +71,7 @@ liveins: - { reg: '$esi', virtual-reg: '%2' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-regallocd-to-stack.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-regallocd-to-stack.mir index 0fe80980e4e6..cb35bd892eea 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-regallocd-to-stack.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-regallocd-to-stack.mir @@ -65,6 +65,7 @@ liveins: - { reg: '$esi', virtual-reg: '%2' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/phi-through-regalloc.mir b/llvm/test/DebugInfo/MIR/InstrRef/phi-through-regalloc.mir index 2a031b295a1e..61dcec49b74c 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/phi-through-regalloc.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/phi-through-regalloc.mir @@ -94,6 +94,7 @@ liveins: - { reg: '$esi', virtual-reg: '%2' } frameInfo: maxAlignment: 1 + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/InstrRef/stack-coloring-dbg-phi.mir b/llvm/test/DebugInfo/MIR/InstrRef/stack-coloring-dbg-phi.mir index 47a7b460e43e..e80ed2e3e8eb 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/stack-coloring-dbg-phi.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/stack-coloring-dbg-phi.mir @@ -106,6 +106,7 @@ liveins: - { reg: '$rdi', virtual-reg: '%15' } frameInfo: maxAlignment: 8 + adjustsStack: true hasCalls: true stack: - { id: 0, size: 8, alignment: 8 } diff --git a/llvm/test/DebugInfo/MIR/InstrRef/survives-livedebugvars.mir b/llvm/test/DebugInfo/MIR/InstrRef/survives-livedebugvars.mir index 3e806e43ca9e..6dbd2cd4faa7 100644 --- a/llvm/test/DebugInfo/MIR/InstrRef/survives-livedebugvars.mir +++ b/llvm/test/DebugInfo/MIR/InstrRef/survives-livedebugvars.mir @@ -113,6 +113,7 @@ liveins: - { reg: '$rdi', virtual-reg: '%2' } - { reg: '$esi', virtual-reg: '%4' } frameInfo: + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir b/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir index 35ab906efc90..5df70096e930 100644 --- a/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir +++ b/llvm/test/DebugInfo/MIR/Mips/livedebugvars-stop-trimming-loc.mir @@ -72,6 +72,8 @@ name: fn2 alignment: 4 tracksRegLiveness: true +frameInfo: + adjustsStack: true registers: - { id: 0, class: gpr32, preferred-register: '' } - { id: 1, class: gpr32, preferred-register: '' } diff --git a/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg-debugonly.mir b/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg-debugonly.mir index 3cb9da8fdfe3..4d48774a78dd 100644 --- a/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg-debugonly.mir +++ b/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg-debugonly.mir @@ -116,7 +116,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 0 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg.mir b/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg.mir index 35d12b52af89..e618f48f527b 100644 --- a/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg.mir +++ b/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg.mir @@ -114,7 +114,7 @@ frameInfo: stackSize: 0 offsetAdjustment: 0 maxAlignment: 0 - adjustsStack: false + adjustsStack: true hasCalls: true stackProtector: '' maxCallFrameSize: 4294967295 diff --git a/llvm/test/DebugInfo/MIR/X86/livedebugvars-crossbb-interval.mir b/llvm/test/DebugInfo/MIR/X86/livedebugvars-crossbb-interval.mir index 037306a2ca95..42ee73dde965 100644 --- a/llvm/test/DebugInfo/MIR/X86/livedebugvars-crossbb-interval.mir +++ b/llvm/test/DebugInfo/MIR/X86/livedebugvars-crossbb-interval.mir @@ -100,6 +100,7 @@ liveins: - { reg: '$rdi', virtual-reg: '%2' } - { reg: '$esi', virtual-reg: '%4' } frameInfo: + adjustsStack: true hasCalls: true machineFunctionInfo: {} body: | diff --git a/llvm/test/DebugInfo/X86/live-debug-vars-intervals.mir b/llvm/test/DebugInfo/X86/live-debug-vars-intervals.mir index c5b6d7381ace..3beaf8996e4f 100644 --- a/llvm/test/DebugInfo/X86/live-debug-vars-intervals.mir +++ b/llvm/test/DebugInfo/X86/live-debug-vars-intervals.mir @@ -99,6 +99,8 @@ --- name: f1 tracksRegLiveness: true +frameInfo: + adjustsStack: true stack: - { id: 0, name: x.addr, type: default, offset: 0, size: 4, alignment: 4, stack-id: default, callee-saved-register: '', callee-saved-restored: true, @@ -127,6 +129,8 @@ body: | --- name: f2 tracksRegLiveness: true +frameInfo: + adjustsStack: true stack: - { id: 0, name: x.addr, type: default, offset: 0, size: 4, alignment: 4, stack-id: default, callee-saved-register: '', callee-saved-restored: true, diff --git a/llvm/test/MachineVerifier/test_adjustsstack.mir b/llvm/test/MachineVerifier/test_adjustsstack.mir new file mode 100644 index 000000000000..d333737e000c --- /dev/null +++ b/llvm/test/MachineVerifier/test_adjustsstack.mir @@ -0,0 +1,26 @@ +# RUN: not --crash llc -o - -start-before=twoaddressinstruction -verify-machineinstrs %s 2>&1 \ +# RUN: | FileCheck %s +# REQUIRES: aarch64-registered-target +--- | + target triple = "aarch64-unknown-linux" + declare i32 @bar(i32) nounwind + define i32 @foo() nounwind { + call i32 @bar(i32 0) + ret i32 0 + } +... +--- +name: foo +registers: + - { id: 0, class: gpr32 } +body: | + bb.0 (%ir-block.0): + ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp + %0 = COPY $wzr + $w0 = COPY %0 + BL @bar, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0 + ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp + $w0 = COPY killed %0 + RET_ReallyLR implicit $w0 +... +# CHECK-LABEL: Bad machine code: AdjustsStack not set in presence of a frame pseudo instruction. -- GitLab From 8cb2d436ca50117026a8dc901c8039d9bd39b507 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Fri, 22 Mar 2024 08:48:22 +0800 Subject: [PATCH 216/296] [Passes] Expose parseSinglePassOption (#86117) BPF and some machine function passes need it. --- llvm/include/llvm/Passes/PassBuilder.h | 7 ++++ llvm/lib/Passes/PassBuilder.cpp | 55 +++++++++++++++----------- 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index 8817a2585646..d1232124d5d8 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -672,6 +672,13 @@ public: return Result; } + /// Handle passes only accept one bool-valued parameter. + /// + /// \return false when Params is empty. + static Expected parseSinglePassOption(StringRef Params, + StringRef OptionName, + StringRef PassName); + private: // O1 pass pipeline FunctionPassManager diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 9d98ae7dde52..c5465da8b4a5 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -600,8 +600,9 @@ Expected parseLoopUnrollOptions(StringRef Params) { return UnrollOpts; } -Expected parseSinglePassOption(StringRef Params, StringRef OptionName, - StringRef PassName) { +Expected PassBuilder::parseSinglePassOption(StringRef Params, + StringRef OptionName, + StringRef PassName) { bool Result = false; while (!Params.empty()) { StringRef ParamName; @@ -620,24 +621,28 @@ Expected parseSinglePassOption(StringRef Params, StringRef OptionName, } Expected parseGlobalDCEPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "vfe-linkage-unit-visibility", "GlobalDCE"); + return PassBuilder::parseSinglePassOption( + Params, "vfe-linkage-unit-visibility", "GlobalDCE"); } Expected parseCGProfilePassOptions(StringRef Params) { - return parseSinglePassOption(Params, "in-lto-post-link", "CGProfile"); + return PassBuilder::parseSinglePassOption(Params, "in-lto-post-link", + "CGProfile"); } Expected parseInlinerPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "only-mandatory", "InlinerPass"); + return PassBuilder::parseSinglePassOption(Params, "only-mandatory", + "InlinerPass"); } Expected parseCoroSplitPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "reuse-storage", "CoroSplitPass"); + return PassBuilder::parseSinglePassOption(Params, "reuse-storage", + "CoroSplitPass"); } Expected parsePostOrderFunctionAttrsPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "skip-non-recursive-function-attrs", - "PostOrderFunctionAttrs"); + return PassBuilder::parseSinglePassOption( + Params, "skip-non-recursive-function-attrs", "PostOrderFunctionAttrs"); } Expected parseCFGuardPassOptions(StringRef Params) { @@ -661,19 +666,21 @@ Expected parseCFGuardPassOptions(StringRef Params) { } Expected parseEarlyCSEPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "memssa", "EarlyCSE"); + return PassBuilder::parseSinglePassOption(Params, "memssa", "EarlyCSE"); } Expected parseEntryExitInstrumenterPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "post-inline", "EntryExitInstrumenter"); + return PassBuilder::parseSinglePassOption(Params, "post-inline", + "EntryExitInstrumenter"); } Expected parseLoopExtractorPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "single", "LoopExtractor"); + return PassBuilder::parseSinglePassOption(Params, "single", "LoopExtractor"); } Expected parseLowerMatrixIntrinsicsPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "minimal", "LowerMatrixIntrinsics"); + return PassBuilder::parseSinglePassOption(Params, "minimal", + "LowerMatrixIntrinsics"); } Expected parseASanPassOptions(StringRef Params) { @@ -1013,13 +1020,13 @@ parseStackLifetimeOptions(StringRef Params) { } Expected parseDependenceAnalysisPrinterOptions(StringRef Params) { - return parseSinglePassOption(Params, "normalized-results", - "DependenceAnalysisPrinter"); + return PassBuilder::parseSinglePassOption(Params, "normalized-results", + "DependenceAnalysisPrinter"); } Expected parseSeparateConstOffsetFromGEPPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "lower-gep", - "SeparateConstOffsetFromGEP"); + return PassBuilder::parseSinglePassOption(Params, "lower-gep", + "SeparateConstOffsetFromGEP"); } Expected @@ -1035,13 +1042,13 @@ parseFunctionSimplificationPipelineOptions(StringRef Params) { } Expected parseMemorySSAPrinterPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "no-ensure-optimized-uses", - "MemorySSAPrinterPass"); + return PassBuilder::parseSinglePassOption(Params, "no-ensure-optimized-uses", + "MemorySSAPrinterPass"); } Expected parseSpeculativeExecutionPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "only-if-divergent-target", - "SpeculativeExecutionPass"); + return PassBuilder::parseSinglePassOption(Params, "only-if-divergent-target", + "SpeculativeExecutionPass"); } Expected parseMemProfUsePassOptions(StringRef Params) { @@ -1062,13 +1069,13 @@ Expected parseMemProfUsePassOptions(StringRef Params) { } Expected parseStructuralHashPrinterPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "detailed", - "StructuralHashPrinterPass"); + return PassBuilder::parseSinglePassOption(Params, "detailed", + "StructuralHashPrinterPass"); } Expected parseWinEHPrepareOptions(StringRef Params) { - return parseSinglePassOption(Params, "demote-catchswitch-only", - "WinEHPreparePass"); + return PassBuilder::parseSinglePassOption(Params, "demote-catchswitch-only", + "WinEHPreparePass"); } Expected parseGlobalMergeOptions(StringRef Params) { -- GitLab From a2dfc9ac7da23ccf0077081c8825a23aed1df0c0 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Fri, 22 Mar 2024 08:49:29 +0800 Subject: [PATCH 217/296] [NewPM][AMDGPU] Add AMDGPUPassRegistry.def (#86095) Move the pass registry to a separate file, prepare for porting dag-isel. --- llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 73 +++++++++++ .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 115 +++--------------- .../AMDGPU/global_atomic_optimizer_fp_rtn.ll | 4 +- .../global_atomics_iterative_scan_fp.ll | 4 +- .../global_atomics_optimizer_fp_no_rtn.ll | 4 +- 5 files changed, 96 insertions(+), 104 deletions(-) create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def new file mode 100644 index 000000000000..90f36fadf359 --- /dev/null +++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def @@ -0,0 +1,73 @@ +//===- AMDGPUPassRegistry.def - Registry of AMDGPU passes -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is used as the registry of passes that are part of the +// AMDGPU backend. +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef MODULE_PASS +#define MODULE_PASS(NAME, CREATE_PASS) +#endif +MODULE_PASS("amdgpu-always-inline", AMDGPUAlwaysInlinePass()) +MODULE_PASS("amdgpu-attributor", AMDGPUAttributorPass(*this)) +MODULE_PASS("amdgpu-lower-buffer-fat-pointers", + AMDGPULowerBufferFatPointersPass(*this)) +MODULE_PASS("amdgpu-lower-ctor-dtor", AMDGPUCtorDtorLoweringPass()) +MODULE_PASS("amdgpu-lower-module-lds", AMDGPULowerModuleLDSPass(*this)) +MODULE_PASS("amdgpu-printf-runtime-binding", AMDGPUPrintfRuntimeBindingPass()) +MODULE_PASS("amdgpu-unify-metadata", AMDGPUUnifyMetadataPass()) +#undef MODULE_PASS + +#ifndef FUNCTION_PASS +#define FUNCTION_PASS(NAME, CREATE_PASS) +#endif +FUNCTION_PASS("amdgpu-codegenprepare", AMDGPUCodeGenPreparePass(*this)) +FUNCTION_PASS("amdgpu-image-intrinsic-opt", + AMDGPUImageIntrinsicOptimizerPass(*this)) +FUNCTION_PASS("amdgpu-lower-kernel-arguments", + AMDGPULowerKernelArgumentsPass(*this)) +FUNCTION_PASS("amdgpu-lower-kernel-attributes", + AMDGPULowerKernelAttributesPass()) +FUNCTION_PASS("amdgpu-simplifylib", AMDGPUSimplifyLibCallsPass()) +FUNCTION_PASS("amdgpu-promote-alloca", AMDGPUPromoteAllocaPass(*this)) +FUNCTION_PASS("amdgpu-promote-alloca-to-vector", + AMDGPUPromoteAllocaToVectorPass(*this)) +FUNCTION_PASS("amdgpu-promote-kernel-arguments", + AMDGPUPromoteKernelArgumentsPass()) +FUNCTION_PASS("amdgpu-rewrite-undef-for-phi", AMDGPURewriteUndefForPHIPass()) +FUNCTION_PASS("amdgpu-unify-divergent-exit-nodes", + AMDGPUUnifyDivergentExitNodesPass()) +FUNCTION_PASS("amdgpu-usenative", AMDGPUUseNativeCallsPass()) +#undef FUNCTION_PASS + +#ifndef FUNCTION_ANALYSIS +#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) +#endif + +#ifndef FUNCTION_ALIAS_ANALYSIS +#define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ + FUNCTION_ANALYSIS(NAME, CREATE_PASS) +#endif +FUNCTION_ALIAS_ANALYSIS("amdgpu-aa", AMDGPUAA()) +#undef FUNCTION_ALIAS_ANALYSIS +#undef FUNCTION_ANALYSIS + +#ifndef FUNCTION_PASS_WITH_PARAMS +#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) +#endif +FUNCTION_PASS_WITH_PARAMS( + "amdgpu-atomic-optimizer", + "AMDGPUAtomicOptimizerPass", + [=](ScanOptions Strategy) { + return AMDGPUAtomicOptimizerPass(*this, Strategy); + }, + parseAMDGPUAtomicOptimizerStrategy, "strategy=dpp|iterative|none") +#undef FUNCTION_PASS_WITH_PARAMS diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index c96625092a76..f7e552177d6f 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -631,107 +631,26 @@ void AMDGPUTargetMachine::registerDefaultAliasAnalyses(AAManager &AAM) { AAM.registerFunctionAnalysis(); } +static Expected +parseAMDGPUAtomicOptimizerStrategy(StringRef Params) { + if (Params.empty()) + return ScanOptions::Iterative; + Params.consume_front("strategy="); + auto Result = StringSwitch>(Params) + .Case("dpp", ScanOptions::DPP) + .Cases("iterative", "", ScanOptions::Iterative) + .Case("none", ScanOptions::None) + .Default(std::nullopt); + if (Result) + return *Result; + return make_error("invalid parameter", inconvertibleErrorCode()); +} + void AMDGPUTargetMachine::registerPassBuilderCallbacks( PassBuilder &PB, bool PopulateClassToPassNames) { - PB.registerPipelineParsingCallback( - [this](StringRef PassName, ModulePassManager &PM, - ArrayRef) { - if (PassName == "amdgpu-attributor") { - PM.addPass(AMDGPUAttributorPass(*this)); - return true; - } - if (PassName == "amdgpu-unify-metadata") { - PM.addPass(AMDGPUUnifyMetadataPass()); - return true; - } - if (PassName == "amdgpu-printf-runtime-binding") { - PM.addPass(AMDGPUPrintfRuntimeBindingPass()); - return true; - } - if (PassName == "amdgpu-always-inline") { - PM.addPass(AMDGPUAlwaysInlinePass()); - return true; - } - if (PassName == "amdgpu-lower-module-lds") { - PM.addPass(AMDGPULowerModuleLDSPass(*this)); - return true; - } - if (PassName == "amdgpu-lower-buffer-fat-pointers") { - PM.addPass(AMDGPULowerBufferFatPointersPass(*this)); - return true; - } - if (PassName == "amdgpu-lower-ctor-dtor") { - PM.addPass(AMDGPUCtorDtorLoweringPass()); - return true; - } - return false; - }); - PB.registerPipelineParsingCallback( - [this](StringRef PassName, FunctionPassManager &PM, - ArrayRef) { - if (PassName == "amdgpu-simplifylib") { - PM.addPass(AMDGPUSimplifyLibCallsPass()); - return true; - } - if (PassName == "amdgpu-image-intrinsic-opt") { - PM.addPass(AMDGPUImageIntrinsicOptimizerPass(*this)); - return true; - } - if (PassName == "amdgpu-usenative") { - PM.addPass(AMDGPUUseNativeCallsPass()); - return true; - } - if (PassName == "amdgpu-promote-alloca") { - PM.addPass(AMDGPUPromoteAllocaPass(*this)); - return true; - } - if (PassName == "amdgpu-promote-alloca-to-vector") { - PM.addPass(AMDGPUPromoteAllocaToVectorPass(*this)); - return true; - } - if (PassName == "amdgpu-lower-kernel-attributes") { - PM.addPass(AMDGPULowerKernelAttributesPass()); - return true; - } - if (PassName == "amdgpu-promote-kernel-arguments") { - PM.addPass(AMDGPUPromoteKernelArgumentsPass()); - return true; - } - if (PassName == "amdgpu-unify-divergent-exit-nodes") { - PM.addPass(AMDGPUUnifyDivergentExitNodesPass()); - return true; - } - if (PassName == "amdgpu-atomic-optimizer") { - PM.addPass( - AMDGPUAtomicOptimizerPass(*this, AMDGPUAtomicOptimizerStrategy)); - return true; - } - if (PassName == "amdgpu-codegenprepare") { - PM.addPass(AMDGPUCodeGenPreparePass(*this)); - return true; - } - if (PassName == "amdgpu-lower-kernel-arguments") { - PM.addPass(AMDGPULowerKernelArgumentsPass(*this)); - return true; - } - if (PassName == "amdgpu-rewrite-undef-for-phi") { - PM.addPass(AMDGPURewriteUndefForPHIPass()); - return true; - } - return false; - }); - - PB.registerAnalysisRegistrationCallback([](FunctionAnalysisManager &FAM) { - FAM.registerPass([&] { return AMDGPUAA(); }); - }); - PB.registerParseAACallback([](StringRef AAName, AAManager &AAM) { - if (AAName == "amdgpu-aa") { - AAM.registerFunctionAnalysis(); - return true; - } - return false; - }); +#define GET_PASS_REGISTRY "AMDGPUPassRegistry.def" +#include "llvm/Passes/TargetPassRegistry.inc" PB.registerPipelineStartEPCallback( [](ModulePassManager &PM, OptimizationLevel Level) { diff --git a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll index 538ef42121b8..b7a91f6fa96d 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -amdgpu-atomic-optimizer-strategy=Iterative -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-ITERATIVE %s -; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -amdgpu-atomic-optimizer-strategy=DPP -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-DPP %s +; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-ITERATIVE %s +; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-DPP %s ; Tests various combinations of uniform/divergent address and uniform/divergent value inputs of various types for atomic operations. ; Optimization remains same for Iterative and DPP strategies when value in uniform. These different scan/reduction diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll index fab24e10f810..86e3d9338e07 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -mtriple=amdgcn-- -amdgpu-atomic-optimizer-strategy=Iterative -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck -check-prefix=IR-ITERATIVE %s -; RUN: opt -S -mtriple=amdgcn-- -amdgpu-atomic-optimizer-strategy=DPP -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck -check-prefix=IR-DPP %s +; RUN: opt -S -mtriple=amdgcn-- -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck -check-prefix=IR-ITERATIVE %s +; RUN: opt -S -mtriple=amdgcn-- -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck -check-prefix=IR-DPP %s declare i32 @llvm.amdgcn.workitem.id.x() define amdgpu_kernel void @global_atomic_fadd_uni_value(ptr addrspace(1) %ptr) #0 { ; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_value( diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll index cc7a45cbb6e3..e70d7347890f 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -amdgpu-atomic-optimizer-strategy=Iterative -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-ITERATIVE %s -; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -amdgpu-atomic-optimizer-strategy=DPP -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-DPP %s +; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-ITERATIVE %s +; RUN: opt -S -mtriple=amdgcn-- -mcpu=gfx906 -passes='amdgpu-atomic-optimizer,verify' %s | FileCheck --check-prefixes=IR,IR-DPP %s ; Tests various combinations of uniform/divergent address and uniform/divergent value inputs of various types for atomic operations. ; Optimization remains same for Iterative and DPP strategies when value in uniform. These different scan/reduction -- GitLab From 3e4caa9da4356247444e973eb470a25adae083b0 Mon Sep 17 00:00:00 2001 From: Freddy Ye Date: Fri, 22 Mar 2024 08:52:40 +0800 Subject: [PATCH 218/296] [X86] Support DomainReassignment for APX NDD instructions (#85737) --- llvm/lib/Target/X86/X86DomainReassignment.cpp | 38 ++++++++++++ .../CodeGen/X86/apx/domain-reassignment.mir | 62 +++++++++---------- 2 files changed, 69 insertions(+), 31 deletions(-) diff --git a/llvm/lib/Target/X86/X86DomainReassignment.cpp b/llvm/lib/Target/X86/X86DomainReassignment.cpp index 53c0486c8697..6289b3a1df1f 100644 --- a/llvm/lib/Target/X86/X86DomainReassignment.cpp +++ b/llvm/lib/Target/X86/X86DomainReassignment.cpp @@ -650,6 +650,16 @@ void X86DomainReassignment::initConverters() { createReplacer(X86::AND16rr, X86::KANDWrr); createReplacer(X86::XOR16rr, X86::KXORWrr); + bool HasNDD = STI->hasNDD(); + if (HasNDD) { + createReplacer(X86::SHR16ri_ND, X86::KSHIFTRWri); + createReplacer(X86::SHL16ri_ND, X86::KSHIFTLWri); + createReplacer(X86::NOT16r_ND, X86::KNOTWrr); + createReplacer(X86::OR16rr_ND, X86::KORWrr); + createReplacer(X86::AND16rr_ND, X86::KANDWrr); + createReplacer(X86::XOR16rr_ND, X86::KXORWrr); + } + if (STI->hasBWI()) { createReplacer(X86::MOV32rm, GET_EGPR_IF_ENABLED(X86::KMOVDkm)); createReplacer(X86::MOV64rm, GET_EGPR_IF_ENABLED(X86::KMOVQkm)); @@ -684,6 +694,23 @@ void X86DomainReassignment::initConverters() { createReplacer(X86::XOR32rr, X86::KXORDrr); createReplacer(X86::XOR64rr, X86::KXORQrr); + if (HasNDD) { + createReplacer(X86::SHR32ri_ND, X86::KSHIFTRDri); + createReplacer(X86::SHL32ri_ND, X86::KSHIFTLDri); + createReplacer(X86::ADD32rr_ND, X86::KADDDrr); + createReplacer(X86::NOT32r_ND, X86::KNOTDrr); + createReplacer(X86::OR32rr_ND, X86::KORDrr); + createReplacer(X86::AND32rr_ND, X86::KANDDrr); + createReplacer(X86::XOR32rr_ND, X86::KXORDrr); + createReplacer(X86::SHR64ri_ND, X86::KSHIFTRQri); + createReplacer(X86::SHL64ri_ND, X86::KSHIFTLQri); + createReplacer(X86::ADD64rr_ND, X86::KADDQrr); + createReplacer(X86::NOT64r_ND, X86::KNOTQrr); + createReplacer(X86::OR64rr_ND, X86::KORQrr); + createReplacer(X86::AND64rr_ND, X86::KANDQrr); + createReplacer(X86::XOR64rr_ND, X86::KXORQrr); + } + // TODO: KTEST is not a replacement for TEST due to flag differences. Need // to prove only Z flag is used. // createReplacer(X86::TEST32rr, X86::KTESTDrr); @@ -713,6 +740,17 @@ void X86DomainReassignment::initConverters() { // createReplacer(X86::TEST16rr, X86::KTESTWrr); createReplacer(X86::XOR8rr, X86::KXORBrr); + + if (HasNDD) { + createReplacer(X86::ADD8rr_ND, X86::KADDBrr); + createReplacer(X86::ADD16rr_ND, X86::KADDWrr); + createReplacer(X86::AND8rr_ND, X86::KANDBrr); + createReplacer(X86::NOT8r_ND, X86::KNOTBrr); + createReplacer(X86::OR8rr_ND, X86::KORBrr); + createReplacer(X86::SHR8ri_ND, X86::KSHIFTRBri); + createReplacer(X86::SHL8ri_ND, X86::KSHIFTLBri); + createReplacer(X86::XOR8rr_ND, X86::KXORBrr); + } } #undef GET_EGPR_IF_ENABLED } diff --git a/llvm/test/CodeGen/X86/apx/domain-reassignment.mir b/llvm/test/CodeGen/X86/apx/domain-reassignment.mir index dcd435619990..7352aa2b307f 100644 --- a/llvm/test/CodeGen/X86/apx/domain-reassignment.mir +++ b/llvm/test/CodeGen/X86/apx/domain-reassignment.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -run-pass x86-domain-reassignment -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512dq -o - %s | FileCheck %s +# RUN: llc -run-pass x86-domain-reassignment -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512dq,+ndd -o - %s | FileCheck %s --- | ; ModuleID = '../test/CodeGen/X86/gpr-to-mask.ll' source_filename = "../test/CodeGen/X86/gpr-to-mask.ll" @@ -302,13 +302,13 @@ body: | %6 = COPY %5 %7 = COPY %6.sub_8bit - %12 = SHR8ri %7, 2, implicit-def dead $eflags - %13 = SHL8ri %12, 1, implicit-def dead $eflags - %14 = NOT8r %13 - %15 = OR8rr %14, %12, implicit-def dead $eflags - %16 = AND8rr %15, %13, implicit-def dead $eflags - %17 = XOR8rr %16, %12, implicit-def dead $eflags - %18 = ADD8rr %17, %14, implicit-def dead $eflags + %12 = SHR8ri_ND %7, 2, implicit-def dead $eflags + %13 = SHL8ri_ND %12, 1, implicit-def dead $eflags + %14 = NOT8r_ND %13 + %15 = OR8rr_ND %14, %12, implicit-def dead $eflags + %16 = AND8rr_ND %15, %13, implicit-def dead $eflags + %17 = XOR8rr_ND %16, %12, implicit-def dead $eflags + %18 = ADD8rr_ND %17, %14, implicit-def dead $eflags %8 = IMPLICIT_DEF %9 = INSERT_SUBREG %8, %18, %subreg.sub_8bit_hi @@ -421,12 +421,12 @@ body: | %6 = COPY %5 %7 = COPY %6.sub_16bit - %12 = SHR16ri %7, 2, implicit-def dead $eflags - %13 = SHL16ri %12, 1, implicit-def dead $eflags - %14 = NOT16r %13 - %15 = OR16rr %14, %12, implicit-def dead $eflags - %16 = AND16rr %15, %13, implicit-def dead $eflags - %17 = XOR16rr %16, %12, implicit-def dead $eflags + %12 = SHR16ri_ND %7, 2, implicit-def dead $eflags + %13 = SHL16ri_ND %12, 1, implicit-def dead $eflags + %14 = NOT16r_ND %13 + %15 = OR16rr_ND %14, %12, implicit-def dead $eflags + %16 = AND16rr_ND %15, %13, implicit-def dead $eflags + %17 = XOR16rr_ND %16, %12, implicit-def dead $eflags %8 = IMPLICIT_DEF %9 = INSERT_SUBREG %8, %17, %subreg.sub_16bit @@ -524,14 +524,14 @@ body: | %2 = COPY $zmm1 %5 = MOV32rm %0, 1, $noreg, 0, $noreg - %6 = SHR32ri %5, 2, implicit-def dead $eflags - %7 = SHL32ri %6, 1, implicit-def dead $eflags - %8 = NOT32r %7 - %9 = OR32rr %8, %6, implicit-def dead $eflags - %10 = AND32rr %9, %7, implicit-def dead $eflags - %11 = XOR32rr %10, %6, implicit-def dead $eflags + %6 = SHR32ri_ND %5, 2, implicit-def dead $eflags + %7 = SHL32ri_ND %6, 1, implicit-def dead $eflags + %8 = NOT32r_ND %7 + %9 = OR32rr_ND %8, %6, implicit-def dead $eflags + %10 = AND32rr_ND %9, %7, implicit-def dead $eflags + %11 = XOR32rr_ND %10, %6, implicit-def dead $eflags %12 = ANDN32rr %11, %9, implicit-def dead $eflags - %13 = ADD32rr %12, %11, implicit-def dead $eflags + %13 = ADD32rr_ND %12, %11, implicit-def dead $eflags %3 = COPY %13 %4 = VMOVDQU16Zrrk %2, killed %3, %1 @@ -627,14 +627,14 @@ body: | %2 = COPY $zmm1 %5 = MOV64rm %0, 1, $noreg, 0, $noreg - %6 = SHR64ri %5, 2, implicit-def dead $eflags - %7 = SHL64ri %6, 1, implicit-def dead $eflags - %8 = NOT64r %7 - %9 = OR64rr %8, %6, implicit-def dead $eflags - %10 = AND64rr %9, %7, implicit-def dead $eflags - %11 = XOR64rr %10, %6, implicit-def dead $eflags + %6 = SHR64ri_ND %5, 2, implicit-def dead $eflags + %7 = SHL64ri_ND %6, 1, implicit-def dead $eflags + %8 = NOT64r_ND %7 + %9 = OR64rr_ND %8, %6, implicit-def dead $eflags + %10 = AND64rr_ND %9, %7, implicit-def dead $eflags + %11 = XOR64rr_ND %10, %6, implicit-def dead $eflags %12 = ANDN64rr %11, %9, implicit-def dead $eflags - %13 = ADD64rr %12, %11, implicit-def dead $eflags + %13 = ADD64rr_ND %12, %11, implicit-def dead $eflags %3 = COPY %13 %4 = VMOVDQU8Zrrk %2, killed %3, %1 @@ -712,7 +712,7 @@ body: | %2 = COPY $zmm1 %5 = MOVZX16rm8 %0, 1, $noreg, 0, $noreg - %6 = NOT16r %5 + %6 = NOT16r_ND %5 %3 = COPY %6 %4 = VMOVAPSZrrk %2, killed %3, %1 @@ -785,7 +785,7 @@ body: | %5 = MOVZX32rm8 %0, 1, $noreg, 0, $noreg %6 = MOVZX32rm16 %0, 1, $noreg, 0, $noreg - %7 = ADD32rr %5, %6, implicit-def dead $eflags + %7 = ADD32rr_ND %5, %6, implicit-def dead $eflags %3 = COPY %7 %4 = VMOVDQU16Zrrk %2, killed %3, %1 @@ -858,7 +858,7 @@ body: | %5 = MOVZX64rm8 %0, 1, $noreg, 0, $noreg %6 = MOVZX64rm16 %0, 1, $noreg, 0, $noreg - %7 = ADD64rr %5, %6, implicit-def dead $eflags + %7 = ADD64rr_ND %5, %6, implicit-def dead $eflags %3 = COPY %7 %4 = VMOVDQU8Zrrk %2, killed %3, %1 -- GitLab From 8d7d581ad2a96ebe54aed0e5a626048d2e2a8d2d Mon Sep 17 00:00:00 2001 From: paperchalice Date: Fri, 22 Mar 2024 08:54:11 +0800 Subject: [PATCH 219/296] Revert "[Passes] Expose parseSinglePassOption" (#86225) Reverts llvm/llvm-project#86117 --- llvm/include/llvm/Passes/PassBuilder.h | 7 ---- llvm/lib/Passes/PassBuilder.cpp | 55 +++++++++++--------------- 2 files changed, 24 insertions(+), 38 deletions(-) diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index d1232124d5d8..8817a2585646 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -672,13 +672,6 @@ public: return Result; } - /// Handle passes only accept one bool-valued parameter. - /// - /// \return false when Params is empty. - static Expected parseSinglePassOption(StringRef Params, - StringRef OptionName, - StringRef PassName); - private: // O1 pass pipeline FunctionPassManager diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index c5465da8b4a5..9d98ae7dde52 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -600,9 +600,8 @@ Expected parseLoopUnrollOptions(StringRef Params) { return UnrollOpts; } -Expected PassBuilder::parseSinglePassOption(StringRef Params, - StringRef OptionName, - StringRef PassName) { +Expected parseSinglePassOption(StringRef Params, StringRef OptionName, + StringRef PassName) { bool Result = false; while (!Params.empty()) { StringRef ParamName; @@ -621,28 +620,24 @@ Expected PassBuilder::parseSinglePassOption(StringRef Params, } Expected parseGlobalDCEPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption( - Params, "vfe-linkage-unit-visibility", "GlobalDCE"); + return parseSinglePassOption(Params, "vfe-linkage-unit-visibility", "GlobalDCE"); } Expected parseCGProfilePassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "in-lto-post-link", - "CGProfile"); + return parseSinglePassOption(Params, "in-lto-post-link", "CGProfile"); } Expected parseInlinerPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "only-mandatory", - "InlinerPass"); + return parseSinglePassOption(Params, "only-mandatory", "InlinerPass"); } Expected parseCoroSplitPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "reuse-storage", - "CoroSplitPass"); + return parseSinglePassOption(Params, "reuse-storage", "CoroSplitPass"); } Expected parsePostOrderFunctionAttrsPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption( - Params, "skip-non-recursive-function-attrs", "PostOrderFunctionAttrs"); + return parseSinglePassOption(Params, "skip-non-recursive-function-attrs", + "PostOrderFunctionAttrs"); } Expected parseCFGuardPassOptions(StringRef Params) { @@ -666,21 +661,19 @@ Expected parseCFGuardPassOptions(StringRef Params) { } Expected parseEarlyCSEPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "memssa", "EarlyCSE"); + return parseSinglePassOption(Params, "memssa", "EarlyCSE"); } Expected parseEntryExitInstrumenterPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "post-inline", - "EntryExitInstrumenter"); + return parseSinglePassOption(Params, "post-inline", "EntryExitInstrumenter"); } Expected parseLoopExtractorPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "single", "LoopExtractor"); + return parseSinglePassOption(Params, "single", "LoopExtractor"); } Expected parseLowerMatrixIntrinsicsPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "minimal", - "LowerMatrixIntrinsics"); + return parseSinglePassOption(Params, "minimal", "LowerMatrixIntrinsics"); } Expected parseASanPassOptions(StringRef Params) { @@ -1020,13 +1013,13 @@ parseStackLifetimeOptions(StringRef Params) { } Expected parseDependenceAnalysisPrinterOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "normalized-results", - "DependenceAnalysisPrinter"); + return parseSinglePassOption(Params, "normalized-results", + "DependenceAnalysisPrinter"); } Expected parseSeparateConstOffsetFromGEPPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "lower-gep", - "SeparateConstOffsetFromGEP"); + return parseSinglePassOption(Params, "lower-gep", + "SeparateConstOffsetFromGEP"); } Expected @@ -1042,13 +1035,13 @@ parseFunctionSimplificationPipelineOptions(StringRef Params) { } Expected parseMemorySSAPrinterPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "no-ensure-optimized-uses", - "MemorySSAPrinterPass"); + return parseSinglePassOption(Params, "no-ensure-optimized-uses", + "MemorySSAPrinterPass"); } Expected parseSpeculativeExecutionPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "only-if-divergent-target", - "SpeculativeExecutionPass"); + return parseSinglePassOption(Params, "only-if-divergent-target", + "SpeculativeExecutionPass"); } Expected parseMemProfUsePassOptions(StringRef Params) { @@ -1069,13 +1062,13 @@ Expected parseMemProfUsePassOptions(StringRef Params) { } Expected parseStructuralHashPrinterPassOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "detailed", - "StructuralHashPrinterPass"); + return parseSinglePassOption(Params, "detailed", + "StructuralHashPrinterPass"); } Expected parseWinEHPrepareOptions(StringRef Params) { - return PassBuilder::parseSinglePassOption(Params, "demote-catchswitch-only", - "WinEHPreparePass"); + return parseSinglePassOption(Params, "demote-catchswitch-only", + "WinEHPreparePass"); } Expected parseGlobalMergeOptions(StringRef Params) { -- GitLab From 718fbbef5f18a2b7e7fc4f842b1452ae9bee581a Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Thu, 21 Mar 2024 18:14:18 -0700 Subject: [PATCH 220/296] [llvm-exegesis] Kill process that recieve a signal (#86069) Before this patch, llvm-exegesis would leave processes lingering that experienced signals like segmentation faults. They would up in a signal-delivery-stop state under the ptrace and never exit. This does not cause problems (or at least many) in llvm-exegesis as they are cleaned up after the main process exits, which usually happens quickly. However, in downstream use, when many blocks are being executed (many of which run into signals) within a single process, these processes stay around and can easily exhaust the process limit on some systems. This patch cleans them up by sending SIGKILL after information about the signal that was sent has been gathered. --- .../llvm-exegesis/lib/BenchmarkRunner.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index 5c9848f3c688..f0452605eb24 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -342,7 +342,7 @@ private: return make_error("Failed to attach to the child process: " + Twine(strerror(errno))); - if (wait(NULL) == -1) { + if (waitpid(ParentOrChildPID, NULL, 0) == -1) { return make_error( "Failed to wait for child process to stop after attaching: " + Twine(strerror(errno))); @@ -361,7 +361,7 @@ private: return SendError; int ChildStatus; - if (wait(&ChildStatus) == -1) { + if (waitpid(ParentOrChildPID, &ChildStatus, 0) == -1) { return make_error( "Waiting for the child process to complete failed: " + Twine(strerror(errno))); @@ -401,6 +401,20 @@ private: Twine(strerror(errno))); } + // Send SIGKILL rather than SIGTERM as the child process has no SIGTERM + // handlers to run, and calling SIGTERM would mean that ptrace will force + // it to block in the signal-delivery-stop for the SIGSEGV/other signals, + // and upon exit. + if (kill(ParentOrChildPID, SIGKILL) == -1) + return make_error("Failed to kill child benchmarking proces: " + + Twine(strerror(errno))); + + // Wait for the process to exit so that there are no zombie processes left + // around. + if (waitpid(ParentOrChildPID, NULL, 0) == -1) + return make_error("Failed to wait for process to die: " + + Twine(strerror(errno))); + if (ChildSignalInfo.si_signo == SIGSEGV) return make_error( reinterpret_cast(ChildSignalInfo.si_addr)); -- GitLab From 4d7f28a2c4b187f0bef3877081100786156defc7 Mon Sep 17 00:00:00 2001 From: paperchalice Date: Fri, 22 Mar 2024 10:55:58 +0800 Subject: [PATCH 221/296] [Passes] Expose parseSinglePassOption (#86226) Reland #86225, adjust the name space. --- llvm/include/llvm/Passes/PassBuilder.h | 7 ++ llvm/lib/Passes/PassBuilder.cpp | 89 ++++++++++++++------------ 2 files changed, 55 insertions(+), 41 deletions(-) diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h index 8817a2585646..d1232124d5d8 100644 --- a/llvm/include/llvm/Passes/PassBuilder.h +++ b/llvm/include/llvm/Passes/PassBuilder.h @@ -672,6 +672,13 @@ public: return Result; } + /// Handle passes only accept one bool-valued parameter. + /// + /// \return false when Params is empty. + static Expected parseSinglePassOption(StringRef Params, + StringRef OptionName, + StringRef PassName); + private: // O1 pass pipeline FunctionPassManager diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 9d98ae7dde52..f60f4eb3f0ef 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -514,6 +514,26 @@ static std::optional parseOptLevel(StringRef S) { .Default(std::nullopt); } +Expected PassBuilder::parseSinglePassOption(StringRef Params, + StringRef OptionName, + StringRef PassName) { + bool Result = false; + while (!Params.empty()) { + StringRef ParamName; + std::tie(ParamName, Params) = Params.split(';'); + + if (ParamName == OptionName) { + Result = true; + } else { + return make_error( + formatv("invalid {1} pass parameter '{0}' ", ParamName, PassName) + .str(), + inconvertibleErrorCode()); + } + } + return Result; +} + namespace { /// Parser of parameters for HardwareLoops pass. @@ -600,44 +620,29 @@ Expected parseLoopUnrollOptions(StringRef Params) { return UnrollOpts; } -Expected parseSinglePassOption(StringRef Params, StringRef OptionName, - StringRef PassName) { - bool Result = false; - while (!Params.empty()) { - StringRef ParamName; - std::tie(ParamName, Params) = Params.split(';'); - - if (ParamName == OptionName) { - Result = true; - } else { - return make_error( - formatv("invalid {1} pass parameter '{0}' ", ParamName, PassName) - .str(), - inconvertibleErrorCode()); - } - } - return Result; -} - Expected parseGlobalDCEPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "vfe-linkage-unit-visibility", "GlobalDCE"); + return PassBuilder::parseSinglePassOption( + Params, "vfe-linkage-unit-visibility", "GlobalDCE"); } Expected parseCGProfilePassOptions(StringRef Params) { - return parseSinglePassOption(Params, "in-lto-post-link", "CGProfile"); + return PassBuilder::parseSinglePassOption(Params, "in-lto-post-link", + "CGProfile"); } Expected parseInlinerPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "only-mandatory", "InlinerPass"); + return PassBuilder::parseSinglePassOption(Params, "only-mandatory", + "InlinerPass"); } Expected parseCoroSplitPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "reuse-storage", "CoroSplitPass"); + return PassBuilder::parseSinglePassOption(Params, "reuse-storage", + "CoroSplitPass"); } Expected parsePostOrderFunctionAttrsPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "skip-non-recursive-function-attrs", - "PostOrderFunctionAttrs"); + return PassBuilder::parseSinglePassOption( + Params, "skip-non-recursive-function-attrs", "PostOrderFunctionAttrs"); } Expected parseCFGuardPassOptions(StringRef Params) { @@ -661,19 +666,21 @@ Expected parseCFGuardPassOptions(StringRef Params) { } Expected parseEarlyCSEPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "memssa", "EarlyCSE"); + return PassBuilder::parseSinglePassOption(Params, "memssa", "EarlyCSE"); } Expected parseEntryExitInstrumenterPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "post-inline", "EntryExitInstrumenter"); + return PassBuilder::parseSinglePassOption(Params, "post-inline", + "EntryExitInstrumenter"); } Expected parseLoopExtractorPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "single", "LoopExtractor"); + return PassBuilder::parseSinglePassOption(Params, "single", "LoopExtractor"); } Expected parseLowerMatrixIntrinsicsPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "minimal", "LowerMatrixIntrinsics"); + return PassBuilder::parseSinglePassOption(Params, "minimal", + "LowerMatrixIntrinsics"); } Expected parseASanPassOptions(StringRef Params) { @@ -1013,13 +1020,13 @@ parseStackLifetimeOptions(StringRef Params) { } Expected parseDependenceAnalysisPrinterOptions(StringRef Params) { - return parseSinglePassOption(Params, "normalized-results", - "DependenceAnalysisPrinter"); + return PassBuilder::parseSinglePassOption(Params, "normalized-results", + "DependenceAnalysisPrinter"); } Expected parseSeparateConstOffsetFromGEPPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "lower-gep", - "SeparateConstOffsetFromGEP"); + return PassBuilder::parseSinglePassOption(Params, "lower-gep", + "SeparateConstOffsetFromGEP"); } Expected @@ -1035,13 +1042,13 @@ parseFunctionSimplificationPipelineOptions(StringRef Params) { } Expected parseMemorySSAPrinterPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "no-ensure-optimized-uses", - "MemorySSAPrinterPass"); + return PassBuilder::parseSinglePassOption(Params, "no-ensure-optimized-uses", + "MemorySSAPrinterPass"); } Expected parseSpeculativeExecutionPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "only-if-divergent-target", - "SpeculativeExecutionPass"); + return PassBuilder::parseSinglePassOption(Params, "only-if-divergent-target", + "SpeculativeExecutionPass"); } Expected parseMemProfUsePassOptions(StringRef Params) { @@ -1062,13 +1069,13 @@ Expected parseMemProfUsePassOptions(StringRef Params) { } Expected parseStructuralHashPrinterPassOptions(StringRef Params) { - return parseSinglePassOption(Params, "detailed", - "StructuralHashPrinterPass"); + return PassBuilder::parseSinglePassOption(Params, "detailed", + "StructuralHashPrinterPass"); } Expected parseWinEHPrepareOptions(StringRef Params) { - return parseSinglePassOption(Params, "demote-catchswitch-only", - "WinEHPreparePass"); + return PassBuilder::parseSinglePassOption(Params, "demote-catchswitch-only", + "WinEHPreparePass"); } Expected parseGlobalMergeOptions(StringRef Params) { -- GitLab From 40beb9b001a3c67c60b98fae9e999dcaa2d88717 Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Thu, 21 Mar 2024 20:14:18 -0700 Subject: [PATCH 222/296] [lldb] Handle clang::Language::CIR (#86234) commit e66b670f3bf9312f696e66c31152ae535207d6bb Author: Nathan Lanza Date: Thu Mar 21 19:53:48 2024 -0400 triggers: lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp:478:16: error: enumeration value 'CIR' not handled in switch [-Werror,-Wswitch] This patch teaches lldb to handle clang::Language::CIR the same way as clang::Language::LLVM_IR. --- lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp index 3ac1cf91932c..ebcc3bc99a80 100644 --- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp +++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp @@ -477,6 +477,7 @@ static void ParseLangArgs(LangOptions &Opts, InputKind IK, const char *triple) { // Based on the base language, pick one. switch (IK.getLanguage()) { case clang::Language::Unknown: + case clang::Language::CIR: case clang::Language::LLVM_IR: case clang::Language::RenderScript: llvm_unreachable("Invalid input kind!"); -- GitLab From 4865dab04cad1c5ce47468b0a52ea968e5a5503b Mon Sep 17 00:00:00 2001 From: Kazu Hirata Date: Thu, 21 Mar 2024 20:21:11 -0700 Subject: [PATCH 223/296] [BOLT] Fix unused variable warnings This patch fixes: bolt/lib/Rewrite/LinuxKernelRewriter.cpp:1664:20: error: unused variable 'TargetAddress' [-Werror,-Wunused-variable] bolt/lib/Rewrite/LinuxKernelRewriter.cpp:1666:20: error: unused variable 'KeyAddress' [-Werror,-Wunused-variable] --- bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp index b028a455a6db..303e8b18fd32 100644 --- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp +++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp @@ -1679,6 +1679,8 @@ Error LinuxKernelRewriter::updateStaticKeysJumpTablePostEmit() { << "\n\tTargetAddress: 0x" << Twine::utohexstr(TargetAddress) << "\n\tKeyAddress: 0x" << Twine::utohexstr(KeyAddress) << '\n'; }); + (void)TargetAddress; + (void)KeyAddress; BinaryFunction *BF = BC.getBinaryFunctionContainingAddress(JumpAddress, -- GitLab From c67ed2f1e12e1b0e16b25606e67b67a47ca848d5 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 21 Mar 2024 20:35:08 -0700 Subject: [PATCH 224/296] [SelectionDAG][RISCV] Use TypeSize version of ComputeValueVTs in TargetLowering::LowerCallTo. (#86166) This is needed to support non-intrinsic functions returning tuple types which are represented as structs with scalable vector types in IR. I suspect this may have been broken since https://reviews.llvm.org/D158115 --- .../SelectionDAG/SelectionDAGBuilder.cpp | 8 +- llvm/test/CodeGen/RISCV/rvv/calling-conv.ll | 76 +++++++++++++++++++ 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 2d63774c75e3..84df98b8a613 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -10500,14 +10500,14 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { CLI.Ins.clear(); Type *OrigRetTy = CLI.RetTy; SmallVector RetTys; - SmallVector Offsets; + SmallVector Offsets; auto &DL = CLI.DAG.getDataLayout(); - ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets, 0); + ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets); if (CLI.IsPostTypeLegalization) { // If we are lowering a libcall after legalization, split the return type. SmallVector OldRetTys; - SmallVector OldOffsets; + SmallVector OldOffsets; RetTys.swap(OldRetTys); Offsets.swap(OldOffsets); @@ -10519,7 +10519,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8; RetTys.append(NumRegs, RegisterVT); for (unsigned j = 0; j != NumRegs; ++j) - Offsets.push_back(Offset + j * RegisterVTByteSZ); + Offsets.push_back(TypeSize::getFixed(Offset + j * RegisterVTByteSZ)); } } diff --git a/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll b/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll index 78385a80b47e..78e8700a9fef 100644 --- a/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll +++ b/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll @@ -86,3 +86,79 @@ define @caller_scalable_vector_split_indirect( @callee_scalable_vector_split_indirect( zeroinitializer, %x) ret %a } + +define {, } @caller_tuple_return() { +; RV32-LABEL: caller_tuple_return: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -16 +; RV32-NEXT: .cfi_def_cfa_offset 16 +; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32-NEXT: .cfi_offset ra, -4 +; RV32-NEXT: call callee_tuple_return +; RV32-NEXT: vmv2r.v v12, v8 +; RV32-NEXT: vmv2r.v v8, v10 +; RV32-NEXT: vmv2r.v v10, v12 +; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 16 +; RV32-NEXT: ret +; +; RV64-LABEL: caller_tuple_return: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -16 +; RV64-NEXT: .cfi_def_cfa_offset 16 +; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: .cfi_offset ra, -8 +; RV64-NEXT: call callee_tuple_return +; RV64-NEXT: vmv2r.v v12, v8 +; RV64-NEXT: vmv2r.v v8, v10 +; RV64-NEXT: vmv2r.v v10, v12 +; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 16 +; RV64-NEXT: ret + %a = call {, } @callee_tuple_return() + %b = extractvalue {, } %a, 0 + %c = extractvalue {, } %a, 1 + %d = insertvalue {, } poison, %c, 0 + %e = insertvalue {, } %d, %b, 1 + ret {, } %e +} + +declare {, } @callee_tuple_return() + +define void @caller_tuple_argument({, } %x) { +; RV32-LABEL: caller_tuple_argument: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -16 +; RV32-NEXT: .cfi_def_cfa_offset 16 +; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32-NEXT: .cfi_offset ra, -4 +; RV32-NEXT: vmv2r.v v12, v8 +; RV32-NEXT: vmv2r.v v8, v10 +; RV32-NEXT: vmv2r.v v10, v12 +; RV32-NEXT: call callee_tuple_argument +; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 16 +; RV32-NEXT: ret +; +; RV64-LABEL: caller_tuple_argument: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -16 +; RV64-NEXT: .cfi_def_cfa_offset 16 +; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: .cfi_offset ra, -8 +; RV64-NEXT: vmv2r.v v12, v8 +; RV64-NEXT: vmv2r.v v8, v10 +; RV64-NEXT: vmv2r.v v10, v12 +; RV64-NEXT: call callee_tuple_argument +; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 16 +; RV64-NEXT: ret + %a = extractvalue {, } %x, 0 + %b = extractvalue {, } %x, 1 + %c = insertvalue {, } poison, %b, 0 + %d = insertvalue {, } %c, %a, 1 + call void @callee_tuple_argument({, } %d) + ret void +} + +declare void @callee_tuple_argument({, }) -- GitLab From e1a8120a63cdb6c9567b0f68d9a0390e4f5da184 Mon Sep 17 00:00:00 2001 From: Pravin Jagtap Date: Fri, 22 Mar 2024 09:25:06 +0530 Subject: [PATCH 225/296] [AMDGPU] Support double type in atomic optimizer. (#84307) Presently the atomic optimizer supports only 32-bit operations. Plan is to extend the atomic optimizer for 64-bit operations for compute and graphics. This patch extends support for double type for `uniform values` only. Going forward, will extend the support for divergent values. Adding support for divergent values requires extending/legalizing readfirstlane, readlane, writelane, etc ops for 64-bit operations to avoid `bitcast` noise that we have currently. --------- Authored-by: Pravin Jagtap --- .../Target/AMDGPU/AMDGPUAtomicOptimizer.cpp | 11 +- .../AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll | 274 +- .../CodeGen/AMDGPU/fp64-atomics-gfx90a.ll | 270 +- .../AMDGPU/global_atomic_optimizer_fp_rtn.ll | 560 ++ .../global_atomics_optimizer_fp_no_rtn.ll | 420 ++ .../AMDGPU/global_atomics_scan_fadd.ll | 5578 +++++++++++++++++ .../AMDGPU/global_atomics_scan_fmax.ll | 3960 ++++++++++++ .../AMDGPU/global_atomics_scan_fmin.ll | 3960 ++++++++++++ .../AMDGPU/global_atomics_scan_fsub.ll | 5576 ++++++++++++++++ 9 files changed, 20497 insertions(+), 112 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp index 9ba74a23e8af..dbb3de76b4dd 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp @@ -209,8 +209,9 @@ void AMDGPUAtomicOptimizerImpl::visitAtomicRMWInst(AtomicRMWInst &I) { break; } - // Only 32-bit floating point atomic ops are supported. - if (AtomicRMWInst::isFPOperation(Op) && !I.getType()->isFloatTy()) { + // Only 32 and 64 bit floating point atomic ops are supported. + if (AtomicRMWInst::isFPOperation(Op) && + !(I.getType()->isFloatTy() || I.getType()->isDoubleTy())) { return; } @@ -920,8 +921,10 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I, Value *BroadcastI = nullptr; if (TyBitWidth == 64) { - Value *const ExtractLo = B.CreateTrunc(PHI, Int32Ty); - Value *const ExtractHi = B.CreateTrunc(B.CreateLShr(PHI, 32), Int32Ty); + Value *CastedPhi = B.CreateBitCast(PHI, IntNTy); + Value *const ExtractLo = B.CreateTrunc(CastedPhi, Int32Ty); + Value *const ExtractHi = + B.CreateTrunc(B.CreateLShr(CastedPhi, 32), Int32Ty); CallInst *const ReadFirstLaneLo = B.CreateIntrinsic(Intrinsic::amdgcn_readfirstlane, {}, ExtractLo); CallInst *const ReadFirstLaneHi = diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll index 255c6dedbd6e..1a76f8cf87ff 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll @@ -1090,18 +1090,29 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %ptr) #1 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB39_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX90A-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v6, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] -; GFX90A-NEXT: .LBB39_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB39_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 +; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] ; GFX90A-NEXT: buffer_wbl2 -; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_wbinvl1_vol @@ -1109,20 +1120,31 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB39_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB39_2 +; GFX90A-NEXT: .LBB39_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB39_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-NEXT: buffer_wbl2 sc0 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] sc1 ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc0 sc1 +; GFX940-NEXT: .LBB39_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 seq_cst @@ -1132,26 +1154,47 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(1) %ptr) #1 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB40_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: v_mov_b32_e32 v2, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol +; GFX90A-NEXT: .LBB40_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_agent: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB40_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-NEXT: buffer_wbl2 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc1 +; GFX940-NEXT: .LBB40_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst @@ -1161,18 +1204,29 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace(1) %ptr) #1 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_system: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB41_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX90A-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v6, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] -; GFX90A-NEXT: .LBB41_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB41_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 +; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] ; GFX90A-NEXT: buffer_wbl2 -; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_wbinvl1_vol @@ -1180,20 +1234,31 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB41_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB41_2 +; GFX90A-NEXT: .LBB41_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_system: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB41_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-NEXT: buffer_wbl2 sc0 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] sc1 ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc0 sc1 +; GFX940-NEXT: .LBB41_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") seq_cst @@ -1203,26 +1268,47 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(1) %ptr) #0 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_flush: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB42_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: v_mov_b32_e32 v2, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol +; GFX90A-NEXT: .LBB42_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_flush: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB42_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-NEXT: buffer_wbl2 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc1 +; GFX940-NEXT: .LBB42_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst @@ -1394,37 +1480,59 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrspace(1) %ptr) { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB49_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX90A-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v6, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] -; GFX90A-NEXT: .LBB49_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB49_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 -; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB49_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB49_2 +; GFX90A-NEXT: .LBB49_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB49_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 ; GFX940-NEXT: buffer_wbl2 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc1 +; GFX940-NEXT: .LBB49_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst @@ -1866,23 +1974,44 @@ main_body: define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr) #1 { ; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB65_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_mov_b32_e32 v2, s0 ; GFX90A-NEXT: ds_add_f64 v2, v[0:1] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: .LBB65_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB65_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: v_mov_b32_e32 v2, s0 ; GFX940-NEXT: ds_add_f64 v2, v[0:1] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: .LBB65_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst @@ -1892,23 +2021,44 @@ main_body: define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3) %ptr) #0 { ; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB66_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_mov_b32_e32 v2, s0 ; GFX90A-NEXT: ds_add_f64 v2, v[0:1] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: .LBB66_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat_flush: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB66_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: v_mov_b32_e32 v2, s0 ; GFX940-NEXT: ds_add_f64 v2, v[0:1] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: .LBB66_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst @@ -1918,44 +2068,66 @@ main_body: define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrspace(3) %ptr) #4 { ; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: s_mov_b32 s4, s3 +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB67_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dword s0, s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_mov_b32_e32 v2, s0 -; GFX90A-NEXT: ds_read_b64 v[0:1], v2 +; GFX90A-NEXT: v_mov_b32_e32 v4, s0 +; GFX90A-NEXT: ds_read_b64 v[2:3], v4 ; GFX90A-NEXT: s_mov_b64 s[0:1], 0 -; GFX90A-NEXT: .LBB67_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB67_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_add_f64 v[4:5], v[0:1], 4.0 -; GFX90A-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5] +; GFX90A-NEXT: v_add_f64 v[6:7], v[2:3], v[0:1] +; GFX90A-NEXT: ds_cmpst_rtn_b64 v[6:7], v4, v[2:3], v[6:7] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1] +; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] ; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1] -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1] +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[6:7], v[6:7] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1] -; GFX90A-NEXT: s_cbranch_execnz .LBB67_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB67_2 +; GFX90A-NEXT: .LBB67_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: s_mov_b32 s4, s3 +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s4, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB67_3 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dword s0, s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_mov_b32_e32 v2, s0 -; GFX940-NEXT: ds_read_b64 v[0:1], v2 +; GFX940-NEXT: v_mov_b32_e32 v4, s0 +; GFX940-NEXT: ds_read_b64 v[2:3], v4 ; GFX940-NEXT: s_mov_b64 s[0:1], 0 -; GFX940-NEXT: .LBB67_1: ; %atomicrmw.start +; GFX940-NEXT: .LBB67_2: ; %atomicrmw.start ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_add_f64 v[4:5], v[0:1], 4.0 -; GFX940-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5] +; GFX940-NEXT: v_add_f64 v[6:7], v[2:3], v[0:1] +; GFX940-NEXT: ds_cmpst_rtn_b64 v[6:7], v4, v[2:3], v[6:7] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1] +; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1] -; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5] +; GFX940-NEXT: v_mov_b64_e32 v[2:3], v[6:7] ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1] -; GFX940-NEXT: s_cbranch_execnz .LBB67_1 -; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX940-NEXT: s_cbranch_execnz .LBB67_2 +; GFX940-NEXT: .LBB67_3: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst diff --git a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll index 767d347bcfaa..a948fab8f1c1 100644 --- a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll +++ b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll @@ -1181,18 +1181,28 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %ptr) #1 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB42_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s6 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 +; GFX90A-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v6, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] -; GFX90A-NEXT: .LBB42_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB42_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 +; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] ; GFX90A-NEXT: buffer_wbl2 -; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_wbinvl1_vol @@ -1200,20 +1210,30 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB42_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB42_2 +; GFX90A-NEXT: .LBB42_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB42_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: buffer_wbl2 sc0 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] sc1 ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc0 sc1 +; GFX940-NEXT: .LBB42_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 seq_cst @@ -1223,26 +1243,45 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(1) %ptr) #1 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB43_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX90A-NEXT: v_mov_b32_e32 v2, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol +; GFX90A-NEXT: .LBB43_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_agent: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB43_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: buffer_wbl2 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc1 +; GFX940-NEXT: .LBB43_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst @@ -1252,18 +1291,28 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace(1) %ptr) #1 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_system: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB44_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s6 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 +; GFX90A-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v6, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] -; GFX90A-NEXT: .LBB44_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB44_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 +; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] ; GFX90A-NEXT: buffer_wbl2 -; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_invl2 ; GFX90A-NEXT: buffer_wbinvl1_vol @@ -1271,20 +1320,30 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB44_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB44_2 +; GFX90A-NEXT: .LBB44_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_system: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB44_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: buffer_wbl2 sc0 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] sc1 ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc0 sc1 +; GFX940-NEXT: .LBB44_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") seq_cst @@ -1294,26 +1353,45 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(1) %ptr) #0 { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_flush: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB45_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 +; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX90A-NEXT: v_mov_b32_e32 v2, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol +; GFX90A-NEXT: .LBB45_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_flush: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB45_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: buffer_wbl2 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc1 +; GFX940-NEXT: .LBB45_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst @@ -1485,37 +1563,57 @@ main_body: define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrspace(1) %ptr) { ; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB52_3 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s6 ; GFX90A-NEXT: s_mov_b64 s[2:3], 0 -; GFX90A-NEXT: v_mov_b32_e32 v4, 0 +; GFX90A-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX90A-NEXT: v_mov_b32_e32 v6, 0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1] -; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start +; GFX90A-NEXT: .LBB52_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 -; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0 -; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc ; GFX90A-NEXT: s_waitcnt vmcnt(0) ; GFX90A-NEXT: buffer_wbinvl1_vol ; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] ; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3] ; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3] -; GFX90A-NEXT: s_cbranch_execnz .LBB52_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB52_2 +; GFX90A-NEXT: .LBB52_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB52_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s2 ; GFX940-NEXT: v_mov_b32_e32 v2, 0 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: buffer_wbl2 sc1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] ; GFX940-NEXT: s_waitcnt vmcnt(0) ; GFX940-NEXT: buffer_inv sc1 +; GFX940-NEXT: .LBB52_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst @@ -2020,23 +2118,42 @@ main_body: define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr) #1 { ; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB70_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_mov_b32_e32 v2, s0 ; GFX90A-NEXT: ds_add_f64 v2, v[0:1] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: .LBB70_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB70_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: v_mov_b32_e32 v2, s0 ; GFX940-NEXT: ds_add_f64 v2, v[0:1] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: .LBB70_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst @@ -2046,23 +2163,42 @@ main_body: define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3) %ptr) #0 { ; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush: ; GFX90A: ; %bb.0: ; %main_body +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB71_2 +; GFX90A-NEXT: ; %bb.1: ; GFX90A-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX90A-NEXT: v_mov_b32_e32 v0, 0 -; GFX90A-NEXT: v_mov_b32_e32 v1, 0x40100000 +; GFX90A-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) ; GFX90A-NEXT: v_mov_b32_e32 v2, s0 ; GFX90A-NEXT: ds_add_f64 v2, v[0:1] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: .LBB71_2: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat_flush: ; GFX940: ; %bb.0: ; %main_body +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB71_2 +; GFX940-NEXT: ; %bb.1: ; GFX940-NEXT: s_load_dword s0, s[0:1], 0x24 -; GFX940-NEXT: v_mov_b64_e32 v[0:1], 4.0 +; GFX940-NEXT: s_bcnt1_i32_b64 s1, s[2:3] +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s1 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) ; GFX940-NEXT: v_mov_b32_e32 v2, s0 ; GFX940-NEXT: ds_add_f64 v2, v[0:1] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: .LBB71_2: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst @@ -2072,46 +2208,66 @@ main_body: define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrspace(3) %ptr) #4 { ; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe: ; GFX90A: ; %bb.0: ; %main_body -; GFX90A-NEXT: s_load_dword s2, s[0:1], 0x24 -; GFX90A-NEXT: s_mov_b64 s[0:1], 0 +; GFX90A-NEXT: s_mov_b64 s[2:3], exec +; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX90A-NEXT: s_cbranch_execz .LBB72_3 +; GFX90A-NEXT: ; %bb.1: +; GFX90A-NEXT: s_load_dword s4, s[0:1], 0x24 +; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[2:3] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_mov_b32_e32 v0, s2 -; GFX90A-NEXT: ds_read_b64 v[0:1], v0 -; GFX90A-NEXT: v_mov_b32_e32 v2, s2 -; GFX90A-NEXT: .LBB72_1: ; %atomicrmw.start +; GFX90A-NEXT: v_mov_b32_e32 v0, s4 +; GFX90A-NEXT: ds_read_b64 v[2:3], v0 +; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 +; GFX90A-NEXT: s_mov_b64 s[0:1], 0 +; GFX90A-NEXT: v_mov_b32_e32 v4, s4 +; GFX90A-NEXT: .LBB72_2: ; %atomicrmw.start ; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_add_f64 v[4:5], v[0:1], 4.0 -; GFX90A-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5] +; GFX90A-NEXT: v_add_f64 v[6:7], v[2:3], v[0:1] +; GFX90A-NEXT: ds_cmpst_rtn_b64 v[6:7], v4, v[2:3], v[6:7] ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) -; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1] +; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] ; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1] -; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1] +; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[6:7], v[6:7] op_sel:[0,1] ; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1] -; GFX90A-NEXT: s_cbranch_execnz .LBB72_1 -; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX90A-NEXT: s_cbranch_execnz .LBB72_2 +; GFX90A-NEXT: .LBB72_3: ; GFX90A-NEXT: s_endpgm ; ; GFX940-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe: ; GFX940: ; %bb.0: ; %main_body -; GFX940-NEXT: s_load_dword s2, s[0:1], 0x24 -; GFX940-NEXT: s_mov_b64 s[0:1], 0 +; GFX940-NEXT: s_mov_b64 s[2:3], exec +; GFX940-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX940-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX940-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX940-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX940-NEXT: s_cbranch_execz .LBB72_3 +; GFX940-NEXT: ; %bb.1: +; GFX940-NEXT: s_load_dword s4, s[0:1], 0x24 +; GFX940-NEXT: s_bcnt1_i32_b64 s0, s[2:3] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_mov_b32_e32 v0, s2 -; GFX940-NEXT: ds_read_b64 v[0:1], v0 -; GFX940-NEXT: v_mov_b32_e32 v2, s2 -; GFX940-NEXT: .LBB72_1: ; %atomicrmw.start +; GFX940-NEXT: v_mov_b32_e32 v0, s4 +; GFX940-NEXT: ds_read_b64 v[2:3], v0 +; GFX940-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX940-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0 +; GFX940-NEXT: s_mov_b64 s[0:1], 0 +; GFX940-NEXT: v_mov_b32_e32 v4, s4 +; GFX940-NEXT: .LBB72_2: ; %atomicrmw.start ; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1 ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_add_f64 v[4:5], v[0:1], 4.0 -; GFX940-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5] +; GFX940-NEXT: v_add_f64 v[6:7], v[2:3], v[0:1] +; GFX940-NEXT: ds_cmpst_rtn_b64 v[6:7], v4, v[2:3], v[6:7] ; GFX940-NEXT: s_waitcnt lgkmcnt(0) -; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1] +; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] ; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1] -; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5] +; GFX940-NEXT: v_mov_b64_e32 v[2:3], v[6:7] ; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1] -; GFX940-NEXT: s_cbranch_execnz .LBB72_1 -; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX940-NEXT: s_cbranch_execnz .LBB72_2 +; GFX940-NEXT: .LBB72_3: ; GFX940-NEXT: s_endpgm main_body: %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst diff --git a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll index b7a91f6fa96d..b71728096093 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll @@ -1058,6 +1058,566 @@ define amdgpu_ps float @global_atomic_fadd_div_address_div_value_system_scope_st ret float %result } +define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() +; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR: 2: +; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) +; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) +; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) +; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) +; IR-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-NEXT: [[TMP11:%.*]] = uitofp i32 [[TMP10]] to double +; IR-NEXT: [[TMP12:%.*]] = fmul double [[VAL:%.*]], [[TMP11]] +; IR-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR: 14: +; IR-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 4 +; IR-NEXT: br label [[TMP16]] +; IR: 16: +; IR-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) +; IR-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) +; IR-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-NEXT: [[TMP27:%.*]] = uitofp i32 [[TMP8]] to double +; IR-NEXT: [[TMP28:%.*]] = fmul double [[VAL]], [[TMP27]] +; IR-NEXT: [[TMP29:%.*]] = fadd double [[TMP26]], [[TMP28]] +; IR-NEXT: br label [[TMP30]] +; IR: 30: +; IR-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-NEXT: ret double [[TMP31]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_scope_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_div_value_scope_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1 { +; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP29:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP28]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: br label [[TMP30]] +; IR-ITERATIVE: 30: +; IR-ITERATIVE-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-ITERATIVE-NEXT: ret double [[TMP31]] +; +; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-DPP-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-DPP-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-DPP-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-DPP-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-DPP-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-DPP-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-DPP-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-DPP-NEXT: [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP29:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP28]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: br label [[TMP30]] +; IR-DPP: 30: +; IR-DPP-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-DPP-NEXT: ret double [[TMP31]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double %val) #1 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 { +; IR-ITERATIVE-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP29:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP28]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: br label [[TMP30]] +; IR-ITERATIVE: 30: +; IR-ITERATIVE-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-ITERATIVE-NEXT: ret double [[TMP31]] +; +; IR-DPP-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-DPP-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-DPP-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-DPP-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-DPP-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-DPP-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-DPP-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-DPP-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-DPP-NEXT: [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP29:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP28]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: br label [[TMP30]] +; IR-DPP: 30: +; IR-DPP-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-DPP-NEXT: ret double [[TMP31]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() +; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP26:%.*]] +; IR: 2: +; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) +; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) +; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) +; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] +; IR: 10: +; IR-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: br label [[TMP12]] +; IR: 12: +; IR-NEXT: [[TMP13:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ] +; IR-NEXT: [[TMP14:%.*]] = bitcast double [[TMP13]] to i64 +; IR-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32 +; IR-NEXT: [[TMP16:%.*]] = lshr i64 [[TMP14]], 32 +; IR-NEXT: [[TMP17:%.*]] = trunc i64 [[TMP16]] to i32 +; IR-NEXT: [[TMP18:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP15]]) +; IR-NEXT: [[TMP19:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP17]]) +; IR-NEXT: [[TMP20:%.*]] = insertelement <2 x i32> poison, i32 [[TMP18]], i32 0 +; IR-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> [[TMP20]], i32 [[TMP19]], i32 1 +; IR-NEXT: [[TMP22:%.*]] = bitcast <2 x i32> [[TMP21]] to double +; IR-NEXT: [[TMP23:%.*]] = uitofp i32 [[TMP8]] to double +; IR-NEXT: [[TMP24:%.*]] = select i1 [[TMP9]], double 0x7FF0000000000000, double [[VAL]] +; IR-NEXT: [[TMP25:%.*]] = call double @llvm.minnum.f64(double [[TMP22]], double [[TMP24]]) +; IR-NEXT: br label [[TMP26]] +; IR: 26: +; IR-NEXT: [[TMP27:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP25]], [[TMP12]] ] +; IR-NEXT: ret double [[TMP27]] +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic__fmax_double_uni_address_uni_value_agent_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1{ +; IR-ITERATIVE-LABEL: @global_atomic__fmax_double_uni_address_uni_value_agent_scope_unsafe_structfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP26:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] +; IR-ITERATIVE: 10: +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP12]] +; IR-ITERATIVE: 12: +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ] +; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = bitcast double [[TMP13]] to i64 +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32 +; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = lshr i64 [[TMP14]], 32 +; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = trunc i64 [[TMP16]] to i32 +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP15]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP17]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = insertelement <2 x i32> poison, i32 [[TMP18]], i32 0 +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> [[TMP20]], i32 [[TMP19]], i32 1 +; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = bitcast <2 x i32> [[TMP21]] to double +; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = select i1 [[TMP9]], double 0xFFF0000000000000, double [[VAL]] +; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP22]], double [[TMP24]], metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: br label [[TMP26]] +; IR-ITERATIVE: 26: +; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP25]], [[TMP12]] ] +; IR-ITERATIVE-NEXT: ret double [[TMP27]] +; +; IR-DPP-LABEL: @global_atomic__fmax_double_uni_address_uni_value_agent_scope_unsafe_structfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP26:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] +; IR-DPP: 10: +; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP12]] +; IR-DPP: 12: +; IR-DPP-NEXT: [[TMP13:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ] +; IR-DPP-NEXT: [[TMP14:%.*]] = bitcast double [[TMP13]] to i64 +; IR-DPP-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32 +; IR-DPP-NEXT: [[TMP16:%.*]] = lshr i64 [[TMP14]], 32 +; IR-DPP-NEXT: [[TMP17:%.*]] = trunc i64 [[TMP16]] to i32 +; IR-DPP-NEXT: [[TMP18:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP15]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP19:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP17]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP20:%.*]] = insertelement <2 x i32> poison, i32 [[TMP18]], i32 0 +; IR-DPP-NEXT: [[TMP21:%.*]] = insertelement <2 x i32> [[TMP20]], i32 [[TMP19]], i32 1 +; IR-DPP-NEXT: [[TMP22:%.*]] = bitcast <2 x i32> [[TMP21]] to double +; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP24:%.*]] = select i1 [[TMP9]], double 0xFFF0000000000000, double [[VAL]] +; IR-DPP-NEXT: [[TMP25:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP22]], double [[TMP24]], metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: br label [[TMP26]] +; IR-DPP: 26: +; IR-DPP-NEXT: [[TMP27:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP25]], [[TMP12]] ] +; IR-DPP-NEXT: ret double [[TMP27]] +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic__fmax_double_uni_address_div_value_agent_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double %val) #1{ +; IR-LABEL: @global_atomic__fmax_double_uni_address_div_value_agent_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 { +; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-ITERATIVE-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-ITERATIVE-NEXT: [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP29:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP28]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: br label [[TMP30]] +; IR-ITERATIVE: 30: +; IR-ITERATIVE-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-ITERATIVE-NEXT: ret double [[TMP31]] +; +; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP30:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ] +; IR-DPP-NEXT: [[TMP18:%.*]] = bitcast double [[TMP17]] to i64 +; IR-DPP-NEXT: [[TMP19:%.*]] = trunc i64 [[TMP18]] to i32 +; IR-DPP-NEXT: [[TMP20:%.*]] = lshr i64 [[TMP18]], 32 +; IR-DPP-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP20]] to i32 +; IR-DPP-NEXT: [[TMP22:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP19]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP23:%.*]] = call i32 @llvm.amdgcn.readfirstlane(i32 [[TMP21]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP24:%.*]] = insertelement <2 x i32> poison, i32 [[TMP22]], i32 0 +; IR-DPP-NEXT: [[TMP25:%.*]] = insertelement <2 x i32> [[TMP24]], i32 [[TMP23]], i32 1 +; IR-DPP-NEXT: [[TMP26:%.*]] = bitcast <2 x i32> [[TMP25]] to double +; IR-DPP-NEXT: [[TMP27:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP28:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP27]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP29:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP26]], double [[TMP28]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: br label [[TMP30]] +; IR-DPP: 30: +; IR-DPP-NEXT: [[TMP31:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP29]], [[TMP16]] ] +; IR-DPP-NEXT: ret double [[TMP31]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_div_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_div_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_div_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_div_address_div_value_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_div_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr, double inreg %val) #1 { +; IR-LABEL: @global_atomic_fadd_double_div_address_uni_value_one_as_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_div_address_div_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr, double %val) #1 { +; IR-LABEL: @global_atomic_fadd_double_div_address_div_value_one_as_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fsub_double_div_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr, double inreg %val) #2 { +; IR-LABEL: @global_atomic_fsub_double_div_address_uni_value_agent_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fsub_double_div_address_div_value_agent_scope_strictfp(ptr addrspace(1) %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fsub_double_div_address_div_value_agent_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fmin_double_div_address_uni_value_agent_scope(ptr addrspace(1) %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_div_address_uni_value_agent_scope( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fmin_double_div_address_div_value_agent_scope(ptr addrspace(1) %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_div_address_div_value_agent_scope( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic__fmax_double_div_address_uni_value_agent_scope_unsafe_structfp(ptr addrspace(1) %ptr, double inreg %val) #1{ +; IR-LABEL: @global_atomic__fmax_double_div_address_uni_value_agent_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic__fmax_double_div_address_div_value_agent_scope_unsafe_structfp(ptr addrspace(1) %ptr, double %val) #1{ +; IR-LABEL: @global_atomic__fmax_double_div_address_div_value_agent_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_div_address_uni_value_system_scope_strictfp(ptr addrspace(1) %ptr, double inreg %val) #2 { +; IR-LABEL: @global_atomic_fadd_double_div_address_uni_value_system_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret double %result +} + +define amdgpu_ps double @global_atomic_fadd_double_div_address_div_value_system_scope_strictfp(ptr addrspace(1) %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fadd_double_div_address_div_value_system_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 +; IR-NEXT: ret double [[RESULT]] +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret double %result +} + attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #1 = { strictfp "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #2 = { strictfp } diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll index e70d7347890f..b9234f47df19 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll @@ -864,6 +864,426 @@ define amdgpu_ps void @global_atomic_fadd_div_address_div_value_system_scope_str ret void } +define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() +; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR: 2: +; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) +; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) +; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) +; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) +; IR-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-NEXT: [[TMP11:%.*]] = uitofp i32 [[TMP10]] to double +; IR-NEXT: [[TMP12:%.*]] = fmul double [[VAL:%.*]], [[TMP11]] +; IR-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR: 14: +; IR-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 4 +; IR-NEXT: br label [[TMP16]] +; IR: 16: +; IR-NEXT: br label [[TMP17]] +; IR: 17: +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_scope_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_div_value_scope_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1 { +; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: br label [[TMP17]] +; IR-ITERATIVE: 17: +; IR-ITERATIVE-NEXT: ret void +; +; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: br label [[TMP17]] +; IR-DPP: 17: +; IR-DPP-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double %val) #1 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 { +; IR-ITERATIVE-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: br label [[TMP17]] +; IR-ITERATIVE: 17: +; IR-ITERATIVE-NEXT: ret void +; +; IR-DPP-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: br label [[TMP17]] +; IR-DPP: 17: +; IR-DPP-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() +; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]] +; IR: 2: +; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) +; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) +; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) +; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] +; IR: 10: +; IR-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: br label [[TMP12]] +; IR: 12: +; IR-NEXT: br label [[TMP13]] +; IR: 13: +; IR-NEXT: ret void +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1{ +; IR-ITERATIVE-LABEL: @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe_structfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] +; IR-ITERATIVE: 10: +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-ITERATIVE-NEXT: br label [[TMP12]] +; IR-ITERATIVE: 12: +; IR-ITERATIVE-NEXT: br label [[TMP13]] +; IR-ITERATIVE: 13: +; IR-ITERATIVE-NEXT: ret void +; +; IR-DPP-LABEL: @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe_structfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]] +; IR-DPP: 10: +; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-DPP-NEXT: br label [[TMP12]] +; IR-DPP: 12: +; IR-DPP-NEXT: br label [[TMP13]] +; IR-DPP: 13: +; IR-DPP-NEXT: ret void +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe_structfp(ptr addrspace(1) inreg %ptr, double %val) #1{ +; IR-LABEL: @global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 { +; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp( +; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]] +; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR-ITERATIVE: 2: +; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]] +; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-ITERATIVE: 14: +; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4 +; IR-ITERATIVE-NEXT: br label [[TMP16]] +; IR-ITERATIVE: 16: +; IR-ITERATIVE-NEXT: br label [[TMP17]] +; IR-ITERATIVE: 17: +; IR-ITERATIVE-NEXT: ret void +; +; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp( +; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]] +; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]] +; IR-DPP: 2: +; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32 +; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32 +; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]] +; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32 +; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]] +; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0 +; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]] +; IR-DPP: 14: +; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4 +; IR-DPP-NEXT: br label [[TMP16]] +; IR-DPP: 16: +; IR-DPP-NEXT: br label [[TMP17]] +; IR-DPP: 17: +; IR-DPP-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_div_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_div_address_uni_value_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_div_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fadd_double_div_address_div_value_agent_scope_unsafe( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 4 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_div_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr, double inreg %val) #1 { +; IR-LABEL: @global_atomic_fadd_double_div_address_uni_value_one_as_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_div_address_div_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr, double %val) #1 { +; IR-LABEL: @global_atomic_fadd_double_div_address_div_value_one_as_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("one-as") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fsub_double_div_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr, double inreg %val) #2 { +; IR-LABEL: @global_atomic_fsub_double_div_address_uni_value_agent_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fsub_double_div_address_div_value_agent_scope_strictfp(ptr addrspace(1) %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fsub_double_div_address_div_value_agent_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmin_double_div_address_uni_value_agent_scope(ptr addrspace(1) %ptr, double inreg %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_div_address_uni_value_agent_scope( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmin_double_div_address_div_value_agent_scope(ptr addrspace(1) %ptr, double %val) #0 { +; IR-LABEL: @global_atomic_fmin_double_div_address_div_value_agent_scope( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmax_double_div_address_uni_value_agent_scope_unsafe_structfp(ptr addrspace(1) %ptr, double inreg %val) #1{ +; IR-LABEL: @global_atomic_fmax_double_div_address_uni_value_agent_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fmax_double_div_address_div_value_agent_scope_unsafe_structfp(ptr addrspace(1) %ptr, double %val) #1{ +; IR-LABEL: @global_atomic_fmax_double_div_address_div_value_agent_scope_unsafe_structfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8 +; IR-NEXT: ret void +; + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_div_address_uni_value_system_scope_strictfp(ptr addrspace(1) %ptr, double inreg %val) #2 { +; IR-LABEL: @global_atomic_fadd_double_div_address_uni_value_system_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret void +} + +define amdgpu_ps void @global_atomic_fadd_double_div_address_div_value_system_scope_strictfp(ptr addrspace(1) %ptr, double %val) #2 { +; IR-LABEL: @global_atomic_fadd_double_div_address_div_value_system_scope_strictfp( +; IR-NEXT: [[RESULT:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] monotonic, align 4 +; IR-NEXT: ret void +; + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4 + ret void +} + attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #1 = { strictfp "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #2 = { strictfp } diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll index 96c615b974ce..4f00d48551be 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll @@ -13,6 +13,7 @@ ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-wavefrontsize64 -amdgpu-atomic-optimizer-strategy=DPP -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1132-DPP %s declare float @div.float.value() +declare double @div.double.value() define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { ; GFX7LESS-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe: @@ -5408,6 +5409,5583 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_defalut_scop ret void } +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB9_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[2:3], s[36:37], 0x0 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX7LESS-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX7LESS-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3 +; GFX7LESS-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], v[41:42] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_2 +; GFX7LESS-NEXT: .LBB9_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], exec +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-NEXT: s_cbranch_execz .LBB9_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB9_2 +; GFX9-NEXT: .LBB9_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB9_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1064-NEXT: .LBB9_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB9_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1032-NEXT: .LBB9_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB9_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1164-NEXT: .LBB9_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b32 s2, exec_lo +; GFX1132-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB9_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1132-NEXT: .LBB9_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX9-DPP-NEXT: .LBB9_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1064-DPP-NEXT: .LBB9_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1032-DPP-NEXT: .LBB9_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1164-DPP-NEXT: .LBB9_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo +; GFX1132-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1132-DPP-NEXT: .LBB9_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v42, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mov_b32_e32 v41, v1 +; GFX7LESS-NEXT: buffer_load_dwordx2 v[0:1], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], v[40:41] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB10_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.float.value() + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s14, -1 +; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s12, s12, s3 +; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0 +; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB11_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX7LESS-NEXT: s_mov_b32 s7, 0x43300000 +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0 +; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 +; GFX7LESS-NEXT: v_add_f64 v[0:1], s[6:7], v[0:1] +; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9 +; GFX7LESS-NEXT: s_mov_b32 s2, -1 +; GFX7LESS-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB11_2 +; GFX7LESS-NEXT: .LBB11_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s10, -1 +; GFX9-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-NEXT: s_add_u32 s8, s8, s3 +; GFX9-NEXT: s_mov_b64 s[2:3], exec +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_cbranch_execz .LBB11_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-NEXT: s_cbranch_execnz .LBB11_2 +; GFX9-NEXT: .LBB11_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s10, -1 +; GFX1064-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB11_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1064-NEXT: .LBB11_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s10, -1 +; GFX1032-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-NEXT: s_mov_b32 s2, 0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB11_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1032-NEXT: .LBB11_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-NEXT: scratch_store_b32 off, v1, off +; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-NEXT: s_cbranch_execz .LBB11_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1164-NEXT: .LBB11_3: +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-NEXT: s_mov_b32 s2, 0 +; GFX1132-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-NEXT: scratch_store_b32 off, v1, off +; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-NEXT: s_cbranch_execz .LBB11_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1132-NEXT: .LBB11_3: +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s10, -1 +; GFX9-DPP-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX9-DPP-NEXT: .LBB11_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1064-DPP-NEXT: .LBB11_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1032-DPP-NEXT: .LBB11_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1164-DPP-NEXT: .LBB11_3: +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1132-DPP-NEXT: .LBB11_3: +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[4:5], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v5 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB12_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB12_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() strictfp + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr) #2{ +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s14, -1 +; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s12, s12, s3 +; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0 +; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB13_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX7LESS-NEXT: s_mov_b32 s7, 0x43300000 +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0 +; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 +; GFX7LESS-NEXT: v_add_f64 v[0:1], s[6:7], v[0:1] +; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9 +; GFX7LESS-NEXT: s_mov_b32 s2, -1 +; GFX7LESS-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB13_2 +; GFX7LESS-NEXT: .LBB13_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s10, -1 +; GFX9-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-NEXT: s_add_u32 s8, s8, s3 +; GFX9-NEXT: s_mov_b64 s[2:3], exec +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_cbranch_execz .LBB13_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-NEXT: s_cbranch_execnz .LBB13_2 +; GFX9-NEXT: .LBB13_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s10, -1 +; GFX1064-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB13_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1064-NEXT: .LBB13_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s10, -1 +; GFX1032-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-NEXT: s_mov_b32 s2, 0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB13_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1032-NEXT: .LBB13_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-NEXT: scratch_store_b32 off, v1, off +; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-NEXT: s_cbranch_execz .LBB13_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1164-NEXT: .LBB13_3: +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-NEXT: s_mov_b32 s2, 0 +; GFX1132-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-NEXT: scratch_store_b32 off, v1, off +; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-NEXT: s_cbranch_execz .LBB13_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1132-NEXT: .LBB13_3: +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s10, -1 +; GFX9-DPP-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX9-DPP-NEXT: .LBB13_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1064-DPP-NEXT: .LBB13_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1032-DPP-NEXT: .LBB13_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1164-DPP-NEXT: .LBB13_3: +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1132-DPP-NEXT: .LBB13_3: +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[4:5], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v5 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB14_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB14_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.float.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.float.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[4:5], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v5 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB15_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.float.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.float.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB15_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.float.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.float.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.float.value() strictfp + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp(ptr addrspace(1) %ptr) #2 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB16_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s1, 0x43300000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[2:3], s[36:37], 0x0 +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX7LESS-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1] +; GFX7LESS-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3 +; GFX7LESS-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], v[41:42] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB16_2 +; GFX7LESS-NEXT: .LBB16_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], exec +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-NEXT: s_cbranch_execz .LBB16_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-NEXT: s_mov_b32 s1, 0x43300000 +; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1] +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB16_2 +; GFX9-NEXT: .LBB16_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB16_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1064-NEXT: .LBB16_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB16_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1032-NEXT: .LBB16_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1164-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-NEXT: s_cbranch_execz .LBB16_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1164-NEXT: .LBB16_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1132-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-NEXT: s_cbranch_execz .LBB16_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1132-NEXT: .LBB16_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1] +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX9-DPP-NEXT: .LBB16_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-DPP-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1064-DPP-NEXT: .LBB16_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1032-DPP-NEXT: .LBB16_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1164-DPP-NEXT: .LBB16_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[41:42] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1132-DPP-NEXT: .LBB16_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp(ptr addrspace(1) %ptr) #2 { +; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v42, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mov_b32_e32 v41, v1 +; GFX7LESS-NEXT: buffer_load_dwordx2 v[0:1], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], v[40:41] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB17_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB17_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], v[40:41] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.float.value() strictfp + %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue monotonic, align 4 + ret void +} + attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #1 = { strictfp "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #2 = { strictfp} diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll index 3cc5a4cd1d0a..622be43e7442 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll @@ -13,6 +13,7 @@ ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-wavefrontsize64 -amdgpu-atomic-optimizer-strategy=DPP -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1132-DPP %s declare float @div.float.value() +declare float @div.double.value() define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { ; GFX7LESS-LABEL: global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe: @@ -3550,6 +3551,3965 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_defalut_scop ret void } +define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB6_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1 +; GFX7LESS-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[2:3], v[0:1], v[0:1] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], 4.0 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB6_2 +; GFX7LESS-NEXT: .LBB6_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-NEXT: s_cbranch_execz .LBB6_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB6_2 +; GFX9-NEXT: .LBB6_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB6_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1064-NEXT: .LBB6_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB6_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1032-NEXT: .LBB6_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB6_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1164-NEXT: .LBB6_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB6_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1132-NEXT: .LBB6_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX9-DPP-NEXT: .LBB6_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1064-DPP-NEXT: .LBB6_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1032-DPP-NEXT: .LBB6_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1164-DPP-NEXT: .LBB6_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1132-DPP-NEXT: .LBB6_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fmax ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v40, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX7LESS-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB7_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB7_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB8_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0 +; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7 +; GFX7LESS-NEXT: s_mov_b32 s2, -1 +; GFX7LESS-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB8_2 +; GFX7LESS-NEXT: .LBB8_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-NEXT: s_cbranch_execz .LBB8_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-NEXT: s_cbranch_execnz .LBB8_2 +; GFX9-NEXT: .LBB8_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB8_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1064-NEXT: .LBB8_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-NEXT: s_mov_b32 s2, 0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_and_saveexec_b32 s3, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB8_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1032-NEXT: .LBB8_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB8_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1164-NEXT: .LBB8_3: +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-NEXT: s_mov_b32 s2, 0 +; GFX1132-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB8_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1132-NEXT: .LBB8_3: +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX9-DPP-NEXT: .LBB8_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1064-DPP-NEXT: .LBB8_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s3, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1032-DPP-NEXT: .LBB8_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1164-DPP-NEXT: .LBB8_3: +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0 +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1132-DPP-NEXT: .LBB8_3: +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fmax ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX7LESS-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX9-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB9_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1064-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1032-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1164-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1132-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX9-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1064-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1032-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %divValue syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB10_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1 +; GFX7LESS-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[2:3], v[0:1], v[0:1] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], 4.0 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_2 +; GFX7LESS-NEXT: .LBB10_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-NEXT: s_cbranch_execz .LBB10_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB10_2 +; GFX9-NEXT: .LBB10_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB10_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1064-NEXT: .LBB10_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB10_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1032-NEXT: .LBB10_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB10_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1164-NEXT: .LBB10_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB10_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1132-NEXT: .LBB10_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX9-DPP-NEXT: .LBB10_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1064-DPP-NEXT: .LBB10_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1032-DPP-NEXT: .LBB10_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1164-DPP-NEXT: .LBB10_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-DPP-NEXT: v_max_f64 v[3:4], v[3:4], 4.0 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1132-DPP-NEXT: .LBB10_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fmax ptr addrspace(1) %ptr, double 4.0 monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v40, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX7LESS-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB11_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB11_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fmax ptr addrspace(1) %ptr, double %divValue monotonic, align 4 + ret void +} + attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } !llvm.module.flags = !{!0} diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll index 314c52a71d93..49d415c9eed7 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll @@ -13,6 +13,7 @@ ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-wavefrontsize64 -amdgpu-atomic-optimizer-strategy=DPP -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1132-DPP %s declare float @div.float.value() +declare float @div.double.value() define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { ; GFX7LESS-LABEL: global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe: @@ -3550,6 +3551,3965 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_defalut_scop ret void } +define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB6_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1 +; GFX7LESS-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[2:3], v[0:1], v[0:1] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_min_f64 v[0:1], v[2:3], 4.0 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB6_2 +; GFX7LESS-NEXT: .LBB6_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-NEXT: s_cbranch_execz .LBB6_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB6_2 +; GFX9-NEXT: .LBB6_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB6_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1064-NEXT: .LBB6_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB6_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1032-NEXT: .LBB6_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB6_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1164-NEXT: .LBB6_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB6_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1132-NEXT: .LBB6_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX9-DPP-NEXT: .LBB6_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1064-DPP-NEXT: .LBB6_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1032-DPP-NEXT: .LBB6_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1164-DPP-NEXT: .LBB6_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB6_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB6_2 +; GFX1132-DPP-NEXT: .LBB6_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fmin ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v40, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX7LESS-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB7_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB7_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB7_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB7_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB8_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0 +; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7 +; GFX7LESS-NEXT: s_mov_b32 s2, -1 +; GFX7LESS-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB8_2 +; GFX7LESS-NEXT: .LBB8_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-NEXT: s_cbranch_execz .LBB8_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-NEXT: s_cbranch_execnz .LBB8_2 +; GFX9-NEXT: .LBB8_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB8_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1064-NEXT: .LBB8_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-NEXT: s_mov_b32 s2, 0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_and_saveexec_b32 s3, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB8_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1032-NEXT: .LBB8_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB8_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1164-NEXT: .LBB8_3: +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-NEXT: s_mov_b32 s2, 0 +; GFX1132-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB8_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1132-NEXT: .LBB8_3: +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX9-DPP-NEXT: .LBB8_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1064-DPP-NEXT: .LBB8_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s3, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1032-DPP-NEXT: .LBB8_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1164-DPP-NEXT: .LBB8_3: +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-DPP-NEXT: .LBB8_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0 +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB8_2 +; GFX1132-DPP-NEXT: .LBB8_3: +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fmin ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX7LESS-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX9-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB9_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1064-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1032-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1164-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1132-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX9-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1064-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[2:3], v40, s[34:35] +; GFX1032-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_one_as_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[2:3], v40, s[34:35] +; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB9_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v40, v[0:3], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %divValue syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB10_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1 +; GFX7LESS-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[2:3], v[0:1], v[0:1] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_min_f64 v[0:1], v[2:3], 4.0 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_2 +; GFX7LESS-NEXT: .LBB10_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-NEXT: s_cbranch_execz .LBB10_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB10_2 +; GFX9-NEXT: .LBB10_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB10_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1064-NEXT: .LBB10_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB10_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1032-NEXT: .LBB10_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB10_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1164-NEXT: .LBB10_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB10_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1132-NEXT: .LBB10_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX9-DPP-NEXT: .LBB10_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1064-DPP-NEXT: .LBB10_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1032-DPP-NEXT: .LBB10_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1164-DPP-NEXT: .LBB10_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_defalut_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB10_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_max_f64 v[3:4], v[1:2], v[1:2] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1132-DPP-NEXT: v_min_f64 v[3:4], v[3:4], 4.0 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB10_2 +; GFX1132-DPP-NEXT: .LBB10_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fmin ptr addrspace(1) %ptr, double 4.0 monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v40, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX7LESS-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB11_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB11_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX9-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1064-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v40, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[2:3], v41, s[42:43] +; GFX1032-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v0, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v3, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1164-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_defalut_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: global_load_b64 v[2:3], v41, s[42:43] +; GFX1132-DPP-NEXT: v_max_f64 v[41:42], v[0:1], v[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB11_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[41:42] +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[2:3], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[0:1], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[2:3], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fmin ptr addrspace(1) %ptr, double %divValue monotonic, align 4 + ret void +} + attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } !llvm.module.flags = !{!0} diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll index bc9125e326c4..7a7ddbe618b0 100644 --- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll +++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll @@ -13,6 +13,7 @@ ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize32,-wavefrontsize64 -amdgpu-atomic-optimizer-strategy=DPP -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX1132-DPP %s declare float @div.float.value() +declare double @div.double.value() define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { ; GFX7LESS-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe: @@ -5616,6 +5617,5581 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_defalut_scop ret void } +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB9_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[2:3], s[36:37], 0x0 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX7LESS-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX7LESS-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3 +; GFX7LESS-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], -v[41:42] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_2 +; GFX7LESS-NEXT: .LBB9_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], exec +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-NEXT: s_cbranch_execz .LBB9_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB9_2 +; GFX9-NEXT: .LBB9_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB9_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1064-NEXT: .LBB9_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB9_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1032-NEXT: .LBB9_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-NEXT: s_cbranch_execz .LBB9_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1164-NEXT: .LBB9_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b32 s2, exec_lo +; GFX1132-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-NEXT: s_cbranch_execz .LBB9_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1132-NEXT: .LBB9_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX9-DPP-NEXT: .LBB9_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1064-DPP-NEXT: .LBB9_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1032-DPP-NEXT: .LBB9_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1164-DPP-NEXT: .LBB9_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo +; GFX1132-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB9_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_mul_f64 v[41:42], v[0:1], 4.0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB9_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_2 +; GFX1132-DPP-NEXT: .LBB9_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v42, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mov_b32_e32 v41, v1 +; GFX7LESS-NEXT: buffer_load_dwordx2 v[0:1], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], -v[40:41] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB10_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB10_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB10_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.float.value() + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s14, -1 +; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s12, s12, s3 +; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0 +; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB11_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX7LESS-NEXT: s_mov_b32 s7, 0x43300000 +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0 +; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 +; GFX7LESS-NEXT: v_add_f64 v[0:1], s[6:7], v[0:1] +; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9 +; GFX7LESS-NEXT: s_mov_b32 s2, -1 +; GFX7LESS-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB11_2 +; GFX7LESS-NEXT: .LBB11_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s10, -1 +; GFX9-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-NEXT: s_add_u32 s8, s8, s3 +; GFX9-NEXT: s_mov_b64 s[2:3], exec +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_cbranch_execz .LBB11_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-NEXT: s_cbranch_execnz .LBB11_2 +; GFX9-NEXT: .LBB11_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s10, -1 +; GFX1064-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB11_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1064-NEXT: .LBB11_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s10, -1 +; GFX1032-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-NEXT: s_mov_b32 s2, 0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB11_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1032-NEXT: .LBB11_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-NEXT: scratch_store_b32 off, v1, off +; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-NEXT: s_cbranch_execz .LBB11_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1164-NEXT: .LBB11_3: +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-NEXT: s_mov_b32 s2, 0 +; GFX1132-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-NEXT: scratch_store_b32 off, v1, off +; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-NEXT: s_cbranch_execz .LBB11_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1132-NEXT: .LBB11_3: +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s10, -1 +; GFX9-DPP-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX9-DPP-NEXT: .LBB11_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1064-DPP-NEXT: .LBB11_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1032-DPP-NEXT: .LBB11_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1164-DPP-NEXT: .LBB11_3: +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB11_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-DPP-NEXT: .LBB11_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_2 +; GFX1132-DPP-NEXT: .LBB11_3: +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") monotonic + ret void +} +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[4:5], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v5 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB12_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB12_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_one_as_scope_unsafe_structfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB12_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB12_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() strictfp + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue syncscope("one-as") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr) #2{ +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s14, -1 +; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s12, s12, s3 +; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0 +; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB13_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[2:3] +; GFX7LESS-NEXT: s_mov_b32 s7, 0x43300000 +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0 +; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000 +; GFX7LESS-NEXT: v_add_f64 v[0:1], s[6:7], v[0:1] +; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9 +; GFX7LESS-NEXT: s_mov_b32 s2, -1 +; GFX7LESS-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3] +; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5] +; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB13_2 +; GFX7LESS-NEXT: .LBB13_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s10, -1 +; GFX9-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-NEXT: s_add_u32 s8, s8, s3 +; GFX9-NEXT: s_mov_b64 s[2:3], exec +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-NEXT: s_cbranch_execz .LBB13_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-NEXT: s_cbranch_execnz .LBB13_2 +; GFX9-NEXT: .LBB13_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s10, -1 +; GFX1064-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB13_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1064-NEXT: .LBB13_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s10, -1 +; GFX1032-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-NEXT: s_mov_b32 s2, 0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB13_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1032-NEXT: .LBB13_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-NEXT: scratch_store_b32 off, v1, off +; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-NEXT: s_cbranch_execz .LBB13_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1164-NEXT: .LBB13_3: +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-NEXT: s_mov_b32 s2, 0 +; GFX1132-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-NEXT: scratch_store_b32 off, v1, off +; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-NEXT: s_cbranch_execz .LBB13_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1132-NEXT: .LBB13_3: +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s10, -1 +; GFX9-DPP-NEXT: s_mov_b32 s11, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX9-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1] +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX9-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX9-DPP-NEXT: .LBB13_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s11, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3] +; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1064-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1064-DPP-NEXT: .LBB13_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s8, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s9, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s10, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s11, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s8, s8, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1032-DPP-NEXT: s_addc_u32 s9, s9, 0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s4, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s4, s3 +; GFX1032-DPP-NEXT: s_mov_b32 s5, 0x43300000 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5 +; GFX1032-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1032-DPP-NEXT: .LBB13_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s2, exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3 +; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0 +; GFX1164-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1164-DPP-NEXT: .LBB13_3: +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s2, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s2 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s2, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off +; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB13_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x24 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1] +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5 +; GFX1132-DPP-NEXT: .LBB13_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0 +; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB13_2 +; GFX1132-DPP-NEXT: .LBB13_3: +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[4:5], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v5 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB14_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB14_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.double.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.double.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB14_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB14_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.double.value() + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic + ret void +} + +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_mov_b32 s32, 0 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s9 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s14, s8 +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s39, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s38, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s2, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s3, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[2:3] +; GFX7LESS-NEXT: s_add_u32 s2, s2, div.float.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s3, s3, div.float.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v31, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s12, s6 +; GFX7LESS-NEXT: s_mov_b32 s13, s7 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX7LESS-NEXT: buffer_load_dwordx2 v[4:5], off, s[36:39], 0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], 0 +; GFX7LESS-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v9, v5 +; GFX7LESS-NEXT: v_mov_b32_e32 v8, v4 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2 +; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5] +; GFX7LESS-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v4, v6 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, v7 +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB15_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s38, -1 +; GFX9-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-NEXT: s_add_u32 s36, s36, s9 +; GFX9-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-NEXT: s_mov_b32 s14, s8 +; GFX9-NEXT: s_add_u32 s8, s2, 44 +; GFX9-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-NEXT: s_getpc_b64 s[2:3] +; GFX9-NEXT: s_add_u32 s2, s2, div.float.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s3, s3, div.float.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-NEXT: s_mov_b32 s12, s6 +; GFX9-NEXT: s_mov_b32 s13, s7 +; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-NEXT: s_mov_b32 s32, 0 +; GFX9-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX9-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-NEXT: s_cbranch_execnz .LBB15_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s38, -1 +; GFX1064-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-NEXT: s_mov_b32 s14, s8 +; GFX1064-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-NEXT: s_getpc_b64 s[4:5] +; GFX1064-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s6 +; GFX1064-NEXT: s_mov_b32 s13, s7 +; GFX1064-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-NEXT: s_mov_b32 s32, 0 +; GFX1064-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1064-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s38, -1 +; GFX1032-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-NEXT: s_mov_b32 s14, s8 +; GFX1032-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-NEXT: s_getpc_b64 s[4:5] +; GFX1032-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s6 +; GFX1032-NEXT: s_mov_b32 s13, s7 +; GFX1032-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-NEXT: s_mov_b32 s32, 0 +; GFX1032-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-NEXT: s_mov_b32 s0, 0 +; GFX1032-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1032-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b32 s14, s8 +; GFX1164-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-NEXT: s_getpc_b64 s[4:5] +; GFX1164-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s32, 0 +; GFX1164-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1164-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-NEXT: s_getpc_b64 s[4:5] +; GFX1132-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1132-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-NEXT: s_mov_b32 s0, 0 +; GFX1132-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1132-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s38, -1 +; GFX9-DPP-NEXT: s_mov_b32 s39, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX9-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX9-DPP-NEXT: s_mov_b32 s14, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX9-DPP-NEXT: s_getpc_b64 s[2:3] +; GFX9-DPP-NEXT: s_add_u32 s2, s2, div.float.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s3, s3, div.float.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX9-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b32 s12, s6 +; GFX9-DPP-NEXT: s_mov_b32 s13, s7 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX9-DPP-NEXT: s_mov_b32 s32, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX9-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX9-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX9-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s39, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1064-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1064-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1064-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1064-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1064-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1064-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1064-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[0:1] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s36, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s37, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s38, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s39, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s36, s36, s9 +; GFX1032-DPP-NEXT: s_addc_u32 s37, s37, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1032-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1032-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[34:35], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1032-DPP-NEXT: v_or3_b32 v31, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1032-DPP-NEXT: global_load_dwordx2 v[4:5], v40, s[34:35] +; GFX1032-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1032-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1032-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s0 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b32 s14, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1164-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1164-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[16:17], s[4:5], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17] +; GFX1164-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], 0 +; GFX1164-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2 +; GFX1164-DPP-NEXT: s_or_b64 s[0:1], vcc, s[0:1] +; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[0:1] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_unsafe_structfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_add_u32 s8, s2, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s3, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[4:5] +; GFX1132-DPP-NEXT: s_getpc_b64 s[4:5] +; GFX1132-DPP-NEXT: s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[34:35], s[2:3], 0x24 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v31, v0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[0:1] +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1132-DPP-NEXT: global_load_b64 v[4:5], v40, s[34:35] +; GFX1132-DPP-NEXT: s_mov_b32 s0, 0 +; GFX1132-DPP-NEXT: .LBB15_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], -v[0:1] +; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[2:3], v40, v[2:5], s[34:35] glc +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2 +; GFX1132-DPP-NEXT: s_or_b32 s0, vcc_lo, s0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB15_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.float.value() strictfp + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic + ret void +} +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp(ptr addrspace(1) %ptr) #2 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s42, -1 +; GFX7LESS-NEXT: s_mov_b32 s43, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s40, s40, s3 +; GFX7LESS-NEXT: s_addc_u32 s41, s41, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s2 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec +; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0 +; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX7LESS-NEXT: s_cbranch_execz .LBB16_3 +; GFX7LESS-NEXT: ; %bb.1: +; GFX7LESS-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x9 +; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX7LESS-NEXT: s_mov_b32 s1, 0x43300000 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_load_dwordx2 s[2:3], s[36:37], 0x0 +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX7LESS-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1] +; GFX7LESS-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], 0 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3 +; GFX7LESS-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], -v[41:42] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s34, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s35, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b32 s12, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v40 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s36 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s37 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[40:43], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[40:43], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB16_2 +; GFX7LESS-NEXT: .LBB16_3: +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s42, -1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-NEXT: s_mov_b64 s[0:1], exec +; GFX9-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-NEXT: s_add_u32 s40, s40, s3 +; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-NEXT: s_mov_b32 s33, s2 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-NEXT: s_cbranch_execz .LBB16_3 +; GFX9-NEXT: ; %bb.1: +; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-NEXT: s_mov_b32 s1, 0x43300000 +; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1] +; GFX9-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX9-NEXT: s_add_u32 s8, s34, 44 +; GFX9-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-NEXT: s_mov_b32 s12, s33 +; GFX9-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-NEXT: s_cbranch_execnz .LBB16_2 +; GFX9-NEXT: .LBB16_3: +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s42, -1 +; GFX1064-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-NEXT: s_mov_b32 s33, s2 +; GFX1064-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-NEXT: s_cbranch_execz .LBB16_3 +; GFX1064-NEXT: ; %bb.1: +; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1064-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1064-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-NEXT: s_mov_b32 s12, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1064-NEXT: .LBB16_3: +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s33, s2 +; GFX1032-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s42, -1 +; GFX1032-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-NEXT: s_mov_b32 s38, 0 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-NEXT: s_cbranch_execz .LBB16_3 +; GFX1032-NEXT: ; %bb.1: +; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1032-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-NEXT: s_mov_b32 s12, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1032-NEXT: .LBB16_3: +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1164-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-NEXT: s_cbranch_execz .LBB16_3 +; GFX1164-NEXT: ; %bb.1: +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-NEXT: s_mov_b32 s33, s2 +; GFX1164-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b32 s12, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1164-NEXT: .LBB16_3: +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo +; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1132-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1132-NEXT: s_mov_b32 s38, 0 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-NEXT: s_cbranch_execz .LBB16_3 +; GFX1132-NEXT: ; %bb.1: +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b32 s12, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1132-NEXT: .LBB16_3: +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s42, -1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX9-DPP-NEXT: s_mov_b32 s43, 0xe00000 +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0 +; GFX9-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0 +; GFX9-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX9-DPP-NEXT: s_mov_b32 s33, s2 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc +; GFX9-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX9-DPP-NEXT: ; %bb.1: +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000 +; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000 +; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1] +; GFX9-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX9-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX9-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: s_mov_b32 s12, s33 +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX9-DPP-NEXT: .LBB16_3: +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s43, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1064-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc +; GFX1064-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1064-DPP-NEXT: ; %bb.1: +; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[2:3] +; GFX1064-DPP-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1064-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1064-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[38:39] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1064-DPP-NEXT: .LBB16_3: +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1032-DPP-NEXT: s_mov_b32 s2, exec_lo +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s40, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s41, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s42, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s43, 0x31c16000 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0 +; GFX1032-DPP-NEXT: s_add_u32 s40, s40, s3 +; GFX1032-DPP-NEXT: s_addc_u32 s41, s41, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1032-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo +; GFX1032-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1032-DPP-NEXT: ; %bb.1: +; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s2 +; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[36:37], s[34:35], 0x24 +; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[36:37], 0x0 +; GFX1032-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1032-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[40:41] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[42:43] +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[40:43], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[40:43], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[4:5] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[40:43], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[40:43], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s38 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1032-DPP-NEXT: .LBB16_3: +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1164-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1164-DPP-NEXT: ; %bb.1: +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1164-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1164-DPP-NEXT: s_mov_b32 s33, s2 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s1 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v40 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s36 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s37 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[38:39], vcc, s[38:39] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[38:39] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1164-DPP-NEXT: .LBB16_3: +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_defalut_scope_strictfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[0:1] +; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo +; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000 +; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:20 +; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off offset:16 +; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off offset:16 +; GFX1132-DPP-NEXT: s_mov_b32 s38, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo +; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2 +; GFX1132-DPP-NEXT: s_cbranch_execz .LBB16_3 +; GFX1132-DPP-NEXT: ; %bb.1: +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1] +; GFX1132-DPP-NEXT: s_load_b64 s[36:37], s[34:35], 0x24 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[36:37], 0x0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_mul_f64 v[41:42], 4.0, v[0:1] +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB16_2: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[41:42] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b32 s12, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s36 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s37 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s38, vcc_lo, s38 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB16_2 +; GFX1132-DPP-NEXT: .LBB16_3: +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 monotonic, align 4 + ret void +} + +define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp(ptr addrspace(1) %ptr) #2 { +; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX7LESS: ; %bb.0: +; GFX7LESS-NEXT: s_movk_i32 s32, 0x800 +; GFX7LESS-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX7LESS-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX7LESS-NEXT: s_mov_b32 s50, -1 +; GFX7LESS-NEXT: s_mov_b32 s51, 0xe8f000 +; GFX7LESS-NEXT: s_add_u32 s48, s48, s9 +; GFX7LESS-NEXT: s_addc_u32 s49, s49, 0 +; GFX7LESS-NEXT: s_mov_b32 s33, s8 +; GFX7LESS-NEXT: s_mov_b32 s40, s7 +; GFX7LESS-NEXT: s_mov_b32 s41, s6 +; GFX7LESS-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX7LESS-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX7LESS-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX7LESS-NEXT: s_load_dwordx2 s[44:45], s[2:3], 0x9 +; GFX7LESS-NEXT: s_mov_b32 s47, 0xf000 +; GFX7LESS-NEXT: s_mov_b32 s46, -1 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX7LESS-NEXT: v_or_b32_e32 v42, v0, v2 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_mov_b32_e32 v40, v0 +; GFX7LESS-NEXT: v_mov_b32_e32 v41, v1 +; GFX7LESS-NEXT: buffer_load_dwordx2 v[0:1], off, s[44:47], 0 +; GFX7LESS-NEXT: s_mov_b64 s[42:43], 0 +; GFX7LESS-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX7LESS-NEXT: s_waitcnt vmcnt(0) +; GFX7LESS-NEXT: v_add_f64 v[2:3], v[0:1], -v[40:41] +; GFX7LESS-NEXT: buffer_store_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: buffer_store_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: s_add_u32 s8, s36, 44 +; GFX7LESS-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:12 +; GFX7LESS-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:8 +; GFX7LESS-NEXT: s_addc_u32 s9, s37, 0 +; GFX7LESS-NEXT: s_getpc_b64 s[0:1] +; GFX7LESS-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX7LESS-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX7LESS-NEXT: s_waitcnt expcnt(2) +; GFX7LESS-NEXT: v_mov_b32_e32 v0, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v5, 8 +; GFX7LESS-NEXT: v_mov_b32_e32 v6, 0 +; GFX7LESS-NEXT: v_mov_b32_e32 v7, 0 +; GFX7LESS-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX7LESS-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX7LESS-NEXT: s_mov_b32 s12, s41 +; GFX7LESS-NEXT: s_mov_b32 s13, s40 +; GFX7LESS-NEXT: s_mov_b32 s14, s33 +; GFX7LESS-NEXT: v_mov_b32_e32 v31, v42 +; GFX7LESS-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX7LESS-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX7LESS-NEXT: s_waitcnt expcnt(0) +; GFX7LESS-NEXT: v_mov_b32_e32 v2, s44 +; GFX7LESS-NEXT: v_mov_b32_e32 v3, s45 +; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0) +; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX7LESS-NEXT: v_and_b32_e32 v2, 1, v0 +; GFX7LESS-NEXT: buffer_load_dword v0, off, s[48:51], 0 +; GFX7LESS-NEXT: buffer_load_dword v1, off, s[48:51], 0 offset:4 +; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2 +; GFX7LESS-NEXT: s_or_b64 s[42:43], vcc, s[42:43] +; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[42:43] +; GFX7LESS-NEXT: s_cbranch_execnz .LBB17_1 +; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX7LESS-NEXT: s_endpgm +; +; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-NEXT: s_mov_b32 s50, -1 +; GFX9-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-NEXT: s_add_u32 s48, s48, s9 +; GFX9-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-NEXT: s_mov_b32 s33, s8 +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-NEXT: s_mov_b32 s40, s7 +; GFX9-NEXT: s_mov_b32 s41, s6 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: s_movk_i32 s32, 0x800 +; GFX9-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX9-NEXT: s_add_u32 s8, s36, 44 +; GFX9-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-NEXT: s_getpc_b64 s[0:1] +; GFX9-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-NEXT: s_mov_b32 s12, s41 +; GFX9-NEXT: s_mov_b32 s13, s40 +; GFX9-NEXT: s_mov_b32 s14, s33 +; GFX9-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-NEXT: s_cbranch_execnz .LBB17_1 +; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-NEXT: s_endpgm +; +; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1064: ; %bb.0: +; GFX1064-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-NEXT: s_mov_b32 s50, -1 +; GFX1064-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-NEXT: s_mov_b32 s33, s8 +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-NEXT: s_mov_b32 s40, s7 +; GFX1064-NEXT: s_mov_b32 s41, s6 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-NEXT: s_waitcnt vmcnt(0) +; GFX1064-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1064-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-NEXT: s_getpc_b64 s[0:1] +; GFX1064-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-NEXT: s_mov_b32 s12, s41 +; GFX1064-NEXT: s_mov_b32 s13, s40 +; GFX1064-NEXT: s_mov_b32 s14, s33 +; GFX1064-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-NEXT: s_clause 0x1 +; GFX1064-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-NEXT: s_endpgm +; +; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1032: ; %bb.0: +; GFX1032-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-NEXT: s_mov_b32 s50, -1 +; GFX1032-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-NEXT: s_mov_b32 s33, s8 +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-NEXT: s_mov_b32 s40, s7 +; GFX1032-NEXT: s_mov_b32 s41, s6 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-NEXT: s_mov_b32 s44, 0 +; GFX1032-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-NEXT: s_waitcnt vmcnt(0) +; GFX1032-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1032-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-NEXT: s_getpc_b64 s[0:1] +; GFX1032-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-NEXT: s_mov_b32 s12, s41 +; GFX1032-NEXT: s_mov_b32 s13, s40 +; GFX1032-NEXT: s_mov_b32 s14, s33 +; GFX1032-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-NEXT: s_clause 0x1 +; GFX1032-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-NEXT: s_endpgm +; +; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1164: ; %bb.0: +; GFX1164-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-NEXT: s_mov_b32 s33, s8 +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s6 +; GFX1164-NEXT: s_mov_b32 s13, s7 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_mov_b32 s32, 32 +; GFX1164-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-NEXT: s_mov_b32 s40, s7 +; GFX1164-NEXT: s_mov_b32 s41, s6 +; GFX1164-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-NEXT: .p2align 6 +; GFX1164-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-NEXT: s_waitcnt vmcnt(0) +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1164-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-NEXT: s_getpc_b64 s[0:1] +; GFX1164-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-NEXT: s_mov_b32 s12, s41 +; GFX1164-NEXT: s_mov_b32 s13, s40 +; GFX1164-NEXT: s_mov_b32 s14, s33 +; GFX1164-NEXT: s_clause 0x1 +; GFX1164-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-NEXT: s_endpgm +; +; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1132: ; %bb.0: +; GFX1132-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-NEXT: s_mov_b32 s40, s14 +; GFX1132-NEXT: s_mov_b32 s41, s13 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s13 +; GFX1132-NEXT: s_mov_b32 s13, s14 +; GFX1132-NEXT: s_mov_b32 s14, s15 +; GFX1132-NEXT: s_mov_b32 s32, 32 +; GFX1132-NEXT: s_mov_b32 s33, s15 +; GFX1132-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-NEXT: s_mov_b32 s44, 0 +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-NEXT: .p2align 6 +; GFX1132-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-NEXT: s_waitcnt vmcnt(0) +; GFX1132-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1132-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-NEXT: s_getpc_b64 s[0:1] +; GFX1132-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-NEXT: s_mov_b32 s12, s41 +; GFX1132-NEXT: s_mov_b32 s13, s40 +; GFX1132-NEXT: s_mov_b32 s14, s33 +; GFX1132-NEXT: s_clause 0x1 +; GFX1132-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-NEXT: s_endpgm +; +; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX9-DPP: ; %bb.0: +; GFX9-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX9-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX9-DPP-NEXT: s_mov_b32 s50, -1 +; GFX9-DPP-NEXT: s_mov_b32 s51, 0xe00000 +; GFX9-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX9-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[36:37], s[2:3] +; GFX9-DPP-NEXT: s_mov_b32 s33, s8 +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_mov_b32 s40, s7 +; GFX9-DPP-NEXT: s_mov_b32 s41, s6 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX9-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX9-DPP-NEXT: s_mov_b64 s[34:35], s[4:5] +; GFX9-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX9-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX9-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX9-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX9-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX9-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX9-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX9-DPP-NEXT: s_add_u32 s8, s36, 44 +; GFX9-DPP-NEXT: s_addc_u32 s9, s37, 0 +; GFX9-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX9-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX9-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX9-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX9-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX9-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX9-DPP-NEXT: s_mov_b64 s[10:11], s[34:35] +; GFX9-DPP-NEXT: s_mov_b32 s12, s41 +; GFX9-DPP-NEXT: s_mov_b32 s13, s40 +; GFX9-DPP-NEXT: s_mov_b32 s14, s33 +; GFX9-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX9-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX9-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX9-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX9-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX9-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX9-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX9-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX9-DPP-NEXT: s_endpgm +; +; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1064-DPP: ; %bb.0: +; GFX1064-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1064-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1064-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1064-DPP-NEXT: s_mov_b32 s51, 0x31e16000 +; GFX1064-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1064-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1064-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1064-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1064-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1064-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1064-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: s_movk_i32 s32, 0x800 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1064-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1064-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1064-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1064-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1064-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1064-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1064-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1064-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1064-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1064-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1064-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1064-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1064-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1064-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1064-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1064-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1064-DPP-NEXT: s_clause 0x1 +; GFX1064-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1064-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1064-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1064-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[44:45] +; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1064-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1064-DPP-NEXT: s_endpgm +; +; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1032-DPP: ; %bb.0: +; GFX1032-DPP-NEXT: s_mov_b32 s48, SCRATCH_RSRC_DWORD0 +; GFX1032-DPP-NEXT: s_mov_b32 s49, SCRATCH_RSRC_DWORD1 +; GFX1032-DPP-NEXT: s_mov_b32 s50, -1 +; GFX1032-DPP-NEXT: s_mov_b32 s51, 0x31c16000 +; GFX1032-DPP-NEXT: s_add_u32 s48, s48, s9 +; GFX1032-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1032-DPP-NEXT: s_addc_u32 s49, s49, 0 +; GFX1032-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1032-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[42:43], s[2:3], 0x24 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v2, 20, v2 +; GFX1032-DPP-NEXT: v_lshlrev_b32_e32 v1, 10, v1 +; GFX1032-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: v_or3_b32 v42, v0, v1, v2 +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: s_movk_i32 s32, 0x400 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1032-DPP-NEXT: global_load_dwordx2 v[1:2], v43, s[42:43] +; GFX1032-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1032-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1032-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1032-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1032-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1032-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1032-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1032-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1032-DPP-NEXT: buffer_store_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: buffer_store_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1032-DPP-NEXT: s_mov_b64 s[0:1], s[48:49] +; GFX1032-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1032-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1032-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1032-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1032-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1032-DPP-NEXT: s_mov_b64 s[2:3], s[50:51] +; GFX1032-DPP-NEXT: buffer_store_dword v4, off, s[48:51], 0 offset:12 +; GFX1032-DPP-NEXT: buffer_store_dword v3, off, s[48:51], 0 offset:8 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[6:7] +; GFX1032-DPP-NEXT: s_clause 0x1 +; GFX1032-DPP-NEXT: buffer_load_dword v1, off, s[48:51], 0 +; GFX1032-DPP-NEXT: buffer_load_dword v2, off, s[48:51], 0 offset:4 +; GFX1032-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1032-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s44 +; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1032-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1032-DPP-NEXT: s_endpgm +; +; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1164-DPP: ; %bb.0: +; GFX1164-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1164-DPP-NEXT: s_mov_b32 s33, s8 +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s6 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v42, v0 +; GFX1164-DPP-NEXT: s_mov_b32 s40, s7 +; GFX1164-DPP-NEXT: s_mov_b32 s41, s6 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v43, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v41, v1 +; GFX1164-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1164-DPP-NEXT: v_mov_b32_e32 v40, v0 +; GFX1164-DPP-NEXT: s_mov_b64 s[44:45], 0 +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1164-DPP-NEXT: .p2align 6 +; GFX1164-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1164-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1164-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1164-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1164-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1164-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1164-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v31, v42 +; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1164-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1164-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1164-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1164-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1164-DPP-NEXT: s_clause 0x1 +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1164-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s42 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s43 +; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0 +; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1164-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1164-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GFX1164-DPP-NEXT: s_or_b64 s[44:45], vcc, s[44:45] +; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[44:45] +; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1164-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1164-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1164-DPP-NEXT: s_endpgm +; +; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_defalut_scope_strictfp: +; GFX1132-DPP: ; %bb.0: +; GFX1132-DPP-NEXT: s_mov_b64 s[34:35], s[2:3] +; GFX1132-DPP-NEXT: s_mov_b64 s[38:39], s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v31, v0 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: s_load_b64 s[42:43], s[2:3], 0x24 +; GFX1132-DPP-NEXT: s_mov_b64 s[36:37], s[4:5] +; GFX1132-DPP-NEXT: s_mov_b32 s40, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s41, s13 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s13 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s14 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s15 +; GFX1132-DPP-NEXT: s_mov_b32 s32, 32 +; GFX1132-DPP-NEXT: s_mov_b32 s33, s15 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v42, v0 :: v_dual_mov_b32 v43, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v41, v1 +; GFX1132-DPP-NEXT: global_load_b64 v[1:2], v43, s[42:43] +; GFX1132-DPP-NEXT: s_mov_b32 s44, 0 +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x1 +; GFX1132-DPP-NEXT: .p2align 6 +; GFX1132-DPP-NEXT: .LBB17_1: ; %atomicrmw.start +; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1 +; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0) +; GFX1132-DPP-NEXT: v_add_f64 v[3:4], v[1:2], -v[40:41] +; GFX1132-DPP-NEXT: s_add_u32 s8, s34, 44 +; GFX1132-DPP-NEXT: s_addc_u32 s9, s35, 0 +; GFX1132-DPP-NEXT: s_getpc_b64 s[0:1] +; GFX1132-DPP-NEXT: s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4 +; GFX1132-DPP-NEXT: s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v31, v42 :: v_dual_mov_b32 v0, 8 +; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0 +; GFX1132-DPP-NEXT: v_mov_b32_e32 v7, 0 +; GFX1132-DPP-NEXT: s_mov_b64 s[4:5], s[38:39] +; GFX1132-DPP-NEXT: s_mov_b64 s[10:11], s[36:37] +; GFX1132-DPP-NEXT: s_mov_b32 s12, s41 +; GFX1132-DPP-NEXT: s_mov_b32 s13, s40 +; GFX1132-DPP-NEXT: s_mov_b32 s14, s33 +; GFX1132-DPP-NEXT: s_clause 0x1 +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[1:2], off +; GFX1132-DPP-NEXT: scratch_store_b64 off, v[3:4], off offset:8 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s42 +; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, s43 :: v_dual_mov_b32 v4, 0 +; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0) +; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[0:1] +; GFX1132-DPP-NEXT: scratch_load_b64 v[1:2], off, off +; GFX1132-DPP-NEXT: v_and_b32_e32 v0, 1, v0 +; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) +; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0 +; GFX1132-DPP-NEXT: s_or_b32 s44, vcc_lo, s44 +; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s44 +; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB17_1 +; GFX1132-DPP-NEXT: ; %bb.2: ; %atomicrmw.end +; GFX1132-DPP-NEXT: s_set_inst_prefetch_distance 0x2 +; GFX1132-DPP-NEXT: s_endpgm + %divValue = call double @div.float.value() strictfp + %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue monotonic, align 4 + ret void +} + attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #1 = { strictfp "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" } attributes #2 = { strictfp} -- GitLab From 4e165dd5ab7f7c022e23b645cd8f4676b03a9ec4 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Thu, 21 Mar 2024 21:39:18 -0700 Subject: [PATCH 226/296] [TableGen] Remove unused CodeGenHwModes argument from RegSizeInfo constructor. NFC --- llvm/utils/TableGen/InfoByHwMode.cpp | 4 ++-- llvm/utils/TableGen/InfoByHwMode.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/utils/TableGen/InfoByHwMode.cpp b/llvm/utils/TableGen/InfoByHwMode.cpp index 4a64421c013c..5496408bb0d3 100644 --- a/llvm/utils/TableGen/InfoByHwMode.cpp +++ b/llvm/utils/TableGen/InfoByHwMode.cpp @@ -115,7 +115,7 @@ ValueTypeByHwMode llvm::getValueTypeByHwMode(Record *Rec, return ValueTypeByHwMode(Rec, llvm::getValueType(Rec)); } -RegSizeInfo::RegSizeInfo(Record *R, const CodeGenHwModes &CGH) { +RegSizeInfo::RegSizeInfo(Record *R) { RegSize = R->getValueAsInt("RegSize"); SpillSize = R->getValueAsInt("SpillSize"); SpillAlignment = R->getValueAsInt("SpillAlignment"); @@ -139,7 +139,7 @@ void RegSizeInfo::writeToStream(raw_ostream &OS) const { RegSizeInfoByHwMode::RegSizeInfoByHwMode(Record *R, const CodeGenHwModes &CGH) { const HwModeSelect &MS = CGH.getHwModeSelect(R); for (const HwModeSelect::PairType &P : MS.Items) { - auto I = Map.insert({P.first, RegSizeInfo(P.second, CGH)}); + auto I = Map.insert({P.first, RegSizeInfo(P.second)}); assert(I.second && "Duplicate entry?"); (void)I; } diff --git a/llvm/utils/TableGen/InfoByHwMode.h b/llvm/utils/TableGen/InfoByHwMode.h index 001509e5317f..1909913c50c6 100644 --- a/llvm/utils/TableGen/InfoByHwMode.h +++ b/llvm/utils/TableGen/InfoByHwMode.h @@ -181,7 +181,7 @@ struct RegSizeInfo { unsigned SpillSize; unsigned SpillAlignment; - RegSizeInfo(Record *R, const CodeGenHwModes &CGH); + RegSizeInfo(Record *R); RegSizeInfo() = default; bool operator<(const RegSizeInfo &I) const; bool operator==(const RegSizeInfo &I) const { -- GitLab From 90454a609894ab278a87be2b9f5c49714caba8df Mon Sep 17 00:00:00 2001 From: Chen Zheng Date: Fri, 22 Mar 2024 13:23:36 +0800 Subject: [PATCH 227/296] [PowerPC][AIX] support explicit sections for -ffunction-sections (#85351) Fix crashes in https://godbolt.org/z/6voEa1o6Y --- llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp | 6 +- .../aix-xcoff-funcsect-explicitsect.ll | 142 ++++++++++++++++++ 2 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect-explicitsect.ll diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp index 64cae1caa643..16942c6893a1 100644 --- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp +++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp @@ -2845,9 +2845,9 @@ void PPCAIXAsmPrinter::emitFunctionDescriptor() { } void PPCAIXAsmPrinter::emitFunctionEntryLabel() { - // It's not necessary to emit the label when we have individual - // function in its own csect. - if (!TM.getFunctionSections()) + // For functions without user defined section, it's not necessary to emit the + // label when we have individual function in its own csect. + if (!TM.getFunctionSections() || MF->getFunction().hasSection()) PPCAsmPrinter::emitFunctionEntryLabel(); // Emit aliasing label for function entry point label. diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect-explicitsect.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect-explicitsect.ll new file mode 100644 index 000000000000..4e94228404d6 --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-funcsect-explicitsect.ll @@ -0,0 +1,142 @@ +; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \ +; RUN: -xcoff-traceback-table=false -filetype=obj -function-sections -o %t.o < %s +; RUN: llvm-readobj -s %t.o | FileCheck %s + +define dso_local signext i32 @foo1() section "sect" { +entry: + ret i32 1 +} + +define dso_local signext i32 @foo2() section "sect2" { +entry: + ret i32 2 +} + +define dso_local signext i32 @foo3() section "sect2" { +entry: + ret i32 3 +} + +define dso_local signext i32 @foo4() { +entry: + ret i32 4 +} + +; CHECK: Symbol {{[{][[:space:]] *}}Index: [[#INDX:]]{{[[:space:]] *}}Name: sect +; CHECK-NEXT: Value (RelocatableAddress): 0x0 +; CHECK-NEXT: Section: .text +; CHECK-NEXT: Type: 0x0 +; CHECK-NEXT: StorageClass: C_HIDEXT (0x6B) +; CHECK-NEXT: NumberOfAuxEntries: 1 +; CHECK-NEXT: CSECT Auxiliary Entry { +; CHECK-NEXT: Index: [[#INDX+1]] +; CHECK-NEXT: SectionLen: 8 +; CHECK-NEXT: ParameterHashIndex: 0x0 +; CHECK-NEXT: TypeChkSectNum: 0x0 +; CHECK-NEXT: SymbolAlignmentLog2: 5 +; CHECK-NEXT: SymbolType: XTY_SD (0x1) +; CHECK-NEXT: StorageMappingClass: XMC_PR (0x0) +; CHECK-NEXT: StabInfoIndex: 0x0 +; CHECK-NEXT: StabSectNum: 0x0 +; CHECK-NEXT: } +; CHECK-NEXT: } +; CHECK-NEXT: Symbol { +; CHECK-NEXT: Index: [[#INDX+2]] +; CHECK-NEXT: Name: .foo1 +; CHECK-NEXT: Value (RelocatableAddress): 0x0 +; CHECK-NEXT: Section: .text +; CHECK-NEXT: Type: 0x0 +; CHECK-NEXT: StorageClass: C_EXT (0x2) +; CHECK-NEXT: NumberOfAuxEntries: 1 +; CHECK-NEXT: CSECT Auxiliary Entry { +; CHECK-NEXT: Index: [[#INDX+3]] +; CHECK-NEXT: ContainingCsectSymbolIndex: [[#INDX]] +; CHECK-NEXT: ParameterHashIndex: 0x0 +; CHECK-NEXT: TypeChkSectNum: 0x0 +; CHECK-NEXT: SymbolAlignmentLog2: 0 +; CHECK-NEXT: SymbolType: XTY_LD (0x2) +; CHECK-NEXT: StorageMappingClass: XMC_PR (0x0) +; CHECK-NEXT: StabInfoIndex: 0x0 +; CHECK-NEXT: StabSectNum: 0x0 +; CHECK-NEXT: } +; CHECK-NEXT: } +; CHECK-NEXT: Symbol { +; CHECK-NEXT: Index: [[#INDX+4]] +; CHECK-NEXT: Name: sect2 +; CHECK-NEXT: Value (RelocatableAddress): 0x20 +; CHECK-NEXT: Section: .text +; CHECK-NEXT: Type: 0x0 +; CHECK-NEXT: StorageClass: C_HIDEXT (0x6B) +; CHECK-NEXT: NumberOfAuxEntries: 1 +; CHECK-NEXT: CSECT Auxiliary Entry { +; CHECK-NEXT: Index: [[#INDX+5]] +; CHECK-NEXT: SectionLen: 24 +; CHECK-NEXT: ParameterHashIndex: 0x0 +; CHECK-NEXT: TypeChkSectNum: 0x0 +; CHECK-NEXT: SymbolAlignmentLog2: 5 +; CHECK-NEXT: SymbolType: XTY_SD (0x1) +; CHECK-NEXT: StorageMappingClass: XMC_PR (0x0) +; CHECK-NEXT: StabInfoIndex: 0x0 +; CHECK-NEXT: StabSectNum: 0x0 +; CHECK-NEXT: } +; CHECK-NEXT: } +; CHECK-NEXT: Symbol { +; CHECK-NEXT: Index: [[#INDX+6]] +; CHECK-NEXT: Name: .foo2 +; CHECK-NEXT: Value (RelocatableAddress): 0x20 +; CHECK-NEXT: Section: .text +; CHECK-NEXT: Type: 0x0 +; CHECK-NEXT: StorageClass: C_EXT (0x2) +; CHECK-NEXT: NumberOfAuxEntries: 1 +; CHECK-NEXT: CSECT Auxiliary Entry { +; CHECK-NEXT: Index: [[#INDX+7]] +; CHECK-NEXT: ContainingCsectSymbolIndex: [[#INDX+4]] +; CHECK-NEXT: ParameterHashIndex: 0x0 +; CHECK-NEXT: TypeChkSectNum: 0x0 +; CHECK-NEXT: SymbolAlignmentLog2: 0 +; CHECK-NEXT: SymbolType: XTY_LD (0x2) +; CHECK-NEXT: StorageMappingClass: XMC_PR (0x0) +; CHECK-NEXT: StabInfoIndex: 0x0 +; CHECK-NEXT: StabSectNum: 0x0 +; CHECK-NEXT: } +; CHECK-NEXT: } +; CHECK-NEXT: Symbol { +; CHECK-NEXT: Index: [[#INDX+8]] +; CHECK-NEXT: Name: .foo3 +; CHECK-NEXT: Value (RelocatableAddress): 0x30 +; CHECK-NEXT: Section: .text +; CHECK-NEXT: Type: 0x0 +; CHECK-NEXT: StorageClass: C_EXT (0x2) +; CHECK-NEXT: NumberOfAuxEntries: 1 +; CHECK-NEXT: CSECT Auxiliary Entry { +; CHECK-NEXT: Index: [[#INDX+9]] +; CHECK-NEXT: ContainingCsectSymbolIndex: [[#INDX+4]] +; CHECK-NEXT: ParameterHashIndex: 0x0 +; CHECK-NEXT: TypeChkSectNum: 0x0 +; CHECK-NEXT: SymbolAlignmentLog2: 0 +; CHECK-NEXT: SymbolType: XTY_LD (0x2) +; CHECK-NEXT: StorageMappingClass: XMC_PR (0x0) +; CHECK-NEXT: StabInfoIndex: 0x0 +; CHECK-NEXT: StabSectNum: 0x0 +; CHECK-NEXT: } +; CHECK-NEXT: } +; CHECK-NEXT: Symbol { +; CHECK-NEXT: Index: [[#INDX+10]] +; CHECK-NEXT: Name: .foo4 +; CHECK-NEXT: Value (RelocatableAddress): 0x40 +; CHECK-NEXT: Section: .text +; CHECK-NEXT: Type: 0x0 +; CHECK-NEXT: StorageClass: C_EXT (0x2) +; CHECK-NEXT: NumberOfAuxEntries: 1 +; CHECK-NEXT: CSECT Auxiliary Entry { +; CHECK-NEXT: Index: 16 +; CHECK-NEXT: SectionLen: 8 +; CHECK-NEXT: ParameterHashIndex: 0x0 +; CHECK-NEXT: TypeChkSectNum: 0x0 +; CHECK-NEXT: SymbolAlignmentLog2: 5 +; CHECK-NEXT: SymbolType: XTY_SD (0x1) +; CHECK-NEXT: StorageMappingClass: XMC_PR (0x0) +; CHECK-NEXT: StabInfoIndex: 0x0 +; CHECK-NEXT: StabSectNum: 0x0 +; CHECK-NEXT: } +; CHECK-NEXT: } -- GitLab From 0289ae51aa375fd297f1d03d27ff517223e5e998 Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Fri, 22 Mar 2024 08:31:17 +0100 Subject: [PATCH 228/296] [MLIR][LLVM][SROA] Support incorrectly typed memory accesses (#85813) This commit relaxes the assumption of type consistency for LLVM dialect load and store operations in SROA. Instead, there is now a check that loads and stores are in the bounds specified by the sub-slot they access. This commit additionally removes the corresponding patterns from the type consistency pass, as they are no longer necessary. Note: It will be necessary to extend Mem2Reg with the logic for differently sized accesses as well. This is non-the-less a strict upgrade for productive flows, as the type consistency pass can produce invalid IR for some odd cases. --- mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td | 6 +- .../LLVMIR/Transforms/TypeConsistency.h | 12 -- .../mlir/Interfaces/MemorySlotInterfaces.h | 3 +- mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp | 108 ++++++++++++-- .../LLVMIR/Transforms/TypeConsistency.cpp | 101 ------------- .../Dialect/MemRef/IR/MemRefMemorySlot.cpp | 7 +- mlir/test/Dialect/LLVMIR/sroa.mlir | 90 +++++++++++ .../test/Dialect/LLVMIR/type-consistency.mlir | 140 ++---------------- 8 files changed, 204 insertions(+), 263 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index b523374f6c06..f8f9264b3889 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -323,7 +323,8 @@ def LLVM_GEPOp : LLVM_Op<"getelementptr", [Pure, } def LLVM_LoadOp : LLVM_MemAccessOpBase<"load", - [DeclareOpInterfaceMethods, + [DeclareOpInterfaceMethods, + DeclareOpInterfaceMethods, DeclareOpInterfaceMethods, DeclareOpInterfaceMethods]> { dag args = (ins LLVM_AnyPointer:$addr, @@ -402,7 +403,8 @@ def LLVM_LoadOp : LLVM_MemAccessOpBase<"load", } def LLVM_StoreOp : LLVM_MemAccessOpBase<"store", - [DeclareOpInterfaceMethods, + [DeclareOpInterfaceMethods, + DeclareOpInterfaceMethods, DeclareOpInterfaceMethods, DeclareOpInterfaceMethods]> { dag args = (ins LLVM_LoadableType:$value, diff --git a/mlir/include/mlir/Dialect/LLVMIR/Transforms/TypeConsistency.h b/mlir/include/mlir/Dialect/LLVMIR/Transforms/TypeConsistency.h index b32ac56d7079..cacb241bfd7a 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/Transforms/TypeConsistency.h +++ b/mlir/include/mlir/Dialect/LLVMIR/Transforms/TypeConsistency.h @@ -29,18 +29,6 @@ namespace LLVM { /// interpret pointee types as consistently as possible. std::unique_ptr createTypeConsistencyPass(); -/// Transforms uses of pointers to a whole struct to uses of pointers to the -/// first element of a struct. This is achieved by inserting a GEP to the first -/// element when possible. -template -class AddFieldGetterToStructDirectUse : public OpRewritePattern { -public: - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(User user, - PatternRewriter &rewriter) const override; -}; - /// Canonicalizes GEPs of which the base type and the pointer's type hint do not /// match. This is done by replacing the original GEP into a GEP with the type /// hint as a base type when an element of the hinted type aligns with the diff --git a/mlir/include/mlir/Interfaces/MemorySlotInterfaces.h b/mlir/include/mlir/Interfaces/MemorySlotInterfaces.h index 56e5e96aecd1..aaa261be6553 100644 --- a/mlir/include/mlir/Interfaces/MemorySlotInterfaces.h +++ b/mlir/include/mlir/Interfaces/MemorySlotInterfaces.h @@ -26,8 +26,7 @@ struct MemorySlot { /// Memory slot attached with information about its destructuring procedure. struct DestructurableMemorySlot : public MemorySlot { - /// Maps an index within the memory slot to the type of the pointer that - /// will be generated to access the element directly. + /// Maps an index within the memory slot to the corresponding subelement type. DenseMap elementPtrs; }; diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp index 00b4559658fd..0ef1d105aca6 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp @@ -13,10 +13,8 @@ #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/Dialect/LLVMIR/LLVMTypes.h" -#include "mlir/IR/IRMapping.h" #include "mlir/IR/Matchers.h" #include "mlir/IR/PatternMatch.h" -#include "mlir/IR/ValueRange.h" #include "mlir/Interfaces/DataLayoutInterfaces.h" #include "mlir/Interfaces/MemorySlotInterfaces.h" #include "llvm/ADT/STLExtras.h" @@ -71,12 +69,8 @@ SmallVector LLVM::AllocaOp::getDestructurableSlots() { if (!destructuredType) return {}; - DenseMap allocaTypeMap; - for (Attribute index : llvm::make_first_range(destructuredType.value())) - allocaTypeMap.insert({index, LLVM::LLVMPointerType::get(getContext())}); - - return { - DestructurableMemorySlot{{getResult(), getElemType()}, {allocaTypeMap}}}; + return {DestructurableMemorySlot{{getResult(), getElemType()}, + *destructuredType}}; } DenseMap @@ -182,17 +176,107 @@ DeletionKind LLVM::StoreOp::removeBlockingUses( return DeletionKind::Delete; } +/// Checks if `slot` can be accessed through the provided access type. +static bool isValidAccessType(const MemorySlot &slot, Type accessType, + const DataLayout &dataLayout) { + return dataLayout.getTypeSize(accessType) <= + dataLayout.getTypeSize(slot.elemType); +} + LogicalResult LLVM::LoadOp::ensureOnlySafeAccesses( const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed, const DataLayout &dataLayout) { - return success(getAddr() != slot.ptr || getType() == slot.elemType); + return success(getAddr() != slot.ptr || + isValidAccessType(slot, getType(), dataLayout)); } LogicalResult LLVM::StoreOp::ensureOnlySafeAccesses( const MemorySlot &slot, SmallVectorImpl &mustBeSafelyUsed, const DataLayout &dataLayout) { return success(getAddr() != slot.ptr || - getValue().getType() == slot.elemType); + isValidAccessType(slot, getValue().getType(), dataLayout)); +} + +/// Returns the subslot's type at the requested index. +static Type getTypeAtIndex(const DestructurableMemorySlot &slot, + Attribute index) { + auto subelementIndexMap = + slot.elemType.cast().getSubelementIndexMap(); + if (!subelementIndexMap) + return {}; + assert(!subelementIndexMap->empty()); + + // Note: Returns a null-type when no entry was found. + return subelementIndexMap->lookup(index); +} + +bool LLVM::LoadOp::canRewire(const DestructurableMemorySlot &slot, + SmallPtrSetImpl &usedIndices, + SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { + if (getVolatile_()) + return false; + + // A load always accesses the first element of the destructured slot. + auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0); + Type subslotType = getTypeAtIndex(slot, index); + if (!subslotType) + return false; + + // The access can only be replaced when the subslot is read within its bounds. + if (dataLayout.getTypeSize(getType()) > dataLayout.getTypeSize(subslotType)) + return false; + + usedIndices.insert(index); + return true; +} + +DeletionKind LLVM::LoadOp::rewire(const DestructurableMemorySlot &slot, + DenseMap &subslots, + RewriterBase &rewriter, + const DataLayout &dataLayout) { + auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0); + auto it = subslots.find(index); + assert(it != subslots.end()); + + rewriter.modifyOpInPlace( + *this, [&]() { getAddrMutable().set(it->getSecond().ptr); }); + return DeletionKind::Keep; +} + +bool LLVM::StoreOp::canRewire(const DestructurableMemorySlot &slot, + SmallPtrSetImpl &usedIndices, + SmallVectorImpl &mustBeSafelyUsed, + const DataLayout &dataLayout) { + if (getVolatile_()) + return false; + + // A store always accesses the first element of the destructured slot. + auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0); + Type subslotType = getTypeAtIndex(slot, index); + if (!subslotType) + return false; + + // The access can only be replaced when the subslot is read within its bounds. + if (dataLayout.getTypeSize(getValue().getType()) > + dataLayout.getTypeSize(subslotType)) + return false; + + usedIndices.insert(index); + return true; +} + +DeletionKind LLVM::StoreOp::rewire(const DestructurableMemorySlot &slot, + DenseMap &subslots, + RewriterBase &rewriter, + const DataLayout &dataLayout) { + auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0); + auto it = subslots.find(index); + assert(it != subslots.end()); + + rewriter.modifyOpInPlace( + *this, [&]() { getAddrMutable().set(it->getSecond().ptr); }); + return DeletionKind::Keep; } //===----------------------------------------------------------------------===// @@ -390,10 +474,8 @@ bool LLVM::GEPOp::canRewire(const DestructurableMemorySlot &slot, auto firstLevelIndex = dyn_cast(getIndices()[1]); if (!firstLevelIndex) return false; - assert(slot.elementPtrs.contains(firstLevelIndex)); - if (!llvm::isa(slot.elementPtrs.at(firstLevelIndex))) - return false; mustBeSafelyUsed.emplace_back({getResult(), reachedType}); + assert(slot.elementPtrs.contains(firstLevelIndex)); usedIndices.insert(firstLevelIndex); return true; } diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp index b25c831bc717..3d700fe94e3b 100644 --- a/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp +++ b/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp @@ -49,104 +49,6 @@ static bool areBitcastCompatible(DataLayout &layout, Type lhs, Type rhs) { layout.getTypeSize(lhs) == layout.getTypeSize(rhs)); } -//===----------------------------------------------------------------------===// -// AddFieldGetterToStructDirectUse -//===----------------------------------------------------------------------===// - -/// Gets the type of the first subelement of `type` if `type` is destructurable, -/// nullptr otherwise. -static Type getFirstSubelementType(Type type) { - auto destructurable = dyn_cast(type); - if (!destructurable) - return nullptr; - - Type subelementType = destructurable.getTypeAtIndex( - IntegerAttr::get(IntegerType::get(type.getContext(), 32), 0)); - if (subelementType) - return subelementType; - - return nullptr; -} - -/// Extracts a pointer to the first field of an `elemType` from the address -/// pointer of the provided MemOp, and rewires the MemOp so it uses that pointer -/// instead. -template -static void insertFieldIndirection(MemOp op, PatternRewriter &rewriter, - Type elemType) { - PatternRewriter::InsertionGuard guard(rewriter); - - rewriter.setInsertionPointAfterValue(op.getAddr()); - SmallVector firstTypeIndices{0, 0}; - - Value properPtr = rewriter.create( - op->getLoc(), LLVM::LLVMPointerType::get(op.getContext()), elemType, - op.getAddr(), firstTypeIndices); - - rewriter.modifyOpInPlace(op, - [&]() { op.getAddrMutable().assign(properPtr); }); -} - -template <> -LogicalResult AddFieldGetterToStructDirectUse::matchAndRewrite( - LoadOp load, PatternRewriter &rewriter) const { - PatternRewriter::InsertionGuard guard(rewriter); - - Type inconsistentElementType = - isElementTypeInconsistent(load.getAddr(), load.getType()); - if (!inconsistentElementType) - return failure(); - Type firstType = getFirstSubelementType(inconsistentElementType); - if (!firstType) - return failure(); - DataLayout layout = DataLayout::closest(load); - if (!areBitcastCompatible(layout, firstType, load.getResult().getType())) - return failure(); - - insertFieldIndirection(load, rewriter, inconsistentElementType); - - // If the load does not use the first type but a type that can be casted from - // it, add a bitcast and change the load type. - if (firstType != load.getResult().getType()) { - rewriter.setInsertionPointAfterValue(load.getResult()); - BitcastOp bitcast = rewriter.create( - load->getLoc(), load.getResult().getType(), load.getResult()); - rewriter.modifyOpInPlace(load, - [&]() { load.getResult().setType(firstType); }); - rewriter.replaceAllUsesExcept(load.getResult(), bitcast.getResult(), - bitcast); - } - - return success(); -} - -template <> -LogicalResult AddFieldGetterToStructDirectUse::matchAndRewrite( - StoreOp store, PatternRewriter &rewriter) const { - PatternRewriter::InsertionGuard guard(rewriter); - - Type inconsistentElementType = - isElementTypeInconsistent(store.getAddr(), store.getValue().getType()); - if (!inconsistentElementType) - return failure(); - Type firstType = getFirstSubelementType(inconsistentElementType); - if (!firstType) - return failure(); - - DataLayout layout = DataLayout::closest(store); - // Check that the first field has the right type or can at least be bitcast - // to the right type. - if (!areBitcastCompatible(layout, firstType, store.getValue().getType())) - return failure(); - - insertFieldIndirection(store, rewriter, inconsistentElementType); - - rewriter.modifyOpInPlace( - store, [&]() { store.getValueMutable().assign(store.getValue()); }); - - return success(); -} - //===----------------------------------------------------------------------===// // CanonicalizeAlignedGep //===----------------------------------------------------------------------===// @@ -684,9 +586,6 @@ struct LLVMTypeConsistencyPass : public LLVM::impl::LLVMTypeConsistencyBase { void runOnOperation() override { RewritePatternSet rewritePatterns(&getContext()); - rewritePatterns.add>(&getContext()); - rewritePatterns.add>( - &getContext()); rewritePatterns.add(&getContext()); rewritePatterns.add(&getContext(), maxVectorSplitSize); rewritePatterns.add(&getContext()); diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp index 7be4056fb2fc..6c5250d527ad 100644 --- a/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp +++ b/mlir/lib/Dialect/MemRef/IR/MemRefMemorySlot.cpp @@ -120,11 +120,8 @@ memref::AllocaOp::getDestructurableSlots() { if (!destructuredType) return {}; - DenseMap indexMap; - for (auto const &[index, type] : *destructuredType) - indexMap.insert({index, MemRefType::get({}, type)}); - - return {DestructurableMemorySlot{{getMemref(), memrefType}, indexMap}}; + return { + DestructurableMemorySlot{{getMemref(), memrefType}, *destructuredType}}; } DenseMap diff --git a/mlir/test/Dialect/LLVMIR/sroa.mlir b/mlir/test/Dialect/LLVMIR/sroa.mlir index 02d25f27f978..ca49b1298b0e 100644 --- a/mlir/test/Dialect/LLVMIR/sroa.mlir +++ b/mlir/test/Dialect/LLVMIR/sroa.mlir @@ -215,3 +215,93 @@ llvm.func @no_nested_dynamic_indexing(%arg: i32) -> i32 { // CHECK: llvm.return %[[RES]] : i32 llvm.return %3 : i32 } + +// ----- + +// CHECK-LABEL: llvm.func @store_first_field +llvm.func @store_first_field(%arg: i32) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i32 + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr + // CHECK-NEXT: llvm.store %{{.*}}, %[[ALLOCA]] : i32 + llvm.store %arg, %1 : i32, !llvm.ptr + llvm.return +} + +// ----- + +// CHECK-LABEL: llvm.func @store_first_field_different_type +// CHECK-SAME: (%[[ARG:.*]]: f32) +llvm.func @store_first_field_different_type(%arg: f32) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i32 + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr + // CHECK-NEXT: llvm.store %[[ARG]], %[[ALLOCA]] : f32 + llvm.store %arg, %1 : f32, !llvm.ptr + llvm.return +} + +// ----- + +// CHECK-LABEL: llvm.func @store_sub_field +// CHECK-SAME: (%[[ARG:.*]]: f32) +llvm.func @store_sub_field(%arg: f32) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i64 + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i64, i32)> : (i32) -> !llvm.ptr + // CHECK-NEXT: llvm.store %[[ARG]], %[[ALLOCA]] : f32 + llvm.store %arg, %1 : f32, !llvm.ptr + llvm.return +} + +// ----- + +// CHECK-LABEL: llvm.func @load_first_field +llvm.func @load_first_field() -> i32 { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i32 + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr + // CHECK-NEXT: %[[RES:.*]] = llvm.load %[[ALLOCA]] : !llvm.ptr -> i32 + %2 = llvm.load %1 : !llvm.ptr -> i32 + // CHECK: llvm.return %[[RES]] : i32 + llvm.return %2 : i32 +} + +// ----- + +// CHECK-LABEL: llvm.func @load_first_field_different_type +llvm.func @load_first_field_different_type() -> f32 { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i32 + %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr + // CHECK-NEXT: %[[RES:.*]] = llvm.load %[[ALLOCA]] : !llvm.ptr -> f32 + %2 = llvm.load %1 : !llvm.ptr -> f32 + // CHECK: llvm.return %[[RES]] : f32 + llvm.return %2 : f32 +} + +// ----- + +// CHECK-LABEL: llvm.func @load_sub_field +llvm.func @load_sub_field() -> i32 { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x i64 : (i32) -> !llvm.ptr + %1 = llvm.alloca %0 x !llvm.struct<(i64, i32)> : (i32) -> !llvm.ptr + // CHECK-NEXT: %[[RES:.*]] = llvm.load %[[ALLOCA]] + %res = llvm.load %1 : !llvm.ptr -> i32 + // CHECK: llvm.return %[[RES]] : i32 + llvm.return %res : i32 +} + +// ----- + +// CHECK-LABEL: llvm.func @vector_store_type_mismatch +// CHECK-SAME: %[[ARG:.*]]: vector<4xi32> +llvm.func @vector_store_type_mismatch(%arg: vector<4xi32>) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x vector<4xf32> + %1 = llvm.alloca %0 x !llvm.struct<"foo", (vector<4xf32>)> : (i32) -> !llvm.ptr + // CHECK-NEXT: llvm.store %[[ARG]], %[[ALLOCA]] + llvm.store %arg, %1 : vector<4xi32>, !llvm.ptr + llvm.return +} diff --git a/mlir/test/Dialect/LLVMIR/type-consistency.mlir b/mlir/test/Dialect/LLVMIR/type-consistency.mlir index 021151b929d8..a6176142f174 100644 --- a/mlir/test/Dialect/LLVMIR/type-consistency.mlir +++ b/mlir/test/Dialect/LLVMIR/type-consistency.mlir @@ -26,63 +26,6 @@ llvm.func @same_address_keep_inbounds(%arg: i32) { // ----- -// CHECK-LABEL: llvm.func @struct_store_instead_of_first_field -llvm.func @struct_store_instead_of_first_field(%arg: i32) { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32, i32)> - // CHECK: llvm.store %{{.*}}, %[[GEP]] : i32 - llvm.store %arg, %1 : i32, !llvm.ptr - llvm.return -} - -// ----- - -// CHECK-LABEL: llvm.func @struct_store_instead_of_first_field_same_size -// CHECK-SAME: (%[[ARG:.*]]: f32) -llvm.func @struct_store_instead_of_first_field_same_size(%arg: f32) { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK-DAG: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr - // CHECK-DAG: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32, i32)> - // CHECK-DAG: %[[BITCAST:.*]] = llvm.bitcast %[[ARG]] : f32 to i32 - // CHECK: llvm.store %[[BITCAST]], %[[GEP]] : i32 - llvm.store %arg, %1 : f32, !llvm.ptr - llvm.return -} - -// ----- - -// CHECK-LABEL: llvm.func @struct_load_instead_of_first_field -llvm.func @struct_load_instead_of_first_field() -> i32 { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32, i32)> - // CHECK: %[[RES:.*]] = llvm.load %[[GEP]] : !llvm.ptr -> i32 - %2 = llvm.load %1 : !llvm.ptr -> i32 - // CHECK: llvm.return %[[RES]] : i32 - llvm.return %2 : i32 -} - -// ----- - -// CHECK-LABEL: llvm.func @struct_load_instead_of_first_field_same_size -llvm.func @struct_load_instead_of_first_field_same_size() -> f32 { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32, i32)> - // CHECK: %[[LOADED:.*]] = llvm.load %[[GEP]] : !llvm.ptr -> i32 - // CHECK: %[[RES:.*]] = llvm.bitcast %[[LOADED]] : i32 to f32 - %2 = llvm.load %1 : !llvm.ptr -> f32 - // CHECK: llvm.return %[[RES]] : f32 - llvm.return %2 : f32 -} - -// ----- - // CHECK-LABEL: llvm.func @index_in_final_padding llvm.func @index_in_final_padding(%arg: i32) { %0 = llvm.mlir.constant(1 : i32) : i32 @@ -135,22 +78,6 @@ llvm.func @index_not_in_padding_because_packed(%arg: i16) { // ----- -// CHECK-LABEL: llvm.func @index_to_struct -// CHECK-SAME: (%[[ARG:.*]]: i32) -llvm.func @index_to_struct(%arg: i32) { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, struct<"bar", (i32, i32)>)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, struct<"bar", (i32, i32)>)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP0:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, struct<"bar", (i32, i32)>)> - // CHECK: %[[GEP1:.*]] = llvm.getelementptr %[[GEP0]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"bar", (i32, i32)> - %7 = llvm.getelementptr %1[4] : (!llvm.ptr) -> !llvm.ptr, i8 - // CHECK: llvm.store %[[ARG]], %[[GEP1]] - llvm.store %arg, %7 : i32, !llvm.ptr - llvm.return -} - -// ----- - // CHECK-LABEL: llvm.func @no_crash_on_negative_gep_index llvm.func @no_crash_on_negative_gep_index() { %0 = llvm.mlir.constant(1.000000e+00 : f16) : f16 @@ -175,10 +102,9 @@ llvm.func @coalesced_store_ints(%arg: i64) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32)> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 - // CHECK: llvm.store %[[TRUNC]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[ALLOCA]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST32]] : i64 // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32)> @@ -225,11 +151,9 @@ llvm.func @coalesced_store_floats(%arg: i64) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (f32, f32)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (f32, f32)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (f32, f32)> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 - // CHECK: %[[BIT_CAST:.*]] = llvm.bitcast %[[TRUNC]] : i32 to f32 - // CHECK: llvm.store %[[BIT_CAST]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[ALLOCA]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST32]] : i64 // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (f32, f32)> @@ -298,10 +222,9 @@ llvm.func @coalesced_store_packed_struct(%arg: i64) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", packed (i16, i32, i16)> %1 = llvm.alloca %0 x !llvm.struct<"foo", packed (i16, i32, i16)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", packed (i16, i32, i16)> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i16 - // CHECK: llvm.store %[[TRUNC]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[ALLOCA]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST16]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", packed (i16, i32, i16)> @@ -328,9 +251,8 @@ llvm.func @vector_write_split(%arg: vector<4xi32>) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32, i32)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, i32, i32, i32)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32, i32, i32)> // CHECK: %[[EXTRACT:.*]] = llvm.extractelement %[[ARG]][%[[CST0]] : i32] : vector<4xi32> - // CHECK: llvm.store %[[EXTRACT]], %[[GEP]] : i32, !llvm.ptr + // CHECK: llvm.store %[[EXTRACT]], %[[ALLOCA]] : i32, !llvm.ptr // CHECK: %[[EXTRACT:.*]] = llvm.extractelement %[[ARG]][%[[CST1]] : i32] : vector<4xi32> // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, i32, i32, i32)> @@ -405,36 +327,6 @@ llvm.func @vector_write_split_struct(%arg: vector<2xi64>) { // ----- -// CHECK-LABEL: llvm.func @type_consistent_vector_store -// CHECK-SAME: %[[ARG:.*]]: vector<4xi32> -llvm.func @type_consistent_vector_store(%arg: vector<4xi32>) { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (vector<4xi32>)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (vector<4xi32>)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (vector<4xi32>)> - // CHECK: llvm.store %[[ARG]], %[[GEP]] - llvm.store %arg, %1 : vector<4xi32>, !llvm.ptr - llvm.return -} - -// ----- - -// CHECK-LABEL: llvm.func @type_consistent_vector_store_other_type -// CHECK-SAME: %[[ARG:.*]]: vector<4xi32> -llvm.func @type_consistent_vector_store_other_type(%arg: vector<4xi32>) { - %0 = llvm.mlir.constant(1 : i32) : i32 - // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (vector<4xf32>)> - %1 = llvm.alloca %0 x !llvm.struct<"foo", (vector<4xf32>)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (vector<4xf32>)> - // CHECK: %[[BIT_CAST:.*]] = llvm.bitcast %[[ARG]] : vector<4xi32> to vector<4xf32> - // CHECK: llvm.store %[[BIT_CAST]], %[[GEP]] - llvm.store %arg, %1 : vector<4xi32>, !llvm.ptr - // CHECK-NOT: llvm.store %[[ARG]], %[[ALLOCA]] - llvm.return -} - -// ----- - // CHECK-LABEL: llvm.func @bitcast_insertion // CHECK-SAME: %[[ARG:.*]]: i32 llvm.func @bitcast_insertion(%arg: i32) { @@ -478,10 +370,9 @@ llvm.func @coalesced_store_ints_subaggregate(%arg: i64) { %3 = llvm.getelementptr %1[0, 1, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i64, struct<(i32, i32)>)> // CHECK: %[[TOP_GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i64, struct<(i32, i32)>)> - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i32, i32)> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 - // CHECK: llvm.store %[[TRUNC]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[TOP_GEP]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST32]] : i64 // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i32, i32)> @@ -520,10 +411,9 @@ llvm.func @overlapping_int_aggregate_store(%arg: i64) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i16 - // CHECK: llvm.store %[[TRUNC]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[ALLOCA]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST16]] : i64 // CHECK: [[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i48 @@ -531,8 +421,7 @@ llvm.func @overlapping_int_aggregate_store(%arg: i64) { // Normal integer splitting of [[TRUNC]] follows: - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16)> - // CHECK: llvm.store %{{.*}}, %[[GEP]] + // CHECK: llvm.store %{{.*}}, %[[TOP_GEP]] // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16)> // CHECK: llvm.store %{{.*}}, %[[GEP]] // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16)> @@ -557,14 +446,12 @@ llvm.func @overlapping_vector_aggregate_store(%arg: vector<4 x i16>) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> // CHECK: %[[EXTRACT:.*]] = llvm.extractelement %[[ARG]][%[[CST0]] : i32] - // CHECK: llvm.store %[[EXTRACT]], %[[GEP]] + // CHECK: llvm.store %[[EXTRACT]], %[[ALLOCA]] // CHECK: %[[EXTRACT:.*]] = llvm.extractelement %[[ARG]][%[[CST1]] : i32] // CHECK: %[[GEP0:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> - // CHECK: %[[GEP1:.*]] = llvm.getelementptr %[[GEP0]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16)> - // CHECK: llvm.store %[[EXTRACT]], %[[GEP1]] + // CHECK: llvm.store %[[EXTRACT]], %[[GEP0]] // CHECK: %[[EXTRACT:.*]] = llvm.extractelement %[[ARG]][%[[CST2]] : i32] // CHECK: %[[GEP0:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i16, struct<(i16, i16, i16)>)> @@ -593,10 +480,9 @@ llvm.func @partially_overlapping_aggregate_store(%arg: i64) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i16, struct<(i16, i16, i16, i16)>)> %1 = llvm.alloca %0 x !llvm.struct<"foo", (i16, struct<(i16, i16, i16, i16)>)> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i16, struct<(i16, i16, i16, i16)>)> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i16 - // CHECK: llvm.store %[[TRUNC]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[ALLOCA]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST16]] : i64 // CHECK: [[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i48 @@ -604,8 +490,7 @@ llvm.func @partially_overlapping_aggregate_store(%arg: i64) { // Normal integer splitting of [[TRUNC]] follows: - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16, i16)> - // CHECK: llvm.store %{{.*}}, %[[GEP]] + // CHECK: llvm.store %{{.*}}, %[[TOP_GEP]] // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16, i16)> // CHECK: llvm.store %{{.*}}, %[[GEP]] // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[TOP_GEP]][0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i16, i16, i16, i16)> @@ -651,10 +536,9 @@ llvm.func @coalesced_store_ints_array(%arg: i64) { // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.array<2 x i32> %1 = llvm.alloca %0 x !llvm.array<2 x i32> : (i32) -> !llvm.ptr - // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.array<2 x i32> // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST0]] // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 - // CHECK: llvm.store %[[TRUNC]], %[[GEP]] + // CHECK: llvm.store %[[TRUNC]], %[[ALLOCA]] // CHECK: %[[SHR:.*]] = llvm.lshr %[[ARG]], %[[CST32]] : i64 // CHECK: %[[TRUNC:.*]] = llvm.trunc %[[SHR]] : i64 to i32 // CHECK: %[[GEP:.*]] = llvm.getelementptr %[[ALLOCA]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.array<2 x i32> -- GitLab From c5f839bd58e7f888acc4cb39a18e9e5bbaa9fb0a Mon Sep 17 00:00:00 2001 From: Guillaume Chatelet Date: Fri, 22 Mar 2024 09:50:42 +0100 Subject: [PATCH 229/296] [reland][libc] Add reverse_iterator comparisons (#86188) This is a reland of #86147 but with a proper `base()` function. https://en.cppreference.com/w/cpp/iterator/reverse_iterator/operator_cmp --- libc/src/__support/CPP/iterator.h | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/libc/src/__support/CPP/iterator.h b/libc/src/__support/CPP/iterator.h index c5bfb1912c7b..b0fd5c9f22ae 100644 --- a/libc/src/__support/CPP/iterator.h +++ b/libc/src/__support/CPP/iterator.h @@ -20,6 +20,7 @@ namespace cpp { template struct iterator_traits; template struct iterator_traits { using reference = T &; + using value_type = T; }; template class reverse_iterator { @@ -27,6 +28,8 @@ template class reverse_iterator { public: using reference = typename iterator_traits::reference; + using value_type = typename iterator_traits::value_type; + using iterator_type = Iter; LIBC_INLINE reverse_iterator() : current() {} LIBC_INLINE constexpr explicit reverse_iterator(Iter it) : current(it) {} @@ -38,6 +41,38 @@ public: LIBC_INLINE constexpr explicit reverse_iterator(const Other &it) : current(it) {} + LIBC_INLINE friend constexpr bool operator==(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() == rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator!=(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() != rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator<(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() > rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator<=(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() >= rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator>(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() < rhs.base(); + } + + LIBC_INLINE friend constexpr bool operator>=(const reverse_iterator &lhs, + const reverse_iterator &rhs) { + return lhs.base() <= rhs.base(); + } + + LIBC_INLINE constexpr iterator_type base() const { return current; } + LIBC_INLINE constexpr reference operator*() const { Iter tmp = current; return *--tmp; -- GitLab From 5f1b2cffe5fab0aa733fc8d5f1546c1c800faac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= Date: Fri, 22 Mar 2024 09:37:43 +0000 Subject: [PATCH 230/296] [mlir][vector] Add support for masks in castAwayContractionLeadingOneDim (#81906) Updates `castAwayContractionLeadingOneDim` to inherit from `MaskableOpRewritePattern` so that this pattern can support masking. Builds on top of #83827 --- .../Vector/Transforms/VectorTransforms.h | 6 +- .../mlir/Dialect/Vector/Utils/VectorUtils.h | 10 +- .../Transforms/VectorDropLeadUnitDim.cpp | 50 +++++---- .../vector-dropleadunitdim-transforms.mlir | 104 +++++++++++++----- 4 files changed, 114 insertions(+), 56 deletions(-) diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h b/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h index 08d3bb157a0e..1f7d6411cd5a 100644 --- a/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h +++ b/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h @@ -110,8 +110,10 @@ void transferOpflowOpt(RewriterBase &rewriter, Operation *rootOp); /// Cast away the leading unit dim, if exists, for the given contract op. /// Return success if the transformation applies; return failure otherwise. -LogicalResult castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, - RewriterBase &rewriter); +FailureOr +castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, + MaskingOpInterface maskingOp, + RewriterBase &rewriter); } // namespace vector } // namespace mlir diff --git a/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h b/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h index 35e76a8b623a..2c548fb67402 100644 --- a/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h +++ b/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h @@ -127,8 +127,8 @@ SmallVector getMixedSizesXfer(bool hasTensorSemantics, /// responsible for providing an updated ("rewritten") version of: /// a. the source Op when mask _is not_ present, /// b. the source Op and the masking Op when mask _is_ present. -/// Note that the return value from `matchAndRewriteMaskableOp` depends on the -/// case above. +/// To use this pattern, implement `matchAndRewriteMaskableOp`. Note that +/// the return value will depend on the case above. template struct MaskableOpRewritePattern : OpRewritePattern { using OpRewritePattern::OpRewritePattern; @@ -162,9 +162,9 @@ private: } public: - // Matches SourceOp that can potentially be masked with `maskingOp`. If the - // latter is present, returns an updated masking op (with a replacement for - // `sourceOp` nested inside). Otherwise, returns an updated `sourceOp`. + // Matches `sourceOp` that can potentially be masked with `maskingOp`. If the + // latter is present, returns a replacement for `maskingOp`. Otherwise, + // returns a replacement for `sourceOp`. virtual FailureOr matchAndRewriteMaskableOp(SourceOp sourceOp, MaskingOpInterface maskingOp, PatternRewriter &rewriter) const = 0; diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp index 74382b027c2f..593c1e53557a 100644 --- a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp +++ b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp @@ -329,12 +329,10 @@ struct CastAwayTransferWriteLeadingOneDim } // namespace -LogicalResult +FailureOr mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, + MaskingOpInterface maskingOp, RewriterBase &rewriter) { - // TODO(#78787): Not supported masked op yet. - if (cast(contractOp.getOperation()).isMasked()) - return failure(); VectorType oldAccType = dyn_cast(contractOp.getAccType()); if (oldAccType == nullptr) return failure(); @@ -368,6 +366,7 @@ mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, SmallVector operands = {contractOp.getLhs(), contractOp.getRhs(), contractOp.getAcc()}; SmallVector newOperands; + auto loc = contractOp.getLoc(); for (const auto &it : llvm::enumerate(oldIndexingMaps)) { // Check if the dim to be dropped exists as a leading dim in the operand @@ -405,7 +404,7 @@ mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, map = AffineMap::get(map.getNumDims(), 0, transposeResults, contractOp.getContext()); operands[it.index()] = rewriter.create( - contractOp.getLoc(), operands[it.index()], perm); + loc, operands[it.index()], perm); } } // We have taken care to have the dim to be dropped be @@ -429,18 +428,29 @@ mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, // Extract if its a valid extraction, otherwise use the operand // without extraction. newOperands.push_back( - validExtract ? rewriter.create(contractOp.getLoc(), - operands[it.index()], - splatZero(dropDim)) + validExtract ? rewriter.create( + loc, operands[it.index()], splatZero(dropDim)) : operands[it.index()]); } - auto newContractOp = rewriter.create( - contractOp.getLoc(), newOperands[0], newOperands[1], newOperands[2], + + // Depending on whether this vector.contract is masked, the replacing Op + // should either be a new vector.contract Op or vector.mask Op. + Operation *newOp = rewriter.create( + loc, newOperands[0], newOperands[1], newOperands[2], rewriter.getAffineMapArrayAttr(newIndexingMaps), rewriter.getArrayAttr(newIteratorTypes), contractOp.getKind()); - rewriter.replaceOpWithNewOp( - contractOp, contractOp->getResultTypes()[0], newContractOp); - return success(); + + if (maskingOp) { + auto newMask = rewriter.create(loc, maskingOp.getMask(), + splatZero(dropDim)); + + newOp = mlir::vector::maskOperation(rewriter, newOp, newMask); + } + + return rewriter + .create(loc, contractOp->getResultTypes()[0], + newOp->getResults()[0]) + .getResult(); } namespace { @@ -450,12 +460,14 @@ namespace { /// 1 dimensions. Also performs tranpose of lhs and rhs operands if required /// prior to extract. struct CastAwayContractionLeadingOneDim - : public OpRewritePattern { - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(vector::ContractionOp contractOp, - PatternRewriter &rewriter) const override { - return castAwayContractionLeadingOneDim(contractOp, rewriter); + : public MaskableOpRewritePattern { + using MaskableOpRewritePattern::MaskableOpRewritePattern; + + FailureOr + matchAndRewriteMaskableOp(vector::ContractionOp contractOp, + MaskingOpInterface maskingOp, + PatternRewriter &rewriter) const override { + return castAwayContractionLeadingOneDim(contractOp, maskingOp, rewriter); } }; diff --git a/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir b/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir index af6e636245b0..4ba51c5953d1 100644 --- a/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir +++ b/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir @@ -30,6 +30,80 @@ func.func @cast_away_contraction_leading_one_dims(%arg0: vector<1x16x8xf32>, %ar } // ----- +// CHECK: #[[$MAP_0:.+]] = affine_map<(d0, d1, d2) -> (d0, d2)> +// CHECK: #[[$MAP_1:.+]] = affine_map<(d0, d1, d2) -> (d2, d1)> +// CHECK: #[[$MAP_2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)> + +// CHECK-LABEL: func.func @cast_away_contraction_leading_one_dim_under_const_mask +// CHECK: %[[MASK:.*]] = vector.constant_mask [15, 15, 8] : vector<16x16x8xi1> +// CHECK: %[[R0:.*]] = vector.extract %{{.*}}[0] : vector<16x8xf32> from vector<1x16x8xf32> +// CHECK: %[[R1:.*]] = vector.extract %{{.*}}[0] : vector<8x16xf32> from vector<1x8x16xf32> +// CHECK: %[[R2:.*]] = vector.extract %{{.*}}[0] : vector<16x16xf32> from vector<1x16x16xf32> +// CHECK: %[[CONTRACT:.*]] = vector.mask %[[MASK]] { +// CHECK-SAME: vector.contract {indexing_maps = [#[[$MAP_0]], #[[$MAP_1]], #[[$MAP_2]]], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} +// CHECK-SAME: %[[R0]], %[[R1]], %[[R2]] : vector<16x8xf32>, vector<8x16xf32> into vector<16x16xf32> +// CHECK-SAME: } : vector<16x16x8xi1> -> vector<16x16xf32> +// CHECK: %[[RES:.*]] = vector.broadcast %[[CONTRACT]] : vector<16x16xf32> to vector<1x16x16xf32> +// CHECK: return %[[RES]] : vector<1x16x16xf32> + +#contraction_accesses0 = [ + affine_map<(l, i, j, k) -> (l, i, k)>, + affine_map<(l, i, j, k) -> (l, k, j)>, + affine_map<(l, i, j, k) -> (l, i, j)> +] +#contraction_trait0 = { + indexing_maps = #contraction_accesses0, + iterator_types = ["parallel", "parallel", "parallel", "reduction"] +} + +func.func @cast_away_contraction_leading_one_dim_under_const_mask(%arg0: vector<1x16x8xf32>, %arg1: vector<1x8x16xf32>, %arg2: vector<1x16x16xf32>) -> vector<1x16x16xf32> { + %mask = vector.constant_mask [1, 15, 15, 8] : vector<1x16x16x8xi1> + %0 = vector.mask %mask { + vector.contract #contraction_trait0 %arg0, %arg1, %arg2 : vector<1x16x8xf32>, vector<1x8x16xf32> into vector<1x16x16xf32> + } : vector<1x16x16x8xi1> -> vector<1x16x16xf32> + return %0 : vector<1x16x16xf32> +} + +// ----- +// CHECK-DAG: #[[$MAP0:.+]] = affine_map<(d0, d1, d2) -> (d0, d2)> +// CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2) -> (d2, d1)> +// CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2) -> (d0, d1)> + +// CHECK-LABEL: func.func @cast_away_contraction_leading_one_dim_under_mask +// CHECK: %[[R0:.*]] = vector.extract %{{.*}} : vector<16x8xf32> from vector<1x16x8xf32> +// CHECK: %[[R1:.*]] = vector.extract %{{.*}} : vector<8x16xf32> from vector<1x8x16xf32> +// CHECK: %[[R2:.*]] = vector.extract %{{.*}} : vector<16x16xf32> from vector<1x16x16xf32> +// CHECK: %[[M:.*]] = vector.extract %{{.*}} : vector<16x16x8xi1> from vector<1x16x16x8xi1> +// CHECK: %[[CONTRACT:.*]] = vector.mask %[[M]] { +// CHECK-SAME: vector.contract {indexing_maps = [#[[$MAP0]], #[[$MAP1]], #[[$MAP2]]], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} +// CHECK-SAME: %[[R0]], %[[R1]], %[[R2]] : vector<16x8xf32>, vector<8x16xf32> into vector<16x16xf32> +// CHECK-SAME: } : vector<16x16x8xi1> -> vector<16x16xf32> +// CHECK-NEXT: %[[RES:.*]] = vector.broadcast %[[CONTRACT]] : vector<16x16xf32> to vector<1x16x16xf32> +// CHECK-NEXT: return %[[RES]] : vector<1x16x16xf32> + +#contraction_accesses0 = [ + affine_map<(l, i, j, k) -> (l, i, k)>, + affine_map<(l, i, j, k) -> (l, k, j)>, + affine_map<(l, i, j, k) -> (l, i, j)> +] +#contraction_trait0 = { + indexing_maps = #contraction_accesses0, + iterator_types = ["parallel", "parallel", "parallel", "reduction"] +} + +func.func @cast_away_contraction_leading_one_dim_under_mask( + %arg0: vector<1x16x8xf32>, + %arg1: vector<1x8x16xf32>, + %arg2: vector<1x16x16xf32>, + %mask: vector<1x16x16x8xi1>) -> vector<1x16x16xf32> { + %0 = vector.mask %mask { + vector.contract #contraction_trait0 %arg0, %arg1, %arg2 : vector<1x16x8xf32>, vector<1x8x16xf32> into vector<1x16x16xf32> + } : vector<1x16x16x8xi1> -> vector<1x16x16xf32> + return %0: vector<1x16x16xf32> +} + +// ----- + // CHECK-DAG: #[[$map0:.*]] = affine_map<(d0, d1) -> (d1)> // CHECK-DAG: #[[$map1:.*]] = affine_map<(d0, d1) -> (d1, d0)> // CHECK-DAG: #[[$map2:.*]] = affine_map<(d0, d1) -> (d0)> @@ -164,36 +238,6 @@ func.func @cast_away_contraction_leading_one_dims_nonleadingunitdim_rank4_acctra return %0: vector<1x1x2x16xf32> } -// ----- - -// CHECK-DAG: #[[MAP0:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)> -// CHECK-DAG: #[[MAP1:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)> -// CHECK-DAG: #[[MAP2:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)> - -// CHECK-LABEL: not_insert_cast_for_contraction_under_mask -// CHECK: %[[MASK:.+]] = vector.constant_mask -// CHECK: %[[CASTED_MASK:.+]] = vector.broadcast %[[MASK]] -// CHECK: %[[RET:.+]] = vector.mask %[[CASTED_MASK]] { -// CHECK-SAME: vector.contract {{.*}} : vector<1x16x8xf32>, vector<1x8x16xf32> into vector<1x16x16xf32> } -// CHECK: return %[[RET]] : vector<1x16x16xf32> - -#contraction_accesses0 = [ - affine_map<(l, i, j, k) -> (l, i, k)>, - affine_map<(l, i, j, k) -> (l, k, j)>, - affine_map<(l, i, j, k) -> (l, i, j)> -] -#contraction_trait0 = { - indexing_maps = #contraction_accesses0, - iterator_types = ["parallel", "parallel", "parallel", "reduction"] -} - -func.func @not_insert_cast_for_contraction_under_mask(%arg0: vector<1x16x8xf32>, %arg1: vector<1x8x16xf32>, %arg2: vector<1x16x16xf32>) -> vector<1x16x16xf32> { - %mask = vector.constant_mask [1, 15, 15, 8] : vector<1x16x16x8xi1> - %0 = vector.mask %mask { - vector.contract #contraction_trait0 %arg0, %arg1, %arg2 : vector<1x16x8xf32>, vector<1x8x16xf32> into vector<1x16x16xf32> - } : vector<1x16x16x8xi1> -> vector<1x16x16xf32> - return %0 : vector<1x16x16xf32> -} // ----- // CHECK-LABEL: func @cast_away_extract_strided_slice_leading_one_dims -- GitLab From 99d8c25b3104fc07f46532bd681515c5f3c71133 Mon Sep 17 00:00:00 2001 From: David Green Date: Fri, 22 Mar 2024 09:55:18 +0000 Subject: [PATCH 231/296] [AArch64] Extra tests for v2i8 concat loads. NFC --- llvm/test/CodeGen/AArch64/insert-subvector.ll | 129 +++++++++++++++++- 1 file changed, 127 insertions(+), 2 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/insert-subvector.ll b/llvm/test/CodeGen/AArch64/insert-subvector.ll index d7656e1cd341..95ad9807ed63 100644 --- a/llvm/test/CodeGen/AArch64/insert-subvector.ll +++ b/llvm/test/CodeGen/AArch64/insert-subvector.ll @@ -374,6 +374,131 @@ define <16 x i8> @load_v16i8_8_2(float %tmp, <16 x i8> %b, ptr %a) { ret <16 x i8> %s2 } +define <8 x i8> @load_v8i8_2_1(float %tmp, <8 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v8i8_2_1: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v2.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: mov v0.16b, v2.16b +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: mov v2.b[1], v0.b[4] +; CHECK-NEXT: fmov d0, d1 +; CHECK-NEXT: mov v0.h[0], v2.h[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <8 x i32> + %s2 = shufflevector <8 x i8> %s1, <8 x i8> %b, <8 x i32> + ret <8 x i8> %s2 +} + +define <8 x i8> @load_v8i8_2_15(float %tmp, <8 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v8i8_2_15: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v0.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: adrp x8, .LCPI33_0 +; CHECK-NEXT: mov v0.b[1], v0.b[4] +; CHECK-NEXT: mov v0.d[1], v1.d[0] +; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI33_0] +; CHECK-NEXT: tbl v0.8b, { v0.16b }, v1.8b +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <8 x i32> + %s2 = shufflevector <8 x i8> %s1, <8 x i8> %b, <8 x i32> + ret <8 x i8> %s2 +} + +define <8 x i8> @load_v8i8_2_2(float %tmp, <8 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v8i8_2_2: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v2.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: mov v0.16b, v2.16b +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: mov v2.b[1], v0.b[4] +; CHECK-NEXT: fmov d0, d1 +; CHECK-NEXT: mov v0.h[1], v2.h[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <8 x i32> + %s2 = shufflevector <8 x i8> %s1, <8 x i8> %b, <8 x i32> + ret <8 x i8> %s2 +} + +define <8 x i8> @load_v8i8_2_3(float %tmp, <8 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v8i8_2_3: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v2.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: mov v0.16b, v2.16b +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: mov v2.b[1], v0.b[4] +; CHECK-NEXT: fmov d0, d1 +; CHECK-NEXT: mov v0.h[2], v2.h[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <8 x i32> + %s2 = shufflevector <8 x i8> %s1, <8 x i8> %b, <8 x i32> + ret <8 x i8> %s2 +} + +define <8 x i8> @load_v8i8_2_4(float %tmp, <8 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v8i8_2_4: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v2.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: mov v0.16b, v2.16b +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: mov v2.b[1], v0.b[4] +; CHECK-NEXT: fmov d0, d1 +; CHECK-NEXT: mov v0.h[3], v2.h[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <8 x i32> + %s2 = shufflevector <8 x i8> %s1, <8 x i8> %b, <8 x i32> + ret <8 x i8> %s2 +} + +define <4 x i8> @load_v4i8_2_1(float %tmp, <4 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v4i8_2_1: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v0.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: uzp1 v0.4h, v0.4h, v0.4h +; CHECK-NEXT: mov v0.s[1], v1.s[1] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <4 x i32> + %s2 = shufflevector <4 x i8> %s1, <4 x i8> %b, <4 x i32> + ret <4 x i8> %s2 +} + +define <4 x i8> @load_v4i8_2_2(float %tmp, <4 x i8> %b, ptr %a) { +; CHECK-LABEL: load_v4i8_2_2: +; CHECK: // %bb.0: +; CHECK-NEXT: ld1 { v0.b }[0], [x0] +; CHECK-NEXT: add x8, x0, #1 +; CHECK-NEXT: ld1 { v0.b }[4], [x8] +; CHECK-NEXT: uzp1 v2.4h, v0.4h, v0.4h +; CHECK-NEXT: fmov d0, d1 +; CHECK-NEXT: mov v0.s[1], v2.s[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: ret + %l = load <2 x i8>, ptr %a + %s1 = shufflevector <2 x i8> %l, <2 x i8> poison, <4 x i32> + %s2 = shufflevector <4 x i8> %s1, <4 x i8> %b, <4 x i32> + ret <4 x i8> %s2 +} + ; i16 define <8 x i16> @load_v8i16_2_1(float %tmp, <8 x i16> %b, ptr %a) { @@ -400,10 +525,10 @@ define <8 x i16> @load_v8i16_2_15(float %tmp, <8 x i16> %b, ptr %a) { ; CHECK-NEXT: add x9, x0, #2 ; CHECK-NEXT: // kill: def $q1 killed $q1 def $q0_q1 ; CHECK-NEXT: fmov s2, w8 -; CHECK-NEXT: adrp x8, .LCPI33_0 +; CHECK-NEXT: adrp x8, .LCPI40_0 ; CHECK-NEXT: ld1 { v2.h }[2], [x9] ; CHECK-NEXT: xtn v0.4h, v2.4s -; CHECK-NEXT: ldr q2, [x8, :lo12:.LCPI33_0] +; CHECK-NEXT: ldr q2, [x8, :lo12:.LCPI40_0] ; CHECK-NEXT: tbl v0.16b, { v0.16b, v1.16b }, v2.16b ; CHECK-NEXT: ret %l = load <2 x i16>, ptr %a -- GitLab From 465ea0bfa69aa48afef58666b084467a1c96c81b Mon Sep 17 00:00:00 2001 From: Crefeda Rodrigues <65665931+cfRod@users.noreply.github.com> Date: Fri, 22 Mar 2024 10:08:03 +0000 Subject: [PATCH 232/296] [mlir][vector] Propagate scalability in TransferWriteNonPermutationLowering (#85632) Updates `extendVectorRank` so that scalability in patterns that use it (in particular, `TransferWriteNonPermutationLowering`), is correctly propagated. Closed related previous PR https://github.com/llvm/llvm-project/pull/85270 --------- Signed-off-by: Crefeda Rodrigues Co-authored-by: Benjamin Maxwell --- .../Vector/Transforms/LowerVectorTransfer.cpp | 8 ++++++-- .../vector-transfer-permutation-lowering.mlir | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp index 4a5e8fcfb6ed..0693aa596cb2 100644 --- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp +++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTransfer.cpp @@ -41,8 +41,12 @@ static Value extendVectorRank(OpBuilder &builder, Location loc, Value vec, SmallVector newShape(addedRank, 1); newShape.append(originalVecType.getShape().begin(), originalVecType.getShape().end()); - VectorType newVecType = - VectorType::get(newShape, originalVecType.getElementType()); + + SmallVector newScalableDims(addedRank, false); + newScalableDims.append(originalVecType.getScalableDims().begin(), + originalVecType.getScalableDims().end()); + VectorType newVecType = VectorType::get( + newShape, originalVecType.getElementType(), newScalableDims); return builder.create(loc, newVecType, vec); } diff --git a/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir b/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir index 13e07f59a72a..31bd19c0be8e 100644 --- a/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir +++ b/mlir/test/Dialect/Vector/vector-transfer-permutation-lowering.mlir @@ -41,6 +41,24 @@ func.func @permutation_with_mask_scalable(%2: memref, %dim_1: index, %d return %1 : vector<8x[4]x2xf32> } +// CHECK: func.func @permutation_with_mask_transfer_write_scalable( +// CHECK-SAME: %[[ARG_0:.*]]: vector<4x[8]xi16>, +// CHECK-SAME: %[[ARG_1:.*]]: memref<1x4x?x1x1x1x1xi16>, +// CHECK-SAME: %[[MASK:.*]]: vector<4x[8]xi1>) { +// CHECK: %[[C0:.*]] = arith.constant 0 : index +// CHECK: %[[BCAST_1:.*]] = vector.broadcast %[[ARG_0]] : vector<4x[8]xi16> to vector<1x1x1x1x4x[8]xi16> +// CHECK: %[[BCAST_2:.*]] = vector.broadcast %[[MASK]] : vector<4x[8]xi1> to vector<1x1x1x1x4x[8]xi1> +// CHECK: %[[TRANSPOSE_1:.*]] = vector.transpose %[[BCAST_2]], [4, 5, 0, 1, 2, 3] : vector<1x1x1x1x4x[8]xi1> to vector<4x[8]x1x1x1x1xi1> +// CHECK: %[[TRANSPOSE_2:.*]] = vector.transpose %[[BCAST_1]], [4, 5, 0, 1, 2, 3] : vector<1x1x1x1x4x[8]xi16> to vector<4x[8]x1x1x1x1xi16> +// CHECK: vector.transfer_write %[[TRANSPOSE_2]], %[[ARG_1]]{{\[}}%[[C0]], %[[C0]], %[[C0]], %[[C0]], %[[C0]], %[[C0]], %[[C0]]], %[[TRANSPOSE_1]] {in_bounds = [true, true, true, true, true, true]} : vector<4x[8]x1x1x1x1xi16>, memref<1x4x?x1x1x1x1xi16> +// CHECK: return +func.func @permutation_with_mask_transfer_write_scalable(%arg0: vector<4x[8]xi16>, %arg1: memref<1x4x?x1x1x1x1xi16>, %mask: vector<4x[8]xi1>){ + %c0 = arith.constant 0 : index + vector.transfer_write %arg0, %arg1[%c0, %c0, %c0, %c0, %c0, %c0, %c0], %mask {in_bounds = [true, true], permutation_map = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d1, d2)> +} : vector<4x[8]xi16>, memref<1x4x?x1x1x1x1xi16> + + return +} module attributes {transform.with_named_sequence} { transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) { %f = transform.structured.match ops{["func.func"]} in %module_op -- GitLab From de7a50fb88faa1dafee33f10149561936214062b Mon Sep 17 00:00:00 2001 From: jeanPerier Date: Fri, 22 Mar 2024 11:13:04 +0100 Subject: [PATCH 233/296] [flang] Fix lowering of host associated cray pointee symbols (#86121) Cray pointee symbols can be host associated from a module or host procedure while the related cray pointer is not explicitly associated. This caused the "not yet implemented: lowering symbol to HLFIR" to fire when lowering a reference to the cray pointee and fetching the cray pointer. This patch: - Ensures cray pointers are always instantiated when instantiating a cray pointee. - Fix internal procedure lowering to deal with cray pointee host association like it does for pointers (the lowering strategy for cray pointee is to create a pointer that is updated with the cray pointer value before being fetched). This should fix the bug reported in https://github.com/llvm/llvm-project/issues/85420. --- flang/include/flang/Lower/ConvertVariable.h | 6 +- flang/include/flang/Semantics/tools.h | 3 + flang/lib/Lower/Bridge.cpp | 5 +- flang/lib/Lower/ConvertExpr.cpp | 10 +- flang/lib/Lower/ConvertExprToHLFIR.cpp | 9 +- flang/lib/Lower/ConvertVariable.cpp | 48 ++++----- flang/lib/Lower/HostAssociations.cpp | 9 +- flang/lib/Lower/PFTBuilder.cpp | 9 ++ flang/lib/Semantics/tools.cpp | 12 +++ flang/test/Lower/HLFIR/cray-pointers.f90 | 114 ++++++++++++++++++-- flang/test/Lower/cray-pointer.f90 | 4 +- 11 files changed, 176 insertions(+), 53 deletions(-) diff --git a/flang/include/flang/Lower/ConvertVariable.h b/flang/include/flang/Lower/ConvertVariable.h index ab30e317d1d9..d70d3268acac 100644 --- a/flang/include/flang/Lower/ConvertVariable.h +++ b/flang/include/flang/Lower/ConvertVariable.h @@ -161,9 +161,9 @@ void genDeclareSymbol(Fortran::lower::AbstractConverter &converter, fir::FortranVariableFlagsEnum::None, bool force = false); -/// For the given Cray pointee symbol return the corresponding -/// Cray pointer symbol. Assert if the pointer symbol cannot be found. -Fortran::semantics::SymbolRef getCrayPointer(Fortran::semantics::SymbolRef sym); +/// Given the Fortran type of a Cray pointee, return the fir.box type used to +/// track the cray pointee as Fortran pointer. +mlir::Type getCrayPointeeBoxType(mlir::Type); } // namespace lower } // namespace Fortran diff --git a/flang/include/flang/Semantics/tools.h b/flang/include/flang/Semantics/tools.h index dc3cd6c894a2..66774b51316c 100644 --- a/flang/include/flang/Semantics/tools.h +++ b/flang/include/flang/Semantics/tools.h @@ -282,6 +282,9 @@ const Symbol *FindExternallyVisibleObject( // specific procedure of the same name, return it instead. const Symbol &BypassGeneric(const Symbol &); +// Given a cray pointee symbol, returns the related cray pointer symbol. +const Symbol &GetCrayPointer(const Symbol &crayPointee); + using SomeExpr = evaluate::Expr; bool ExprHasTypeCategory( diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp index c3cb9ba6a47e..0b54ee818e3c 100644 --- a/flang/lib/Lower/Bridge.cpp +++ b/flang/lib/Lower/Bridge.cpp @@ -3995,11 +3995,12 @@ private: sym->Rank() == 0) { // get the corresponding Cray pointer - auto ptrSym = Fortran::lower::getCrayPointer(*sym); + const Fortran::semantics::Symbol &ptrSym = + Fortran::semantics::GetCrayPointer(*sym); fir::ExtendedValue ptr = getSymbolExtendedValue(ptrSym, nullptr); mlir::Value ptrVal = fir::getBase(ptr); - mlir::Type ptrTy = genType(*ptrSym); + mlir::Type ptrTy = genType(ptrSym); fir::ExtendedValue pte = getSymbolExtendedValue(*sym, nullptr); diff --git a/flang/lib/Lower/ConvertExpr.cpp b/flang/lib/Lower/ConvertExpr.cpp index d157db2cde49..fb7807718ff8 100644 --- a/flang/lib/Lower/ConvertExpr.cpp +++ b/flang/lib/Lower/ConvertExpr.cpp @@ -862,7 +862,8 @@ public: addr); } else if (sym->test(Fortran::semantics::Symbol::Flag::CrayPointee)) { // get the corresponding Cray pointer - auto ptrSym = Fortran::lower::getCrayPointer(sym); + Fortran::semantics::SymbolRef ptrSym{ + Fortran::semantics::GetCrayPointer(sym)}; ExtValue ptr = gen(ptrSym); mlir::Value ptrVal = fir::getBase(ptr); mlir::Type ptrTy = converter.genType(*ptrSym); @@ -1537,8 +1538,8 @@ public: auto baseSym = getFirstSym(aref); if (baseSym.test(Fortran::semantics::Symbol::Flag::CrayPointee)) { // get the corresponding Cray pointer - auto ptrSym = Fortran::lower::getCrayPointer(baseSym); - + Fortran::semantics::SymbolRef ptrSym{ + Fortran::semantics::GetCrayPointer(baseSym)}; fir::ExtendedValue ptr = gen(ptrSym); mlir::Value ptrVal = fir::getBase(ptr); mlir::Type ptrTy = ptrVal.getType(); @@ -6946,7 +6947,8 @@ private: ComponentPath &components) { mlir::Value ptrVal = nullptr; if (x.test(Fortran::semantics::Symbol::Flag::CrayPointee)) { - auto ptrSym = Fortran::lower::getCrayPointer(x); + Fortran::semantics::SymbolRef ptrSym{ + Fortran::semantics::GetCrayPointer(x)}; ExtValue ptr = converter.getSymbolExtendedValue(ptrSym); ptrVal = fir::getBase(ptr); } diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp index c5bfbdf6b8c1..fe5ce4b17b25 100644 --- a/flang/lib/Lower/ConvertExprToHLFIR.cpp +++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp @@ -284,7 +284,7 @@ private: // value of the Cray pointer variable. fir::FirOpBuilder &builder = getBuilder(); fir::FortranVariableOpInterface ptrVar = - gen(Fortran::lower::getCrayPointer(symbolRef)); + gen(Fortran::semantics::GetCrayPointer(symbolRef)); mlir::Value ptrAddr = ptrVar.getBase(); // Reinterpret the reference to a Cray pointer so that @@ -306,9 +306,16 @@ private: } return *varDef; } + llvm::errs() << *symbolRef << "\n"; TODO(getLoc(), "lowering symbol to HLFIR"); } + fir::FortranVariableOpInterface + gen(const Fortran::semantics::Symbol &symbol) { + Fortran::evaluate::SymbolRef symref{symbol}; + return gen(symref); + } + fir::FortranVariableOpInterface gen(const Fortran::evaluate::Component &component) { if (Fortran::semantics::IsAllocatableOrPointer(component.GetLastSymbol())) diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp index 94d849862099..e07ae42dc749 100644 --- a/flang/lib/Lower/ConvertVariable.cpp +++ b/flang/lib/Lower/ConvertVariable.cpp @@ -1554,6 +1554,11 @@ fir::FortranVariableFlagsAttr Fortran::lower::translateSymbolAttributes( mlir::MLIRContext *mlirContext, const Fortran::semantics::Symbol &sym, fir::FortranVariableFlagsEnum extraFlags) { fir::FortranVariableFlagsEnum flags = extraFlags; + if (sym.test(Fortran::semantics::Symbol::Flag::CrayPointee)) { + // CrayPointee are represented as pointers. + flags = flags | fir::FortranVariableFlagsEnum::pointer; + return fir::FortranVariableFlagsAttr::get(mlirContext, flags); + } const auto &attrs = sym.attrs(); if (attrs.test(Fortran::semantics::Attr::ALLOCATABLE)) flags = flags | fir::FortranVariableFlagsEnum::allocatable; @@ -1615,8 +1620,6 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter, (!Fortran::semantics::IsProcedure(sym) || Fortran::semantics::IsPointer(sym)) && !sym.detailsIf()) { - bool isCrayPointee = - sym.test(Fortran::semantics::Symbol::Flag::CrayPointee); fir::FirOpBuilder &builder = converter.getFirOpBuilder(); const mlir::Location loc = genLocation(converter, sym); mlir::Value shapeOrShift; @@ -1636,31 +1639,21 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter, Fortran::lower::translateSymbolCUDADataAttribute(builder.getContext(), sym); - if (isCrayPointee) { - mlir::Type baseType = - hlfir::getFortranElementOrSequenceType(base.getType()); - if (auto seqType = mlir::dyn_cast(baseType)) { - // The pointer box's sequence type must be with unknown shape. - llvm::SmallVector shape(seqType.getDimension(), - fir::SequenceType::getUnknownExtent()); - baseType = fir::SequenceType::get(shape, seqType.getEleTy()); - } - fir::BoxType ptrBoxType = - fir::BoxType::get(fir::PointerType::get(baseType)); + if (sym.test(Fortran::semantics::Symbol::Flag::CrayPointee)) { + mlir::Type ptrBoxType = + Fortran::lower::getCrayPointeeBoxType(base.getType()); mlir::Value boxAlloc = builder.createTemporary(loc, ptrBoxType); // Declare a local pointer variable. - attributes = fir::FortranVariableFlagsAttr::get( - builder.getContext(), fir::FortranVariableFlagsEnum::pointer); auto newBase = builder.create( loc, boxAlloc, name, /*shape=*/nullptr, lenParams, attributes); - mlir::Value nullAddr = - builder.createNullConstant(loc, ptrBoxType.getEleTy()); + mlir::Value nullAddr = builder.createNullConstant( + loc, llvm::cast(ptrBoxType).getEleTy()); // If the element type is known-length character, then // EmboxOp does not need the length parameters. if (auto charType = mlir::dyn_cast( - fir::unwrapSequenceType(baseType))) + hlfir::getFortranElementType(base.getType()))) if (!charType.hasDynamicLen()) lenParams.clear(); @@ -2346,16 +2339,13 @@ void Fortran::lower::createRuntimeTypeInfoGlobal( defineGlobal(converter, var, globalName, linkage); } -Fortran::semantics::SymbolRef -Fortran::lower::getCrayPointer(Fortran::semantics::SymbolRef sym) { - assert(!sym->GetUltimate().owner().crayPointers().empty() && - "empty Cray pointer/pointee map"); - for (const auto &[pointee, pointer] : - sym->GetUltimate().owner().crayPointers()) { - if (pointee == sym->name()) { - Fortran::semantics::SymbolRef v{pointer.get()}; - return v; - } +mlir::Type Fortran::lower::getCrayPointeeBoxType(mlir::Type fortranType) { + mlir::Type baseType = hlfir::getFortranElementOrSequenceType(fortranType); + if (auto seqType = mlir::dyn_cast(baseType)) { + // The pointer box's sequence type must be with unknown shape. + llvm::SmallVector shape(seqType.getDimension(), + fir::SequenceType::getUnknownExtent()); + baseType = fir::SequenceType::get(shape, seqType.getEleTy()); } - llvm_unreachable("corresponding Cray pointer cannot be found"); + return fir::BoxType::get(fir::PointerType::get(baseType)); } diff --git a/flang/lib/Lower/HostAssociations.cpp b/flang/lib/Lower/HostAssociations.cpp index 414673b00f44..8eb548eb2bd5 100644 --- a/flang/lib/Lower/HostAssociations.cpp +++ b/flang/lib/Lower/HostAssociations.cpp @@ -315,7 +315,11 @@ class CapturedAllocatableAndPointer public: static mlir::Type getType(Fortran::lower::AbstractConverter &converter, const Fortran::semantics::Symbol &sym) { - return fir::ReferenceType::get(converter.genType(sym)); + mlir::Type baseType = converter.genType(sym); + if (sym.GetUltimate().test(Fortran::semantics::Symbol::Flag::CrayPointee)) + return fir::ReferenceType::get( + Fortran::lower::getCrayPointeeBoxType(baseType)); + return fir::ReferenceType::get(baseType); } static void instantiateHostTuple(const InstantiateHostTuple &args, Fortran::lower::AbstractConverter &converter, @@ -507,7 +511,8 @@ walkCaptureCategories(T visitor, Fortran::lower::AbstractConverter &converter, if (Fortran::semantics::IsProcedure(sym)) return CapturedProcedure::visit(visitor, converter, sym, ba); ba.analyze(sym); - if (Fortran::semantics::IsAllocatableOrPointer(sym)) + if (Fortran::semantics::IsAllocatableOrPointer(sym) || + sym.GetUltimate().test(Fortran::semantics::Symbol::Flag::CrayPointee)) return CapturedAllocatableAndPointer::visit(visitor, converter, sym, ba); if (ba.isArray()) return CapturedArrays::visit(visitor, converter, sym, ba); diff --git a/flang/lib/Lower/PFTBuilder.cpp b/flang/lib/Lower/PFTBuilder.cpp index 1dacd5cf64cd..f196b9c5a0cb 100644 --- a/flang/lib/Lower/PFTBuilder.cpp +++ b/flang/lib/Lower/PFTBuilder.cpp @@ -1594,6 +1594,11 @@ private: if (!s->has()) depth = std::max(analyze(s) + 1, depth); } + + // Make sure cray pointer is instantiated even if it is not visible. + if (ultimate.test(Fortran::semantics::Symbol::Flag::CrayPointee)) + depth = std::max( + analyze(Fortran::semantics::GetCrayPointer(ultimate)) + 1, depth); adjustSize(depth + 1); bool global = lower::symbolIsGlobal(sym); layeredVarList[depth].emplace_back(sym, global, depth); @@ -2002,6 +2007,10 @@ struct SymbolVisitor { } } } + // - CrayPointer needs to be available whenever a CrayPointee is used. + if (symbol.GetUltimate().test( + Fortran::semantics::Symbol::Flag::CrayPointee)) + visitSymbol(Fortran::semantics::GetCrayPointer(symbol)); } template diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp index 0484baae93cd..2230047abd72 100644 --- a/flang/lib/Semantics/tools.cpp +++ b/flang/lib/Semantics/tools.cpp @@ -403,6 +403,18 @@ const Symbol &BypassGeneric(const Symbol &symbol) { return symbol; } +const Symbol &GetCrayPointer(const Symbol &crayPointee) { + const Symbol *found{nullptr}; + for (const auto &[pointee, pointer] : + crayPointee.GetUltimate().owner().crayPointers()) { + if (pointee == crayPointee.name()) { + found = &pointer.get(); + break; + } + } + return DEREF(found); +} + bool ExprHasTypeCategory( const SomeExpr &expr, const common::TypeCategory &type) { auto dynamicType{expr.GetType()}; diff --git a/flang/test/Lower/HLFIR/cray-pointers.f90 b/flang/test/Lower/HLFIR/cray-pointers.f90 index d1f1a5647ff1..d969aa5d747a 100644 --- a/flang/test/Lower/HLFIR/cray-pointers.f90 +++ b/flang/test/Lower/HLFIR/cray-pointers.f90 @@ -204,14 +204,14 @@ end subroutine test7 ! CHECK: %[[VAL_3:.*]] = fir.alloca !fir.array<5xi32> {bindc_name = "arr", uniq_name = "_QFtest7Earr"} ! CHECK: %[[VAL_4:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1> ! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_3]](%[[VAL_4]]) {uniq_name = "_QFtest7Earr"} : (!fir.ref>, !fir.shape<1>) -> (!fir.ref>, !fir.ref>) +! CHECK: %[[VAL_12:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest7Eptr"} +! CHECK: %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFtest7Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_6:.*]] = arith.constant 5 : index ! CHECK: %[[VAL_8:.*]] = fir.shape %[[VAL_6]] : (index) -> !fir.shape<1> ! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest7Epte"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) ! CHECK: %[[VAL_10:.*]] = fir.zero_bits !fir.ptr> ! CHECK: %[[VAL_11:.*]] = fir.embox %[[VAL_10]](%[[VAL_8]]) : (!fir.ptr>, !fir.shape<1>) -> !fir.box>> ! CHECK: fir.store %[[VAL_11]] to %[[VAL_9]]#0 : !fir.ref>>> -! CHECK: %[[VAL_12:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest7Eptr"} -! CHECK: %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFtest7Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_14:.*]] = fir.convert %[[VAL_13]]#0 : (!fir.ref) -> !fir.ref> ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_14]] : !fir.ref> ! CHECK: %[[VAL_16:.*]] = fir.convert %[[VAL_9]]#0 : (!fir.ref>>>) -> !fir.ref> @@ -226,14 +226,14 @@ subroutine test8() end subroutine test8 ! CHECK-LABEL: func.func @_QPtest8( ! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.box>> +! CHECK: %[[VAL_8:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest8Eptr"} +! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFtest8Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_2:.*]] = arith.constant 5 : index ! CHECK: %[[VAL_4:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1> ! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest8Epte"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) ! CHECK: %[[VAL_6:.*]] = fir.zero_bits !fir.ptr> ! CHECK: %[[VAL_7:.*]] = fir.embox %[[VAL_6]](%[[VAL_4]]) : (!fir.ptr>, !fir.shape<1>) -> !fir.box>> ! CHECK: fir.store %[[VAL_7]] to %[[VAL_5]]#0 : !fir.ref>>> -! CHECK: %[[VAL_8:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest8Eptr"} -! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFtest8Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_9]]#0 : (!fir.ref) -> !fir.ref> ! CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_10]] : !fir.ref> ! CHECK: %[[VAL_12:.*]] = fir.convert %[[VAL_5]]#0 : (!fir.ref>>>) -> !fir.ref> @@ -256,14 +256,14 @@ subroutine test9() end subroutine test9 ! CHECK-LABEL: func.func @_QPtest9( ! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.box>> +! CHECK: %[[VAL_8:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest9Eptr"} +! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFtest9Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_2:.*]] = arith.constant 5 : index ! CHECK: %[[VAL_4:.*]] = fir.shape %[[VAL_2]] : (index) -> !fir.shape<1> ! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest9Epte"} : (!fir.ref>>>) -> (!fir.ref>>>, !fir.ref>>>) ! CHECK: %[[VAL_6:.*]] = fir.zero_bits !fir.ptr> ! CHECK: %[[VAL_7:.*]] = fir.embox %[[VAL_6]](%[[VAL_4]]) : (!fir.ptr>, !fir.shape<1>) -> !fir.box>> ! CHECK: fir.store %[[VAL_7]] to %[[VAL_5]]#0 : !fir.ref>>> -! CHECK: %[[VAL_8:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest9Eptr"} -! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_8]] {uniq_name = "_QFtest9Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_9]]#0 : (!fir.ref) -> !fir.ref> ! CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_10]] : !fir.ref> ! CHECK: %[[VAL_12:.*]] = fir.convert %[[VAL_5]]#0 : (!fir.ref>>>) -> !fir.ref> @@ -287,12 +287,12 @@ subroutine test10() end subroutine test10 ! CHECK-LABEL: func.func @_QPtest10( ! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.box> +! CHECK: %[[VAL_6:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest10Eptr"} +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFtest10Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest10Epte"} : (!fir.ref>>) -> (!fir.ref>>, !fir.ref>>) ! CHECK: %[[VAL_4:.*]] = fir.zero_bits !fir.ptr ! CHECK: %[[VAL_5:.*]] = fir.embox %[[VAL_4]] : (!fir.ptr) -> !fir.box> ! CHECK: fir.store %[[VAL_5]] to %[[VAL_3]]#0 : !fir.ref>> -! CHECK: %[[VAL_6:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest10Eptr"} -! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFtest10Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref) -> !fir.ref> ! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_8]] : !fir.ref> ! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_3]]#0 : (!fir.ref>>) -> !fir.ref> @@ -315,12 +315,12 @@ subroutine test11() end subroutine test11 ! CHECK-LABEL: func.func @_QPtest11( ! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.box> +! CHECK: %[[VAL_6:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest11Eptr"} +! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFtest11Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_1]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest11Epte"} : (!fir.ref>>) -> (!fir.ref>>, !fir.ref>>) ! CHECK: %[[VAL_4:.*]] = fir.zero_bits !fir.ptr ! CHECK: %[[VAL_5:.*]] = fir.embox %[[VAL_4]] : (!fir.ptr) -> !fir.box> ! CHECK: fir.store %[[VAL_5]] to %[[VAL_3]]#0 : !fir.ref>> -! CHECK: %[[VAL_6:.*]] = fir.alloca i64 {bindc_name = "ptr", uniq_name = "_QFtest11Eptr"} -! CHECK: %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]] {uniq_name = "_QFtest11Eptr"} : (!fir.ref) -> (!fir.ref, !fir.ref) ! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref) -> !fir.ref> ! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_8]] : !fir.ref> ! CHECK: %[[VAL_10:.*]] = fir.convert %[[VAL_3]]#0 : (!fir.ref>>) -> !fir.ref> @@ -330,3 +330,97 @@ end subroutine test11 ! CHECK: %[[VAL_14:.*]] = fir.box_addr %[[VAL_13]] : (!fir.box>) -> !fir.ptr ! CHECK: %[[VAL_15:.*]] = fir.load %[[VAL_14]] : !fir.ptr ! CHECK: fir.call @_QPsub2(%[[VAL_15]]) fastmath : (i32) -> () + +module test_mod + integer(8) :: cray_pointer + real :: cray_pointee + pointer(cray_pointer, cray_pointee) +end module + +subroutine test_hidden_pointer + ! Only the pointee is accessed, yet the pointer is needed + ! for lowering. + use test_mod, only : cray_pointee + call takes_real(cray_pointee) +end +! CHECK-LABEL: func.func @_QPtest_hidden_pointer() { +! CHECK: %[[VAL_0:.*]] = fir.alloca !fir.box> +! CHECK: %[[VAL_1:.*]] = fir.address_of(@_QMtest_modEcray_pointer) : !fir.ref +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "_QMtest_modEcray_pointer"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QMtest_modEcray_pointee"} : (!fir.ref>>) -> (!fir.ref>>, !fir.ref>>) +! CHECK: %[[VAL_4:.*]] = fir.zero_bits !fir.ptr +! CHECK: %[[VAL_5:.*]] = fir.embox %[[VAL_4]] : (!fir.ptr) -> !fir.box> +! CHECK: fir.store %[[VAL_5]] to %[[VAL_3]]#0 : !fir.ref>> +! CHECK: %[[VAL_6:.*]] = fir.convert %[[VAL_2]]#0 : (!fir.ref) -> !fir.ref> +! CHECK: %[[VAL_7:.*]] = fir.load %[[VAL_6]] : !fir.ref> +! CHECK: %[[VAL_8:.*]] = fir.convert %[[VAL_3]]#0 : (!fir.ref>>) -> !fir.ref> +! CHECK: %[[VAL_9:.*]] = fir.convert %[[VAL_7]] : (!fir.ptr) -> !fir.llvm_ptr +! CHECK: %[[VAL_10:.*]] = fir.call @_FortranAPointerAssociateScalar(%[[VAL_8]], %[[VAL_9]]) fastmath : (!fir.ref>, !fir.llvm_ptr) -> none +! CHECK: %[[VAL_11:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref>> +! CHECK: %[[VAL_12:.*]] = fir.box_addr %[[VAL_11]] : (!fir.box>) -> !fir.ptr +! CHECK: %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (!fir.ptr) -> !fir.ref +! CHECK: fir.call @_QPtakes_real(%[[VAL_13]]) fastmath : (!fir.ref) -> () +! CHECK: return +! CHECK: } + + + +subroutine test_craypointer_capture(n) + integer :: n + character(n) :: cray_pointee + integer(8) :: cray_pointer + pointer(cray_pointer, cray_pointee) + call internal() + contains +subroutine internal() + call takes_character(cray_pointee) +end subroutine +end subroutine +! CHECK-LABEL: func.func @_QPtest_craypointer_capture( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref {fir.bindc_name = "n"}) { +! CHECK: %[[VAL_1:.*]] = fir.alloca !fir.box>> +! CHECK: %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFtest_craypointer_captureEn"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_3:.*]] = fir.alloca i64 {bindc_name = "cray_pointer", uniq_name = "_QFtest_craypointer_captureEcray_pointer"} +! CHECK: %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_3]] {uniq_name = "_QFtest_craypointer_captureEcray_pointer"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_5:.*]] = fir.load %[[VAL_2]]#0 : !fir.ref +! CHECK: %[[VAL_6:.*]] = arith.constant 0 : i32 +! CHECK: %[[VAL_7:.*]] = arith.cmpi sgt, %[[VAL_5]], %[[VAL_6]] : i32 +! CHECK: %[[VAL_8:.*]] = arith.select %[[VAL_7]], %[[VAL_5]], %[[VAL_6]] : i32 +! CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_1]] typeparams %[[VAL_8]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_craypointer_captureEcray_pointee"} : (!fir.ref>>>, i32) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_10:.*]] = fir.zero_bits !fir.ptr> +! CHECK: %[[VAL_11:.*]] = fir.embox %[[VAL_10]] typeparams %[[VAL_8]] : (!fir.ptr>, i32) -> !fir.box>> +! CHECK: fir.store %[[VAL_11]] to %[[VAL_9]]#0 : !fir.ref>>> +! CHECK: %[[VAL_12:.*]] = fir.alloca tuple>>>, !fir.ref> +! CHECK: %[[VAL_13:.*]] = arith.constant 0 : i32 +! CHECK: %[[VAL_14:.*]] = fir.coordinate_of %[[VAL_12]], %[[VAL_13]] : (!fir.ref>>>, !fir.ref>>, i32) -> !fir.llvm_ptr>>>> +! CHECK: fir.store %[[VAL_9]]#1 to %[[VAL_14]] : !fir.llvm_ptr>>>> +! CHECK: %[[VAL_15:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_16:.*]] = fir.coordinate_of %[[VAL_12]], %[[VAL_15]] : (!fir.ref>>>, !fir.ref>>, i32) -> !fir.llvm_ptr> +! CHECK: fir.store %[[VAL_4]]#1 to %[[VAL_16]] : !fir.llvm_ptr> +! CHECK: fir.call @_QFtest_craypointer_capturePinternal(%[[VAL_12]]) fastmath : (!fir.ref>>>, !fir.ref>>) -> () +! CHECK: return +! CHECK: } + +! CHECK-LABEL: func.func private @_QFtest_craypointer_capturePinternal( +! CHECK-SAME: %[[VAL_0:.*]]: !fir.ref>>>, !fir.ref>> {fir.host_assoc}) +! CHECK: %[[VAL_1:.*]] = arith.constant 0 : i32 +! CHECK: %[[VAL_2:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_1]] : (!fir.ref>>>, !fir.ref>>, i32) -> !fir.llvm_ptr>>>> +! CHECK: %[[VAL_3:.*]] = fir.load %[[VAL_2]] : !fir.llvm_ptr>>>> +! CHECK: %[[VAL_4:.*]] = fir.load %[[VAL_3]] : !fir.ref>>> +! CHECK: %[[VAL_5:.*]] = fir.box_elesize %[[VAL_4]] : (!fir.box>>) -> index +! CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_3]] typeparams %[[VAL_5]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_craypointer_captureEcray_pointee"} : (!fir.ref>>>, index) -> (!fir.ref>>>, !fir.ref>>>) +! CHECK: %[[VAL_7:.*]] = arith.constant 1 : i32 +! CHECK: %[[VAL_8:.*]] = fir.coordinate_of %[[VAL_0]], %[[VAL_7]] : (!fir.ref>>>, !fir.ref>>, i32) -> !fir.llvm_ptr> +! CHECK: %[[VAL_9:.*]] = fir.load %[[VAL_8]] : !fir.llvm_ptr> +! CHECK: %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_9]] {fortran_attrs = #fir.var_attrs, uniq_name = "_QFtest_craypointer_captureEcray_pointer"} : (!fir.ref) -> (!fir.ref, !fir.ref) +! CHECK: %[[VAL_11:.*]] = fir.convert %[[VAL_10]]#0 : (!fir.ref) -> !fir.ref> +! CHECK: %[[VAL_12:.*]] = fir.load %[[VAL_11]] : !fir.ref> +! CHECK: %[[VAL_13:.*]] = fir.convert %[[VAL_6]]#0 : (!fir.ref>>>) -> !fir.ref> +! CHECK: %[[VAL_14:.*]] = fir.convert %[[VAL_12]] : (!fir.ptr) -> !fir.llvm_ptr +! CHECK: %[[VAL_15:.*]] = fir.call @_FortranAPointerAssociateScalar(%[[VAL_13]], %[[VAL_14]]) fastmath : (!fir.ref>, !fir.llvm_ptr) -> none +! CHECK: %[[VAL_16:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref>>> +! CHECK: %[[VAL_17:.*]] = fir.box_addr %[[VAL_16]] : (!fir.box>>) -> !fir.ptr> +! CHECK: %[[VAL_18:.*]] = fir.emboxchar %[[VAL_17]], %[[VAL_5]] : (!fir.ptr>, index) -> !fir.boxchar<1> +! CHECK: fir.call @_QPtakes_character(%[[VAL_18]]) fastmath : (!fir.boxchar<1>) -> () +! CHECK: return +! CHECK: } diff --git a/flang/test/Lower/cray-pointer.f90 b/flang/test/Lower/cray-pointer.f90 index 4e9f49daab4e..06910bce35a1 100644 --- a/flang/test/Lower/cray-pointer.f90 +++ b/flang/test/Lower/cray-pointer.f90 @@ -264,8 +264,8 @@ subroutine cray_array() ! CHECK: %[[data:.*]] = fir.alloca !fir.array<5xi32> {{.*}} ! CHECK: %[[c3:.*]] = arith.constant 3 : index ! CHECK: %[[k:.*]] = fir.alloca !fir.array<3xi32> {{.*}} -! CHECK: %[[c31:.*]] = arith.constant 3 : index ! CHECK: %[[ptr:.*]] = fir.alloca i64 {{.*}} +! CHECK: %[[c31:.*]] = arith.constant 3 : index ! CHECK: %[[c2:.*]] = arith.constant 2 : i64 ! CHECK: %[[c1:.*]] = arith.constant 1 : i64 ! CHECK: %[[sub:.*]] = arith.subi %[[c2]], %[[c1]] : i64 @@ -327,8 +327,8 @@ subroutine cray_arraySection() ! CHECK: %[[data:.*]] = fir.alloca !fir.array<5xi32> {{.*}} ! CHECK: %[[c2:.*]] = arith.constant 2 : index ! CHECK: %[[k:.*]] = fir.alloca !fir.array<2xi32> {{.*}} -! CHECK: %[[c3:.*]] = arith.constant 3 : index ! CHECK: %[[ptr:.*]] = fir.alloca i64 {{.*}} +! CHECK: %[[c3:.*]] = arith.constant 3 : index ! CHECK: %[[c1:.*]] = arith.constant 2 : i64 ! CHECK: %[[c0:.*]] = arith.constant 1 : i64 ! CHECK: %[[sub:.*]] = arith.subi %[[c1]], %[[c0]] : i64 -- GitLab From 66f88de80599ec4461b0fdac3d1e396b6e83052d Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Fri, 22 Mar 2024 18:24:23 +0800 Subject: [PATCH 234/296] [RISCV] Support RISC-V Profiles in -march option (#76357) This PR implements the draft https://github.com/riscv-non-isa/riscv-toolchain-conventions/pull/36. Currently, we replace specified profile in `-march` with standard arch string. --- clang/docs/ReleaseNotes.rst | 1 + clang/test/Driver/riscv-profiles.c | 312 +++++++++++++++++++++++++++++ llvm/lib/Support/RISCVISAInfo.cpp | 64 ++++++ 3 files changed, 377 insertions(+) create mode 100644 clang/test/Driver/riscv-profiles.c diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index fd12bb41be47..005cdebc0d8a 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -520,6 +520,7 @@ RISC-V Support ^^^^^^^^^^^^^^ - ``__attribute__((rvv_vector_bits(N)))`` is now supported for RVV vbool*_t types. +- Profile names in ``-march`` option are now supported. CUDA/HIP Language Changes ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/test/Driver/riscv-profiles.c b/clang/test/Driver/riscv-profiles.c new file mode 100644 index 000000000000..904f0c371f44 --- /dev/null +++ b/clang/test/Driver/riscv-profiles.c @@ -0,0 +1,312 @@ +// RUN: %clang -### -c %s 2>&1 -march=rvi20u32 | FileCheck -check-prefix=RVI20U32 %s +// RVI20U32: "-target-feature" "-a" +// RVI20U32: "-target-feature" "-c" +// RVI20U32: "-target-feature" "-d" +// RVI20U32: "-target-feature" "-f" +// RVI20U32: "-target-feature" "-m" + +// RUN: %clang -### -c %s 2>&1 -march=rvi20u64 | FileCheck -check-prefix=RVI20U64 %s +// RVI20U64: "-target-feature" "-a" +// RVI20U64: "-target-feature" "-c" +// RVI20U64: "-target-feature" "-d" +// RVI20U64: "-target-feature" "-f" +// RVI20U64: "-target-feature" "-m" + +// RUN: %clang -### -c %s 2>&1 -march=rva20u64 | FileCheck -check-prefix=RVA20U64 %s +// RVA20U64: "-target-feature" "+m" +// RVA20U64: "-target-feature" "+a" +// RVA20U64: "-target-feature" "+f" +// RVA20U64: "-target-feature" "+d" +// RVA20U64: "-target-feature" "+c" +// RVA20U64: "-target-feature" "+ziccamoa" +// RVA20U64: "-target-feature" "+ziccif" +// RVA20U64: "-target-feature" "+zicclsm" +// RVA20U64: "-target-feature" "+ziccrse" +// RVA20U64: "-target-feature" "+zicntr" +// RVA20U64: "-target-feature" "+zicsr" +// RVA20U64: "-target-feature" "+za128rs" + +// RUN: %clang -### -c %s 2>&1 -march=rva20s64 | FileCheck -check-prefix=RVA20S64 %s +// RVA20S64: "-target-feature" "+m" +// RVA20S64: "-target-feature" "+a" +// RVA20S64: "-target-feature" "+f" +// RVA20S64: "-target-feature" "+d" +// RVA20S64: "-target-feature" "+c" +// RVA20S64: "-target-feature" "+ziccamoa" +// RVA20S64: "-target-feature" "+ziccif" +// RVA20S64: "-target-feature" "+zicclsm" +// RVA20S64: "-target-feature" "+ziccrse" +// RVA20S64: "-target-feature" "+zicntr" +// RVA20S64: "-target-feature" "+zicsr" +// RVA20S64: "-target-feature" "+zifencei" +// RVA20S64: "-target-feature" "+za128rs" +// RVA20S64: "-target-feature" "+ssccptr" +// RVA20S64: "-target-feature" "+sstvala" +// RVA20S64: "-target-feature" "+sstvecd" +// RVA20S64: "-target-feature" "+svade" +// RVA20S64: "-target-feature" "+svbare" + +// RUN: %clang -### -c %s 2>&1 -march=rva22u64 | FileCheck -check-prefix=RVA22U64 %s +// RVA22U64: "-target-feature" "+m" +// RVA22U64: "-target-feature" "+a" +// RVA22U64: "-target-feature" "+f" +// RVA22U64: "-target-feature" "+d" +// RVA22U64: "-target-feature" "+c" +// RVA22U64: "-target-feature" "+zic64b" +// RVA22U64: "-target-feature" "+zicbom" +// RVA22U64: "-target-feature" "+zicbop" +// RVA22U64: "-target-feature" "+zicboz" +// RVA22U64: "-target-feature" "+ziccamoa" +// RVA22U64: "-target-feature" "+ziccif" +// RVA22U64: "-target-feature" "+zicclsm" +// RVA22U64: "-target-feature" "+ziccrse" +// RVA22U64: "-target-feature" "+zicntr" +// RVA22U64: "-target-feature" "+zicsr" +// RVA22U64: "-target-feature" "+zihintpause" +// RVA22U64: "-target-feature" "+zihpm" +// RVA22U64: "-target-feature" "+za64rs" +// RVA22U64: "-target-feature" "+zfhmin" +// RVA22U64: "-target-feature" "+zba" +// RVA22U64: "-target-feature" "+zbb" +// RVA22U64: "-target-feature" "+zbs" +// RVA22U64: "-target-feature" "+zkt" + +// RUN: %clang -### -c %s 2>&1 -march=rva22s64 | FileCheck -check-prefix=RVA22S64 %s +// RVA22S64: "-target-feature" "+m" +// RVA22S64: "-target-feature" "+a" +// RVA22S64: "-target-feature" "+f" +// RVA22S64: "-target-feature" "+d" +// RVA22S64: "-target-feature" "+c" +// RVA22S64: "-target-feature" "+zic64b" +// RVA22S64: "-target-feature" "+zicbom" +// RVA22S64: "-target-feature" "+zicbop" +// RVA22S64: "-target-feature" "+zicboz" +// RVA22S64: "-target-feature" "+ziccamoa" +// RVA22S64: "-target-feature" "+ziccif" +// RVA22S64: "-target-feature" "+zicclsm" +// RVA22S64: "-target-feature" "+ziccrse" +// RVA22S64: "-target-feature" "+zicntr" +// RVA22S64: "-target-feature" "+zicsr" +// RVA22S64: "-target-feature" "+zifencei" +// RVA22S64: "-target-feature" "+zihintpause" +// RVA22S64: "-target-feature" "+zihpm" +// RVA22S64: "-target-feature" "+za64rs" +// RVA22S64: "-target-feature" "+zfhmin" +// RVA22S64: "-target-feature" "+zba" +// RVA22S64: "-target-feature" "+zbb" +// RVA22S64: "-target-feature" "+zbs" +// RVA22S64: "-target-feature" "+zkt" +// RVA22S64: "-target-feature" "+ssccptr" +// RVA22S64: "-target-feature" "+sscounterenw" +// RVA22S64: "-target-feature" "+sstvala" +// RVA22S64: "-target-feature" "+sstvecd" +// RVA22S64: "-target-feature" "+svade" +// RVA22S64: "-target-feature" "+svbare" +// RVA22S64: "-target-feature" "+svinval" +// RVA22S64: "-target-feature" "+svpbmt" + +// RUN: %clang -### -c %s 2>&1 -march=rva23u64 -menable-experimental-extensions | FileCheck -check-prefix=RVA23U64 %s +// RVA23U64: "-target-feature" "+m" +// RVA23U64: "-target-feature" "+a" +// RVA23U64: "-target-feature" "+f" +// RVA23U64: "-target-feature" "+d" +// RVA23U64: "-target-feature" "+c" +// RVA23U64: "-target-feature" "+v" +// RVA23U64: "-target-feature" "+zic64b" +// RVA23U64: "-target-feature" "+zicbom" +// RVA23U64: "-target-feature" "+zicbop" +// RVA23U64: "-target-feature" "+zicboz" +// RVA23U64: "-target-feature" "+ziccamoa" +// RVA23U64: "-target-feature" "+ziccif" +// RVA23U64: "-target-feature" "+zicclsm" +// RVA23U64: "-target-feature" "+ziccrse" +// RVA23U64: "-target-feature" "+zicntr" +// RVA23U64: "-target-feature" "+zicond" +// RVA23U64: "-target-feature" "+zicsr" +// RVA23U64: "-target-feature" "+zihintntl" +// RVA23U64: "-target-feature" "+zihintpause" +// RVA23U64: "-target-feature" "+zihpm" +// RVA23U64: "-target-feature" "+experimental-zimop" +// RVA23U64: "-target-feature" "+za64rs" +// RVA23U64: "-target-feature" "+zawrs" +// RVA23U64: "-target-feature" "+zfa" +// RVA23U64: "-target-feature" "+zfhmin" +// RVA23U64: "-target-feature" "+zcb" +// RVA23U64: "-target-feature" "+experimental-zcmop" +// RVA23U64: "-target-feature" "+zba" +// RVA23U64: "-target-feature" "+zbb" +// RVA23U64: "-target-feature" "+zbs" +// RVA23U64: "-target-feature" "+zkt" +// RVA23U64: "-target-feature" "+zvbb" +// RVA23U64: "-target-feature" "+zvfhmin" +// RVA23U64: "-target-feature" "+zvkt" + +// RUN: %clang -### -c %s 2>&1 -march=rva23s64 -menable-experimental-extensions | FileCheck -check-prefix=RVA23S64 %s +// RVA23S64: "-target-feature" "+m" +// RVA23S64: "-target-feature" "+a" +// RVA23S64: "-target-feature" "+f" +// RVA23S64: "-target-feature" "+d" +// RVA23S64: "-target-feature" "+c" +// RVA23S64: "-target-feature" "+v" +// RVA23S64: "-target-feature" "+h" +// RVA23S64: "-target-feature" "+zic64b" +// RVA23S64: "-target-feature" "+zicbom" +// RVA23S64: "-target-feature" "+zicbop" +// RVA23S64: "-target-feature" "+zicboz" +// RVA23S64: "-target-feature" "+ziccamoa" +// RVA23S64: "-target-feature" "+ziccif" +// RVA23S64: "-target-feature" "+zicclsm" +// RVA23S64: "-target-feature" "+ziccrse" +// RVA23S64: "-target-feature" "+zicntr" +// RVA23S64: "-target-feature" "+zicond" +// RVA23S64: "-target-feature" "+zicsr" +// RVA23S64: "-target-feature" "+zifencei" +// RVA23S64: "-target-feature" "+zihintntl" +// RVA23S64: "-target-feature" "+zihintpause" +// RVA23S64: "-target-feature" "+zihpm" +// RVA23S64: "-target-feature" "+experimental-zimop" +// RVA23S64: "-target-feature" "+za64rs" +// RVA23S64: "-target-feature" "+zawrs" +// RVA23S64: "-target-feature" "+zfa" +// RVA23S64: "-target-feature" "+zfhmin" +// RVA23S64: "-target-feature" "+zcb" +// RVA23S64: "-target-feature" "+experimental-zcmop" +// RVA23S64: "-target-feature" "+zba" +// RVA23S64: "-target-feature" "+zbb" +// RVA23S64: "-target-feature" "+zbs" +// RVA23S64: "-target-feature" "+zkt" +// RVA23S64: "-target-feature" "+zvbb" +// RVA23S64: "-target-feature" "+zvfhmin" +// RVA23S64: "-target-feature" "+zvkt" +// RVA23S64: "-target-feature" "+shcounterenw" +// RVA23S64: "-target-feature" "+shgatpa" +// RVA23S64: "-target-feature" "+shtvala" +// RVA23S64: "-target-feature" "+shvsatpa" +// RVA23S64: "-target-feature" "+shvstvala" +// RVA23S64: "-target-feature" "+shvstvecd" +// RVA23S64: "-target-feature" "+ssccptr" +// RVA23S64: "-target-feature" "+sscofpmf" +// RVA23S64: "-target-feature" "+sscounterenw" +// RVA23S64: "-target-feature" "+experimental-ssnpm" +// RVA23S64: "-target-feature" "+ssstateen" +// RVA23S64: "-target-feature" "+sstc" +// RVA23S64: "-target-feature" "+sstvala" +// RVA23S64: "-target-feature" "+sstvecd" +// RVA23S64: "-target-feature" "+ssu64xl" +// RVA23S64: "-target-feature" "+svade" +// RVA23S64: "-target-feature" "+svbare" +// RVA23S64: "-target-feature" "+svinval" +// RVA23S64: "-target-feature" "+svnapot" +// RVA23S64: "-target-feature" "+svpbmt" + +// RUN: %clang -### -c %s 2>&1 -march=rvb23u64 -menable-experimental-extensions | FileCheck -check-prefix=RVB23U64 %s +// RVB23U64: "-target-feature" "+m" +// RVB23U64: "-target-feature" "+a" +// RVB23U64: "-target-feature" "+f" +// RVB23U64: "-target-feature" "+d" +// RVB23U64: "-target-feature" "+c" +// RVB23U64: "-target-feature" "+zic64b" +// RVB23U64: "-target-feature" "+zicbom" +// RVB23U64: "-target-feature" "+zicbop" +// RVB23U64: "-target-feature" "+zicboz" +// RVB23U64: "-target-feature" "+ziccamoa" +// RVB23U64: "-target-feature" "+ziccif" +// RVB23U64: "-target-feature" "+zicclsm" +// RVB23U64: "-target-feature" "+ziccrse" +// RVB23U64: "-target-feature" "+zicntr" +// RVB23U64: "-target-feature" "+zicond" +// RVB23U64: "-target-feature" "+zicsr" +// RVB23U64: "-target-feature" "+zihintntl" +// RVB23U64: "-target-feature" "+zihintpause" +// RVB23U64: "-target-feature" "+zihpm" +// RVB23U64: "-target-feature" "+experimental-zimop" +// RVB23U64: "-target-feature" "+za64rs" +// RVB23U64: "-target-feature" "+zawrs" +// RVB23U64: "-target-feature" "+zfa" +// RVB23U64: "-target-feature" "+zcb" +// RVB23U64: "-target-feature" "+experimental-zcmop" +// RVB23U64: "-target-feature" "+zba" +// RVB23U64: "-target-feature" "+zbb" +// RVB23U64: "-target-feature" "+zbs" +// RVB23U64: "-target-feature" "+zkt" + +// RUN: %clang -### -c %s 2>&1 -march=rvb23s64 -menable-experimental-extensions | FileCheck -check-prefix=RVB23S64 %s +// RVB23S64: "-target-feature" "+m" +// RVB23S64: "-target-feature" "+a" +// RVB23S64: "-target-feature" "+f" +// RVB23S64: "-target-feature" "+d" +// RVB23S64: "-target-feature" "+c" +// RVB23S64: "-target-feature" "+zic64b" +// RVB23S64: "-target-feature" "+zicbom" +// RVB23S64: "-target-feature" "+zicbop" +// RVB23S64: "-target-feature" "+zicboz" +// RVB23S64: "-target-feature" "+ziccamoa" +// RVB23S64: "-target-feature" "+ziccif" +// RVB23S64: "-target-feature" "+zicclsm" +// RVB23S64: "-target-feature" "+ziccrse" +// RVB23S64: "-target-feature" "+zicntr" +// RVB23S64: "-target-feature" "+zicond" +// RVB23S64: "-target-feature" "+zicsr" +// RVB23S64: "-target-feature" "+zifencei" +// RVB23S64: "-target-feature" "+zihintntl" +// RVB23S64: "-target-feature" "+zihintpause" +// RVB23S64: "-target-feature" "+zihpm" +// RVB23S64: "-target-feature" "+experimental-zimop" +// RVB23S64: "-target-feature" "+za64rs" +// RVB23S64: "-target-feature" "+zawrs" +// RVB23S64: "-target-feature" "+zfa" +// RVB23S64: "-target-feature" "+zcb" +// RVB23S64: "-target-feature" "+experimental-zcmop" +// RVB23S64: "-target-feature" "+zba" +// RVB23S64: "-target-feature" "+zbb" +// RVB23S64: "-target-feature" "+zbs" +// RVB23S64: "-target-feature" "+zkt" +// RVB23S64: "-target-feature" "+ssccptr" +// RVB23S64: "-target-feature" "+sscofpmf" +// RVB23S64: "-target-feature" "+sscounterenw" +// RVB23S64: "-target-feature" "+sstc" +// RVB23S64: "-target-feature" "+sstvala" +// RVB23S64: "-target-feature" "+sstvecd" +// RVB23S64: "-target-feature" "+ssu64xl" +// RVB23S64: "-target-feature" "+svade" +// RVB23S64: "-target-feature" "+svbare" +// RVB23S64: "-target-feature" "+svinval" +// RVB23S64: "-target-feature" "+svnapot" +// RVB23S64: "-target-feature" "+svpbmt" + +// RUN: %clang -### -c %s 2>&1 -march=rvm23u32 -menable-experimental-extensions | FileCheck -check-prefix=RVM23U32 %s +// RVM23U32: "-target-feature" "+m" +// RVM23U32: "-target-feature" "+zicbop" +// RVM23U32: "-target-feature" "+zicond" +// RVM23U32: "-target-feature" "+zicsr" +// RVM23U32: "-target-feature" "+zihintntl" +// RVM23U32: "-target-feature" "+zihintpause" +// RVM23U32: "-target-feature" "+experimental-zimop" +// RVM23U32: "-target-feature" "+zce" +// RVM23U32: "-target-feature" "+experimental-zcmop" +// RVM23U32: "-target-feature" "+zba" +// RVM23U32: "-target-feature" "+zbb" +// RVM23U32: "-target-feature" "+zbs" + +// RUN: %clang -### -c %s 2>&1 -march=rva22u64_zfa | FileCheck -check-prefix=PROFILE-WITH-ADDITIONAL %s +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+m" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+a" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+f" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+d" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+c" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicbom" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicbop" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicboz" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zihintpause" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zfa" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zfhmin" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zba" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zbb" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zbs" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zkt" + +// RUN: not %clang -### -c %s 2>&1 -march=rva19u64_zfa | FileCheck -check-prefix=INVALID-PROFILE %s +// INVALID-PROFILE: error: invalid arch name 'rva19u64_zfa', unsupported profile + +// RUN: not %clang -### -c %s 2>&1 -march=rva22u64zfa | FileCheck -check-prefix=INVALID-ADDITIONAL %s +// INVALID-ADDITIONAL: error: invalid arch name 'rva22u64zfa', additional extensions must be after separator '_' diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp index 39235ace4724..67e6e5b962b1 100644 --- a/llvm/lib/Support/RISCVISAInfo.cpp +++ b/llvm/lib/Support/RISCVISAInfo.cpp @@ -36,6 +36,11 @@ struct RISCVSupportedExtension { } }; +struct RISCVProfile { + StringLiteral Name; + StringLiteral MArch; +}; + } // end anonymous namespace static constexpr StringLiteral AllStdExts = "mafdqlcbkjtpvnh"; @@ -244,6 +249,42 @@ static const RISCVSupportedExtension SupportedExperimentalExtensions[] = { }; // clang-format on +static constexpr RISCVProfile SupportedProfiles[] = { + {"rvi20u32", "rv32i"}, + {"rvi20u64", "rv64i"}, + {"rva20u64", "rv64imafdc_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_za128rs"}, + {"rva20s64", "rv64imafdc_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_zifencei_" + "za128rs_ssccptr_sstvala_sstvecd_svade_svbare"}, + {"rva22u64", + "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zihintpause_zihpm_za64rs_zfhmin_zba_zbb_zbs_zkt"}, + {"rva22s64", + "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zifencei_zihintpause_zihpm_za64rs_zfhmin_zba_zbb_zbs_zkt_ssccptr_" + "sscounterenw_sstvala_sstvecd_svade_svbare_svinval_svpbmt"}, + {"rva23u64", + "rv64imafdcv_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zicond_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_zfa_" + "zfhmin_zcb_zcmop0p2_zba_zbb_zbs_zkt_zvbb_zvfhmin_zvkt"}, + {"rva23s64", + "rv64imafdcvh_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zicond_zifencei_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_" + "zfa_zfhmin_zcb_zcmop0p2_zba_zbb_zbs_zkt_zvbb_zvfhmin_zvkt_shcounterenw_" + "shgatpa_shtvala_shvsatpa_shvstvala_shvstvecd_ssccptr_sscofpmf_" + "sscounterenw_ssnpm0p8_ssstateen_sstc_sstvala_sstvecd_ssu64xl_svade_" + "svbare_svinval_svnapot_svpbmt"}, + {"rvb23u64", "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_" + "zicclsm_ziccrse_zicntr_zicond_zihintntl_zihintpause_zihpm_" + "zimop0p1_za64rs_zawrs_zfa_zcb_zcmop0p2_zba_zbb_zbs_zkt"}, + {"rvb23s64", + "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zicond_zifencei_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_" + "zfa_zcb_zcmop0p2_zba_zbb_zbs_zkt_ssccptr_sscofpmf_sscounterenw_sstc_" + "sstvala_sstvecd_ssu64xl_svade_svbare_svinval_svnapot_svpbmt"}, + {"rvm23u32", "rv32im_zicbop_zicond_zicsr_zihintntl_zihintpause_zimop0p1_" + "zca_zcb_zce_zcmop0p2_zcmp_zcmt_zba_zbb_zbs"}, +}; + static void verifyTables() { #ifndef NDEBUG static std::atomic TableChecked(false); @@ -857,6 +898,29 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension, "string must be lowercase"); } + if (Arch.starts_with("rvi") || Arch.starts_with("rva") || + Arch.starts_with("rvb") || Arch.starts_with("rvm")) { + const auto *FoundProfile = + llvm::find_if(SupportedProfiles, [Arch](const RISCVProfile &Profile) { + return Arch.starts_with(Profile.Name); + }); + + if (FoundProfile == std::end(SupportedProfiles)) + return createStringError(errc::invalid_argument, "unsupported profile"); + + std::string NewArch = FoundProfile->MArch.str(); + StringRef ArchWithoutProfile = Arch.substr(FoundProfile->Name.size()); + if (!ArchWithoutProfile.empty()) { + if (!ArchWithoutProfile.starts_with("_")) + return createStringError( + errc::invalid_argument, + "additional extensions must be after separator '_'"); + NewArch += ArchWithoutProfile.str(); + } + return parseArchString(NewArch, EnableExperimentalExtension, + ExperimentalExtensionVersionCheck, IgnoreUnknown); + } + bool HasRV64 = Arch.starts_with("rv64"); // ISA string must begin with rv32 or rv64. if (!(Arch.starts_with("rv32") || HasRV64) || (Arch.size() < 5)) { -- GitLab From cb4453dc69d75064c9a82e9a6a9bf0d0ded4b204 Mon Sep 17 00:00:00 2001 From: XChy Date: Fri, 22 Mar 2024 18:35:20 +0800 Subject: [PATCH 235/296] [SelectionDAG] Prevent combination on inconsistent type in `combineCarryDiamond` (#84888) Fixes #84831 When matching carry pattern with `getAsCarry`, it may produce different type of carryout. This patch checks such case and does early exit. I'm new to DAG, any suggestion is appreciated. --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 7 +++++- llvm/test/CodeGen/X86/addcarry.ll | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 7009f375df11..db81f9199170 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -3473,6 +3473,11 @@ static SDValue combineCarryDiamond(SelectionDAG &DAG, const TargetLowering &TLI, return SDValue(); if (Opcode != ISD::UADDO && Opcode != ISD::USUBO) return SDValue(); + // Guarantee identical type of CarryOut + EVT CarryOutType = N->getValueType(0); + if (CarryOutType != Carry0.getValue(1).getValueType() || + CarryOutType != Carry1.getValue(1).getValueType()) + return SDValue(); // Canonicalize the add/sub of A and B (the top node in the above ASCII art) // as Carry0 and the add/sub of the carry in as Carry1 (the middle node). @@ -3520,7 +3525,7 @@ static SDValue combineCarryDiamond(SelectionDAG &DAG, const TargetLowering &TLI, // TODO: match other operations that can merge flags (ADD, etc) DAG.ReplaceAllUsesOfValueWith(Carry1.getValue(0), Merged.getValue(0)); if (N->getOpcode() == ISD::AND) - return DAG.getConstant(0, DL, MVT::i1); + return DAG.getConstant(0, DL, CarryOutType); return Merged.getValue(1); } diff --git a/llvm/test/CodeGen/X86/addcarry.ll b/llvm/test/CodeGen/X86/addcarry.ll index 3fc4ed99fad0..f8d32fc2d292 100644 --- a/llvm/test/CodeGen/X86/addcarry.ll +++ b/llvm/test/CodeGen/X86/addcarry.ll @@ -1490,3 +1490,26 @@ define { i64, i64 } @addcarry_commutative_2(i64 %x0, i64 %x1, i64 %y0, i64 %y1) %r1 = insertvalue { i64, i64 } %r0, i64 %b1s, 1 ret { i64, i64 } %r1 } + +define i1 @pr84831(i64 %arg) { +; CHECK-LABEL: pr84831: +; CHECK: # %bb.0: +; CHECK-NEXT: testq %rdi, %rdi +; CHECK-NEXT: setne %al +; CHECK-NEXT: xorl %ecx, %ecx +; CHECK-NEXT: addb $-1, %al +; CHECK-NEXT: adcq $1, %rcx +; CHECK-NEXT: setb %al +; CHECK-NEXT: retq + %a = icmp ult i64 0, %arg + %add1 = add i64 0, 1 + %carryout1 = icmp ult i64 %add1, 0 + %b = zext i1 %a to i64 + %add2 = add i64 %add1, %b + %carryout2 = icmp ult i64 %add2, %add1 + %zc1 = zext i1 %carryout1 to i63 + %zc2 = zext i1 %carryout2 to i63 + %or = or i63 %zc1, %zc2 + %trunc = trunc i63 %or to i1 + ret i1 %trunc +} -- GitLab From 6e755c51a916dc521ffe89738bcab47a5442ad06 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Fri, 22 Mar 2024 18:48:35 +0800 Subject: [PATCH 236/296] Revert "[RISCV] Support RISC-V Profiles in -march option (#76357)" This reverts commit 66f88de80599ec4461b0fdac3d1e396b6e83052d as there are some failures. --- clang/docs/ReleaseNotes.rst | 1 - clang/test/Driver/riscv-profiles.c | 312 ----------------------------- llvm/lib/Support/RISCVISAInfo.cpp | 64 ------ 3 files changed, 377 deletions(-) delete mode 100644 clang/test/Driver/riscv-profiles.c diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 005cdebc0d8a..fd12bb41be47 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -520,7 +520,6 @@ RISC-V Support ^^^^^^^^^^^^^^ - ``__attribute__((rvv_vector_bits(N)))`` is now supported for RVV vbool*_t types. -- Profile names in ``-march`` option are now supported. CUDA/HIP Language Changes ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/test/Driver/riscv-profiles.c b/clang/test/Driver/riscv-profiles.c deleted file mode 100644 index 904f0c371f44..000000000000 --- a/clang/test/Driver/riscv-profiles.c +++ /dev/null @@ -1,312 +0,0 @@ -// RUN: %clang -### -c %s 2>&1 -march=rvi20u32 | FileCheck -check-prefix=RVI20U32 %s -// RVI20U32: "-target-feature" "-a" -// RVI20U32: "-target-feature" "-c" -// RVI20U32: "-target-feature" "-d" -// RVI20U32: "-target-feature" "-f" -// RVI20U32: "-target-feature" "-m" - -// RUN: %clang -### -c %s 2>&1 -march=rvi20u64 | FileCheck -check-prefix=RVI20U64 %s -// RVI20U64: "-target-feature" "-a" -// RVI20U64: "-target-feature" "-c" -// RVI20U64: "-target-feature" "-d" -// RVI20U64: "-target-feature" "-f" -// RVI20U64: "-target-feature" "-m" - -// RUN: %clang -### -c %s 2>&1 -march=rva20u64 | FileCheck -check-prefix=RVA20U64 %s -// RVA20U64: "-target-feature" "+m" -// RVA20U64: "-target-feature" "+a" -// RVA20U64: "-target-feature" "+f" -// RVA20U64: "-target-feature" "+d" -// RVA20U64: "-target-feature" "+c" -// RVA20U64: "-target-feature" "+ziccamoa" -// RVA20U64: "-target-feature" "+ziccif" -// RVA20U64: "-target-feature" "+zicclsm" -// RVA20U64: "-target-feature" "+ziccrse" -// RVA20U64: "-target-feature" "+zicntr" -// RVA20U64: "-target-feature" "+zicsr" -// RVA20U64: "-target-feature" "+za128rs" - -// RUN: %clang -### -c %s 2>&1 -march=rva20s64 | FileCheck -check-prefix=RVA20S64 %s -// RVA20S64: "-target-feature" "+m" -// RVA20S64: "-target-feature" "+a" -// RVA20S64: "-target-feature" "+f" -// RVA20S64: "-target-feature" "+d" -// RVA20S64: "-target-feature" "+c" -// RVA20S64: "-target-feature" "+ziccamoa" -// RVA20S64: "-target-feature" "+ziccif" -// RVA20S64: "-target-feature" "+zicclsm" -// RVA20S64: "-target-feature" "+ziccrse" -// RVA20S64: "-target-feature" "+zicntr" -// RVA20S64: "-target-feature" "+zicsr" -// RVA20S64: "-target-feature" "+zifencei" -// RVA20S64: "-target-feature" "+za128rs" -// RVA20S64: "-target-feature" "+ssccptr" -// RVA20S64: "-target-feature" "+sstvala" -// RVA20S64: "-target-feature" "+sstvecd" -// RVA20S64: "-target-feature" "+svade" -// RVA20S64: "-target-feature" "+svbare" - -// RUN: %clang -### -c %s 2>&1 -march=rva22u64 | FileCheck -check-prefix=RVA22U64 %s -// RVA22U64: "-target-feature" "+m" -// RVA22U64: "-target-feature" "+a" -// RVA22U64: "-target-feature" "+f" -// RVA22U64: "-target-feature" "+d" -// RVA22U64: "-target-feature" "+c" -// RVA22U64: "-target-feature" "+zic64b" -// RVA22U64: "-target-feature" "+zicbom" -// RVA22U64: "-target-feature" "+zicbop" -// RVA22U64: "-target-feature" "+zicboz" -// RVA22U64: "-target-feature" "+ziccamoa" -// RVA22U64: "-target-feature" "+ziccif" -// RVA22U64: "-target-feature" "+zicclsm" -// RVA22U64: "-target-feature" "+ziccrse" -// RVA22U64: "-target-feature" "+zicntr" -// RVA22U64: "-target-feature" "+zicsr" -// RVA22U64: "-target-feature" "+zihintpause" -// RVA22U64: "-target-feature" "+zihpm" -// RVA22U64: "-target-feature" "+za64rs" -// RVA22U64: "-target-feature" "+zfhmin" -// RVA22U64: "-target-feature" "+zba" -// RVA22U64: "-target-feature" "+zbb" -// RVA22U64: "-target-feature" "+zbs" -// RVA22U64: "-target-feature" "+zkt" - -// RUN: %clang -### -c %s 2>&1 -march=rva22s64 | FileCheck -check-prefix=RVA22S64 %s -// RVA22S64: "-target-feature" "+m" -// RVA22S64: "-target-feature" "+a" -// RVA22S64: "-target-feature" "+f" -// RVA22S64: "-target-feature" "+d" -// RVA22S64: "-target-feature" "+c" -// RVA22S64: "-target-feature" "+zic64b" -// RVA22S64: "-target-feature" "+zicbom" -// RVA22S64: "-target-feature" "+zicbop" -// RVA22S64: "-target-feature" "+zicboz" -// RVA22S64: "-target-feature" "+ziccamoa" -// RVA22S64: "-target-feature" "+ziccif" -// RVA22S64: "-target-feature" "+zicclsm" -// RVA22S64: "-target-feature" "+ziccrse" -// RVA22S64: "-target-feature" "+zicntr" -// RVA22S64: "-target-feature" "+zicsr" -// RVA22S64: "-target-feature" "+zifencei" -// RVA22S64: "-target-feature" "+zihintpause" -// RVA22S64: "-target-feature" "+zihpm" -// RVA22S64: "-target-feature" "+za64rs" -// RVA22S64: "-target-feature" "+zfhmin" -// RVA22S64: "-target-feature" "+zba" -// RVA22S64: "-target-feature" "+zbb" -// RVA22S64: "-target-feature" "+zbs" -// RVA22S64: "-target-feature" "+zkt" -// RVA22S64: "-target-feature" "+ssccptr" -// RVA22S64: "-target-feature" "+sscounterenw" -// RVA22S64: "-target-feature" "+sstvala" -// RVA22S64: "-target-feature" "+sstvecd" -// RVA22S64: "-target-feature" "+svade" -// RVA22S64: "-target-feature" "+svbare" -// RVA22S64: "-target-feature" "+svinval" -// RVA22S64: "-target-feature" "+svpbmt" - -// RUN: %clang -### -c %s 2>&1 -march=rva23u64 -menable-experimental-extensions | FileCheck -check-prefix=RVA23U64 %s -// RVA23U64: "-target-feature" "+m" -// RVA23U64: "-target-feature" "+a" -// RVA23U64: "-target-feature" "+f" -// RVA23U64: "-target-feature" "+d" -// RVA23U64: "-target-feature" "+c" -// RVA23U64: "-target-feature" "+v" -// RVA23U64: "-target-feature" "+zic64b" -// RVA23U64: "-target-feature" "+zicbom" -// RVA23U64: "-target-feature" "+zicbop" -// RVA23U64: "-target-feature" "+zicboz" -// RVA23U64: "-target-feature" "+ziccamoa" -// RVA23U64: "-target-feature" "+ziccif" -// RVA23U64: "-target-feature" "+zicclsm" -// RVA23U64: "-target-feature" "+ziccrse" -// RVA23U64: "-target-feature" "+zicntr" -// RVA23U64: "-target-feature" "+zicond" -// RVA23U64: "-target-feature" "+zicsr" -// RVA23U64: "-target-feature" "+zihintntl" -// RVA23U64: "-target-feature" "+zihintpause" -// RVA23U64: "-target-feature" "+zihpm" -// RVA23U64: "-target-feature" "+experimental-zimop" -// RVA23U64: "-target-feature" "+za64rs" -// RVA23U64: "-target-feature" "+zawrs" -// RVA23U64: "-target-feature" "+zfa" -// RVA23U64: "-target-feature" "+zfhmin" -// RVA23U64: "-target-feature" "+zcb" -// RVA23U64: "-target-feature" "+experimental-zcmop" -// RVA23U64: "-target-feature" "+zba" -// RVA23U64: "-target-feature" "+zbb" -// RVA23U64: "-target-feature" "+zbs" -// RVA23U64: "-target-feature" "+zkt" -// RVA23U64: "-target-feature" "+zvbb" -// RVA23U64: "-target-feature" "+zvfhmin" -// RVA23U64: "-target-feature" "+zvkt" - -// RUN: %clang -### -c %s 2>&1 -march=rva23s64 -menable-experimental-extensions | FileCheck -check-prefix=RVA23S64 %s -// RVA23S64: "-target-feature" "+m" -// RVA23S64: "-target-feature" "+a" -// RVA23S64: "-target-feature" "+f" -// RVA23S64: "-target-feature" "+d" -// RVA23S64: "-target-feature" "+c" -// RVA23S64: "-target-feature" "+v" -// RVA23S64: "-target-feature" "+h" -// RVA23S64: "-target-feature" "+zic64b" -// RVA23S64: "-target-feature" "+zicbom" -// RVA23S64: "-target-feature" "+zicbop" -// RVA23S64: "-target-feature" "+zicboz" -// RVA23S64: "-target-feature" "+ziccamoa" -// RVA23S64: "-target-feature" "+ziccif" -// RVA23S64: "-target-feature" "+zicclsm" -// RVA23S64: "-target-feature" "+ziccrse" -// RVA23S64: "-target-feature" "+zicntr" -// RVA23S64: "-target-feature" "+zicond" -// RVA23S64: "-target-feature" "+zicsr" -// RVA23S64: "-target-feature" "+zifencei" -// RVA23S64: "-target-feature" "+zihintntl" -// RVA23S64: "-target-feature" "+zihintpause" -// RVA23S64: "-target-feature" "+zihpm" -// RVA23S64: "-target-feature" "+experimental-zimop" -// RVA23S64: "-target-feature" "+za64rs" -// RVA23S64: "-target-feature" "+zawrs" -// RVA23S64: "-target-feature" "+zfa" -// RVA23S64: "-target-feature" "+zfhmin" -// RVA23S64: "-target-feature" "+zcb" -// RVA23S64: "-target-feature" "+experimental-zcmop" -// RVA23S64: "-target-feature" "+zba" -// RVA23S64: "-target-feature" "+zbb" -// RVA23S64: "-target-feature" "+zbs" -// RVA23S64: "-target-feature" "+zkt" -// RVA23S64: "-target-feature" "+zvbb" -// RVA23S64: "-target-feature" "+zvfhmin" -// RVA23S64: "-target-feature" "+zvkt" -// RVA23S64: "-target-feature" "+shcounterenw" -// RVA23S64: "-target-feature" "+shgatpa" -// RVA23S64: "-target-feature" "+shtvala" -// RVA23S64: "-target-feature" "+shvsatpa" -// RVA23S64: "-target-feature" "+shvstvala" -// RVA23S64: "-target-feature" "+shvstvecd" -// RVA23S64: "-target-feature" "+ssccptr" -// RVA23S64: "-target-feature" "+sscofpmf" -// RVA23S64: "-target-feature" "+sscounterenw" -// RVA23S64: "-target-feature" "+experimental-ssnpm" -// RVA23S64: "-target-feature" "+ssstateen" -// RVA23S64: "-target-feature" "+sstc" -// RVA23S64: "-target-feature" "+sstvala" -// RVA23S64: "-target-feature" "+sstvecd" -// RVA23S64: "-target-feature" "+ssu64xl" -// RVA23S64: "-target-feature" "+svade" -// RVA23S64: "-target-feature" "+svbare" -// RVA23S64: "-target-feature" "+svinval" -// RVA23S64: "-target-feature" "+svnapot" -// RVA23S64: "-target-feature" "+svpbmt" - -// RUN: %clang -### -c %s 2>&1 -march=rvb23u64 -menable-experimental-extensions | FileCheck -check-prefix=RVB23U64 %s -// RVB23U64: "-target-feature" "+m" -// RVB23U64: "-target-feature" "+a" -// RVB23U64: "-target-feature" "+f" -// RVB23U64: "-target-feature" "+d" -// RVB23U64: "-target-feature" "+c" -// RVB23U64: "-target-feature" "+zic64b" -// RVB23U64: "-target-feature" "+zicbom" -// RVB23U64: "-target-feature" "+zicbop" -// RVB23U64: "-target-feature" "+zicboz" -// RVB23U64: "-target-feature" "+ziccamoa" -// RVB23U64: "-target-feature" "+ziccif" -// RVB23U64: "-target-feature" "+zicclsm" -// RVB23U64: "-target-feature" "+ziccrse" -// RVB23U64: "-target-feature" "+zicntr" -// RVB23U64: "-target-feature" "+zicond" -// RVB23U64: "-target-feature" "+zicsr" -// RVB23U64: "-target-feature" "+zihintntl" -// RVB23U64: "-target-feature" "+zihintpause" -// RVB23U64: "-target-feature" "+zihpm" -// RVB23U64: "-target-feature" "+experimental-zimop" -// RVB23U64: "-target-feature" "+za64rs" -// RVB23U64: "-target-feature" "+zawrs" -// RVB23U64: "-target-feature" "+zfa" -// RVB23U64: "-target-feature" "+zcb" -// RVB23U64: "-target-feature" "+experimental-zcmop" -// RVB23U64: "-target-feature" "+zba" -// RVB23U64: "-target-feature" "+zbb" -// RVB23U64: "-target-feature" "+zbs" -// RVB23U64: "-target-feature" "+zkt" - -// RUN: %clang -### -c %s 2>&1 -march=rvb23s64 -menable-experimental-extensions | FileCheck -check-prefix=RVB23S64 %s -// RVB23S64: "-target-feature" "+m" -// RVB23S64: "-target-feature" "+a" -// RVB23S64: "-target-feature" "+f" -// RVB23S64: "-target-feature" "+d" -// RVB23S64: "-target-feature" "+c" -// RVB23S64: "-target-feature" "+zic64b" -// RVB23S64: "-target-feature" "+zicbom" -// RVB23S64: "-target-feature" "+zicbop" -// RVB23S64: "-target-feature" "+zicboz" -// RVB23S64: "-target-feature" "+ziccamoa" -// RVB23S64: "-target-feature" "+ziccif" -// RVB23S64: "-target-feature" "+zicclsm" -// RVB23S64: "-target-feature" "+ziccrse" -// RVB23S64: "-target-feature" "+zicntr" -// RVB23S64: "-target-feature" "+zicond" -// RVB23S64: "-target-feature" "+zicsr" -// RVB23S64: "-target-feature" "+zifencei" -// RVB23S64: "-target-feature" "+zihintntl" -// RVB23S64: "-target-feature" "+zihintpause" -// RVB23S64: "-target-feature" "+zihpm" -// RVB23S64: "-target-feature" "+experimental-zimop" -// RVB23S64: "-target-feature" "+za64rs" -// RVB23S64: "-target-feature" "+zawrs" -// RVB23S64: "-target-feature" "+zfa" -// RVB23S64: "-target-feature" "+zcb" -// RVB23S64: "-target-feature" "+experimental-zcmop" -// RVB23S64: "-target-feature" "+zba" -// RVB23S64: "-target-feature" "+zbb" -// RVB23S64: "-target-feature" "+zbs" -// RVB23S64: "-target-feature" "+zkt" -// RVB23S64: "-target-feature" "+ssccptr" -// RVB23S64: "-target-feature" "+sscofpmf" -// RVB23S64: "-target-feature" "+sscounterenw" -// RVB23S64: "-target-feature" "+sstc" -// RVB23S64: "-target-feature" "+sstvala" -// RVB23S64: "-target-feature" "+sstvecd" -// RVB23S64: "-target-feature" "+ssu64xl" -// RVB23S64: "-target-feature" "+svade" -// RVB23S64: "-target-feature" "+svbare" -// RVB23S64: "-target-feature" "+svinval" -// RVB23S64: "-target-feature" "+svnapot" -// RVB23S64: "-target-feature" "+svpbmt" - -// RUN: %clang -### -c %s 2>&1 -march=rvm23u32 -menable-experimental-extensions | FileCheck -check-prefix=RVM23U32 %s -// RVM23U32: "-target-feature" "+m" -// RVM23U32: "-target-feature" "+zicbop" -// RVM23U32: "-target-feature" "+zicond" -// RVM23U32: "-target-feature" "+zicsr" -// RVM23U32: "-target-feature" "+zihintntl" -// RVM23U32: "-target-feature" "+zihintpause" -// RVM23U32: "-target-feature" "+experimental-zimop" -// RVM23U32: "-target-feature" "+zce" -// RVM23U32: "-target-feature" "+experimental-zcmop" -// RVM23U32: "-target-feature" "+zba" -// RVM23U32: "-target-feature" "+zbb" -// RVM23U32: "-target-feature" "+zbs" - -// RUN: %clang -### -c %s 2>&1 -march=rva22u64_zfa | FileCheck -check-prefix=PROFILE-WITH-ADDITIONAL %s -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+m" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+a" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+f" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+d" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+c" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicbom" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicbop" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicboz" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zihintpause" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zfa" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zfhmin" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zba" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zbb" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zbs" -// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zkt" - -// RUN: not %clang -### -c %s 2>&1 -march=rva19u64_zfa | FileCheck -check-prefix=INVALID-PROFILE %s -// INVALID-PROFILE: error: invalid arch name 'rva19u64_zfa', unsupported profile - -// RUN: not %clang -### -c %s 2>&1 -march=rva22u64zfa | FileCheck -check-prefix=INVALID-ADDITIONAL %s -// INVALID-ADDITIONAL: error: invalid arch name 'rva22u64zfa', additional extensions must be after separator '_' diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp index 67e6e5b962b1..39235ace4724 100644 --- a/llvm/lib/Support/RISCVISAInfo.cpp +++ b/llvm/lib/Support/RISCVISAInfo.cpp @@ -36,11 +36,6 @@ struct RISCVSupportedExtension { } }; -struct RISCVProfile { - StringLiteral Name; - StringLiteral MArch; -}; - } // end anonymous namespace static constexpr StringLiteral AllStdExts = "mafdqlcbkjtpvnh"; @@ -249,42 +244,6 @@ static const RISCVSupportedExtension SupportedExperimentalExtensions[] = { }; // clang-format on -static constexpr RISCVProfile SupportedProfiles[] = { - {"rvi20u32", "rv32i"}, - {"rvi20u64", "rv64i"}, - {"rva20u64", "rv64imafdc_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_za128rs"}, - {"rva20s64", "rv64imafdc_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_zifencei_" - "za128rs_ssccptr_sstvala_sstvecd_svade_svbare"}, - {"rva22u64", - "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" - "zicntr_zihintpause_zihpm_za64rs_zfhmin_zba_zbb_zbs_zkt"}, - {"rva22s64", - "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" - "zicntr_zifencei_zihintpause_zihpm_za64rs_zfhmin_zba_zbb_zbs_zkt_ssccptr_" - "sscounterenw_sstvala_sstvecd_svade_svbare_svinval_svpbmt"}, - {"rva23u64", - "rv64imafdcv_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" - "zicntr_zicond_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_zfa_" - "zfhmin_zcb_zcmop0p2_zba_zbb_zbs_zkt_zvbb_zvfhmin_zvkt"}, - {"rva23s64", - "rv64imafdcvh_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" - "zicntr_zicond_zifencei_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_" - "zfa_zfhmin_zcb_zcmop0p2_zba_zbb_zbs_zkt_zvbb_zvfhmin_zvkt_shcounterenw_" - "shgatpa_shtvala_shvsatpa_shvstvala_shvstvecd_ssccptr_sscofpmf_" - "sscounterenw_ssnpm0p8_ssstateen_sstc_sstvala_sstvecd_ssu64xl_svade_" - "svbare_svinval_svnapot_svpbmt"}, - {"rvb23u64", "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_" - "zicclsm_ziccrse_zicntr_zicond_zihintntl_zihintpause_zihpm_" - "zimop0p1_za64rs_zawrs_zfa_zcb_zcmop0p2_zba_zbb_zbs_zkt"}, - {"rvb23s64", - "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" - "zicntr_zicond_zifencei_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_" - "zfa_zcb_zcmop0p2_zba_zbb_zbs_zkt_ssccptr_sscofpmf_sscounterenw_sstc_" - "sstvala_sstvecd_ssu64xl_svade_svbare_svinval_svnapot_svpbmt"}, - {"rvm23u32", "rv32im_zicbop_zicond_zicsr_zihintntl_zihintpause_zimop0p1_" - "zca_zcb_zce_zcmop0p2_zcmp_zcmt_zba_zbb_zbs"}, -}; - static void verifyTables() { #ifndef NDEBUG static std::atomic TableChecked(false); @@ -898,29 +857,6 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension, "string must be lowercase"); } - if (Arch.starts_with("rvi") || Arch.starts_with("rva") || - Arch.starts_with("rvb") || Arch.starts_with("rvm")) { - const auto *FoundProfile = - llvm::find_if(SupportedProfiles, [Arch](const RISCVProfile &Profile) { - return Arch.starts_with(Profile.Name); - }); - - if (FoundProfile == std::end(SupportedProfiles)) - return createStringError(errc::invalid_argument, "unsupported profile"); - - std::string NewArch = FoundProfile->MArch.str(); - StringRef ArchWithoutProfile = Arch.substr(FoundProfile->Name.size()); - if (!ArchWithoutProfile.empty()) { - if (!ArchWithoutProfile.starts_with("_")) - return createStringError( - errc::invalid_argument, - "additional extensions must be after separator '_'"); - NewArch += ArchWithoutProfile.str(); - } - return parseArchString(NewArch, EnableExperimentalExtension, - ExperimentalExtensionVersionCheck, IgnoreUnknown); - } - bool HasRV64 = Arch.starts_with("rv64"); // ISA string must begin with rv32 or rv64. if (!(Arch.starts_with("rv32") || HasRV64) || (Arch.size() < 5)) { -- GitLab From a62441d4bb6bd0cd8eccab8c5692340c5a2c60bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20=C3=81lvarez=20Ayll=C3=B3n?= Date: Fri, 22 Mar 2024 11:50:34 +0100 Subject: [PATCH 237/296] [clang][analyzer][NFC] UnixAPIMisuseChecker inherits from Checker (#83027) --- .../Checkers/UnixAPIChecker.cpp | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp index 19f1ca2dc824..599e5e6cedc6 100644 --- a/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp @@ -17,6 +17,7 @@ #include "clang/StaticAnalyzer/Core/BugReporter/CommonBugCategories.h" #include "clang/StaticAnalyzer/Core/Checker.h" #include "clang/StaticAnalyzer/Core/CheckerManager.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h" #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h" #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h" #include "llvm/ADT/STLExtras.h" @@ -41,8 +42,7 @@ enum class OpenVariant { namespace { class UnixAPIMisuseChecker - : public Checker, - check::ASTDecl> { + : public Checker> { const BugType BT_open{this, "Improper use of 'open'", categories::UnixAPI}; const BugType BT_pthreadOnce{this, "Improper use of 'pthread_once'", categories::UnixAPI}; @@ -52,14 +52,14 @@ public: void checkASTDecl(const TranslationUnitDecl *TU, AnalysisManager &Mgr, BugReporter &BR) const; - void checkPreStmt(const CallExpr *CE, CheckerContext &C) const; + void checkPreCall(const CallEvent &Call, CheckerContext &C) const; - void CheckOpen(CheckerContext &C, const CallExpr *CE) const; - void CheckOpenAt(CheckerContext &C, const CallExpr *CE) const; - void CheckPthreadOnce(CheckerContext &C, const CallExpr *CE) const; + void CheckOpen(CheckerContext &C, const CallEvent &Call) const; + void CheckOpenAt(CheckerContext &C, const CallEvent &Call) const; + void CheckPthreadOnce(CheckerContext &C, const CallEvent &Call) const; - void CheckOpenVariant(CheckerContext &C, - const CallExpr *CE, OpenVariant Variant) const; + void CheckOpenVariant(CheckerContext &C, const CallEvent &Call, + OpenVariant Variant) const; void ReportOpenBug(CheckerContext &C, ProgramStateRef State, const char *Msg, SourceRange SR) const; @@ -113,9 +113,9 @@ void UnixAPIMisuseChecker::checkASTDecl(const TranslationUnitDecl *TU, // "open" (man 2 open) //===----------------------------------------------------------------------===/ -void UnixAPIMisuseChecker::checkPreStmt(const CallExpr *CE, +void UnixAPIMisuseChecker::checkPreCall(const CallEvent &Call, CheckerContext &C) const { - const FunctionDecl *FD = C.getCalleeDecl(CE); + const FunctionDecl *FD = dyn_cast_if_present(Call.getDecl()); if (!FD || FD->getKind() != Decl::Function) return; @@ -130,13 +130,13 @@ void UnixAPIMisuseChecker::checkPreStmt(const CallExpr *CE, return; if (FName == "open") - CheckOpen(C, CE); + CheckOpen(C, Call); else if (FName == "openat") - CheckOpenAt(C, CE); + CheckOpenAt(C, Call); else if (FName == "pthread_once") - CheckPthreadOnce(C, CE); + CheckPthreadOnce(C, Call); } void UnixAPIMisuseChecker::ReportOpenBug(CheckerContext &C, ProgramStateRef State, @@ -152,17 +152,17 @@ void UnixAPIMisuseChecker::ReportOpenBug(CheckerContext &C, } void UnixAPIMisuseChecker::CheckOpen(CheckerContext &C, - const CallExpr *CE) const { - CheckOpenVariant(C, CE, OpenVariant::Open); + const CallEvent &Call) const { + CheckOpenVariant(C, Call, OpenVariant::Open); } void UnixAPIMisuseChecker::CheckOpenAt(CheckerContext &C, - const CallExpr *CE) const { - CheckOpenVariant(C, CE, OpenVariant::OpenAt); + const CallEvent &Call) const { + CheckOpenVariant(C, Call, OpenVariant::OpenAt); } void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, - const CallExpr *CE, + const CallEvent &Call, OpenVariant Variant) const { // The index of the argument taking the flags open flags (O_RDONLY, // O_WRONLY, O_CREAT, etc.), @@ -191,11 +191,11 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, ProgramStateRef state = C.getState(); - if (CE->getNumArgs() < MinArgCount) { + if (Call.getNumArgs() < MinArgCount) { // The frontend should issue a warning for this case. Just return. return; - } else if (CE->getNumArgs() == MaxArgCount) { - const Expr *Arg = CE->getArg(CreateModeArgIndex); + } else if (Call.getNumArgs() == MaxArgCount) { + const Expr *Arg = Call.getArgExpr(CreateModeArgIndex); QualType QT = Arg->getType(); if (!QT->isIntegerType()) { SmallString<256> SBuf; @@ -209,7 +209,7 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, Arg->getSourceRange()); return; } - } else if (CE->getNumArgs() > MaxArgCount) { + } else if (Call.getNumArgs() > MaxArgCount) { SmallString<256> SBuf; llvm::raw_svector_ostream OS(SBuf); OS << "Call to '" << VariantName << "' with more than " << MaxArgCount @@ -217,7 +217,7 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, ReportOpenBug(C, state, SBuf.c_str(), - CE->getArg(MaxArgCount)->getSourceRange()); + Call.getArgExpr(MaxArgCount)->getSourceRange()); return; } @@ -226,8 +226,8 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, } // Now check if oflags has O_CREAT set. - const Expr *oflagsEx = CE->getArg(FlagsArgIndex); - const SVal V = C.getSVal(oflagsEx); + const Expr *oflagsEx = Call.getArgExpr(FlagsArgIndex); + const SVal V = Call.getArgSVal(FlagsArgIndex); if (!isa(V)) { // The case where 'V' can be a location can only be due to a bad header, // so in this case bail out. @@ -253,7 +253,7 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, if (!(trueState && !falseState)) return; - if (CE->getNumArgs() < MaxArgCount) { + if (Call.getNumArgs() < MaxArgCount) { SmallString<256> SBuf; llvm::raw_svector_ostream OS(SBuf); OS << "Call to '" << VariantName << "' requires a " @@ -271,18 +271,18 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, //===----------------------------------------------------------------------===// void UnixAPIMisuseChecker::CheckPthreadOnce(CheckerContext &C, - const CallExpr *CE) const { + const CallEvent &Call) const { // This is similar to 'CheckDispatchOnce' in the MacOSXAPIChecker. // They can possibly be refactored. - if (CE->getNumArgs() < 1) + if (Call.getNumArgs() < 1) return; // Check if the first argument is stack allocated. If so, issue a warning // because that's likely to be bad news. ProgramStateRef state = C.getState(); - const MemRegion *R = C.getSVal(CE->getArg(0)).getAsRegion(); + const MemRegion *R = Call.getArgSVal(0).getAsRegion(); if (!R || !isa(R->getMemorySpace())) return; @@ -304,7 +304,7 @@ void UnixAPIMisuseChecker::CheckPthreadOnce(CheckerContext &C, auto report = std::make_unique(BT_pthreadOnce, os.str(), N); - report->addRange(CE->getArg(0)->getSourceRange()); + report->addRange(Call.getArgExpr(0)->getSourceRange()); C.emitReport(std::move(report)); } -- GitLab From 730ca47a0cc7380def6df1d25b30c1378fd8bf14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20=C3=81lvarez=20Ayll=C3=B3n?= Date: Fri, 22 Mar 2024 11:50:34 +0100 Subject: [PATCH 238/296] [clang][analyzer] Model getline/getdelim preconditions and evaluation (#83027) According to POSIX 2018. 1. lineptr, n and stream can not be NULL. 2. If *n is non-zero, *lineptr must point to a region of at least *n bytes, or be a NULL pointer. Additionally, if *lineptr is not NULL, *n must not be undefined. --- .../Core/PathSensitive/CheckerHelpers.h | 4 +- .../StaticAnalyzer/Checkers/MallocChecker.cpp | 8 +- .../StaticAnalyzer/Checkers/StreamChecker.cpp | 23 +- .../Checkers/UnixAPIChecker.cpp | 132 ++++++- .../StaticAnalyzer/Core/CheckerHelpers.cpp | 5 +- clang/test/Analysis/getline-unixapi.c | 322 ++++++++++++++++++ clang/test/Analysis/stream.c | 73 ++++ 7 files changed, 554 insertions(+), 13 deletions(-) create mode 100644 clang/test/Analysis/getline-unixapi.c diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h index 60421e5437d8..d053a9718912 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h @@ -15,7 +15,6 @@ #include "ProgramState_Fwd.h" #include "SVals.h" - #include "clang/AST/OperationKinds.h" #include "clang/AST/Stmt.h" #include "clang/Basic/OperatorKinds.h" @@ -113,8 +112,7 @@ public: OperatorKind operationKindFromOverloadedOperator(OverloadedOperatorKind OOK, bool IsBinary); -std::optional getPointeeDefVal(SVal PtrSVal, - ProgramStateRef State); +std::optional getPointeeVal(SVal PtrSVal, ProgramStateRef State); } // namespace ento diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp index 03cb7696707f..c2d96f592609 100644 --- a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp @@ -1441,7 +1441,7 @@ void MallocChecker::preGetdelim(const CallEvent &Call, return; ProgramStateRef State = C.getState(); - const auto LinePtr = getPointeeDefVal(Call.getArgSVal(0), State); + const auto LinePtr = getPointeeVal(Call.getArgSVal(0), State); if (!LinePtr) return; @@ -1470,8 +1470,10 @@ void MallocChecker::checkGetdelim(const CallEvent &Call, SValBuilder &SVB = C.getSValBuilder(); - const auto LinePtr = getPointeeDefVal(Call.getArgSVal(0), State); - const auto Size = getPointeeDefVal(Call.getArgSVal(1), State); + const auto LinePtr = + getPointeeVal(Call.getArgSVal(0), State)->getAs(); + const auto Size = + getPointeeVal(Call.getArgSVal(1), State)->getAs(); if (!LinePtr || !Size || !LinePtr->getAsRegion()) return; diff --git a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp index 10972158f398..902c42a2799b 100644 --- a/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/StreamChecker.cpp @@ -1200,10 +1200,25 @@ void StreamChecker::evalGetdelim(const FnDescription *Desc, // Add transition for the successful state. NonLoc RetVal = makeRetVal(C, E.CE).castAs(); - ProgramStateRef StateNotFailed = - State->BindExpr(E.CE, C.getLocationContext(), RetVal); + ProgramStateRef StateNotFailed = E.bindReturnValue(State, C, RetVal); StateNotFailed = E.assumeBinOpNN(StateNotFailed, BO_GE, RetVal, E.getZeroVal(Call)); + + // On success, a buffer is allocated. + auto NewLinePtr = getPointeeVal(Call.getArgSVal(0), State); + if (NewLinePtr && isa(*NewLinePtr)) + StateNotFailed = StateNotFailed->assume( + NewLinePtr->castAs(), true); + + // The buffer size `*n` must be enough to hold the whole line, and + // greater than the return value, since it has to account for '\0'. + SVal SizePtrSval = Call.getArgSVal(1); + auto NVal = getPointeeVal(SizePtrSval, State); + if (NVal && isa(*NVal)) { + StateNotFailed = E.assumeBinOpNN(StateNotFailed, BO_GT, + NVal->castAs(), RetVal); + StateNotFailed = E.bindReturnValue(StateNotFailed, C, RetVal); + } if (!StateNotFailed) return; C.addTransition(StateNotFailed); @@ -1217,6 +1232,10 @@ void StreamChecker::evalGetdelim(const FnDescription *Desc, E.isStreamEof() ? ErrorFEof : ErrorFEof | ErrorFError; StateFailed = E.setStreamState( StateFailed, StreamState::getOpened(Desc, NewES, !NewES.isFEof())); + // On failure, the content of the buffer is undefined. + if (auto NewLinePtr = getPointeeVal(Call.getArgSVal(0), State)) + StateFailed = StateFailed->bindLoc(*NewLinePtr, UndefinedVal(), + C.getLocationContext()); C.addTransition(StateFailed, E.getFailureNoteTag(this, C)); } diff --git a/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp index 599e5e6cedc6..da2d16ca9b5d 100644 --- a/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp @@ -20,6 +20,7 @@ #include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h" #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h" #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerHelpers.h" +#include "clang/StaticAnalyzer/Core/PathSensitive/DynamicExtent.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringExtras.h" @@ -44,10 +45,23 @@ namespace { class UnixAPIMisuseChecker : public Checker> { const BugType BT_open{this, "Improper use of 'open'", categories::UnixAPI}; + const BugType BT_getline{this, "Improper use of getdelim", + categories::UnixAPI}; const BugType BT_pthreadOnce{this, "Improper use of 'pthread_once'", categories::UnixAPI}; + const BugType BT_ArgumentNull{this, "NULL pointer", categories::UnixAPI}; mutable std::optional Val_O_CREAT; + ProgramStateRef + EnsurePtrNotNull(SVal PtrVal, const Expr *PtrExpr, CheckerContext &C, + ProgramStateRef State, const StringRef PtrDescr, + std::optional> BT = + std::nullopt) const; + + ProgramStateRef EnsureGetdelimBufferAndSizeCorrect( + SVal LinePtrPtrSVal, SVal SizePtrSVal, const Expr *LinePtrPtrExpr, + const Expr *SizePtrExpr, CheckerContext &C, ProgramStateRef State) const; + public: void checkASTDecl(const TranslationUnitDecl *TU, AnalysisManager &Mgr, BugReporter &BR) const; @@ -56,6 +70,7 @@ public: void CheckOpen(CheckerContext &C, const CallEvent &Call) const; void CheckOpenAt(CheckerContext &C, const CallEvent &Call) const; + void CheckGetDelim(CheckerContext &C, const CallEvent &Call) const; void CheckPthreadOnce(CheckerContext &C, const CallEvent &Call) const; void CheckOpenVariant(CheckerContext &C, const CallEvent &Call, @@ -95,6 +110,30 @@ private: } // end anonymous namespace +ProgramStateRef UnixAPIMisuseChecker::EnsurePtrNotNull( + SVal PtrVal, const Expr *PtrExpr, CheckerContext &C, ProgramStateRef State, + const StringRef PtrDescr, + std::optional> BT) const { + const auto Ptr = PtrVal.getAs(); + if (!Ptr) + return State; + + const auto [PtrNotNull, PtrNull] = State->assume(*Ptr); + if (!PtrNotNull && PtrNull) { + if (ExplodedNode *N = C.generateErrorNode(PtrNull)) { + auto R = std::make_unique( + BT.value_or(std::cref(BT_ArgumentNull)), + (PtrDescr + " pointer might be NULL.").str(), N); + if (PtrExpr) + bugreporter::trackExpressionValue(N, PtrExpr, *R); + C.emitReport(std::move(R)); + } + return nullptr; + } + + return PtrNotNull; +} + void UnixAPIMisuseChecker::checkASTDecl(const TranslationUnitDecl *TU, AnalysisManager &Mgr, BugReporter &) const { @@ -137,6 +176,9 @@ void UnixAPIMisuseChecker::checkPreCall(const CallEvent &Call, else if (FName == "pthread_once") CheckPthreadOnce(C, Call); + + else if (is_contained({"getdelim", "getline"}, FName)) + CheckGetDelim(C, Call); } void UnixAPIMisuseChecker::ReportOpenBug(CheckerContext &C, ProgramStateRef State, @@ -215,8 +257,7 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, OS << "Call to '" << VariantName << "' with more than " << MaxArgCount << " arguments"; - ReportOpenBug(C, state, - SBuf.c_str(), + ReportOpenBug(C, state, SBuf.c_str(), Call.getArgExpr(MaxArgCount)->getSourceRange()); return; } @@ -266,6 +307,93 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C, } } +//===----------------------------------------------------------------------===// +// getdelim and getline +//===----------------------------------------------------------------------===// + +ProgramStateRef UnixAPIMisuseChecker::EnsureGetdelimBufferAndSizeCorrect( + SVal LinePtrPtrSVal, SVal SizePtrSVal, const Expr *LinePtrPtrExpr, + const Expr *SizePtrExpr, CheckerContext &C, ProgramStateRef State) const { + static constexpr llvm::StringLiteral SizeGreaterThanBufferSize = + "The buffer from the first argument is smaller than the size " + "specified by the second parameter"; + static constexpr llvm::StringLiteral SizeUndef = + "The buffer from the first argument is not NULL, but the size specified " + "by the second parameter is undefined."; + + auto EmitBugReport = [this, &C, SizePtrExpr, LinePtrPtrExpr]( + ProgramStateRef BugState, StringRef ErrMsg) { + if (ExplodedNode *N = C.generateErrorNode(BugState)) { + auto R = std::make_unique(BT_getline, ErrMsg, N); + bugreporter::trackExpressionValue(N, SizePtrExpr, *R); + bugreporter::trackExpressionValue(N, LinePtrPtrExpr, *R); + C.emitReport(std::move(R)); + } + }; + + // We have a pointer to a pointer to the buffer, and a pointer to the size. + // We want what they point at. + auto LinePtrSVal = getPointeeVal(LinePtrPtrSVal, State)->getAs(); + auto NSVal = getPointeeVal(SizePtrSVal, State); + if (!LinePtrSVal || !NSVal || NSVal->isUnknown()) + return nullptr; + + assert(LinePtrPtrExpr && SizePtrExpr); + + const auto [LinePtrNotNull, LinePtrNull] = State->assume(*LinePtrSVal); + if (LinePtrNotNull && !LinePtrNull) { + // If `*lineptr` is not null, but `*n` is undefined, there is UB. + if (NSVal->isUndef()) { + EmitBugReport(LinePtrNotNull, SizeUndef); + return nullptr; + } + + // If it is defined, and known, its size must be less than or equal to + // the buffer size. + auto NDefSVal = NSVal->getAs(); + auto &SVB = C.getSValBuilder(); + auto LineBufSize = + getDynamicExtent(LinePtrNotNull, LinePtrSVal->getAsRegion(), SVB); + auto LineBufSizeGtN = SVB.evalBinOp(LinePtrNotNull, BO_GE, LineBufSize, + *NDefSVal, SVB.getConditionType()) + .getAs(); + if (!LineBufSizeGtN) + return LinePtrNotNull; + if (auto LineBufSizeOk = LinePtrNotNull->assume(*LineBufSizeGtN, true)) + return LineBufSizeOk; + + EmitBugReport(LinePtrNotNull, SizeGreaterThanBufferSize); + return nullptr; + } + return State; +} + +void UnixAPIMisuseChecker::CheckGetDelim(CheckerContext &C, + const CallEvent &Call) const { + ProgramStateRef State = C.getState(); + + // The parameter `n` must not be NULL. + SVal SizePtrSval = Call.getArgSVal(1); + State = EnsurePtrNotNull(SizePtrSval, Call.getArgExpr(1), C, State, "Size"); + if (!State) + return; + + // The parameter `lineptr` must not be NULL. + SVal LinePtrPtrSVal = Call.getArgSVal(0); + State = + EnsurePtrNotNull(LinePtrPtrSVal, Call.getArgExpr(0), C, State, "Line"); + if (!State) + return; + + State = EnsureGetdelimBufferAndSizeCorrect(LinePtrPtrSVal, SizePtrSval, + Call.getArgExpr(0), + Call.getArgExpr(1), C, State); + if (!State) + return; + + C.addTransition(State); +} + //===----------------------------------------------------------------------===// // pthread_once //===----------------------------------------------------------------------===// diff --git a/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp b/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp index 364c87e910b7..d7137a915b3d 100644 --- a/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp +++ b/clang/lib/StaticAnalyzer/Core/CheckerHelpers.cpp @@ -183,10 +183,9 @@ OperatorKind operationKindFromOverloadedOperator(OverloadedOperatorKind OOK, } } -std::optional getPointeeDefVal(SVal PtrSVal, - ProgramStateRef State) { +std::optional getPointeeVal(SVal PtrSVal, ProgramStateRef State) { if (const auto *Ptr = PtrSVal.getAsRegion()) { - return State->getSVal(Ptr).getAs(); + return State->getSVal(Ptr); } return std::nullopt; } diff --git a/clang/test/Analysis/getline-unixapi.c b/clang/test/Analysis/getline-unixapi.c new file mode 100644 index 000000000000..86635ed84997 --- /dev/null +++ b/clang/test/Analysis/getline-unixapi.c @@ -0,0 +1,322 @@ +// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix,debug.ExprInspection -verify %s + +#include "Inputs/system-header-simulator.h" +#include "Inputs/system-header-simulator-for-malloc.h" +#include "Inputs/system-header-simulator-for-valist.h" + +void clang_analyzer_eval(int); +void clang_analyzer_dump_int(int); +void clang_analyzer_dump_ptr(void*); +void clang_analyzer_warnIfReached(); + +void test_getline_null_lineptr() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char **buffer = NULL; + size_t n = 0; + getline(buffer, &n, F1); // expected-warning {{Line pointer might be NULL}} + fclose(F1); +} + +void test_getline_null_size() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + getline(&buffer, NULL, F1); // expected-warning {{Size pointer might be NULL}} + fclose(F1); +} + +void test_getline_null_buffer_size_gt0() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + size_t n = 8; + getline(&buffer, &n, F1); // ok since posix 2018 + free(buffer); + fclose(F1); +} + +void test_getline_null_buffer_size_gt0_2(size_t n) { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + if (n > 0) { + getline(&buffer, &n, F1); // ok since posix 2018 + } + free(buffer); + fclose(F1); +} + +void test_getline_null_buffer_unknown_size(size_t n) { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + + getline(&buffer, &n, F1); // ok + fclose(F1); + free(buffer); +} + +void test_getline_null_buffer_undef_size() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char *buffer = NULL; + size_t n; + + getline(&buffer, &n, F1); // ok since posix 2018 + fclose(F1); + free(buffer); +} + +void test_getline_buffer_size_0() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char *buffer = malloc(10); + size_t n = 0; + if (buffer != NULL) + getline(&buffer, &n, F1); // ok, the buffer is enough for 0 character + fclose(F1); + free(buffer); +} + +void test_getline_buffer_bad_size() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char *buffer = malloc(10); + size_t n = 100; + if (buffer != NULL) + getline(&buffer, &n, F1); // expected-warning {{The buffer from the first argument is smaller than the size specified by the second parameter}} + fclose(F1); + free(buffer); +} + +void test_getline_buffer_smaller_size() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char *buffer = malloc(100); + size_t n = 10; + if (buffer != NULL) + getline(&buffer, &n, F1); // ok, there is enough space for 10 characters + fclose(F1); + free(buffer); +} + +void test_getline_buffer_undef_size() { + FILE *F1 = tmpfile(); + if (!F1) + return; + + char *buffer = malloc(100); + size_t n; + if (buffer != NULL) + getline(&buffer, &n, F1); // expected-warning {{The buffer from the first argument is not NULL, but the size specified by the second parameter is undefined}} + fclose(F1); + free(buffer); +} + + +void test_getline_null_buffer() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + size_t n = 0; + ssize_t r = getline(&buffer, &n, F1); + // getline returns -1 on failure, number of char reads on success (>= 0) + if (r < -1) { + clang_analyzer_warnIfReached(); // must not happen + } else { + // The buffer could be allocated both on failure and success + clang_analyzer_dump_int(n); // expected-warning {{conj_$}} + clang_analyzer_dump_ptr(buffer); // expected-warning {{conj_$}} + } + free(buffer); + fclose(F1); +} + +void test_getdelim_null_size() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + getdelim(&buffer, NULL, ',', F1); // expected-warning {{Size pointer might be NULL}} + fclose(F1); +} + +void test_getdelim_null_buffer_size_gt0() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + size_t n = 8; + getdelim(&buffer, &n, ';', F1); // ok since posix 2018 + free(buffer); + fclose(F1); +} + +void test_getdelim_null_buffer_size_gt0_2(size_t n) { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + if (n > 0) { + getdelim(&buffer, &n, ' ', F1); // ok since posix 2018 + } + free(buffer); + fclose(F1); +} + +void test_getdelim_null_buffer_unknown_size(size_t n) { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + getdelim(&buffer, &n, '-', F1); // ok + fclose(F1); + free(buffer); +} + +void test_getdelim_null_buffer() { + FILE *F1 = tmpfile(); + if (!F1) + return; + char *buffer = NULL; + size_t n = 0; + ssize_t r = getdelim(&buffer, &n, '\r', F1); + // getdelim returns -1 on failure, number of char reads on success (>= 0) + if (r < -1) { + clang_analyzer_warnIfReached(); // must not happen + } + else { + // The buffer could be allocated both on failure and success + clang_analyzer_dump_int(n); // expected-warning {{conj_$}} + clang_analyzer_dump_ptr(buffer); // expected-warning {{conj_$}} + } + free(buffer); + fclose(F1); +} + +void test_getline_while() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + char *line = NULL; + size_t len = 0; + ssize_t read; + + while ((read = getline(&line, &len, file)) != -1) { + printf("%s\n", line); + } + + free(line); + fclose(file); +} + +void test_getline_return_check() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + char *line = NULL; + size_t len = 0; + ssize_t r = getline(&line, &len, file); + + if (r != -1) { + if (line[0] == '\0') {} // ok + } + free(line); + fclose(file); +} + +void test_getline_clear_eof() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + size_t n = 10; + char *buffer = malloc(n); + ssize_t read = fread(buffer, n, 1, file); + if (feof(file)) { + clearerr(file); + getline(&buffer, &n, file); // ok + } + fclose(file); + free(buffer); +} + +void test_getline_not_null(char **buffer, size_t *size) { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + getline(buffer, size, file); + fclose(file); + + if (size == NULL || buffer == NULL) { + clang_analyzer_warnIfReached(); // must not happen + } +} + +void test_getline_size_constraint(size_t size) { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + size_t old_size = size; + char *buffer = malloc(10); + if (buffer != NULL) { + ssize_t r = getline(&buffer, &size, file); + if (r >= 0) { + // Since buffer has a size of 10, old_size must be less than or equal to 10. + // Otherwise, there would be UB. + clang_analyzer_eval(old_size <= 10); // expected-warning{{TRUE}} + } + } + fclose(file); + free(buffer); +} + +void test_getline_negative_buffer() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + char *buffer = NULL; + size_t n = -1; + getline(&buffer, &n, file); // ok since posix 2018 + free(buffer); + fclose(file); +} + +void test_getline_negative_buffer_2(char *buffer) { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + size_t n = -1; + (void)getline(&buffer, &n, file); // ok + free(buffer); + fclose(file); +} diff --git a/clang/test/Analysis/stream.c b/clang/test/Analysis/stream.c index 7ba27740a937..ba5e66a4102e 100644 --- a/clang/test/Analysis/stream.c +++ b/clang/test/Analysis/stream.c @@ -4,6 +4,7 @@ // RUN: %clang_analyze_cc1 -triple=hexagon -analyzer-checker=core,alpha.unix.Stream,debug.ExprInspection -verify %s #include "Inputs/system-header-simulator.h" +#include "Inputs/system-header-simulator-for-malloc.h" #include "Inputs/system-header-simulator-for-valist.h" void clang_analyzer_eval(int); @@ -376,3 +377,75 @@ void fflush_on_open_failed_stream(void) { } fclose(F); } + +void getline_null_file() { + char *buffer = NULL; + size_t n = 0; + getline(&buffer, &n, NULL); // expected-warning {{Stream pointer might be NULL}} +} + +void getdelim_null_file() { + char *buffer = NULL; + size_t n = 0; + getdelim(&buffer, &n, '\n', NULL); // expected-warning {{Stream pointer might be NULL}} +} + +void getline_buffer_on_error() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + char *line = NULL; + size_t len = 0; + if (getline(&line, &len, file) == -1) { + if (line[0] == '\0') {} // expected-warning {{The left operand of '==' is a garbage value}} + } else { + if (line[0] == '\0') {} // no warning + } + + free(line); + fclose(file); +} + +void getline_ret_value() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + size_t n = 0; + char *buffer = NULL; + ssize_t r = getline(&buffer, &n, file); + + if (r > -1) { + // The return value does *not* include the terminating null byte. + // The buffer must be large enough to include it. + clang_analyzer_eval(n > r); // expected-warning{{TRUE}} + clang_analyzer_eval(buffer != NULL); // expected-warning{{TRUE}} + } + + fclose(file); + free(buffer); +} + + +void getline_buffer_size_negative() { + FILE *file = fopen("file.txt", "r"); + if (file == NULL) { + return; + } + + size_t n = -1; + clang_analyzer_eval((ssize_t)n >= 0); // expected-warning{{FALSE}} + char *buffer = NULL; + ssize_t r = getline(&buffer, &n, file); + + if (r > -1) { + clang_analyzer_eval((ssize_t)n > r); // expected-warning{{TRUE}} + clang_analyzer_eval(buffer != NULL); // expected-warning{{TRUE}} + } + + free(buffer); + fclose(file); +} -- GitLab From d8e5c0b4e546c73b2d10956a9517f1f2727702ae Mon Sep 17 00:00:00 2001 From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com> Date: Fri, 22 Mar 2024 07:01:01 -0400 Subject: [PATCH 239/296] [DXIL] Complete abs lowering (#86158) This change completes #86155 - `DXIL.td` - lowering `fabs` intrinsic to the float dxil op. - `DXILIntrinsicExpansion.cpp` - Add intrinsic expansion for the abs case. --- llvm/lib/Target/DirectX/DXIL.td | 2 + .../Target/DirectX/DXILIntrinsicExpansion.cpp | 23 +++++++++++ llvm/test/CodeGen/DirectX/abs-vec.ll | 34 +++++++++++++++++ llvm/test/CodeGen/DirectX/abs.ll | 38 +++++++++++++++++++ llvm/test/CodeGen/DirectX/fabs.ll | 32 ++++++++++++++++ 5 files changed, 129 insertions(+) create mode 100644 llvm/test/CodeGen/DirectX/abs-vec.ll create mode 100644 llvm/test/CodeGen/DirectX/abs.ll create mode 100644 llvm/test/CodeGen/DirectX/fabs.ll diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index 36eb29d53766..ba7cd6ae9183 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -255,6 +255,8 @@ class DXILOpMapping; def IsInf : DXILOpMapping<9, isSpecialFloat, int_dx_isinf, "Determines if the specified value is infinite.", [llvm_i1_ty, llvm_halforfloat_ty]>; diff --git a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp index 0db42bc0a0fb..b46564702c7a 100644 --- a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp +++ b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp @@ -33,6 +33,7 @@ using namespace llvm; static bool isIntrinsicExpansion(Function &F) { switch (F.getIntrinsicID()) { + case Intrinsic::abs: case Intrinsic::exp: case Intrinsic::dx_any: case Intrinsic::dx_clamp: @@ -46,6 +47,26 @@ static bool isIntrinsicExpansion(Function &F) { return false; } +static bool expandAbs(CallInst *Orig) { + Value *X = Orig->getOperand(0); + IRBuilder<> Builder(Orig->getParent()); + Builder.SetInsertPoint(Orig); + Type *Ty = X->getType(); + Type *EltTy = Ty->getScalarType(); + Constant *Zero = Ty->isVectorTy() + ? ConstantVector::getSplat( + ElementCount::getFixed( + cast(Ty)->getNumElements()), + ConstantInt::get(EltTy, 0)) + : ConstantInt::get(EltTy, 0); + auto *V = Builder.CreateSub(Zero, X); + auto *MaxCall = + Builder.CreateIntrinsic(Ty, Intrinsic::smax, {X, V}, nullptr, "dx.max"); + Orig->replaceAllUsesWith(MaxCall); + Orig->eraseFromParent(); + return true; +} + static bool expandIntegerDot(CallInst *Orig, Intrinsic::ID DotIntrinsic) { assert(DotIntrinsic == Intrinsic::dx_sdot || DotIntrinsic == Intrinsic::dx_udot); @@ -213,6 +234,8 @@ static bool expandClampIntrinsic(CallInst *Orig, Intrinsic::ID ClampIntrinsic) { static bool expandIntrinsic(Function &F, CallInst *Orig) { switch (F.getIntrinsicID()) { + case Intrinsic::abs: + return expandAbs(Orig); case Intrinsic::exp: return expandExpIntrinsic(Orig); case Intrinsic::dx_any: diff --git a/llvm/test/CodeGen/DirectX/abs-vec.ll b/llvm/test/CodeGen/DirectX/abs-vec.ll new file mode 100644 index 000000000000..1c40555eb390 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/abs-vec.ll @@ -0,0 +1,34 @@ +; RUN: opt -S -dxil-intrinsic-expansion < %s | FileCheck %s + +; Make sure dxil operation function calls for abs are generated for int vectors. + +; CHECK-LABEL: abs_i16Vec2 +define noundef <2 x i16> @abs_i16Vec2(<2 x i16> noundef %a) #0 { +entry: +; CHECK: sub <2 x i16> zeroinitializer, %a +; CHECK: call <2 x i16> @llvm.smax.v2i16(<2 x i16> %a, <2 x i16> %{{.*}}) + %elt.abs = call <2 x i16> @llvm.abs.v2i16(<2 x i16> %a, i1 false) + ret <2 x i16> %elt.abs +} + +; CHECK-LABEL: abs_i32Vec3 +define noundef <3 x i32> @abs_i32Vec3(<3 x i32> noundef %a) #0 { +entry: +; CHECK: sub <3 x i32> zeroinitializer, %a +; CHECK: call <3 x i32> @llvm.smax.v3i32(<3 x i32> %a, <3 x i32> %{{.*}}) + %elt.abs = call <3 x i32> @llvm.abs.v3i32(<3 x i32> %a, i1 false) + ret <3 x i32> %elt.abs +} + +; CHECK-LABEL: abs_i64Vec4 +define noundef <4 x i64> @abs_i64Vec4(<4 x i64> noundef %a) #0 { +entry: +; CHECK: sub <4 x i64> zeroinitializer, %a +; CHECK: call <4 x i64> @llvm.smax.v4i64(<4 x i64> %a, <4 x i64> %{{.*}}) + %elt.abs = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a, i1 false) + ret <4 x i64> %elt.abs +} + +declare <2 x i16> @llvm.abs.v2i16(<2 x i16>, i1 immarg) +declare <3 x i32> @llvm.abs.v3i32(<3 x i32>, i1 immarg) +declare <4 x i64> @llvm.abs.v4i64(<4 x i64>, i1 immarg) diff --git a/llvm/test/CodeGen/DirectX/abs.ll b/llvm/test/CodeGen/DirectX/abs.ll new file mode 100644 index 000000000000..822580e8c089 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/abs.ll @@ -0,0 +1,38 @@ +; RUN: opt -S -dxil-intrinsic-expansion < %s | FileCheck %s --check-prefixes=CHECK,EXPCHECK +; RUN: opt -S -dxil-op-lower < %s | FileCheck %s --check-prefixes=CHECK,DOPCHECK + +; Make sure dxil operation function calls for abs are generated for int16_t/int/int64_t. + +; CHECK-LABEL: abs_i16 +define noundef i16 @abs_i16(i16 noundef %a) { +entry: +; CHECK: sub i16 0, %a +; EXPCHECK: call i16 @llvm.smax.i16(i16 %a, i16 %{{.*}}) +; DOPCHECK: call i16 @dx.op.binary.i16(i32 37, i16 %a, i16 %{{.*}}) + %elt.abs = call i16 @llvm.abs.i16(i16 %a, i1 false) + ret i16 %elt.abs +} + +; CHECK-LABEL: abs_i32 +define noundef i32 @abs_i32(i32 noundef %a) { +entry: +; CHECK: sub i32 0, %a +; EXPCHECK: call i32 @llvm.smax.i32(i32 %a, i32 %{{.*}}) +; DOPCHECK: call i32 @dx.op.binary.i32(i32 37, i32 %a, i32 %{{.*}}) + %elt.abs = call i32 @llvm.abs.i32(i32 %a, i1 false) + ret i32 %elt.abs +} + +; CHECK-LABEL: abs_i64 +define noundef i64 @abs_i64(i64 noundef %a) { +entry: +; CHECK: sub i64 0, %a +; EXPCHECK: call i64 @llvm.smax.i64(i64 %a, i64 %{{.*}}) +; DOPCHECK: call i64 @dx.op.binary.i64(i32 37, i64 %a, i64 %{{.*}}) + %elt.abs = call i64 @llvm.abs.i64(i64 %a, i1 false) + ret i64 %elt.abs +} + +declare i16 @llvm.abs.i16(i16, i1 immarg) +declare i32 @llvm.abs.i32(i32, i1 immarg) +declare i64 @llvm.abs.i64(i64, i1 immarg) diff --git a/llvm/test/CodeGen/DirectX/fabs.ll b/llvm/test/CodeGen/DirectX/fabs.ll new file mode 100644 index 000000000000..3b3f8aa9a4a9 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/fabs.ll @@ -0,0 +1,32 @@ +; RUN: opt -S -dxil-op-lower < %s | FileCheck %s + +; Make sure dxil operation function calls for abs are generated for float, half, and double. + + +; CHECK-LABEL: fabs_half +define noundef half @fabs_half(half noundef %a) { +entry: + ; CHECK: call half @dx.op.unary.f16(i32 6, half %{{.*}}) + %elt.abs = call half @llvm.fabs.f16(half %a) + ret half %elt.abs +} + +; CHECK-LABEL: fabs_float +define noundef float @fabs_float(float noundef %a) { +entry: +; CHECK: call float @dx.op.unary.f32(i32 6, float %{{.*}}) + %elt.abs = call float @llvm.fabs.f32(float %a) + ret float %elt.abs +} + +; CHECK-LABEL: fabs_double +define noundef double @fabs_double(double noundef %a) { +entry: +; CHECK: call double @dx.op.unary.f64(i32 6, double %{{.*}}) + %elt.abs = call double @llvm.fabs.f64(double %a) + ret double %elt.abs +} + +declare half @llvm.fabs.f16(half) +declare float @llvm.fabs.f32(float) +declare double @llvm.fabs.f64(double) -- GitLab From 79c32eb03d9ee4dd0a913c4130bc87c5e8ce7908 Mon Sep 17 00:00:00 2001 From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com> Date: Fri, 22 Mar 2024 07:02:47 -0400 Subject: [PATCH 240/296] [DXIL] Add lowerings for cosine and floor (#86173) Completes #86170 Completes #86172 - `DXIL.td` - Add changes to lower the cosine and floor intrinsics to dxilOps. --- llvm/lib/Target/DirectX/DXIL.td | 6 ++++++ llvm/test/CodeGen/DirectX/cos.ll | 20 ++++++++++++++++++++ llvm/test/CodeGen/DirectX/cos_error.ll | 10 ++++++++++ llvm/test/CodeGen/DirectX/floor.ll | 20 ++++++++++++++++++++ llvm/test/CodeGen/DirectX/floor_error.ll | 10 ++++++++++ 5 files changed, 66 insertions(+) create mode 100644 llvm/test/CodeGen/DirectX/cos.ll create mode 100644 llvm/test/CodeGen/DirectX/cos_error.ll create mode 100644 llvm/test/CodeGen/DirectX/floor.ll create mode 100644 llvm/test/CodeGen/DirectX/floor_error.ll diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td index ba7cd6ae9183..f7e69ebae15b 100644 --- a/llvm/lib/Target/DirectX/DXIL.td +++ b/llvm/lib/Target/DirectX/DXIL.td @@ -260,6 +260,9 @@ def Abs : DXILOpMapping<6, unary, int_fabs, def IsInf : DXILOpMapping<9, isSpecialFloat, int_dx_isinf, "Determines if the specified value is infinite.", [llvm_i1_ty, llvm_halforfloat_ty]>; +def Cos : DXILOpMapping<12, unary, int_cos, + "Returns cosine(theta) for theta in radians.", + [llvm_halforfloat_ty, LLVMMatchType<0>]>; def Sin : DXILOpMapping<13, unary, int_sin, "Returns sine(theta) for theta in radians.", [llvm_halforfloat_ty, LLVMMatchType<0>]>; @@ -279,6 +282,9 @@ def Round : DXILOpMapping<26, unary, int_round, "Returns the input rounded to the nearest integer" "within a floating-point type.", [llvm_halforfloat_ty, LLVMMatchType<0>]>; +def Floor : DXILOpMapping<27, unary, int_floor, + "Returns the largest integer that is less than or equal to the input.", + [llvm_halforfloat_ty, LLVMMatchType<0>]>; def FMax : DXILOpMapping<35, binary, int_maxnum, "Float maximum. FMax(a,b) = a > b ? a : b">; def FMin : DXILOpMapping<36, binary, int_minnum, diff --git a/llvm/test/CodeGen/DirectX/cos.ll b/llvm/test/CodeGen/DirectX/cos.ll new file mode 100644 index 000000000000..00f2e2c3f6e5 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/cos.ll @@ -0,0 +1,20 @@ +; RUN: opt -S -dxil-op-lower < %s | FileCheck %s + +; Make sure dxil operation function calls for cos are generated for float and half. + +define noundef float @cos_float(float noundef %a) #0 { +entry: +; CHECK:call float @dx.op.unary.f32(i32 12, float %{{.*}}) + %elt.cos = call float @llvm.cos.f32(float %a) + ret float %elt.cos +} + +define noundef half @cos_half(half noundef %a) #0 { +entry: +; CHECK:call half @dx.op.unary.f16(i32 12, half %{{.*}}) + %elt.cos = call half @llvm.cos.f16(half %a) + ret half %elt.cos +} + +declare half @llvm.cos.f16(half) +declare float @llvm.cos.f32(float) diff --git a/llvm/test/CodeGen/DirectX/cos_error.ll b/llvm/test/CodeGen/DirectX/cos_error.ll new file mode 100644 index 000000000000..a074f5b493df --- /dev/null +++ b/llvm/test/CodeGen/DirectX/cos_error.ll @@ -0,0 +1,10 @@ +; RUN: not opt -S -dxil-op-lower %s 2>&1 | FileCheck %s + +; DXIL operation cos does not support double overload type +; CHECK: LLVM ERROR: Invalid Overload Type + +define noundef double @cos_double(double noundef %a) { +entry: + %elt.cos = call double @llvm.cos.f64(double %a) + ret double %elt.cos +} diff --git a/llvm/test/CodeGen/DirectX/floor.ll b/llvm/test/CodeGen/DirectX/floor.ll new file mode 100644 index 000000000000..b033e2eaa491 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/floor.ll @@ -0,0 +1,20 @@ +; RUN: opt -S -dxil-op-lower < %s | FileCheck %s + +; Make sure dxil operation function calls for floor are generated for float and half. + +define noundef float @floor_float(float noundef %a) #0 { +entry: +; CHECK:call float @dx.op.unary.f32(i32 27, float %{{.*}}) + %elt.floor = call float @llvm.floor.f32(float %a) + ret float %elt.floor +} + +define noundef half @floor_half(half noundef %a) #0 { +entry: +; CHECK:call half @dx.op.unary.f16(i32 27, half %{{.*}}) + %elt.floor = call half @llvm.floor.f16(half %a) + ret half %elt.floor +} + +declare half @llvm.floor.f16(half) +declare float @llvm.floor.f32(float) diff --git a/llvm/test/CodeGen/DirectX/floor_error.ll b/llvm/test/CodeGen/DirectX/floor_error.ll new file mode 100644 index 000000000000..3b51a4b543b7 --- /dev/null +++ b/llvm/test/CodeGen/DirectX/floor_error.ll @@ -0,0 +1,10 @@ +; RUN: not opt -S -dxil-op-lower %s 2>&1 | FileCheck %s + +; DXIL operation floor does not support double overload type +; CHECK: LLVM ERROR: Invalid Overload Type + +define noundef double @floor_double(double noundef %a) { +entry: + %elt.floor = call double @llvm.floor.f64(double %a) + ret double %elt.floor +} -- GitLab From db33444ffa7e210e7040d8def958a14171f52eef Mon Sep 17 00:00:00 2001 From: "Oleksandr \"Alex\" Zinenko" Date: Fri, 22 Mar 2024 12:03:28 +0100 Subject: [PATCH 241/296] CODEOWNERS: extend scope of MLIR transform dialect There are a bunch of related directories in another dialects. --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index fea132c8fe78..77ba81c58c5d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -96,6 +96,7 @@ clang/test/AST/Interp/ @tbaederr # Transform Dialect in MLIR. /mlir/include/mlir/Dialect/Transform/* @ftynse @nicolasvasilache /mlir/lib/Dialect/Transform/* @ftynse @nicolasvasilache +/mlir/**/*TransformOps* @ftynse @nicolasvasilache # SPIR-V Dialect in MLIR. /mlir/**/SPIRV/ @antiagainst @kuhar -- GitLab From e925968e7815ac3810fdb54bb884b8a8bed02eb5 Mon Sep 17 00:00:00 2001 From: Balazs Benics Date: Fri, 22 Mar 2024 12:04:44 +0100 Subject: [PATCH 242/296] [analyzer] Support C++23 static operator calls (#84972) Made by following: https://github.com/llvm/llvm-project/pull/83585#issuecomment-1980340866 Thanks for the details Tomek! CPP-5080 --- clang/docs/ReleaseNotes.rst | 1 + .../Core/PathSensitive/CallEvent.h | 72 +++++++++++++++++++ clang/lib/StaticAnalyzer/Core/CallEvent.cpp | 5 +- .../Core/ExprEngineCallAndReturn.cpp | 1 + clang/test/Analysis/cxx23-static-operator.cpp | 38 ++++++++++ 5 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 clang/test/Analysis/cxx23-static-operator.cpp diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index fd12bb41be47..45b2e01af997 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -576,6 +576,7 @@ Static Analyzer - Fixed crashing on loops if the loop variable was declared in switch blocks but not under any case blocks if ``unroll-loops=true`` analyzer config is set. (#GH68819) +- Support C++23 static operator calls. (#GH84972) New features ^^^^^^^^^^^^ diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h index 0d36587484bf..549c864dc91e 100644 --- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h +++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h @@ -59,6 +59,7 @@ namespace ento { enum CallEventKind { CE_Function, + CE_CXXStaticOperator, CE_CXXMember, CE_CXXMemberOperator, CE_CXXDestructor, @@ -709,6 +710,77 @@ public: } }; +/// Represents a static C++ operator call. +/// +/// "A" in this example. +/// However, "B" and "C" are represented by SimpleFunctionCall. +/// \code +/// struct S { +/// int pad; +/// static void operator()(int x, int y); +/// }; +/// S s{10}; +/// void (*fptr)(int, int) = &S::operator(); +/// +/// s(1, 2); // A +/// S::operator()(1, 2); // B +/// fptr(1, 2); // C +/// \endcode +class CXXStaticOperatorCall : public SimpleFunctionCall { + friend class CallEventManager; + +protected: + CXXStaticOperatorCall(const CXXOperatorCallExpr *CE, ProgramStateRef St, + const LocationContext *LCtx, + CFGBlock::ConstCFGElementRef ElemRef) + : SimpleFunctionCall(CE, St, LCtx, ElemRef) {} + CXXStaticOperatorCall(const CXXStaticOperatorCall &Other) = default; + + void cloneTo(void *Dest) const override { + new (Dest) CXXStaticOperatorCall(*this); + } + +public: + const CXXOperatorCallExpr *getOriginExpr() const override { + return cast(SimpleFunctionCall::getOriginExpr()); + } + + unsigned getNumArgs() const override { + // Ignore the object parameter that is not used for static member functions. + assert(getOriginExpr()->getNumArgs() > 0); + return getOriginExpr()->getNumArgs() - 1; + } + + const Expr *getArgExpr(unsigned Index) const override { + // Ignore the object parameter that is not used for static member functions. + return getOriginExpr()->getArg(Index + 1); + } + + std::optional + getAdjustedParameterIndex(unsigned ASTArgumentIndex) const override { + // Ignore the object parameter that is not used for static member functions. + if (ASTArgumentIndex == 0) + return std::nullopt; + return ASTArgumentIndex - 1; + } + + unsigned getASTArgumentIndex(unsigned CallArgumentIndex) const override { + // Account for the object parameter for the static member function. + return CallArgumentIndex + 1; + } + + OverloadedOperatorKind getOverloadedOperator() const { + return getOriginExpr()->getOperator(); + } + + Kind getKind() const override { return CE_CXXStaticOperator; } + StringRef getKindAsString() const override { return "CXXStaticOperatorCall"; } + + static bool classof(const CallEvent *CA) { + return CA->getKind() == CE_CXXStaticOperator; + } +}; + /// Represents a non-static C++ member function call. /// /// Example: \c obj.fun() diff --git a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp index bc14aea27f67..0e317ec765ec 100644 --- a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp +++ b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp @@ -1408,9 +1408,12 @@ CallEventManager::getSimpleCall(const CallExpr *CE, ProgramStateRef State, if (const auto *OpCE = dyn_cast(CE)) { const FunctionDecl *DirectCallee = OpCE->getDirectCallee(); - if (const auto *MD = dyn_cast(DirectCallee)) + if (const auto *MD = dyn_cast(DirectCallee)) { if (MD->isImplicitObjectMemberFunction()) return create(OpCE, State, LCtx, ElemRef); + if (MD->isStatic()) + return create(OpCE, State, LCtx, ElemRef); + } } else if (CE->getCallee()->getType()->isBlockPointerType()) { return create(CE, State, LCtx, ElemRef); diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp index 4755b6bfa6dc..9d3e4fc944fb 100644 --- a/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp +++ b/clang/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp @@ -846,6 +846,7 @@ ExprEngine::mayInlineCallKind(const CallEvent &Call, const ExplodedNode *Pred, const StackFrameContext *CallerSFC = CurLC->getStackFrame(); switch (Call.getKind()) { case CE_Function: + case CE_CXXStaticOperator: case CE_Block: break; case CE_CXXMember: diff --git a/clang/test/Analysis/cxx23-static-operator.cpp b/clang/test/Analysis/cxx23-static-operator.cpp new file mode 100644 index 000000000000..f380bd0dfa42 --- /dev/null +++ b/clang/test/Analysis/cxx23-static-operator.cpp @@ -0,0 +1,38 @@ +// RUN: %clang_analyze_cc1 -std=c++2b -verify %s \ +// RUN: -analyzer-checker=core,debug.ExprInspection + +template void clang_analyzer_dump(T); + +struct Adder { + int data; + static int operator()(int x, int y) { + clang_analyzer_dump(x); // expected-warning {{1}} + clang_analyzer_dump(y); // expected-warning {{2}} + return x + y; + } +}; + +void static_operator_call_inlines() { + Adder s{10}; + clang_analyzer_dump(s(1, 2)); // expected-warning {{3}} +} + +struct DataWithCtor { + int x; + int y; + DataWithCtor(int parm) : x(parm + 10), y(parm + 20) { + clang_analyzer_dump(this); // expected-warning {{&v}} + } +}; + +struct StaticSubscript { + static void operator[](DataWithCtor v) { + clang_analyzer_dump(v.x); // expected-warning {{20}} + clang_analyzer_dump(v.y); // expected-warning {{30}} + } +}; + +void top() { + StaticSubscript s; + s[DataWithCtor{10}]; +} -- GitLab From 74c3150ffc86a149abc68acdf8af1eed1ea0f038 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 22 Mar 2024 11:23:25 +0000 Subject: [PATCH 243/296] [X86] Add shuffle tests from Issue #86076 SLP should be doing a better job, but both shuffles lower to poorer codegen than necessary --- .../CodeGen/X86/vector-shuffle-512-v16.ll | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll index dfa7f2dbdaee..c981d973fef3 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll @@ -177,6 +177,36 @@ define <16 x float> @shuffle_v16f32_02_03_16_17_06_07_20_21_10_11_24_25_14_15_28 ret <16 x float> %shuffle } +; PR86076 +define <16 x float> @shuffle_f32_v16f32_00_08_01_09_02_10_03_11_04_12_05_13_06_14_07_15(float %a0, float %a1) { +; ALL-LABEL: shuffle_f32_v16f32_00_08_01_09_02_10_03_11_04_12_05_13_06_14_07_15: +; ALL: # %bb.0: +; ALL-NEXT: vbroadcastss %xmm0, %ymm0 +; ALL-NEXT: vbroadcastss %xmm1, %ymm1 +; ALL-NEXT: vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] +; ALL-NEXT: vinsertf64x4 $1, %ymm0, %zmm0, %zmm0 +; ALL-NEXT: retq + %v0 = insertelement <8 x float> poison, float %a0, i64 0 + %v1 = insertelement <8 x float> poison, float %a1, i64 0 + %b0 = shufflevector <8 x float> %v0, <8 x float> poison, <8 x i32> zeroinitializer + %b1 = shufflevector <8 x float> %v1, <8 x float> poison, <8 x i32> zeroinitializer + %r = shufflevector <8 x float> %b0, <8 x float> %b1, <16 x i32> + ret <16 x float> %r +} + +; PR86076 +define <16 x float> @shuffle_f32_v16f32_00_08_00_08_00_08_00_08_00_08_00_08_00_08_00_08(float %a0, float %a1) { +; ALL-LABEL: shuffle_f32_v16f32_00_08_00_08_00_08_00_08_00_08_00_08_00_08_00_08: +; ALL: # %bb.0: +; ALL-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],zero,zero +; ALL-NEXT: vbroadcastsd %xmm0, %zmm0 +; ALL-NEXT: retq + %v0 = insertelement <8 x float> poison, float %a0, i64 0 + %v1 = insertelement <8 x float> poison, float %a1, i64 0 + %sv = shufflevector <8 x float> %v0, <8 x float> %v1, <16 x i32> + ret <16 x float> %sv +} + define <16 x i32> @shuffle_v16i32_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00(<16 x i32> %a, <16 x i32> %b) { ; ALL-LABEL: shuffle_v16i32_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00: ; ALL: # %bb.0: -- GitLab From ceabaa7e7a2d02b20cbd2b31e8336dedb1d4d9f5 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 22 Mar 2024 11:48:03 +0000 Subject: [PATCH 244/296] [DAG] Fix some missing formatting when I rewrote the SUB(MAX,MIN) -> ABD patterns. NFC. --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index db81f9199170..dcd0310734ad 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -4024,13 +4024,13 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { } // smax(a,b) - smin(a,b) --> abds(a,b) - if (hasOperation(ISD::ABDS, VT) && + if (hasOperation(ISD::ABDS, VT) && sd_match(N0, m_SMax(m_Value(A), m_Value(B))) && sd_match(N1, m_SMin(m_Specific(A), m_Specific(B)))) return DAG.getNode(ISD::ABDS, DL, VT, A, B); // umax(a,b) - umin(a,b) --> abdu(a,b) - if (hasOperation(ISD::ABDU, VT) && + if (hasOperation(ISD::ABDU, VT) && sd_match(N0, m_UMax(m_Value(A), m_Value(B))) && sd_match(N1, m_UMin(m_Specific(A), m_Specific(B)))) return DAG.getNode(ISD::ABDU, DL, VT, A, B); -- GitLab From c41286af3f30e099556c6edbef0001466afaefcb Mon Sep 17 00:00:00 2001 From: Pablo Antonio Martinez Date: Fri, 22 Mar 2024 11:53:29 +0000 Subject: [PATCH 245/296] [mlir][linalg] Emit a warning when tile_using_forall generates non thread-safe code (#80813) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Description** The documentation of `transform.structured.tile_using_forall` says: _"It is the user’s responsibility to ensure that num_threads/tile_sizes is a valid tiling specification (i.e. that only tiles parallel dimensions, e.g. in the Linalg case)."_ In other words, tiling a non-parallel dimension would generate code with data races which is not safe to parallelize. For example, consider this example (included in the tests in this PR): ``` func.func @tile_thread_safety2(%arg0: tensor<100x300x8xf32>, %arg1: tensor<300x8xf32>) -> tensor<300x8xf32> { %0 = scf.forall (%arg2) in (8) shared_outs(%arg3 = %arg1) -> (tensor<300x8xf32>) { %1 = affine.min #map(%arg2) %2 = affine.max #map1(%1) %3 = affine.apply #map2(%arg2) %extracted_slice = tensor.extract_slice %arg0[%3, 0, 0] [%2, 300, 8] [1, 1, 1] : tensor<100x300x8xf32> to tensor %4 = linalg.generic {indexing_maps = [#map3, #map4], iterator_types = ["reduction", "parallel", "parallel"]} ins(%extracted_slice : tensor) outs(%arg3 : tensor<300x8xf32>) { ^bb0(%in: f32, %out: f32): %5 = arith.addf %in, %out : f32 linalg.yield %5 : f32 } -> tensor<300x8xf32> scf.forall.in_parallel { tensor.parallel_insert_slice %4 into %arg3[0, 0] [300, 8] [1, 1] : tensor<300x8xf32> into tensor<300x8xf32> } } return %0 : tensor<300x8xf32> } ``` We can easily see that this is not safe to parallelize because all threads would be writing to the same position in `%arg3` (in the `scf.forall.in_parallel`. This PR detects wether it's safe to `tile_using_forall` and emits a warning in the case it is not. **Brief explanation** It first generates a vector of affine expressions representing the tile values and stores it in `dimExprs`. These affine expressions are compared with the affine expressions coming from the results of the affine map of each output in the linalg op. So going back to the previous example, the original transform is: ``` #map = affine_map<(d0, d1, d2) -> (d0, d1, d2)> #map1 = affine_map<(d0, d1, d2) -> (d1, d2)> func.func @tile_thread_safety2(%arg0: tensor<100x300x8xf32>, %arg1: tensor<300x8xf32>) -> tensor<300x8xf32> { // expected-warning@+1 {{tiling is not thread safe at axis #0}} %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["reduction", "parallel", "parallel"]} ins(%arg0 : tensor<100x300x8xf32>) outs(%arg1 : tensor<300x8xf32>) { ^bb0(%in: f32, %out: f32): %1 = arith.addf %in, %out : f32 linalg.yield %1 : f32 } -> tensor<300x8xf32> return %0 : tensor<300x8xf32> } module attributes {transform.with_named_sequence} { transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op %forall, %tiled_generic = transform.structured.tile_using_forall %0 num_threads [8] : (!transform.any_op) -> (!transform.any_op, !transform.any_op) transform.yield } } ``` The `num_threads` attribute would be represented as `(d0)`. Because the linalg op has only one output (`arg1`) it would only check against the results of `#map1`, which are `(d1, d2)`. The idea is to check that all affine expressions in `dimExprs` are present in the output affine map. In this example, `d0` is not in `(d1, d2)`, so tiling that axis is considered not thread safe. --- .../Linalg/TransformOps/LinalgTransformOps.td | 4 +- mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp | 38 ++++- mlir/test/Dialect/Linalg/tile-to-forall.mlir | 141 ++++++++++++++++++ 3 files changed, 180 insertions(+), 3 deletions(-) diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td index 4f34016066b4..c260fe3f7a46 100644 --- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td +++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td @@ -1918,7 +1918,9 @@ def TileUsingForallOp : It is the user's responsibility to ensure that `num_threads/tile_sizes` is a valid tiling specification (i.e. that only tiles parallel dimensions, - e.g. in the Linalg case). + e.g. in the Linalg case). If the dimension is not parallelizable, a warning + is issued to notify the user that the generated code is not safe to + parallelize. If non-empty, the `mapping` is added as an attribute to the resulting `scf.forall`. diff --git a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp index 30aed850bed8..462f692615fa 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp @@ -304,6 +304,28 @@ static void calculateTileOffsetsAndSizes( } } +/// Returns a vector of bools representing if, for each axis, `op` can be tiled +/// without incurring in a race condition and thus it is thread-safe to do the +/// tiling. This is checked by iterating over numThreads and ensuring that the +/// corresponding iterator type is "parallel". If it is not, then we know that +/// such dimension is unsafe to tile. +SmallVector safeToTileToForall(mlir::MLIRContext *ctx, LinalgOp linalgOp, + ArrayRef numThreads) { + auto iterators = linalgOp.getIteratorTypesArray(); + SmallVector safeToTile(numThreads.size(), true); + + for (unsigned i = 0, e = numThreads.size(); i != e; i++) { + if (auto attr = llvm::dyn_cast_if_present(numThreads[i])) { + if (cast(attr).getValue().getSExtValue() > 1) { + safeToTile[i] = iterators[i] == utils::IteratorType::parallel; + } + } else { + safeToTile[i] = iterators[i] == utils::IteratorType::parallel; + } + } + return safeToTile; +} + /// Rewrite a TilingInterface `op` to a tiled `scf.forall`. The /// tiling is specified by the number of tiles/threads `numThreads` and the /// optional nominal tile size `nominalTileSizes`. If `nominalTilSizes` is @@ -314,8 +336,10 @@ static void calculateTileOffsetsAndSizes( /// size of data. /// It is the user's responsibility to ensure that `numThreads` is a valid /// tiling specification (i.e. that only tiles parallel dimensions, e.g. in the -/// Linalg case). If `omitTileOffsetBoundsCheck` is true, then the function will -/// assume that `tileSize[i] * (numThread[i] -1) <= dimSize[i]` holds. +/// Linalg case). If the dimension is not parallelizable, a warning is issued to +/// notify the user that the generated code is not safe to parallelize. If +/// `omitTileOffsetBoundsCheck` is true, then the function will assume that +/// `tileSize[i] * (numThread[i] -1) <= dimSize[i]` holds. static FailureOr tileToForallOpImpl( RewriterBase &b, TilingInterface op, ArrayRef numThreads, std::optional> nominalTileSizes, @@ -344,6 +368,16 @@ static FailureOr tileToForallOpImpl( return getValueOrCreateConstantIndexOp(b, loc, ofr); })); + LinalgOp linalgOp = dyn_cast(op.getOperation()); + if (linalgOp) { + // Check if tiling is thread safe and print a warning if not. + SmallVector tilingSafety = + safeToTileToForall(b.getContext(), linalgOp, numThreads); + for (size_t i = 0; i < tilingSafety.size(); i++) + if (!tilingSafety[i]) + op.emitWarning() << "tiling is not thread safe at axis #" << i; + } + // 1. Create the ForallOp. We don't use the lambda body-builder // version because we require the use of RewriterBase in the body, so we // manually move the insertion point to the body below. diff --git a/mlir/test/Dialect/Linalg/tile-to-forall.mlir b/mlir/test/Dialect/Linalg/tile-to-forall.mlir index abd807b3e4d3..12e2dea5530b 100644 --- a/mlir/test/Dialect/Linalg/tile-to-forall.mlir +++ b/mlir/test/Dialect/Linalg/tile-to-forall.mlir @@ -586,3 +586,144 @@ module attributes {transform.with_named_sequence} { transform.yield } } + +// ----- + +#map = affine_map<(d0, d1) -> (d0, d1)> +#map1 = affine_map<(d0, d1) -> (d0)> + +func.func @tile_thread_safety1(%arg0: tensor<100x300xf32>, %arg1: tensor<100xf32>) -> tensor<100xf32> { + // expected-warning@below {{tiling is not thread safe at axis #1}} + %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "reduction"]} ins(%arg0 : tensor<100x300xf32>) outs(%arg1 : tensor<100xf32>) { + ^bb0(%in: f32, %out: f32): + %1 = arith.addf %in, %out : f32 + linalg.yield %1 : f32 + } -> tensor<100xf32> + return %0 : tensor<100xf32> +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %forall, %tiled_generic = transform.structured.tile_using_forall %0 num_threads [4, 2] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} + +// ----- + +#map = affine_map<(d0, d1, d2) -> (d0, d1, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d1, d2)> + +func.func @tile_thread_safety2(%arg0: tensor<100x300x8xf32>, %arg1: tensor<300x8xf32>) -> tensor<300x8xf32> { + // expected-warning@below {{tiling is not thread safe at axis #0}} + %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["reduction", "parallel", "parallel"]} ins(%arg0 : tensor<100x300x8xf32>) outs(%arg1 : tensor<300x8xf32>) { + ^bb0(%in: f32, %out: f32): + %1 = arith.addf %in, %out : f32 + linalg.yield %1 : f32 + } -> tensor<300x8xf32> + return %0 : tensor<300x8xf32> +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %forall, %tiled_generic = transform.structured.tile_using_forall %0 num_threads [8] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} + +// ----- + +#map = affine_map<(d0, d1, d2) -> (d0, d1, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d0, d2)> + +func.func @tile_thread_safety3(%arg0: tensor<100x300x8xf32>, %arg1: tensor<100x8xf32>) -> tensor<100x8xf32> { + // expected-warning@below {{tiling is not thread safe at axis #1}} + %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "reduction", "parallel"]} ins(%arg0 : tensor<100x300x8xf32>) outs(%arg1 : tensor<100x8xf32>) { + ^bb0(%in: f32, %out: f32): + %1 = arith.addf %in, %out : f32 + linalg.yield %1 : f32 + } -> tensor<100x8xf32> + return %0 : tensor<100x8xf32> +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %forall, %tiled_generic = transform.structured.tile_using_forall %0 num_threads [8, 4, 2] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} + +// ----- + +#map = affine_map<(d0, d1, d2) -> (d0, d1, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d0, d2)> +#map2 = affine_map<(d0, d1, d2) -> (d2)> + +func.func @tile_thread_safety4(%arg0: tensor<100x300x8xf32>, %arg1: tensor<100x8xf32>, %arg2 : tensor<8xf32>) -> (tensor<100x8xf32>, tensor<8xf32>) { + // expected-warning@+2 {{tiling is not thread safe at axis #0}} + // expected-warning@below {{tiling is not thread safe at axis #1}} + %0:2 = linalg.generic {indexing_maps = [#map, #map1, #map2], iterator_types = ["reduction", "reduction", "parallel"]} ins(%arg0 : tensor<100x300x8xf32>) outs(%arg1, %arg2 : tensor<100x8xf32>, tensor<8xf32>) { + ^bb0(%in: f32, %out1: f32, %out2: f32): + %1 = arith.addf %in, %out1 : f32 + %2 = arith.addf %in, %out2 : f32 + linalg.yield %1, %2 : f32, f32 + } -> (tensor<100x8xf32>, tensor<8xf32>) + return %0#0, %0#1 : tensor<100x8xf32>, tensor<8xf32> +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %forall, %tiled_generic = transform.structured.tile_using_forall %0 num_threads [8, 4, 2] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} + +// ----- + +#map = affine_map<(d0, d1) -> (d0, d1)> +#map1 = affine_map<(d0, d1) -> (d0)> + +func.func @tile_thread_safety5(%arg0: tensor<100x300xf32>, %arg1: tensor<100xf32>) -> tensor<100xf32> { + // expected-warning@below {{tiling is not thread safe at axis #1}} + %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "reduction"]} ins(%arg0 : tensor<100x300xf32>) outs(%arg1 : tensor<100xf32>) { + ^bb0(%in: f32, %out: f32): + %1 = arith.addf %in, %out : f32 + linalg.yield %1 : f32 + } -> tensor<100xf32> + return %0 : tensor<100xf32> +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %forall, %tiled_generic = transform.structured.tile_using_forall %0 tile_sizes [10, 1] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} + +// ----- + +func.func @tile_thread_safety6(%A: tensor, %B: tensor, %C: tensor) -> tensor { + // expected-warning@below {{tiling is not thread safe at axis #2}} + %0 = linalg.matmul ins(%A, %B : tensor, tensor) + outs(%C : tensor) -> (tensor) + return %0 : tensor +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.matmul"]} in %arg0 : (!transform.any_op) -> !transform.any_op + %forall, %tiled_generic = transform.structured.tile_using_forall %0 num_threads [2, 0, 8] + : (!transform.any_op) -> (!transform.any_op, !transform.any_op) + transform.yield + } +} -- GitLab From d03f470cbdbae3f86469ea4d79bb54d3ef680512 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Fri, 22 Mar 2024 06:54:33 -0500 Subject: [PATCH 246/296] [Clang] Make '-frtlib-add-rpath' include the standard library directory (#86217) Summary: The original intention of the `openmp-add-rpath` option was to add the rpath to the language runtime directory. However, the current implementation only adds it to the compiler's resource directory. This patch adds support for appending the `-rpath` to the compiler's standard library directory as well. Currently this is `/../lib/`. --- clang/lib/Driver/ToolChains/CommonArgs.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 447886531363..6b1fbba7abd0 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -1142,7 +1142,11 @@ void tools::addArchSpecificRPath(const ToolChain &TC, const ArgList &Args, options::OPT_fno_rtlib_add_rpath, false)) return; - for (const auto &CandidateRPath : TC.getArchSpecificLibPaths()) { + SmallVector CandidateRPaths(TC.getArchSpecificLibPaths()); + if (const auto CandidateRPath = TC.getStdlibPath()) + CandidateRPaths.emplace_back(*CandidateRPath); + + for (const auto &CandidateRPath : CandidateRPaths) { if (TC.getVFS().exists(CandidateRPath)) { CmdArgs.push_back("-rpath"); CmdArgs.push_back(Args.MakeArgString(CandidateRPath)); -- GitLab From d51f1c442b1dc999267726a33e25b7d019726c89 Mon Sep 17 00:00:00 2001 From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com> Date: Fri, 22 Mar 2024 08:14:26 -0400 Subject: [PATCH 247/296] [DirectX][Docs] Add DXILIntrinsicExpansion Pass to DXILArchitecture.rst (#86198) Completes #84839 --------- Co-authored-by: Farzon Lotfi --- llvm/docs/DirectX/DXILArchitecture.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/llvm/docs/DirectX/DXILArchitecture.rst b/llvm/docs/DirectX/DXILArchitecture.rst index d6712bea4f77..32b1e72deae7 100644 --- a/llvm/docs/DirectX/DXILArchitecture.rst +++ b/llvm/docs/DirectX/DXILArchitecture.rst @@ -61,6 +61,19 @@ on the utilities described in "Common Code" above in order to share logic with both the DirectX backend and with Clang's codegen of HLSL support as much as possible. +The DirectX Intrinsic Expansion Pass +==================================== +There are intrinsics that don't map directly to DXIL Ops. In some cases +an intrinsic needs to be expanded to a set of LLVM IR instructions. In +other cases an intrinsic needs modifications to the arguments or return +values of a DXIL Op. The `DXILIntrinsicExpansion` pass handles all +the cases where our intrinsics don't have a one to one mapping. This +pass may also be used when the expansion is specific to DXIL to keep +implementation details out of CodeGen. Finally, there is an expectation +that we maintain vector types through this pass. Therefore, best +practice would be to avoid scalarization in this pass. + + The DirectX Backend =================== -- GitLab From fe64b26df9429c82f706424dcdae3d65723c3e5e Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 22 Mar 2024 12:20:52 +0000 Subject: [PATCH 248/296] NFC Rename LoadBitcodeIntoNewDbgInforFormat to LoadBitcodeIntoNewDbgInfoFormat (drop additional 'r' before Format) --- llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 3fc8141381c6..6ee93f17792b 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -104,7 +104,7 @@ static cl::opt ExpandConstantExprs( /// of debug intrinsics). UNSET is treated as FALSE, so the default action /// is to do nothing. Individual tools can override this to incrementally add /// support for the RemoveDIs format. -cl::opt LoadBitcodeIntoNewDbgInforFormat( +cl::opt LoadBitcodeIntoNewDbgInfoFormat( "load-bitcode-into-experimental-debuginfo-iterators", cl::Hidden, cl::desc("Load bitcode directly into the new debug info format (regardless " "of input format)")); @@ -4300,11 +4300,11 @@ Error BitcodeReader::parseGlobalIndirectSymbolRecord( Error BitcodeReader::parseModule(uint64_t ResumeBit, bool ShouldLazyLoadMetadata, ParserCallbacks Callbacks) { - // Load directly into RemoveDIs format if LoadBitcodeIntoNewDbgInforFormat + // Load directly into RemoveDIs format if LoadBitcodeIntoNewDbgInfoFormat // has been set to true (default action: load into the old debug format). TheModule->IsNewDbgInfoFormat = UseNewDbgInfoFormat && - LoadBitcodeIntoNewDbgInforFormat == cl::boolOrDefault::BOU_TRUE; + LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_TRUE; this->ValueTypeCallback = std::move(Callbacks.ValueType); if (ResumeBit) { -- GitLab From e1f50fdc03efecb5da39c1df4fc08d2ce5da90e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingo=20M=C3=BCller?= Date: Fri, 22 Mar 2024 13:35:36 +0100 Subject: [PATCH 249/296] [mlir] Remove unused and untested `shouldSplitInputFile`. (#85622) This was changed by #84765 but turned out to be buggy. Since it isn't used and isn't tested, it is probably best to remove it. --- mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h | 1 - 1 file changed, 1 deletion(-) diff --git a/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h b/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h index 8adc80908de1..4f7f83cdb473 100644 --- a/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h +++ b/mlir/include/mlir/Tools/mlir-opt/MlirOptMain.h @@ -144,7 +144,6 @@ public: splitInputFileFlag = std::move(splitMarker); return *this; } - bool shouldSplitInputFile() const { return splitInputFileFlag.empty(); } StringRef inputSplitMarker() const { return splitInputFileFlag; } /// Set whether to merge the output chunks into one file using the given -- GitLab From 83da7b6338053ca04cf0afe3c70ef5b8a9f6d300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingo=20M=C3=BCller?= Date: Fri, 22 Mar 2024 13:36:07 +0100 Subject: [PATCH 250/296] [mlir] Extend split marker tests of `mlir-opt` and `mlir-pdll`. (#85620) Recently #84765 made the split markers of various tools configurable but did not test *not* using the split markers for two of them. This PR adds those tests. --- mlir/test/mlir-opt/split-markers.mlir | 32 ++++++++++++++++---------- mlir/test/mlir-pdll/split-markers.pdll | 12 ++++++++++ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/mlir/test/mlir-opt/split-markers.mlir b/mlir/test/mlir-opt/split-markers.mlir index 665a37f31770..f372654bcc8d 100644 --- a/mlir/test/mlir-opt/split-markers.mlir +++ b/mlir/test/mlir-opt/split-markers.mlir @@ -1,12 +1,17 @@ // Check near-miss mechanics: // RUN: mlir-opt --split-input-file --verify-diagnostics %s 2> %t \ -// RUN: && FileCheck --input-file %t %s +// RUN: && FileCheck --input-file %t --check-prefix=CHECK-DEFAULT %s // RUN: cat %t // Check that (1) custom input splitter and (2) custom output splitters work. -// RUN: mlir-opt %s -split-input-file="// CHECK: ""----" \ +// RUN: mlir-opt %s -split-input-file="// CHECK-DEFAULT: ""----" \ // RUN: -output-split-marker="// ---- next split ----" \ -// RUN: | FileCheck --check-prefix=CHECK-SPLITTERS %s +// RUN: | FileCheck --check-prefix=CHECK-CUSTOM %s + +// Check that (3) the input is not split if `-split-input-file` is not given. +// RUN: mlir-opt %s 2> %t \ +// RUN: || FileCheck --input-file %t --check-prefix=CHECK-NOSPLIT %s +// RUN: cat %t func.func @main() {return} @@ -14,22 +19,25 @@ func.func @main() {return} // expected-note @+1 {{see existing symbol definition here}} func.func @foo() { return } -// CHECK: warning: near miss with file split marker -// CHECK: ---- +// CHECK-DEFAULT: warning: near miss with file split marker +// CHECK-DEFAULT: ---- // ---- +// CHECK-NOSPLIT: error: redefinition of symbol named 'main' +func.func @main() {return} + // expected-error @+1 {{redefinition of symbol named 'foo'}} func.func @foo() { return } -// CHECK: warning: near miss with file split marker -// CHECK: ---- +// CHECK-DEFAULT: warning: near miss with file split marker +// CHECK-DEFAULT: ---- // ---- func.func @bar2() {return } // No error flagged at the end for a near miss. // ---- -// CHECK-SPLITTERS: module -// CHECK-SPLITTERS: ---- next split ---- -// CHECK-SPLITTERS: module -// CHECK-SPLITTERS: ---- next split ---- -// CHECK-SPLITTERS: module +// CHECK-CUSTOM: module +// CHECK-CUSTOM: ---- next split ---- +// CHECK-CUSTOM: module +// CHECK-CUSTOM: ---- next split ---- +// CHECK-CUSTOM: module diff --git a/mlir/test/mlir-pdll/split-markers.pdll b/mlir/test/mlir-pdll/split-markers.pdll index 45e409a83836..2b314538004f 100644 --- a/mlir/test/mlir-pdll/split-markers.pdll +++ b/mlir/test/mlir-pdll/split-markers.pdll @@ -9,6 +9,10 @@ // RUN: -split-input-file="// ""=====" -output-split-marker "// #####" \ // RUN: | FileCheck -check-prefix=CHECK-CUSTOM %s +// Check that (5) the input is not split if `-split-input-file` is not given. +// RUN: mlir-pdll %s \ +// RUN: | FileCheck -check-prefix=CHECK-NOSPLIT %s + // CHECK-DEFAULT: Module // CHECK-DEFAULT-NEXT: PatternDecl // CHECK-DEFAULT-NOT: PatternDecl @@ -25,6 +29,14 @@ // CHECK-CUSTOM-NEXT: PatternDecl // CHECK-CUSTOM-NOT: PatternDecl +// CHECK-NOSPLIT: Module +// CHECK-NOSPLIT-NEXT: PatternDecl +// CHECK-NOSPLIT-NOT: Module +// CHECK-NOSPLIT: PatternDecl +// CHECK-NOSPLIT-NOT: Module +// CHECK-NOSPLIT: PatternDecl +// CHECK-NOSPLIT-NOT: Module + Pattern => erase op; // ----- -- GitLab From 04a6e0f1634f9a53120c27a30250d26dff4ada1c Mon Sep 17 00:00:00 2001 From: Paul Robinson Date: Fri, 22 Mar 2024 05:35:09 -0700 Subject: [PATCH 251/296] [X86][Headers] change 'yields' to 'returns' in more places --- clang/lib/Headers/mmintrin.h | 12 +++++----- clang/lib/Headers/smmintrin.h | 4 ++-- clang/lib/Headers/xmmintrin.h | 44 +++++++++++++++++------------------ 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/clang/lib/Headers/mmintrin.h b/clang/lib/Headers/mmintrin.h index 962d24738e7a..4e154e2d8593 100644 --- a/clang/lib/Headers/mmintrin.h +++ b/clang/lib/Headers/mmintrin.h @@ -1141,7 +1141,7 @@ _mm_xor_si64(__m64 __m1, __m64 __m2) /// [8 x i8] to determine if the element of the first vector is equal to the /// corresponding element of the second vector. /// -/// The comparison yields 0 for false, 0xFF for true. +/// Each comparison returns 0 for false, 0xFF for true. /// /// \headerfile /// @@ -1163,7 +1163,7 @@ _mm_cmpeq_pi8(__m64 __m1, __m64 __m2) /// [4 x i16] to determine if the element of the first vector is equal to the /// corresponding element of the second vector. /// -/// The comparison yields 0 for false, 0xFFFF for true. +/// Each comparison returns 0 for false, 0xFFFF for true. /// /// \headerfile /// @@ -1185,7 +1185,7 @@ _mm_cmpeq_pi16(__m64 __m1, __m64 __m2) /// [2 x i32] to determine if the element of the first vector is equal to the /// corresponding element of the second vector. /// -/// The comparison yields 0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0 for false, 0xFFFFFFFF for true. /// /// \headerfile /// @@ -1207,7 +1207,7 @@ _mm_cmpeq_pi32(__m64 __m1, __m64 __m2) /// [8 x i8] to determine if the element of the first vector is greater than /// the corresponding element of the second vector. /// -/// The comparison yields 0 for false, 0xFF for true. +/// Each comparison returns 0 for false, 0xFF for true. /// /// \headerfile /// @@ -1229,7 +1229,7 @@ _mm_cmpgt_pi8(__m64 __m1, __m64 __m2) /// [4 x i16] to determine if the element of the first vector is greater than /// the corresponding element of the second vector. /// -/// The comparison yields 0 for false, 0xFFFF for true. +/// Each comparison returns 0 for false, 0xFFFF for true. /// /// \headerfile /// @@ -1251,7 +1251,7 @@ _mm_cmpgt_pi16(__m64 __m1, __m64 __m2) /// [2 x i32] to determine if the element of the first vector is greater than /// the corresponding element of the second vector. /// -/// The comparison yields 0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0 for false, 0xFFFFFFFF for true. /// /// \headerfile /// diff --git a/clang/lib/Headers/smmintrin.h b/clang/lib/Headers/smmintrin.h index 9fb9cc9b0134..b3fec474e35a 100644 --- a/clang/lib/Headers/smmintrin.h +++ b/clang/lib/Headers/smmintrin.h @@ -1188,7 +1188,7 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_testnzc_si128(__m128i __M, /// Compares each of the corresponding 64-bit values of the 128-bit /// integer vectors for equality. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. /// /// \headerfile /// @@ -2303,7 +2303,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_minpos_epu16(__m128i __V) { /// integer vectors to determine if the values in the first operand are /// greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. /// /// \headerfile /// diff --git a/clang/lib/Headers/xmmintrin.h b/clang/lib/Headers/xmmintrin.h index 040194786a27..1ef89de9c9f5 100644 --- a/clang/lib/Headers/xmmintrin.h +++ b/clang/lib/Headers/xmmintrin.h @@ -484,7 +484,7 @@ _mm_xor_ps(__m128 __a, __m128 __b) /// Compares two 32-bit float values in the low-order bits of both /// operands for equality. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// The comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector [4 x float]. /// If either value in a comparison is NaN, returns false. /// @@ -509,7 +509,7 @@ _mm_cmpeq_ss(__m128 __a, __m128 __b) /// Compares each of the corresponding 32-bit float values of the /// 128-bit vectors of [4 x float] for equality. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns false. /// /// \headerfile @@ -531,7 +531,7 @@ _mm_cmpeq_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is less than the /// corresponding value in the second operand. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// The comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns false. /// @@ -557,7 +557,7 @@ _mm_cmplt_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are less than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. /// If either value in a comparison is NaN, returns false. /// /// \headerfile @@ -579,7 +579,7 @@ _mm_cmplt_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is less than or /// equal to the corresponding value in the second operand. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true, in +/// The comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true, in /// the low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns false. /// @@ -605,7 +605,7 @@ _mm_cmple_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are less than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns false. /// /// \headerfile @@ -627,7 +627,7 @@ _mm_cmple_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is greater than /// the corresponding value in the second operand. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// The comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns false. /// @@ -655,7 +655,7 @@ _mm_cmpgt_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns false. /// /// \headerfile @@ -677,7 +677,7 @@ _mm_cmpgt_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is greater than /// or equal to the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns false. /// @@ -705,7 +705,7 @@ _mm_cmpge_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are greater than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFFFFFFFFFF for true. /// If either value in a comparison is NaN, returns false. /// /// \headerfile @@ -726,7 +726,7 @@ _mm_cmpge_ps(__m128 __a, __m128 __b) /// Compares two 32-bit float values in the low-order bits of both operands /// for inequality. /// -/// The comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// The comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns true. /// @@ -752,7 +752,7 @@ _mm_cmpneq_ss(__m128 __a, __m128 __b) /// Compares each of the corresponding 32-bit float values of the /// 128-bit vectors of [4 x float] for inequality. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns true. /// /// \headerfile @@ -775,7 +775,7 @@ _mm_cmpneq_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is not less than /// the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns true. /// @@ -802,7 +802,7 @@ _mm_cmpnlt_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are not less than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns true. /// /// \headerfile @@ -825,7 +825,7 @@ _mm_cmpnlt_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is not less than /// or equal to the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns true. /// @@ -852,7 +852,7 @@ _mm_cmpnle_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are not less than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns true. /// /// \headerfile @@ -875,7 +875,7 @@ _mm_cmpnle_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is not greater /// than the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns true. /// @@ -904,7 +904,7 @@ _mm_cmpngt_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are not greater than those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns true. /// /// \headerfile @@ -927,7 +927,7 @@ _mm_cmpngt_ps(__m128 __a, __m128 __b) /// operands to determine if the value in the first operand is not greater /// than or equal to the corresponding value in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true, in the +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true, in the /// low-order bits of a vector of [4 x float]. /// If either value in a comparison is NaN, returns true. /// @@ -956,7 +956,7 @@ _mm_cmpnge_ss(__m128 __a, __m128 __b) /// 128-bit vectors of [4 x float] to determine if the values in the first /// operand are not greater than or equal to those in the second operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, returns true. /// /// \headerfile @@ -3061,7 +3061,7 @@ _mm_movemask_ps(__m128 __a) /// [4 x float], using the operation specified by the immediate integer /// operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, comparisons that are ordered /// return false, and comparisons that are unordered return true. /// @@ -3096,7 +3096,7 @@ _mm_movemask_ps(__m128 __a) /// vectors of [4 x float], using the operation specified by the immediate /// integer operand. /// -/// Each comparison yields 0x0 for false, 0xFFFFFFFF for true. +/// Each comparison returns 0x0 for false, 0xFFFFFFFF for true. /// If either value in a comparison is NaN, comparisons that are ordered /// return false, and comparisons that are unordered return true. /// -- GitLab From 31a9a4b83720af79110941824abe28fc6ff42355 Mon Sep 17 00:00:00 2001 From: yronglin Date: Fri, 22 Mar 2024 20:45:17 +0800 Subject: [PATCH 252/296] [libc++] Implement LWG3528 (`make_from_tuple` can perform (the equivalent of) a C-style cast) (#85263) Implement [LWG3528](https://wg21.link/LWG3528). Based on LWG3528(https://wg21.link/LWG3528) and http://eel.is/c++draft/description#structure.requirements-9, the standard allows to impose requirements, we constraint `std::make_from_tuple` to make `std::make_from_tuple` SFINAE friendly and also avoid worse diagnostic messages. We still keep the constraints of `std::__make_from_tuple_impl` so that `std::__make_from_tuple_impl` will have the same advantages when used alone. --------- Signed-off-by: yronglin --- libcxx/docs/Status/Cxx23Issues.csv | 2 +- libcxx/include/tuple | 28 +++++- .../tuple.apply/make_from_tuple.pass.cpp | 88 +++++++++++++++++++ 3 files changed, 116 insertions(+), 2 deletions(-) diff --git a/libcxx/docs/Status/Cxx23Issues.csv b/libcxx/docs/Status/Cxx23Issues.csv index 8de265f4d1b6..ebdc4a745c9f 100644 --- a/libcxx/docs/Status/Cxx23Issues.csv +++ b/libcxx/docs/Status/Cxx23Issues.csv @@ -77,7 +77,7 @@ `3523 `__,"``iota_view::sentinel`` is not always ``iota_view``'s sentinel","June 2021","","","|ranges|" `3526 `__,"Return types of ``uses_allocator_construction_args`` unspecified","June 2021","","" `3527 `__,"``uses_allocator_construction_args`` handles rvalue pairs of rvalue references incorrectly","June 2021","","" -`3528 `__,"``make_from_tuple`` can perform (the equivalent of) a C-style cast","June 2021","","" +`3528 `__,"``make_from_tuple`` can perform (the equivalent of) a C-style cast","June 2021","|Complete|","19.0" `3529 `__,"``priority_queue(first, last)`` should construct ``c`` with ``(first, last)``","June 2021","|Complete|","14.0" `3530 `__,"``BUILTIN-PTR-MEOW`` should not opt the type out of syntactic checks","June 2021","","" `3532 `__,"``split_view::inner-iterator::operator++(int)`` should depend on ``Base``","June 2021","","","|ranges|" diff --git a/libcxx/include/tuple b/libcxx/include/tuple index f78db061b844..a9f0d680fe0e 100644 --- a/libcxx/include/tuple +++ b/libcxx/include/tuple @@ -1377,15 +1377,41 @@ inline _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) apply(_Fn&& __f, _Tuple&& std::forward<_Tuple>(__t), typename __make_tuple_indices>>::type{})) +#if _LIBCPP_STD_VER >= 20 template inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp __make_from_tuple_impl(_Tuple&& __t, __tuple_indices<_Idx...>) + noexcept(noexcept(_Tp(std::get<_Idx>(std::forward<_Tuple>(__t))...))) + requires is_constructible_v<_Tp, decltype(std::get<_Idx>(std::forward<_Tuple>(__t)))...> { + return _Tp(std::get<_Idx>(std::forward<_Tuple>(__t))...); +} +#else +template +inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp __make_from_tuple_impl(_Tuple&& __t, __tuple_indices<_Idx...>, + enable_if_t(std::forward<_Tuple>(__t)))...>> * = nullptr) _LIBCPP_NOEXCEPT_RETURN(_Tp(std::get<_Idx>(std::forward<_Tuple>(__t))...)) +#endif // _LIBCPP_STD_VER >= 20 + +template >>::type, class = void> +inline constexpr bool __can_make_from_tuple = false; +template +inline constexpr bool __can_make_from_tuple<_Tp, _Tuple, __tuple_indices<_Idx...>, + enable_if_t(std::declval<_Tuple>()))...>>> = true; + +// Based on LWG3528(https://wg21.link/LWG3528) and http://eel.is/c++draft/description#structure.requirements-9, +// the standard allows to impose requirements, we constraint std::make_from_tuple to make std::make_from_tuple +// SFINAE friendly and also avoid worse diagnostic messages. We still keep the constraints of std::__make_from_tuple_impl +// so that std::__make_from_tuple_impl will have the same advantages when used alone. +#if _LIBCPP_STD_VER >= 20 template + requires __can_make_from_tuple<_Tp, _Tuple> // strengthen +#else +template >> // strengthen +#endif // _LIBCPP_STD_VER >= 20 inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp make_from_tuple(_Tuple&& __t) _LIBCPP_NOEXCEPT_RETURN(std::__make_from_tuple_impl<_Tp>( std::forward<_Tuple>(__t), typename __make_tuple_indices>>::type{})) - # undef _LIBCPP_NOEXCEPT_RETURN # endif // _LIBCPP_STD_VER >= 17 diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.pass.cpp index e3a21149c21e..d7374351afa8 100644 --- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.pass.cpp +++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.pass.cpp @@ -195,6 +195,94 @@ void test_noexcept() { } } +namespace LWG3528 { +template +auto test_make_from_tuple(T&&, Tuple&& t) -> decltype(std::make_from_tuple(t), uint8_t()) { + return 0; +} +template +uint32_t test_make_from_tuple(...) { + return 0; +} + +template +static constexpr bool can_make_from_tuple = + std::is_same_v(T{}, Tuple{})), uint8_t>; + +template +auto test_make_from_tuple_impl(T&&, Tuple&& t) + -> decltype(std::__make_from_tuple_impl( + t, typename std::__make_tuple_indices< std::tuple_size_v>>::type{}), + uint8_t()) { + return 0; +} +template +uint32_t test_make_from_tuple_impl(...) { + return 0; +} + +template +static constexpr bool can_make_from_tuple_impl = + std::is_same_v(T{}, Tuple{})), uint8_t>; + +struct A { + int a; +}; +struct B : public A {}; + +struct C { + C(const B&) {} +}; + +enum class D { + ONE, + TWO, +}; + +// Test std::make_from_tuple constraints. + +// reinterpret_cast +static_assert(!can_make_from_tuple>); +static_assert(can_make_from_tuple>); + +// const_cast +static_assert(!can_make_from_tuple>); +static_assert(!can_make_from_tuple>); +static_assert(can_make_from_tuple>); +static_assert(can_make_from_tuple>); +static_assert(can_make_from_tuple>); +static_assert(can_make_from_tuple>); + +// static_cast +static_assert(!can_make_from_tuple>); +static_assert(!can_make_from_tuple>); +static_assert(can_make_from_tuple>); +static_assert(can_make_from_tuple>); +static_assert(can_make_from_tuple>); + +// Test std::__make_from_tuple_impl constraints. + +// reinterpret_cast +static_assert(!can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); + +// const_cast +static_assert(!can_make_from_tuple_impl>); +static_assert(!can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); + +// static_cast +static_assert(!can_make_from_tuple_impl>); +static_assert(!can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); +static_assert(can_make_from_tuple_impl>); + +} // namespace LWG3528 + int main(int, char**) { test_constexpr_construction(); -- GitLab From 3b3de48fd84b8269d5f45ee0a9dc6b7448368424 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Fri, 22 Mar 2024 06:07:17 -0700 Subject: [PATCH 253/296] [BOLT] Add BB index to BAT (#86044) --- bolt/docs/BAT.md | 11 ++--- .../bolt/Profile/BoltAddressTranslation.h | 7 +++- bolt/lib/Profile/BoltAddressTranslation.cpp | 39 ++++++++++++----- .../X86/bolt-address-translation-yaml.test | 2 +- bolt/test/X86/bolt-address-translation.test | 2 +- clang/lib/Driver/ToolChains/Clang.cpp | 4 +- clang/test/Driver/unsupported-option-gpu.c | 1 - lld/MachO/Driver.cpp | 42 +++++++++++++++++-- lld/MachO/InputSection.cpp | 38 ----------------- lld/MachO/InputSection.h | 3 -- lld/MachO/ObjC.cpp | 16 ++++--- lld/MachO/SyntheticSections.cpp | 4 +- 12 files changed, 92 insertions(+), 77 deletions(-) diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md index 186b0e5ea89d..436593478a39 100644 --- a/bolt/docs/BAT.md +++ b/bolt/docs/BAT.md @@ -90,11 +90,12 @@ current function. ### Address translation table Delta encoding means that only the difference with the previous corresponding entry is encoded. Input offsets implicitly start at zero. -| Entry | Encoding | Description | -| ------ | ------| ----------- | -| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | -| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | -| `BBHash` | Optional, 8b | Basic block entries only: basic block hash in input binary | +| Entry | Encoding | Description | Branch/BB | +| ------ | ------| ----------- | ------ | +| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | Both | +| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | Both | +| `BBHash` | Optional, 8b | Basic block hash in input binary | BB | +| `BBIdx` | Optional, Delta, ULEB128 | Basic block index in input binary | BB | `BRANCHENTRY` bit denotes whether a given offset pair is a control flow source (branch or call instruction). If not set, it signifies a control flow target diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index 1f53f6d344ad..eda2b318f0d0 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -122,6 +122,10 @@ public: /// Returns BF hash by function output address (after BOLT). size_t getBFHash(uint64_t OutputAddress) const; + /// Returns BB index by function output address (after BOLT) and basic block + /// input offset. + unsigned getBBIndex(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const; + /// True if a given \p Address is a function with translation table entry. bool isBATFunction(uint64_t Address) const { return Maps.count(Address); } @@ -154,7 +158,8 @@ private: std::map Maps; - using BBHashMap = std::unordered_map; + /// Map basic block input offset to a basic block index and hash pair. + using BBHashMap = std::unordered_map>; std::unordered_map> FuncHashes; /// Links outlined cold bocks to their original function diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index 1d61a1b735b4..8fe976cc00e5 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -45,6 +45,8 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", getBBHash(HotFuncAddress, BBInputOffset))); (void)HotFuncAddress; + LLVM_DEBUG(dbgs() << formatv(" Index: {0}\n", + getBBIndex(HotFuncAddress, BBInputOffset))); // In case of conflicts (same Key mapping to different Vals), the last // update takes precedence. Of course it is not ideal to have conflicts and // those happen when we have an empty BB that either contained only @@ -217,6 +219,7 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, } size_t Index = 0; uint64_t InOffset = 0; + size_t PrevBBIndex = 0; // Output and Input addresses and delta-encoded for (std::pair &KeyVal : Map) { const uint64_t OutputAddress = KeyVal.first + Address; @@ -226,11 +229,15 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, encodeSLEB128(KeyVal.second - InOffset, OS); InOffset = KeyVal.second; // Keeping InOffset as if BRANCHENTRY is encoded if ((InOffset & BRANCHENTRY) == 0) { - // Basic block hash - size_t BBHash = FuncHashPair.second[InOffset >> 1]; + unsigned BBIndex; + size_t BBHash; + std::tie(BBIndex, BBHash) = FuncHashPair.second[InOffset >> 1]; OS.write(reinterpret_cast(&BBHash), 8); - LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x}\n", KeyVal.first, - InOffset >> 1, BBHash)); + // Basic block index in the input binary + encodeULEB128(BBIndex - PrevBBIndex, OS); + PrevBBIndex = BBIndex; + LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x} {3}\n", KeyVal.first, + InOffset >> 1, BBHash, BBIndex)); } } } @@ -316,6 +323,7 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, LLVM_DEBUG(dbgs() << "Parsing " << NumEntries << " entries for 0x" << Twine::utohexstr(Address) << "\n"); uint64_t InputOffset = 0; + size_t BBIndex = 0; for (uint32_t J = 0; J < NumEntries; ++J) { const uint64_t OutputDelta = DE.getULEB128(&Offset, &Err); const uint64_t OutputAddress = PrevAddress + OutputDelta; @@ -330,19 +338,25 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, } Map.insert(std::pair(OutputOffset, InputOffset)); size_t BBHash = 0; + size_t BBIndexDelta = 0; const bool IsBranchEntry = InputOffset & BRANCHENTRY; if (!IsBranchEntry) { BBHash = DE.getU64(&Offset, &Err); + BBIndexDelta = DE.getULEB128(&Offset, &Err); + BBIndex += BBIndexDelta; // Map basic block hash to hot fragment by input offset - FuncHashes[HotAddress].second.emplace(InputOffset >> 1, BBHash); + FuncHashes[HotAddress].second.emplace(InputOffset >> 1, + std::pair(BBIndex, BBHash)); } LLVM_DEBUG({ dbgs() << formatv( "{0:x} -> {1:x} ({2}/{3}b -> {4}/{5}b), {6:x}", OutputOffset, InputOffset, OutputDelta, getULEB128Size(OutputDelta), InputDelta, (J < EqualElems) ? 0 : getSLEB128Size(InputDelta), OutputAddress); - if (BBHash) - dbgs() << formatv(" {0:x}", BBHash); + if (!IsBranchEntry) { + dbgs() << formatv(" {0:x} {1}/{2}b", BBHash, BBIndex, + getULEB128Size(BBIndexDelta)); + } dbgs() << '\n'; }); } @@ -494,14 +508,19 @@ void BoltAddressTranslation::saveMetadata(BinaryContext &BC) { FuncHashes[BF.getAddress()].first = BF.computeHash(); BF.computeBlockHashes(); for (const BinaryBasicBlock &BB : BF) - FuncHashes[BF.getAddress()].second.emplace(BB.getInputOffset(), - BB.getHash()); + FuncHashes[BF.getAddress()].second.emplace( + BB.getInputOffset(), std::pair(BB.getIndex(), BB.getHash())); } } +unsigned BoltAddressTranslation::getBBIndex(uint64_t FuncOutputAddress, + uint32_t BBInputOffset) const { + return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).first; +} + size_t BoltAddressTranslation::getBBHash(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const { - return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset); + return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).second; } size_t BoltAddressTranslation::getBFHash(uint64_t OutputAddress) const { diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test index 25ff4e7fbfcc..4516a662697a 100644 --- a/bolt/test/X86/bolt-address-translation-yaml.test +++ b/bolt/test/X86/bolt-address-translation-yaml.test @@ -18,7 +18,7 @@ RUN: | FileCheck --check-prefix CHECK-BOLT-YAML %s WRITE-BAT-CHECK: BOLT-INFO: Wrote 5 BAT maps WRITE-BAT-CHECK: BOLT-INFO: Wrote 4 function and 22 basic block hashes -WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 344 +WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 376 READ-BAT-CHECK-NOT: BOLT-ERROR: unable to save profile in YAML format for input file processed by BOLT READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries diff --git a/bolt/test/X86/bolt-address-translation.test b/bolt/test/X86/bolt-address-translation.test index 4277b4e0d0fe..5c1db89e3c6b 100644 --- a/bolt/test/X86/bolt-address-translation.test +++ b/bolt/test/X86/bolt-address-translation.test @@ -37,7 +37,7 @@ # CHECK: BOLT: 3 out of 7 functions were overwritten. # CHECK: BOLT-INFO: Wrote 6 BAT maps # CHECK: BOLT-INFO: Wrote 3 function and 58 basic block hashes -# CHECK: BOLT-INFO: BAT section size (bytes): 816 +# CHECK: BOLT-INFO: BAT section size (bytes): 920 # # usqrt mappings (hot part). We match against any key (left side containing # the bolted binary offsets) because BOLT may change where it puts instructions diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index 86a287db72a4..bc9cc8ce6cf5 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -5863,8 +5863,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, } else if (Triple.getArch() == llvm::Triple::x86_64) { Ok = llvm::is_contained({"small", "kernel", "medium", "large", "tiny"}, CM); - } else if (Triple.isNVPTX() || Triple.isAMDGPU() || Triple.isSPIRV()) { - // NVPTX/AMDGPU/SPIRV does not care about the code model and will accept + } else if (Triple.isNVPTX() || Triple.isAMDGPU()) { + // NVPTX/AMDGPU does not care about the code model and will accept // whatever works for the host. Ok = true; } else if (Triple.isSPARC64()) { diff --git a/clang/test/Driver/unsupported-option-gpu.c b/clang/test/Driver/unsupported-option-gpu.c index 5618b2cba72e..f23cb71ebfb0 100644 --- a/clang/test/Driver/unsupported-option-gpu.c +++ b/clang/test/Driver/unsupported-option-gpu.c @@ -2,5 +2,4 @@ // DEFINE: %{check} = %clang -### --target=x86_64-linux-gnu -c -mcmodel=medium // RUN: %{check} -x cuda %s --cuda-path=%S/Inputs/CUDA/usr/local/cuda --offload-arch=sm_60 --no-cuda-version-check -fbasic-block-sections=all -// RUN: %{check} -x hip %s --offload=spirv64 -nogpulib -nogpuinc // RUN: %{check} -x hip %s --rocm-path=%S/Inputs/rocm -nogpulib -nogpuinc diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp index 919a14b8bcf0..36248925d65a 100644 --- a/lld/MachO/Driver.cpp +++ b/lld/MachO/Driver.cpp @@ -612,7 +612,7 @@ static void replaceCommonSymbols() { if (!osec) osec = ConcatOutputSection::getOrCreateForInput(isec); isec->parent = osec; - addInputSection(isec); + inputSections.push_back(isec); // FIXME: CommonSymbol should store isReferencedDynamically, noDeadStrip // and pass them on here. @@ -1220,18 +1220,53 @@ static void createFiles(const InputArgList &args) { static void gatherInputSections() { TimeTraceScope timeScope("Gathering input sections"); + int inputOrder = 0; for (const InputFile *file : inputFiles) { for (const Section *section : file->sections) { // Compact unwind entries require special handling elsewhere. (In // contrast, EH frames are handled like regular ConcatInputSections.) if (section->name == section_names::compactUnwind) continue; - for (const Subsection &subsection : section->subsections) - addInputSection(subsection.isec); + ConcatOutputSection *osec = nullptr; + for (const Subsection &subsection : section->subsections) { + if (auto *isec = dyn_cast(subsection.isec)) { + if (isec->isCoalescedWeak()) + continue; + if (config->emitInitOffsets && + sectionType(isec->getFlags()) == S_MOD_INIT_FUNC_POINTERS) { + in.initOffsets->addInput(isec); + continue; + } + isec->outSecOff = inputOrder++; + if (!osec) + osec = ConcatOutputSection::getOrCreateForInput(isec); + isec->parent = osec; + inputSections.push_back(isec); + } else if (auto *isec = + dyn_cast(subsection.isec)) { + if (isec->getName() == section_names::objcMethname) { + if (in.objcMethnameSection->inputOrder == UnspecifiedInputOrder) + in.objcMethnameSection->inputOrder = inputOrder++; + in.objcMethnameSection->addInput(isec); + } else { + if (in.cStringSection->inputOrder == UnspecifiedInputOrder) + in.cStringSection->inputOrder = inputOrder++; + in.cStringSection->addInput(isec); + } + } else if (auto *isec = + dyn_cast(subsection.isec)) { + if (in.wordLiteralSection->inputOrder == UnspecifiedInputOrder) + in.wordLiteralSection->inputOrder = inputOrder++; + in.wordLiteralSection->addInput(isec); + } else { + llvm_unreachable("unexpected input section kind"); + } + } } if (!file->objCImageInfo.empty()) in.objCImageInfo->addFile(file); } + assert(inputOrder <= UnspecifiedInputOrder); } static void foldIdenticalLiterals() { @@ -1387,7 +1422,6 @@ bool link(ArrayRef argsArr, llvm::raw_ostream &stdoutOS, concatOutputSections.clear(); inputFiles.clear(); inputSections.clear(); - inputSectionsOrder = 0; loadedArchives.clear(); loadedObjectFrameworks.clear(); missingAutolinkWarnings.clear(); diff --git a/lld/MachO/InputSection.cpp b/lld/MachO/InputSection.cpp index 22930d52dd1d..8f5affb1dc21 100644 --- a/lld/MachO/InputSection.cpp +++ b/lld/MachO/InputSection.cpp @@ -37,44 +37,6 @@ static_assert(sizeof(void *) != 8 || "instances of it"); std::vector macho::inputSections; -int macho::inputSectionsOrder = 0; - -// Call this function to add a new InputSection and have it routed to the -// appropriate container. Depending on its type and current config, it will -// either be added to 'inputSections' vector or to a synthetic section. -void lld::macho::addInputSection(InputSection *inputSection) { - if (auto *isec = dyn_cast(inputSection)) { - if (isec->isCoalescedWeak()) - return; - if (config->emitInitOffsets && - sectionType(isec->getFlags()) == S_MOD_INIT_FUNC_POINTERS) { - in.initOffsets->addInput(isec); - return; - } - isec->outSecOff = inputSectionsOrder++; - auto *osec = ConcatOutputSection::getOrCreateForInput(isec); - isec->parent = osec; - inputSections.push_back(isec); - } else if (auto *isec = dyn_cast(inputSection)) { - if (isec->getName() == section_names::objcMethname) { - if (in.objcMethnameSection->inputOrder == UnspecifiedInputOrder) - in.objcMethnameSection->inputOrder = inputSectionsOrder++; - in.objcMethnameSection->addInput(isec); - } else { - if (in.cStringSection->inputOrder == UnspecifiedInputOrder) - in.cStringSection->inputOrder = inputSectionsOrder++; - in.cStringSection->addInput(isec); - } - } else if (auto *isec = dyn_cast(inputSection)) { - if (in.wordLiteralSection->inputOrder == UnspecifiedInputOrder) - in.wordLiteralSection->inputOrder = inputSectionsOrder++; - in.wordLiteralSection->addInput(isec); - } else { - llvm_unreachable("unexpected input section kind"); - } - - assert(inputSectionsOrder <= UnspecifiedInputOrder); -} uint64_t InputSection::getFileSize() const { return isZeroFill(getFlags()) ? 0 : getSize(); diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h index 694bdf734907..b25f0638f4c6 100644 --- a/lld/MachO/InputSection.h +++ b/lld/MachO/InputSection.h @@ -302,8 +302,6 @@ bool isEhFrameSection(const InputSection *); bool isGccExceptTabSection(const InputSection *); extern std::vector inputSections; -// This is used as a counter for specyfing input order for input sections -extern int inputSectionsOrder; namespace section_names { @@ -371,7 +369,6 @@ constexpr const char addrSig[] = "__llvm_addrsig"; } // namespace section_names -void addInputSection(InputSection *inputSection); } // namespace macho std::string toString(const macho::InputSection *); diff --git a/lld/MachO/ObjC.cpp b/lld/MachO/ObjC.cpp index 5902b82d30f5..40df2243b26f 100644 --- a/lld/MachO/ObjC.cpp +++ b/lld/MachO/ObjC.cpp @@ -790,7 +790,7 @@ void ObjcCategoryMerger::emitAndLinkProtocolList( infoCategoryWriter.catPtrListInfo.align); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; listSec->live = true; - addInputSection(listSec); + allInputSections.push_back(listSec); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; @@ -848,7 +848,7 @@ void ObjcCategoryMerger::emitAndLinkPointerList( infoCategoryWriter.catPtrListInfo.align); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; listSec->live = true; - addInputSection(listSec); + allInputSections.push_back(listSec); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; @@ -889,7 +889,7 @@ ObjcCategoryMerger::emitCatListEntrySec(const std::string &forCateogryName, bodyData, infoCategoryWriter.catListInfo.align); newCatList->parent = infoCategoryWriter.catListInfo.outputSection; newCatList->live = true; - addInputSection(newCatList); + allInputSections.push_back(newCatList); newCatList->parent = infoCategoryWriter.catListInfo.outputSection; @@ -927,7 +927,7 @@ Defined *ObjcCategoryMerger::emitCategoryBody(const std::string &name, bodyData, infoCategoryWriter.catBodyInfo.align); newBodySec->parent = infoCategoryWriter.catBodyInfo.outputSection; newBodySec->live = true; - addInputSection(newBodySec); + allInputSections.push_back(newBodySec); std::string symName = objc::symbol_names::category + baseClassName + "_$_(" + name + ")"; @@ -1132,7 +1132,7 @@ void ObjcCategoryMerger::generateCatListForNonErasedCategories( infoCategoryWriter.catListInfo.align); listSec->parent = infoCategoryWriter.catListInfo.outputSection; listSec->live = true; - addInputSection(listSec); + allInputSections.push_back(listSec); std::string slotSymName = "<__objc_catlist slot for category "; slotSymName += nonErasedCatBody->getName(); @@ -1221,11 +1221,9 @@ void ObjcCategoryMerger::doCleanup() { generatedSectionData.clear(); } StringRef ObjcCategoryMerger::newStringData(const char *str) { uint32_t len = strlen(str); - uint32_t bufSize = len + 1; - auto &data = newSectionData(bufSize); + auto &data = newSectionData(len + 1); char *strData = reinterpret_cast(data.data()); - // Copy the string chars and null-terminator - memcpy(strData, str, bufSize); + strncpy(strData, str, len); return StringRef(strData, len); } diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp index 1b3694528de1..7ee3261ce307 100644 --- a/lld/MachO/SyntheticSections.cpp +++ b/lld/MachO/SyntheticSections.cpp @@ -793,7 +793,7 @@ void StubHelperSection::setUp() { in.imageLoaderCache->parent = ConcatOutputSection::getOrCreateForInput(in.imageLoaderCache); - addInputSection(in.imageLoaderCache); + inputSections.push_back(in.imageLoaderCache); // Since this isn't in the symbol table or in any input file, the noDeadStrip // argument doesn't matter. dyldPrivate = @@ -855,7 +855,7 @@ ConcatInputSection *ObjCSelRefsSection::makeSelRef(StringRef methname) { /*addend=*/static_cast(methnameOffset), /*referent=*/in.objcMethnameSection->isec}); objcSelref->parent = ConcatOutputSection::getOrCreateForInput(objcSelref); - addInputSection(objcSelref); + inputSections.push_back(objcSelref); objcSelref->isFinal = true; methnameToSelref[CachedHashStringRef(methname)] = objcSelref; return objcSelref; -- GitLab From b3f98dff75469b115e3d4b1f10cbf270c8ee81af Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 22 Mar 2024 13:52:11 +0000 Subject: [PATCH 254/296] [RemoveDIs] Load into new debug info format by default in llvm-lto and llvm-lto2 (#86271) Directly load all bitcode into the new debug info format in `llvm-lto` and `llvm-lto2`. This means that new-mode bitcode no longer round-trips back to old-mode after parsing, and that old-mode bitcode gets auto-upgraded to new-mode debug info (which is the current in-memory default in LLVM). --- llvm/lib/LTO/LTO.cpp | 4 +++- llvm/tools/llvm-lto/llvm-lto.cpp | 4 ++++ llvm/tools/llvm-lto2/llvm-lto2.cpp | 4 ++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index b58418c64a11..53060df7f503 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -579,7 +579,9 @@ LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel, const Config &Conf) : ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel), Ctx(Conf), CombinedModule(std::make_unique("ld-temp.o", Ctx)), - Mover(std::make_unique(*CombinedModule)) {} + Mover(std::make_unique(*CombinedModule)) { + CombinedModule->IsNewDbgInfoFormat = UseNewDbgInfoFormat; +} LTO::ThinLTOState::ThinLTOState(ThinBackend Backend) : Backend(Backend), CombinedIndex(/*HaveGVs*/ false) { diff --git a/llvm/tools/llvm-lto/llvm-lto.cpp b/llvm/tools/llvm-lto/llvm-lto.cpp index 7943d6952b82..3c452b650cee 100644 --- a/llvm/tools/llvm-lto/llvm-lto.cpp +++ b/llvm/tools/llvm-lto/llvm-lto.cpp @@ -270,6 +270,7 @@ static cl::opt TryUseNewDbgInfoFormat( cl::init(false), cl::Hidden); extern cl::opt UseNewDbgInfoFormat; +extern cl::opt LoadBitcodeIntoNewDbgInfoFormat; namespace { @@ -943,6 +944,9 @@ int main(int argc, char **argv) { InitLLVM X(argc, argv); cl::HideUnrelatedOptions({<OCategory, &getColorCategory()}); cl::ParseCommandLineOptions(argc, argv, "llvm LTO linker\n"); + // Load bitcode into the new debug info format by default. + if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_TRUE; // RemoveDIs debug-info transition: tests may request that we /try/ to use the // new debug-info format. diff --git a/llvm/tools/llvm-lto2/llvm-lto2.cpp b/llvm/tools/llvm-lto2/llvm-lto2.cpp index d5de4f6b1a27..f222d02bd7ce 100644 --- a/llvm/tools/llvm-lto2/llvm-lto2.cpp +++ b/llvm/tools/llvm-lto2/llvm-lto2.cpp @@ -193,6 +193,7 @@ static cl::opt TryUseNewDbgInfoFormat( cl::init(false), cl::Hidden); extern cl::opt UseNewDbgInfoFormat; +extern cl::opt LoadBitcodeIntoNewDbgInfoFormat; static void check(Error E, std::string Msg) { if (!E) @@ -228,6 +229,9 @@ static int usage() { static int run(int argc, char **argv) { cl::ParseCommandLineOptions(argc, argv, "Resolution-based LTO test harness"); + // Load bitcode into the new debug info format by default. + if (LoadBitcodeIntoNewDbgInfoFormat == cl::boolOrDefault::BOU_UNSET) + LoadBitcodeIntoNewDbgInfoFormat = cl::boolOrDefault::BOU_TRUE; // RemoveDIs debug-info transition: tests may request that we /try/ to use the // new debug-info format. -- GitLab From a277dd82d89a17849cf99fb335660ea0b8894878 Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Fri, 22 Mar 2024 13:43:58 +0000 Subject: [PATCH 255/296] [X86] vector-half-conversions.ll - add v4f16->v4i32 fptosi/fptoui test coverage --- .../CodeGen/X86/vector-half-conversions.ll | 254 ++++++++++++++++++ 1 file changed, 254 insertions(+) diff --git a/llvm/test/CodeGen/X86/vector-half-conversions.ll b/llvm/test/CodeGen/X86/vector-half-conversions.ll index ba21af231985..563cf0165013 100644 --- a/llvm/test/CodeGen/X86/vector-half-conversions.ll +++ b/llvm/test/CodeGen/X86/vector-half-conversions.ll @@ -4989,3 +4989,257 @@ define <4 x i32> @fptosi_2f16_to_4i32(<2 x half> %a) nounwind { %ext = shufflevector <2 x i32> %cvt, <2 x i32> zeroinitializer, <4 x i32> ret <4 x i32> %ext } + +define <4 x i32> @fptosi_4f16_to_4i32(<4 x half> %a) nounwind { +; AVX-LABEL: fptosi_4f16_to_4i32: +; AVX: # %bb.0: +; AVX-NEXT: subq $72, %rsp +; AVX-NEXT: vmovdqa %xmm0, %xmm1 +; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3] +; AVX-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; AVX-NEXT: vpsrlq $48, %xmm1, %xmm0 +; AVX-NEXT: callq __extendhfsf2@PLT +; AVX-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload +; AVX-NEXT: callq __extendhfsf2@PLT +; AVX-NEXT: vinsertps $16, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[2,3] +; AVX-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX-NEXT: callq __extendhfsf2@PLT +; AVX-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX-NEXT: callq __extendhfsf2@PLT +; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; AVX-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX-NEXT: vunpcklpd (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX-NEXT: # xmm0 = xmm0[0],mem[0] +; AVX-NEXT: addq $72, %rsp +; AVX-NEXT: retq +; +; F16C-LABEL: fptosi_4f16_to_4i32: +; F16C: # %bb.0: +; F16C-NEXT: vcvtph2ps %xmm0, %ymm0 +; F16C-NEXT: vcvttps2dq %ymm0, %ymm0 +; F16C-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 +; F16C-NEXT: vzeroupper +; F16C-NEXT: retq +; +; AVX512-LABEL: fptosi_4f16_to_4i32: +; AVX512: # %bb.0: +; AVX512-NEXT: vcvtph2ps %xmm0, %ymm0 +; AVX512-NEXT: vcvttps2dq %ymm0, %ymm0 +; AVX512-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 +; AVX512-NEXT: vzeroupper +; AVX512-NEXT: retq + %cvt = fptosi <4 x half> %a to <4 x i32> + ret <4 x i32> %cvt +} + +define <4 x i32> @fptoui_2f16_to_4i32(<2 x half> %a) nounwind { +; AVX1-LABEL: fptoui_2f16_to_4i32: +; AVX1: # %bb.0: +; AVX1-NEXT: subq $40, %rsp +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX1-NEXT: callq __extendhfsf2@PLT +; AVX1-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill +; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX1-NEXT: callq __extendhfsf2@PLT +; AVX1-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload +; AVX1-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; AVX1-NEXT: vcvttps2dq %xmm0, %xmm1 +; AVX1-NEXT: vpsrad $31, %xmm1, %xmm2 +; AVX1-NEXT: vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX1-NEXT: addq $40, %rsp +; AVX1-NEXT: retq +; +; AVX2-LABEL: fptoui_2f16_to_4i32: +; AVX2: # %bb.0: +; AVX2-NEXT: subq $40, %rsp +; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX2-NEXT: callq __extendhfsf2@PLT +; AVX2-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill +; AVX2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX2-NEXT: callq __extendhfsf2@PLT +; AVX2-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload +; AVX2-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; AVX2-NEXT: vcvttps2dq %xmm0, %xmm1 +; AVX2-NEXT: vpsrad $31, %xmm1, %xmm2 +; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9] +; AVX2-NEXT: vsubps %xmm3, %xmm0, %xmm0 +; AVX2-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX2-NEXT: addq $40, %rsp +; AVX2-NEXT: retq +; +; F16C-LABEL: fptoui_2f16_to_4i32: +; F16C: # %bb.0: +; F16C-NEXT: vpsrld $16, %xmm0, %xmm1 +; F16C-NEXT: vcvtph2ps %xmm1, %xmm1 +; F16C-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; F16C-NEXT: vcvtph2ps %xmm0, %xmm0 +; F16C-NEXT: vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; F16C-NEXT: vcvttps2dq %xmm0, %xmm1 +; F16C-NEXT: vpsrad $31, %xmm1, %xmm2 +; F16C-NEXT: vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; F16C-NEXT: vcvttps2dq %xmm0, %xmm0 +; F16C-NEXT: vpand %xmm2, %xmm0, %xmm0 +; F16C-NEXT: vpor %xmm0, %xmm1, %xmm0 +; F16C-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; F16C-NEXT: retq +; +; AVX512F-LABEL: fptoui_2f16_to_4i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX512F-NEXT: vcvtph2ps %xmm1, %xmm1 +; AVX512F-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX512F-NEXT: vcvtph2ps %xmm0, %xmm0 +; AVX512F-NEXT: vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; AVX512F-NEXT: vcvttps2udq %zmm0, %zmm0 +; AVX512F-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX512F-NEXT: vzeroupper +; AVX512F-NEXT: retq +; +; AVX512-FASTLANE-LABEL: fptoui_2f16_to_4i32: +; AVX512-FASTLANE: # %bb.0: +; AVX512-FASTLANE-NEXT: vpsrld $16, %xmm0, %xmm1 +; AVX512-FASTLANE-NEXT: vcvtph2ps %xmm1, %xmm1 +; AVX512-FASTLANE-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX512-FASTLANE-NEXT: vcvtph2ps %xmm0, %xmm0 +; AVX512-FASTLANE-NEXT: vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; AVX512-FASTLANE-NEXT: vcvttps2udq %xmm0, %xmm0 +; AVX512-FASTLANE-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX512-FASTLANE-NEXT: retq + %cvt = fptoui <2 x half> %a to <2 x i32> + %ext = shufflevector <2 x i32> %cvt, <2 x i32> zeroinitializer, <4 x i32> + ret <4 x i32> %ext +} + +define <4 x i32> @fptoui_4f16_to_4i32(<4 x half> %a) nounwind { +; AVX1-LABEL: fptoui_4f16_to_4i32: +; AVX1: # %bb.0: +; AVX1-NEXT: subq $72, %rsp +; AVX1-NEXT: vmovdqa %xmm0, %xmm1 +; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX1-NEXT: vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3] +; AVX1-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; AVX1-NEXT: vpsrlq $48, %xmm1, %xmm0 +; AVX1-NEXT: callq __extendhfsf2@PLT +; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX1-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload +; AVX1-NEXT: callq __extendhfsf2@PLT +; AVX1-NEXT: vinsertps $16, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX1-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[2,3] +; AVX1-NEXT: vcvttps2dq %xmm0, %xmm1 +; AVX1-NEXT: vpsrad $31, %xmm1, %xmm2 +; AVX1-NEXT: vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill +; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX1-NEXT: callq __extendhfsf2@PLT +; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX1-NEXT: callq __extendhfsf2@PLT +; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; AVX1-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; AVX1-NEXT: vcvttps2dq %xmm0, %xmm1 +; AVX1-NEXT: vpsrad $31, %xmm1, %xmm2 +; AVX1-NEXT: vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX1-NEXT: vpunpcklqdq (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX1-NEXT: # xmm0 = xmm0[0],mem[0] +; AVX1-NEXT: addq $72, %rsp +; AVX1-NEXT: retq +; +; AVX2-LABEL: fptoui_4f16_to_4i32: +; AVX2: # %bb.0: +; AVX2-NEXT: subq $72, %rsp +; AVX2-NEXT: vmovdqa %xmm0, %xmm1 +; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX2-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX2-NEXT: vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3] +; AVX2-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill +; AVX2-NEXT: vpsrlq $48, %xmm1, %xmm0 +; AVX2-NEXT: callq __extendhfsf2@PLT +; AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX2-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload +; AVX2-NEXT: callq __extendhfsf2@PLT +; AVX2-NEXT: vinsertps $16, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX2-NEXT: # xmm0 = xmm0[0],mem[0],xmm0[2,3] +; AVX2-NEXT: vcvttps2dq %xmm0, %xmm1 +; AVX2-NEXT: vpsrad $31, %xmm1, %xmm2 +; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9] +; AVX2-NEXT: vsubps %xmm3, %xmm0, %xmm0 +; AVX2-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill +; AVX2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX2-NEXT: callq __extendhfsf2@PLT +; AVX2-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; AVX2-NEXT: callq __extendhfsf2@PLT +; AVX2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; AVX2-NEXT: vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[2,3] +; AVX2-NEXT: vcvttps2dq %xmm0, %xmm1 +; AVX2-NEXT: vpsrad $31, %xmm1, %xmm2 +; AVX2-NEXT: vbroadcastss {{.*#+}} xmm3 = [2.14748365E+9,2.14748365E+9,2.14748365E+9,2.14748365E+9] +; AVX2-NEXT: vsubps %xmm3, %xmm0, %xmm0 +; AVX2-NEXT: vcvttps2dq %xmm0, %xmm0 +; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm0 +; AVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 +; AVX2-NEXT: vpunpcklqdq (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload +; AVX2-NEXT: # xmm0 = xmm0[0],mem[0] +; AVX2-NEXT: addq $72, %rsp +; AVX2-NEXT: retq +; +; F16C-LABEL: fptoui_4f16_to_4i32: +; F16C: # %bb.0: +; F16C-NEXT: vcvtph2ps %xmm0, %ymm0 +; F16C-NEXT: vcvttps2dq %ymm0, %ymm1 +; F16C-NEXT: vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 +; F16C-NEXT: vcvttps2dq %ymm0, %ymm0 +; F16C-NEXT: vorps %ymm0, %ymm1, %ymm0 +; F16C-NEXT: vblendvps %ymm1, %ymm0, %ymm1, %ymm0 +; F16C-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 +; F16C-NEXT: vzeroupper +; F16C-NEXT: retq +; +; AVX512F-LABEL: fptoui_4f16_to_4i32: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vcvtph2ps %xmm0, %ymm0 +; AVX512F-NEXT: vcvttps2udq %zmm0, %zmm0 +; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 +; AVX512F-NEXT: vzeroupper +; AVX512F-NEXT: retq +; +; AVX512-FASTLANE-LABEL: fptoui_4f16_to_4i32: +; AVX512-FASTLANE: # %bb.0: +; AVX512-FASTLANE-NEXT: vcvtph2ps %xmm0, %ymm0 +; AVX512-FASTLANE-NEXT: vcvttps2udq %ymm0, %ymm0 +; AVX512-FASTLANE-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0 +; AVX512-FASTLANE-NEXT: vzeroupper +; AVX512-FASTLANE-NEXT: retq + %cvt = fptoui <4 x half> %a to <4 x i32> + ret <4 x i32> %cvt +} -- GitLab From f82d0187a7e581d4f8f825021dbcb08e8eb37d61 Mon Sep 17 00:00:00 2001 From: David Green Date: Fri, 22 Mar 2024 14:00:21 +0000 Subject: [PATCH 256/296] [AArch64] Add a test to show incorrect latencies into Bundle instructions. NFC --- llvm/test/CodeGen/AArch64/misched-bundle.mir | 195 +++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 llvm/test/CodeGen/AArch64/misched-bundle.mir diff --git a/llvm/test/CodeGen/AArch64/misched-bundle.mir b/llvm/test/CodeGen/AArch64/misched-bundle.mir new file mode 100644 index 000000000000..a947c04a4229 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/misched-bundle.mir @@ -0,0 +1,195 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 +# RUN: llc -mtriple=aarch64-none-linux-gnu -mcpu=cortex-a510 -run-pass=machine-scheduler -debug-only=machine-scheduler %s -o - 2>&1 | FileCheck %s +# REQUIRES: asserts + +# CHECK: SU(0): renamable $z0 = LD1H renamable $p0, renamable $x1, renamable $x10 :: (load unknown-size, align 1) +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 4 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 3 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 7 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(6): Out Latency=1 +# CHECK-NEXT: SU(6): Data Latency=3 Reg=$z0 +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(1): renamable $z1 = LD1H renamable $p0, renamable $x2, renamable $x10 :: (load unknown-size, align 1) +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 4 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 3 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 7 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(7): Out Latency=1 +# CHECK-NEXT: SU(6): Data Latency=3 Reg=$z1 +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(2): renamable $z2 = LD1H renamable $p0, renamable $x0, renamable $x10 :: (load unknown-size, align 1) +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 3 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 3 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 7 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(6): Data Latency=3 Reg=$z2 +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(3): renamable $z3 = LD1H renamable $p0, renamable $x11, renamable $x10 :: (load unknown-size, align 1) +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 3 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 3 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 0 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(7): Data Latency=0 Reg=$z3 +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(4): renamable $z4 = LD1H renamable $p0, renamable $x12, renamable $x10 :: (load unknown-size, align 1) +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 3 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 3 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 0 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(7): Data Latency=0 Reg=$z4 +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(5): renamable $z5 = LD1H renamable $p0, renamable $x13, renamable $x10 :: (load unknown-size, align 1) +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 3 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 3 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 0 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(7): Data Latency=0 Reg=$z5 +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(6): $z0 = FMAD_ZPmZZ_H renamable $p0, killed $z0(tied-def 0), killed renamable $z1, killed renamable $z2 +# CHECK-NEXT: # preds left : 4 +# CHECK-NEXT: # succs left : 2 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 4 +# CHECK-NEXT: Depth : 3 +# CHECK-NEXT: Height : 4 +# CHECK-NEXT: Predecessors: +# CHECK-NEXT: SU(2): Data Latency=3 Reg=$z2 +# CHECK-NEXT: SU(1): Data Latency=3 Reg=$z1 +# CHECK-NEXT: SU(0): Out Latency=1 +# CHECK-NEXT: SU(0): Data Latency=3 Reg=$z0 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(8): Data Latency=4 Reg=$z0 +# CHECK-NEXT: SU(7): Anti Latency=0 +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(7): BUNDLE implicit-def $z1, implicit-def $q1, implicit-def $d1, implicit-def $s1, implicit-def $h1, implicit-def $b1, implicit $z5, implicit $p0, implicit killed $z4, implicit killed $z3 +# CHECK-NEXT: # preds left : 5 +# CHECK-NEXT: # succs left : 1 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 1 +# CHECK-NEXT: Depth : 3 +# CHECK-NEXT: Height : 0 +# CHECK-NEXT: Predecessors: +# CHECK-NEXT: SU(6): Anti Latency=0 +# CHECK-NEXT: SU(5): Data Latency=0 Reg=$z5 +# CHECK-NEXT: SU(4): Data Latency=0 Reg=$z4 +# CHECK-NEXT: SU(3): Data Latency=0 Reg=$z3 +# CHECK-NEXT: SU(1): Out Latency=1 +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(9): Data Latency=0 Reg=$z1 +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(8): ST1H killed renamable $z0, renamable $p0, renamable $x0, renamable $x10 :: (store unknown-size, align 1) +# CHECK-NEXT: # preds left : 7 +# CHECK-NEXT: # succs left : 1 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 1 +# CHECK-NEXT: Depth : 7 +# CHECK-NEXT: Height : 0 +# CHECK-NEXT: Predecessors: +# CHECK-NEXT: SU(6): Data Latency=4 Reg=$z0 +# CHECK-NEXT: SU(5): Ord Latency=0 Memory +# CHECK-NEXT: SU(4): Ord Latency=0 Memory +# CHECK-NEXT: SU(3): Ord Latency=0 Memory +# CHECK-NEXT: SU(2): Ord Latency=0 Memory +# CHECK-NEXT: SU(1): Ord Latency=0 Memory +# CHECK-NEXT: SU(0): Ord Latency=0 Memory +# CHECK-NEXT: Successors: +# CHECK-NEXT: SU(9): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: SU(9): ST1H killed renamable $z1, renamable $p0, renamable $x13, renamable $x10 :: (store unknown-size, align 1) +# CHECK-NEXT: # preds left : 8 +# CHECK-NEXT: # succs left : 0 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 1 +# CHECK-NEXT: Depth : 7 +# CHECK-NEXT: Height : 0 +# CHECK-NEXT: Predecessors: +# CHECK-NEXT: SU(8): Ord Latency=0 Memory +# CHECK-NEXT: SU(7): Data Latency=0 Reg=$z1 +# CHECK-NEXT: SU(5): Ord Latency=0 Memory +# CHECK-NEXT: SU(4): Ord Latency=0 Memory +# CHECK-NEXT: SU(3): Ord Latency=0 Memory +# CHECK-NEXT: SU(2): Ord Latency=0 Memory +# CHECK-NEXT: SU(1): Ord Latency=0 Memory +# CHECK-NEXT: SU(0): Ord Latency=0 Memory +# CHECK-NEXT: Single Issue : false; +# CHECK-NEXT: ExitSU: RET_ReallyLR +# CHECK-NEXT: # preds left : 0 +# CHECK-NEXT: # succs left : 0 +# CHECK-NEXT: # rdefs left : 0 +# CHECK-NEXT: Latency : 0 +# CHECK-NEXT: Depth : 0 +# CHECK-NEXT: Height : 0 + +--- +name: test +alignment: 4 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $p0, $x0, $x1, $x2, $x10, $x11, $x12, $x13 + + ; CHECK-LABEL: name: test + ; CHECK: liveins: $p0, $x0, $x1, $x2, $x10, $x11, $x12, $x13 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: renamable $z0 = LD1H renamable $p0, renamable $x1, renamable $x10 :: (load unknown-size, align 1) + ; CHECK-NEXT: renamable $z1 = LD1H renamable $p0, renamable $x2, renamable $x10 :: (load unknown-size, align 1) + ; CHECK-NEXT: renamable $z2 = LD1H renamable $p0, renamable $x0, renamable $x10 :: (load unknown-size, align 1) + ; CHECK-NEXT: $z0 = FMAD_ZPmZZ_H renamable $p0, killed $z0, renamable $z1, killed renamable $z2 + ; CHECK-NEXT: renamable $z3 = LD1H renamable $p0, renamable $x11, renamable $x10 :: (load unknown-size, align 1) + ; CHECK-NEXT: renamable $z4 = LD1H renamable $p0, renamable $x12, renamable $x10 :: (load unknown-size, align 1) + ; CHECK-NEXT: renamable $z5 = LD1H renamable $p0, renamable $x13, renamable $x10 :: (load unknown-size, align 1) + ; CHECK-NEXT: ST1H killed renamable $z0, renamable $p0, renamable $x0, renamable $x10 :: (store unknown-size, align 1) + ; CHECK-NEXT: BUNDLE implicit-def $z1, implicit-def $q1, implicit-def $d1, implicit-def $s1, implicit-def $h1, implicit-def $b1, implicit $z5, implicit $p0, implicit $z4, implicit $z3 { + ; CHECK-NEXT: $z1 = MOVPRFX_ZZ $z5 + ; CHECK-NEXT: $z1 = FMLA_ZPmZZ_H renamable $p0, internal $z1, renamable $z4, renamable $z3 + ; CHECK-NEXT: } + ; CHECK-NEXT: ST1H renamable $z1, renamable $p0, renamable $x13, renamable $x10 :: (store unknown-size, align 1) + ; CHECK-NEXT: RET_ReallyLR + + renamable $z0 = LD1H renamable $p0, renamable $x1, renamable $x10 :: (load unknown-size) + renamable $z1 = LD1H renamable $p0, renamable $x2, renamable $x10 :: (load unknown-size) + renamable $z2 = LD1H renamable $p0, renamable $x0, renamable $x10 :: (load unknown-size) + renamable $z3 = LD1H renamable $p0, renamable $x11, renamable $x10 :: (load unknown-size) + renamable $z4 = LD1H renamable $p0, renamable $x12, renamable $x10 :: (load unknown-size) + renamable $z5 = LD1H renamable $p0, renamable $x13, renamable $x10 :: (load unknown-size) + $z0 = FMAD_ZPmZZ_H renamable $p0, killed $z0, killed renamable $z1, killed renamable $z2 + BUNDLE implicit-def $z1, implicit-def $q1, implicit-def $d1, implicit-def $s1, implicit-def $h1, implicit-def $b1, implicit $z5, implicit $p0, implicit killed $z4, implicit killed $z3 { + $z1 = MOVPRFX_ZZ $z5 + $z1 = FMLA_ZPmZZ_H renamable $p0, internal killed $z1, killed renamable $z4, killed renamable $z3 + } + ST1H killed renamable $z0, renamable $p0, renamable $x0, renamable $x10 :: (store unknown-size) + ST1H killed renamable $z1, renamable $p0, renamable $x13, renamable $x10 :: (store unknown-size) + RET_ReallyLR + +... -- GitLab From e54af608160350baa7ae1b8069f916eb625beadd Mon Sep 17 00:00:00 2001 From: Hirofumi Nakamura Date: Fri, 22 Mar 2024 23:11:36 +0900 Subject: [PATCH 257/296] [clang-format] Added AlignConsecutiveTableGenBreakingDAGArgColons option. (#86150) The option to specify the style of alignment of the colons inside TableGen's DAGArg. --- clang/docs/ClangFormatStyleOptions.rst | 145 ++++++++++++++++++ clang/include/clang/Format/Format.h | 17 ++ clang/lib/Format/Format.cpp | 3 + clang/lib/Format/FormatToken.h | 1 + clang/lib/Format/TokenAnnotator.cpp | 18 ++- clang/lib/Format/WhitespaceManager.cpp | 6 + clang/lib/Format/WhitespaceManager.h | 3 + clang/unittests/Format/FormatTestTableGen.cpp | 32 ++++ clang/unittests/Format/TokenAnnotatorTest.cpp | 16 ++ 9 files changed, 236 insertions(+), 5 deletions(-) diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst index be021dfc5c08..2ee36f24d7ce 100644 --- a/clang/docs/ClangFormatStyleOptions.rst +++ b/clang/docs/ClangFormatStyleOptions.rst @@ -955,6 +955,151 @@ the configuration (without a prefix: ``Auto``). } +.. _AlignConsecutiveTableGenBreakingDAGArgColons: + +**AlignConsecutiveTableGenBreakingDAGArgColons** (``AlignConsecutiveStyle``) :versionbadge:`clang-format 19` :ref:`¶ ` + Style of aligning consecutive TableGen DAGArg operator colons. + If enabled, align the colon inside DAGArg which have line break inside. + This works only when TableGenBreakInsideDAGArg is BreakElements or + BreakAll and the DAGArg is not excepted by + TableGenBreakingDAGArgOperators's effect. + + .. code-block:: c++ + + let dagarg = (ins + a :$src1, + aa :$src2, + aaa:$src3 + ) + + Nested configuration flags: + + Alignment options. + + They can also be read as a whole for compatibility. The choices are: + - None + - Consecutive + - AcrossEmptyLines + - AcrossComments + - AcrossEmptyLinesAndComments + + For example, to align across empty lines and not across comments, either + of these work. + + .. code-block:: c++ + + AlignConsecutiveTableGenBreakingDAGArgColons: AcrossEmptyLines + + AlignConsecutiveTableGenBreakingDAGArgColons: + Enabled: true + AcrossEmptyLines: true + AcrossComments: false + + * ``bool Enabled`` Whether aligning is enabled. + + .. code-block:: c++ + + #define SHORT_NAME 42 + #define LONGER_NAME 0x007f + #define EVEN_LONGER_NAME (2) + #define foo(x) (x * x) + #define bar(y, z) (y + z) + + int a = 1; + int somelongname = 2; + double c = 3; + + int aaaa : 1; + int b : 12; + int ccc : 8; + + int aaaa = 12; + float b = 23; + std::string ccc; + + * ``bool AcrossEmptyLines`` Whether to align across empty lines. + + .. code-block:: c++ + + true: + int a = 1; + int somelongname = 2; + double c = 3; + + int d = 3; + + false: + int a = 1; + int somelongname = 2; + double c = 3; + + int d = 3; + + * ``bool AcrossComments`` Whether to align across comments. + + .. code-block:: c++ + + true: + int d = 3; + /* A comment. */ + double e = 4; + + false: + int d = 3; + /* A comment. */ + double e = 4; + + * ``bool AlignCompound`` Only for ``AlignConsecutiveAssignments``. Whether compound assignments + like ``+=`` are aligned along with ``=``. + + .. code-block:: c++ + + true: + a &= 2; + bbb = 2; + + false: + a &= 2; + bbb = 2; + + * ``bool AlignFunctionPointers`` Only for ``AlignConsecutiveDeclarations``. Whether function pointers are + aligned. + + .. code-block:: c++ + + true: + unsigned i; + int &r; + int *p; + int (*f)(); + + false: + unsigned i; + int &r; + int *p; + int (*f)(); + + * ``bool PadOperators`` Only for ``AlignConsecutiveAssignments``. Whether short assignment + operators are left-padded to the same length as long ones in order to + put all assignment operators to the right of the left hand side. + + .. code-block:: c++ + + true: + a >>= 2; + bbb = 2; + + a = 2; + bbb >>= 2; + + false: + a >>= 2; + bbb = 2; + + a = 2; + bbb >>= 2; + + .. _AlignConsecutiveTableGenCondOperatorColons: **AlignConsecutiveTableGenCondOperatorColons** (``AlignConsecutiveStyle``) :versionbadge:`clang-format 19` :ref:`¶ ` diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h index 7ad2579bf777..0720c8283cd7 100644 --- a/clang/include/clang/Format/Format.h +++ b/clang/include/clang/Format/Format.h @@ -414,6 +414,21 @@ struct FormatStyle { /// \version 17 ShortCaseStatementsAlignmentStyle AlignConsecutiveShortCaseStatements; + /// Style of aligning consecutive TableGen DAGArg operator colons. + /// If enabled, align the colon inside DAGArg which have line break inside. + /// This works only when TableGenBreakInsideDAGArg is BreakElements or + /// BreakAll and the DAGArg is not excepted by + /// TableGenBreakingDAGArgOperators's effect. + /// \code + /// let dagarg = (ins + /// a :$src1, + /// aa :$src2, + /// aaa:$src3 + /// ) + /// \endcode + /// \version 19 + AlignConsecutiveStyle AlignConsecutiveTableGenBreakingDAGArgColons; + /// Style of aligning consecutive TableGen cond operator colons. /// Align the colons of cases inside !cond operators. /// \code @@ -4879,6 +4894,8 @@ struct FormatStyle { AlignConsecutiveMacros == R.AlignConsecutiveMacros && AlignConsecutiveShortCaseStatements == R.AlignConsecutiveShortCaseStatements && + AlignConsecutiveTableGenBreakingDAGArgColons == + R.AlignConsecutiveTableGenBreakingDAGArgColons && AlignConsecutiveTableGenCondOperatorColons == R.AlignConsecutiveTableGenCondOperatorColons && AlignConsecutiveTableGenDefinitionColons == diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp index 63ec3a88978d..46ed5baaeace 100644 --- a/clang/lib/Format/Format.cpp +++ b/clang/lib/Format/Format.cpp @@ -895,6 +895,8 @@ template <> struct MappingTraits { IO.mapOptional("AlignConsecutiveMacros", Style.AlignConsecutiveMacros); IO.mapOptional("AlignConsecutiveShortCaseStatements", Style.AlignConsecutiveShortCaseStatements); + IO.mapOptional("AlignConsecutiveTableGenBreakingDAGArgColons", + Style.AlignConsecutiveTableGenBreakingDAGArgColons); IO.mapOptional("AlignConsecutiveTableGenCondOperatorColons", Style.AlignConsecutiveTableGenCondOperatorColons); IO.mapOptional("AlignConsecutiveTableGenDefinitionColons", @@ -1408,6 +1410,7 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) { LLVMStyle.AlignConsecutiveDeclarations = {}; LLVMStyle.AlignConsecutiveMacros = {}; LLVMStyle.AlignConsecutiveShortCaseStatements = {}; + LLVMStyle.AlignConsecutiveTableGenBreakingDAGArgColons = {}; LLVMStyle.AlignConsecutiveTableGenCondOperatorColons = {}; LLVMStyle.AlignConsecutiveTableGenDefinitionColons = {}; LLVMStyle.AlignEscapedNewlines = FormatStyle::ENAS_Right; diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h index 06f567059c35..2ddcd5259446 100644 --- a/clang/lib/Format/FormatToken.h +++ b/clang/lib/Format/FormatToken.h @@ -152,6 +152,7 @@ namespace format { TYPE(TableGenCondOperatorComma) \ TYPE(TableGenDAGArgCloser) \ TYPE(TableGenDAGArgListColon) \ + TYPE(TableGenDAGArgListColonToAlign) \ TYPE(TableGenDAGArgListComma) \ TYPE(TableGenDAGArgListCommaToBreak) \ TYPE(TableGenDAGArgOpener) \ diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp index 94d2266555f6..a5cafcbacaa5 100644 --- a/clang/lib/Format/TokenAnnotator.cpp +++ b/clang/lib/Format/TokenAnnotator.cpp @@ -975,12 +975,15 @@ private: // DagArg ::= Value [":" TokVarName] | TokVarName // Appears as a part of SimpleValue6. - bool parseTableGenDAGArg() { + bool parseTableGenDAGArg(bool AlignColon = false) { if (tryToParseTableGenTokVar()) return true; if (parseTableGenValue()) { if (CurrentToken && CurrentToken->is(tok::colon)) { - CurrentToken->setType(TT_TableGenDAGArgListColon); + if (AlignColon) + CurrentToken->setType(TT_TableGenDAGArgListColonToAlign); + else + CurrentToken->setType(TT_TableGenDAGArgListColon); skipToNextNonComment(); return tryToParseTableGenTokVar(); } @@ -1051,8 +1054,11 @@ private: skipToNextNonComment(); return true; } - if (!parseTableGenDAGArg()) + if (!parseTableGenDAGArg( + BreakInside && + Style.AlignConsecutiveTableGenBreakingDAGArgColons.Enabled)) { return false; + } FirstDAGArgListElm = false; } return false; @@ -5130,8 +5136,10 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line, if (Left.is(tok::r_brace) && Right.is(tok::r_square)) return true; // Do not insert around colon in DAGArg and cond operator. - if (Right.is(TT_TableGenDAGArgListColon) || - Left.is(TT_TableGenDAGArgListColon)) { + if (Right.isOneOf(TT_TableGenDAGArgListColon, + TT_TableGenDAGArgListColonToAlign) || + Left.isOneOf(TT_TableGenDAGArgListColon, + TT_TableGenDAGArgListColonToAlign)) { return false; } if (Right.is(TT_TableGenCondOperatorColon)) diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp index 753be25bfd67..fef85abf79a3 100644 --- a/clang/lib/Format/WhitespaceManager.cpp +++ b/clang/lib/Format/WhitespaceManager.cpp @@ -112,6 +112,7 @@ const tooling::Replacements &WhitespaceManager::generateReplacements() { alignConsecutiveBitFields(); alignConsecutiveAssignments(); if (Style.isTableGen()) { + alignConsecutiveTableGenBreakingDAGArgColons(); alignConsecutiveTableGenCondOperatorColons(); alignConsecutiveTableGenDefinitions(); } @@ -981,6 +982,11 @@ void WhitespaceManager::alignConsecutiveShortCaseStatements() { Changes); } +void WhitespaceManager::alignConsecutiveTableGenBreakingDAGArgColons() { + alignConsecutiveColons(Style.AlignConsecutiveTableGenBreakingDAGArgColons, + TT_TableGenDAGArgListColonToAlign); +} + void WhitespaceManager::alignConsecutiveTableGenCondOperatorColons() { alignConsecutiveColons(Style.AlignConsecutiveTableGenCondOperatorColons, TT_TableGenCondOperatorColon); diff --git a/clang/lib/Format/WhitespaceManager.h b/clang/lib/Format/WhitespaceManager.h index 0ebc6cf8377c..98cf4a260cc4 100644 --- a/clang/lib/Format/WhitespaceManager.h +++ b/clang/lib/Format/WhitespaceManager.h @@ -235,6 +235,9 @@ private: /// Align consecutive short case statements over all \c Changes. void alignConsecutiveShortCaseStatements(); + /// Align consecutive TableGen DAGArg colon over all \c Changes. + void alignConsecutiveTableGenBreakingDAGArgColons(); + /// Align consecutive TableGen cond operator colon over all \c Changes. void alignConsecutiveTableGenCondOperatorColons(); diff --git a/clang/unittests/Format/FormatTestTableGen.cpp b/clang/unittests/Format/FormatTestTableGen.cpp index c96866f0840f..8ca6bf97e5a6 100644 --- a/clang/unittests/Format/FormatTestTableGen.cpp +++ b/clang/unittests/Format/FormatTestTableGen.cpp @@ -411,6 +411,38 @@ TEST_F(FormatTestTableGen, DAGArgBreakAll) { Style); } +TEST_F(FormatTestTableGen, DAGArgAlignment) { + FormatStyle Style = getGoogleStyle(FormatStyle::LK_TableGen); + Style.ColumnLimit = 60; + Style.TableGenBreakInsideDAGArg = FormatStyle::DAS_BreakAll; + Style.TableGenBreakingDAGArgOperators = {"ins", "outs"}; + verifyFormat("def Def : Parent {\n" + " let dagarg = (ins\n" + " a:$src1,\n" + " aa:$src2,\n" + " aaa:$src3\n" + " )\n" + "}\n", + Style); + verifyFormat("def Def : Parent {\n" + " let dagarg = (not a:$src1, aa:$src2, aaa:$src2)\n" + "}\n", + Style); + Style.AlignConsecutiveTableGenBreakingDAGArgColons.Enabled = true; + verifyFormat("def Def : Parent {\n" + " let dagarg = (ins\n" + " a :$src1,\n" + " aa :$src2,\n" + " aaa:$src3\n" + " )\n" + "}\n", + Style); + verifyFormat("def Def : Parent {\n" + " let dagarg = (not a:$src1, aa:$src2, aaa:$src2)\n" + "}\n", + Style); +} + TEST_F(FormatTestTableGen, CondOperatorAlignment) { FormatStyle Style = getGoogleStyle(FormatStyle::LK_TableGen); Style.ColumnLimit = 60; diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp index 1aa855b34198..2539d3d76ef0 100644 --- a/clang/unittests/Format/TokenAnnotatorTest.cpp +++ b/clang/unittests/Format/TokenAnnotatorTest.cpp @@ -2424,6 +2424,22 @@ TEST_F(TokenAnnotatorTest, UnderstandTableGenTokens) { EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown); // other EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListComma); EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser); + + // If TableGenBreakingDAGArgOperators is enabled, it uses + // TT_TableGenDAGArgListColonToAlign to annotate the colon to align. + Style.AlignConsecutiveTableGenBreakingDAGArgColons.Enabled = true; + Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)"); + ASSERT_EQ(Tokens.size(), 10u) << Tokens; + EXPECT_TOKEN(Tokens[1], tok::identifier, + TT_TableGenDAGArgOperatorToBreak); // ins + EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColonToAlign); + EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColonToAlign); + + Tokens = AnnotateValue("(other type1:$src1, type2:$src2)"); + ASSERT_EQ(Tokens.size(), 10u) << Tokens; + EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown); // other + EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColon); + EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColon); } TEST_F(TokenAnnotatorTest, UnderstandConstructors) { -- GitLab From 46b853a82ce64e5213f8dfa2c12c6e6a950018a0 Mon Sep 17 00:00:00 2001 From: Billy Laws Date: Fri, 22 Mar 2024 14:17:06 +0000 Subject: [PATCH 258/296] [MC][COFF][AArch64] Treat ARM64EC/X as ARM64 for relocations (#86019) Since ARM64EC/X objects use regular ARM64 relocations, any special handling must be done for them too. --- llvm/lib/MC/WinCOFFObjectWriter.cpp | 4 ++-- llvm/test/MC/AArch64/coff-relocations.s | 12 +++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/llvm/lib/MC/WinCOFFObjectWriter.cpp b/llvm/lib/MC/WinCOFFObjectWriter.cpp index f265fafa59e7..3c9ff71b6b06 100644 --- a/llvm/lib/MC/WinCOFFObjectWriter.cpp +++ b/llvm/lib/MC/WinCOFFObjectWriter.cpp @@ -266,7 +266,7 @@ WinCOFFWriter::WinCOFFWriter(WinCOFFObjectWriter &OWriter, // limited range for the immediate offset (+/- 1 MB); create extra offset // label symbols with regular intervals to allow referencing a // non-temporary symbol that is close enough. - UseOffsetLabels = Header.Machine == COFF::IMAGE_FILE_MACHINE_ARM64; + UseOffsetLabels = COFF::isAnyArm64(Header.Machine); } COFFSymbol *WinCOFFWriter::createSymbol(StringRef Name) { @@ -954,7 +954,7 @@ void WinCOFFWriter::recordRelocation(MCAssembler &Asm, Reloc.Data.Type == COFF::IMAGE_REL_I386_REL32) || (Header.Machine == COFF::IMAGE_FILE_MACHINE_ARMNT && Reloc.Data.Type == COFF::IMAGE_REL_ARM_REL32) || - (Header.Machine == COFF::IMAGE_FILE_MACHINE_ARM64 && + (COFF::isAnyArm64(Header.Machine) && Reloc.Data.Type == COFF::IMAGE_REL_ARM64_REL32)) FixedValue += 4; diff --git a/llvm/test/MC/AArch64/coff-relocations.s b/llvm/test/MC/AArch64/coff-relocations.s index fb67a21992c6..2370fd9fb436 100644 --- a/llvm/test/MC/AArch64/coff-relocations.s +++ b/llvm/test/MC/AArch64/coff-relocations.s @@ -1,7 +1,11 @@ // RUN: llvm-mc -triple aarch64-windows -filetype obj -o %t.obj %s -// RUN: llvm-readobj -r %t.obj | FileCheck %s +// RUN: llvm-mc -triple arm64ec-windows -filetype obj -o %t-ec.obj %s +// RUN: llvm-readobj -r %t.obj | FileCheck %s --check-prefixes=CHECK,CHECK-ARM64 +// RUN: llvm-readobj -r %t-ec.obj | FileCheck %s --check-prefixes=CHECK,CHECK-ARM64EC // RUN: llvm-objdump --no-print-imm-hex -d %t.obj | FileCheck %s --check-prefix=DISASM +// RUN: llvm-objdump --no-print-imm-hex -d %t-ec.obj | FileCheck %s --check-prefix=DISASM // RUN: llvm-objdump -s %t.obj | FileCheck %s --check-prefix=DATA +// RUN: llvm-objdump -s %t-ec.obj | FileCheck %s --check-prefix=DATA // IMAGE_REL_ARM64_ADDR32 .Linfo_foo: @@ -71,8 +75,10 @@ tbz x0, #0, target // IMAGE_REL_ARM64_REL32 because IMAGE_REL_ARM64_REL64 does not exist. .xword .Linfo_foo - .Ltable -// CHECK: Format: COFF-ARM64 -// CHECK: Arch: aarch64 +// CHECK-ARM64: Format: COFF-ARM64 +// CHECK-ARM64EC: Format: COFF-ARM64EC +// CHECK-ARM64: Arch: aarch64 +// CHECK-ARM64EC: Arch: aarch64 // CHECK: AddressSize: 64bit // CHECK: Relocations [ // CHECK: Section (1) .text { -- GitLab From d231e3b10ead90e4360f7ceb88e4bca9d42d7d04 Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Fri, 22 Mar 2024 10:16:27 -0400 Subject: [PATCH 259/296] [C11] Add test & update status of N1282 and DR087 Our existing diagnostics for catching unsequenced modifications handles test coverage for N1282, which is correcting the standard based on the resolution of DR087. --- clang/test/C/C11/n1282.c | 20 ++++++++++++++++++++ clang/test/C/drs/dr0xx.c | 4 ++++ clang/www/c_dr_status.html | 2 +- clang/www/c_status.html | 2 +- 4 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 clang/test/C/C11/n1282.c diff --git a/clang/test/C/C11/n1282.c b/clang/test/C/C11/n1282.c new file mode 100644 index 000000000000..ed952790c883 --- /dev/null +++ b/clang/test/C/C11/n1282.c @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 -verify -Wunsequenced -Wno-unused-value %s + +/* WG14 N1282: Yes + * Clarification of Expressions + */ + +int g; + +int f(int i) { + g = i; + return 0; +} + +int main(void) { + int x; + x = (10, g = 1, 20) + (30, g = 2, 40); /* Line A */ // expected-warning {{multiple unsequenced modifications to 'g'}} + x = (10, f(1), 20) + (30, f(2), 40); /* Line B */ + x = (g = 1) + (g = 2); /* Line C */ // expected-warning {{multiple unsequenced modifications to 'g'}} + return 0; +} diff --git a/clang/test/C/drs/dr0xx.c b/clang/test/C/drs/dr0xx.c index c93cfb63d604..36de32a93da9 100644 --- a/clang/test/C/drs/dr0xx.c +++ b/clang/test/C/drs/dr0xx.c @@ -73,6 +73,10 @@ * WG14 DR085: yes * Returning from main * + * WG14 DR087: yes + * Order of evaluation + * Note: this DR is covered by C/C11/n1282.c + * * WG14 DR086: yes * Object-like macros in system headers * diff --git a/clang/www/c_dr_status.html b/clang/www/c_dr_status.html index fa2ceb1be58b..ed45123ffd0e 100644 --- a/clang/www/c_dr_status.html +++ b/clang/www/c_dr_status.html @@ -577,7 +577,7 @@ conformance.

87 NAD Order of evaluation - Unknown + Yes 88 diff --git a/clang/www/c_status.html b/clang/www/c_status.html index b1f5ab4cbc4f..0069da74cbd5 100644 --- a/clang/www/c_status.html +++ b/clang/www/c_status.html @@ -401,7 +401,7 @@ conformance.

Clarification of expressions N1282 - Unknown + Yes Extending the lifetime of temporary objects (factored approach) -- GitLab From 8612fa0d84c730a753d04de012a8372ba5a10677 Mon Sep 17 00:00:00 2001 From: agozillon Date: Fri, 22 Mar 2024 15:32:39 +0100 Subject: [PATCH 260/296] [MLIR][OpenMP] Refactor bounds offsetting and fix to apply to all directives (#84349) This PR refactors bounds offsetting by combining the two differing implementations (one applying to initial derived type member map implementation for descriptors and the other for regular arrays, effectively allocatable array vs regular array in fortran) now that it's a little simpler to do. The PR also moves the utilization of createAlteredByCaptureMap into genMapInfoOp, where it will be correctly applied to all MapInfoData, appropriately offsetting and altering Pointer data set in the kernel argument structure on the host. This primarily means bounds offsets will now correctly apply to enter/exit/update map clauses as opposed to just the Target directive that is currently the case. A few fortran runtime tests have been added to verify this new behavior. This PR depends on: https://github.com/llvm/llvm-project/pull/84328 and is an extraction of the larger derived type member map PR stack (so a requirement for it to land). --- .../OpenMP/OpenMPToLLVMIRTranslation.cpp | 374 +++++++++++------- ...target-fortran-allocatable-types-host.mlir | 11 +- mlir/test/Target/LLVMIR/omptarget-llvm.mlir | 25 +- .../fortran/target-map-enter-exit-array-2.f90 | 39 ++ .../target-map-enter-exit-array-bounds.f90 | 44 +++ .../fortran/target-map-enter-exit-scalar.f90 | 33 ++ 6 files changed, 360 insertions(+), 166 deletions(-) create mode 100644 openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-2.f90 create mode 100644 openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-bounds.f90 create mode 100644 openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-scalar.f90 diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp index 7df33470ea06..646d0ed73084 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp @@ -1787,6 +1787,20 @@ getDeclareTargetRefPtrSuffix(LLVM::GlobalOp globalOp, return suffix; } +static bool isDeclareTargetLink(mlir::Value value) { + if (auto addressOfOp = + llvm::dyn_cast_if_present(value.getDefiningOp())) { + auto modOp = addressOfOp->getParentOfType(); + Operation *gOp = modOp.lookupSymbol(addressOfOp.getGlobalName()); + if (auto declareTargetGlobal = + llvm::dyn_cast(gOp)) + if (declareTargetGlobal.getDeclareTargetCaptureClause() == + mlir::omp::DeclareTargetCaptureClause::link) + return true; + } + return false; +} + // Returns the reference pointer generated by the lowering of the declare target // operation in cases where the link clause is used or the to clause is used in // USM mode. @@ -1982,6 +1996,99 @@ void collectMapDataFromMapOperands(MapInfoData &mapData, } } +/// This function calculates the array/pointer offset for map data provided +/// with bounds operations, e.g. when provided something like the following: +/// +/// Fortran +/// map(tofrom: array(2:5, 3:2)) +/// or +/// C++ +/// map(tofrom: array[1:4][2:3]) +/// We must calculate the initial pointer offset to pass across, this function +/// performs this using bounds. +/// +/// NOTE: which while specified in row-major order it currently needs to be +/// flipped for Fortran's column order array allocation and access (as +/// opposed to C++'s row-major, hence the backwards processing where order is +/// important). This is likely important to keep in mind for the future when +/// we incorporate a C++ frontend, both frontends will need to agree on the +/// ordering of generated bounds operations (one may have to flip them) to +/// make the below lowering frontend agnostic. The offload size +/// calcualtion may also have to be adjusted for C++. +std::vector +calculateBoundsOffset(LLVM::ModuleTranslation &moduleTranslation, + llvm::IRBuilderBase &builder, bool isArrayTy, + mlir::OperandRange bounds) { + std::vector idx; + // There's no bounds to calculate an offset from, we can safely + // ignore and return no indices. + if (bounds.empty()) + return idx; + + // If we have an array type, then we have its type so can treat it as a + // normal GEP instruction where the bounds operations are simply indexes + // into the array. We currently do reverse order of the bounds, which + // I believe leans more towards Fortran's column-major in memory. + if (isArrayTy) { + idx.push_back(builder.getInt64(0)); + for (int i = bounds.size() - 1; i >= 0; --i) { + if (auto boundOp = mlir::dyn_cast_if_present( + bounds[i].getDefiningOp())) { + idx.push_back(moduleTranslation.lookupValue(boundOp.getLowerBound())); + } + } + } else { + // If we do not have an array type, but we have bounds, then we're dealing + // with a pointer that's being treated like an array and we have the + // underlying type e.g. an i32, or f64 etc, e.g. a fortran descriptor base + // address (pointer pointing to the actual data) so we must caclulate the + // offset using a single index which the following two loops attempts to + // compute. + + // Calculates the size offset we need to make per row e.g. first row or + // column only needs to be offset by one, but the next would have to be + // the previous row/column offset multiplied by the extent of current row. + // + // For example ([1][10][100]): + // + // - First row/column we move by 1 for each index increment + // - Second row/column we move by 1 (first row/column) * 10 (extent/size of + // current) for 10 for each index increment + // - Third row/column we would move by 10 (second row/column) * + // (extent/size of current) 100 for 1000 for each index increment + std::vector dimensionIndexSizeOffset{builder.getInt64(1)}; + for (size_t i = 1; i < bounds.size(); ++i) { + if (auto boundOp = mlir::dyn_cast_if_present( + bounds[i].getDefiningOp())) { + dimensionIndexSizeOffset.push_back(builder.CreateMul( + moduleTranslation.lookupValue(boundOp.getExtent()), + dimensionIndexSizeOffset[i - 1])); + } + } + + // Now that we have calculated how much we move by per index, we must + // multiply each lower bound offset in indexes by the size offset we + // have calculated in the previous and accumulate the results to get + // our final resulting offset. + for (int i = bounds.size() - 1; i >= 0; --i) { + if (auto boundOp = mlir::dyn_cast_if_present( + bounds[i].getDefiningOp())) { + if (idx.empty()) + idx.emplace_back(builder.CreateMul( + moduleTranslation.lookupValue(boundOp.getLowerBound()), + dimensionIndexSizeOffset[i])); + else + idx.back() = builder.CreateAdd( + idx.back(), builder.CreateMul(moduleTranslation.lookupValue( + boundOp.getLowerBound()), + dimensionIndexSizeOffset[i])); + } + } + } + + return idx; +} + // This creates two insertions into the MapInfosTy data structure for the // "parent" of a set of members, (usually a container e.g. // class/structure/derived type) when subsequent members have also been @@ -2057,6 +2164,27 @@ static llvm::omp::OpenMPOffloadMappingFlags mapParentWithMembers( return memberOfFlag; } +// The intent is to verify if the mapped data being passed is a +// pointer -> pointee that requires special handling in certain cases, +// e.g. applying the OMP_MAP_PTR_AND_OBJ map type. +// +// There may be a better way to verify this, but unfortunately with +// opaque pointers we lose the ability to easily check if something is +// a pointer whilst maintaining access to the underlying type. +static bool checkIfPointerMap(mlir::omp::MapInfoOp mapOp) { + // If we have a varPtrPtr field assigned then the underlying type is a pointer + if (mapOp.getVarPtrPtr()) + return true; + + // If the map data is declare target with a link clause, then it's represented + // as a pointer when we lower it to LLVM-IR even if at the MLIR level it has + // no relation to pointers. + if (isDeclareTargetLink(mapOp.getVarPtr())) + return true; + + return false; +} + // This function is intended to add explicit mappings of members static void processMapMembersWithParent( LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder, @@ -2083,8 +2211,11 @@ static void processMapMembersWithParent( auto mapFlag = llvm::omp::OpenMPOffloadMappingFlags(memberClause.getMapType().value()); mapFlag &= ~llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM; + mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF; ompBuilder.setCorrectMemberOfFlag(mapFlag, memberOfFlag); - mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ; + if (checkIfPointerMap(memberClause)) + mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ; + combinedInfo.Types.emplace_back(mapFlag); combinedInfo.DevicePointers.emplace_back( llvm::OpenMPIRBuilder::DeviceInfoTy::None); @@ -2092,55 +2223,7 @@ static void processMapMembersWithParent( LLVM::createMappingInformation(memberClause.getLoc(), ompBuilder)); combinedInfo.BasePointers.emplace_back(mapData.BasePointers[memberDataIdx]); - - std::vector idx{builder.getInt64(0)}; - llvm::Value *offsetAddress = nullptr; - if (!memberClause.getBounds().empty()) { - if (mapData.BaseType[memberDataIdx]->isArrayTy()) { - for (int i = memberClause.getBounds().size() - 1; i >= 0; --i) { - if (auto boundOp = mlir::dyn_cast_if_present( - memberClause.getBounds()[i].getDefiningOp())) { - idx.push_back( - moduleTranslation.lookupValue(boundOp.getLowerBound())); - } - } - } else { - std::vector dimensionIndexSizeOffset{ - builder.getInt64(1)}; - for (size_t i = 1; i < memberClause.getBounds().size(); ++i) { - if (auto boundOp = mlir::dyn_cast_if_present( - memberClause.getBounds()[i].getDefiningOp())) { - dimensionIndexSizeOffset.push_back(builder.CreateMul( - moduleTranslation.lookupValue(boundOp.getExtent()), - dimensionIndexSizeOffset[i - 1])); - } - } - - for (int i = memberClause.getBounds().size() - 1; i >= 0; --i) { - if (auto boundOp = mlir::dyn_cast_if_present( - memberClause.getBounds()[i].getDefiningOp())) { - if (!offsetAddress) - offsetAddress = builder.CreateMul( - moduleTranslation.lookupValue(boundOp.getLowerBound()), - dimensionIndexSizeOffset[i]); - else - offsetAddress = builder.CreateAdd( - offsetAddress, - builder.CreateMul( - moduleTranslation.lookupValue(boundOp.getLowerBound()), - dimensionIndexSizeOffset[i])); - } - } - } - } - - llvm::Value *memberIdx = - builder.CreateLoad(builder.getPtrTy(), mapData.Pointers[memberDataIdx]); - memberIdx = builder.CreateInBoundsGEP( - mapData.BaseType[memberDataIdx], memberIdx, - offsetAddress ? std::vector{offsetAddress} : idx, - "member_idx"); - combinedInfo.Pointers.emplace_back(memberIdx); + combinedInfo.Pointers.emplace_back(mapData.Pointers[memberDataIdx]); combinedInfo.Sizes.emplace_back(mapData.Sizes[memberDataIdx]); } } @@ -2158,6 +2241,76 @@ static void processMapWithMembersOf( memberOfParentFlag); } +// This is a variation on Clang's GenerateOpenMPCapturedVars, which +// generates different operation (e.g. load/store) combinations for +// arguments to the kernel, based on map capture kinds which are then +// utilised in the combinedInfo in place of the original Map value. +static void +createAlteredByCaptureMap(MapInfoData &mapData, + LLVM::ModuleTranslation &moduleTranslation, + llvm::IRBuilderBase &builder) { + for (size_t i = 0; i < mapData.MapClause.size(); ++i) { + // if it's declare target, skip it, it's handled seperately. + if (!mapData.IsDeclareTarget[i]) { + auto mapOp = + mlir::dyn_cast_if_present(mapData.MapClause[i]); + mlir::omp::VariableCaptureKind captureKind = + mapOp.getMapCaptureType().value_or( + mlir::omp::VariableCaptureKind::ByRef); + bool isPtrTy = checkIfPointerMap(mapOp); + + // Currently handles array sectioning lowerbound case, but more + // logic may be required in the future. Clang invokes EmitLValue, + // which has specialised logic for special Clang types such as user + // defines, so it is possible we will have to extend this for + // structures or other complex types. As the general idea is that this + // function mimics some of the logic from Clang that we require for + // kernel argument passing from host -> device. + switch (captureKind) { + case mlir::omp::VariableCaptureKind::ByRef: { + llvm::Value *newV = mapData.Pointers[i]; + std::vector offsetIdx = calculateBoundsOffset( + moduleTranslation, builder, mapData.BaseType[i]->isArrayTy(), + mapOp.getBounds()); + if (isPtrTy) + newV = builder.CreateLoad(builder.getPtrTy(), newV); + + if (!offsetIdx.empty()) + newV = builder.CreateInBoundsGEP(mapData.BaseType[i], newV, offsetIdx, + "array_offset"); + mapData.Pointers[i] = newV; + } break; + case mlir::omp::VariableCaptureKind::ByCopy: { + llvm::Type *type = mapData.BaseType[i]; + llvm::Value *newV; + if (mapData.Pointers[i]->getType()->isPointerTy()) + newV = builder.CreateLoad(type, mapData.Pointers[i]); + else + newV = mapData.Pointers[i]; + + if (!isPtrTy) { + auto curInsert = builder.saveIP(); + builder.restoreIP(findAllocaInsertPoint(builder, moduleTranslation)); + auto *memTempAlloc = + builder.CreateAlloca(builder.getPtrTy(), nullptr, ".casted"); + builder.restoreIP(curInsert); + + builder.CreateStore(newV, memTempAlloc); + newV = builder.CreateLoad(builder.getPtrTy(), memTempAlloc); + } + + mapData.Pointers[i] = newV; + mapData.BasePointers[i] = newV; + } break; + case mlir::omp::VariableCaptureKind::This: + case mlir::omp::VariableCaptureKind::VLAType: + mapData.MapClause[i]->emitOpError("Unhandled capture kind"); + break; + } + } + } +} + // Generate all map related information and fill the combinedInfo. static void genMapInfos(llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, @@ -2167,6 +2320,20 @@ static void genMapInfos(llvm::IRBuilderBase &builder, const SmallVector &devPtrOperands = {}, const SmallVector &devAddrOperands = {}, bool isTargetParams = false) { + // We wish to modify some of the methods in which arguments are + // passed based on their capture type by the target region, this can + // involve generating new loads and stores, which changes the + // MLIR value to LLVM value mapping, however, we only wish to do this + // locally for the current function/target and also avoid altering + // ModuleTranslation, so we remap the base pointer or pointer stored + // in the map infos corresponding MapInfoData, which is later accessed + // by genMapInfos and createTarget to help generate the kernel and + // kernel arg structure. It primarily becomes relevant in cases like + // bycopy, or byref range'd arrays. In the default case, we simply + // pass thee pointer byref as both basePointer and pointer. + if (!moduleTranslation.getOpenMPBuilder()->Config.isTargetDevice()) + createAlteredByCaptureMap(mapData, moduleTranslation, builder); + llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); auto fail = [&combinedInfo]() -> void { @@ -2200,19 +2367,20 @@ static void genMapInfos(llvm::IRBuilderBase &builder, continue; } - // Declare Target Mappings are excluded from being marked as - // OMP_MAP_TARGET_PARAM as they are not passed as parameters, they're - // marked with OMP_MAP_PTR_AND_OBJ instead. auto mapFlag = mapData.Types[i]; - if (mapData.IsDeclareTarget[i]) + bool isPtrTy = checkIfPointerMap(mapInfoOp); + if (isPtrTy) mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ; - else if (isTargetParams) + + // Declare Target Mappings are excluded from being marked as + // OMP_MAP_TARGET_PARAM as they are not passed as parameters. + if (isTargetParams && !mapData.IsDeclareTarget[i]) mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM; if (auto mapInfoOp = dyn_cast(mapData.MapClause[i])) if (mapInfoOp.getMapCaptureType().value() == mlir::omp::VariableCaptureKind::ByCopy && - !mapInfoOp.getVarType().isa()) + !isPtrTy) mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL; combinedInfo.BasePointers.emplace_back(mapData.BasePointers[i]); @@ -2662,86 +2830,6 @@ createDeviceArgumentAccessor(MapInfoData &mapData, llvm::Argument &arg, return builder.saveIP(); } -// This is a variation on Clang's GenerateOpenMPCapturedVars, which -// generates different operation (e.g. load/store) combinations for -// arguments to the kernel, based on map capture kinds which are then -// utilised in the combinedInfo in place of the original Map value. -static void -createAlteredByCaptureMap(MapInfoData &mapData, - LLVM::ModuleTranslation &moduleTranslation, - llvm::IRBuilderBase &builder) { - for (size_t i = 0; i < mapData.MapClause.size(); ++i) { - // if it's declare target, skip it, it's handled seperately. - if (!mapData.IsDeclareTarget[i]) { - mlir::omp::VariableCaptureKind captureKind = - mlir::omp::VariableCaptureKind::ByRef; - - if (auto mapOp = mlir::dyn_cast_if_present( - mapData.MapClause[i])) { - captureKind = mapOp.getMapCaptureType().value_or( - mlir::omp::VariableCaptureKind::ByRef); - } - - switch (captureKind) { - case mlir::omp::VariableCaptureKind::ByRef: { - // Currently handles array sectioning lowerbound case, but more - // logic may be required in the future. Clang invokes EmitLValue, - // which has specialised logic for special Clang types such as user - // defines, so it is possible we will have to extend this for - // structures or other complex types. As the general idea is that this - // function mimics some of the logic from Clang that we require for - // kernel argument passing from host -> device. - if (auto mapOp = mlir::dyn_cast_if_present( - mapData.MapClause[i])) { - if (!mapOp.getBounds().empty() && mapData.BaseType[i]->isArrayTy()) { - - std::vector idx = - std::vector{builder.getInt64(0)}; - for (int i = mapOp.getBounds().size() - 1; i >= 0; --i) { - if (auto boundOp = - mlir::dyn_cast_if_present( - mapOp.getBounds()[i].getDefiningOp())) { - idx.push_back( - moduleTranslation.lookupValue(boundOp.getLowerBound())); - } - } - - mapData.Pointers[i] = builder.CreateInBoundsGEP( - mapData.BaseType[i], mapData.Pointers[i], idx); - } - } - } break; - case mlir::omp::VariableCaptureKind::ByCopy: { - llvm::Type *type = mapData.BaseType[i]; - llvm::Value *newV; - if (mapData.Pointers[i]->getType()->isPointerTy()) - newV = builder.CreateLoad(type, mapData.Pointers[i]); - else - newV = mapData.Pointers[i]; - - if (!type->isPointerTy()) { - auto curInsert = builder.saveIP(); - builder.restoreIP(findAllocaInsertPoint(builder, moduleTranslation)); - auto *memTempAlloc = - builder.CreateAlloca(builder.getPtrTy(), nullptr, ".casted"); - builder.restoreIP(curInsert); - - builder.CreateStore(newV, memTempAlloc); - newV = builder.CreateLoad(builder.getPtrTy(), memTempAlloc); - } - - mapData.Pointers[i] = newV; - mapData.BasePointers[i] = newV; - } break; - case mlir::omp::VariableCaptureKind::This: - case mlir::omp::VariableCaptureKind::VLAType: - mapData.MapClause[i]->emitOpError("Unhandled capture kind"); - break; - } - } - } -} - static LogicalResult convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) { @@ -2810,20 +2898,6 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder, collectMapDataFromMapOperands(mapData, mapOperands, moduleTranslation, dl, builder); - // We wish to modify some of the methods in which kernel arguments are - // passed based on their capture type by the target region, this can - // involve generating new loads and stores, which changes the - // MLIR value to LLVM value mapping, however, we only wish to do this - // locally for the current function/target and also avoid altering - // ModuleTranslation, so we remap the base pointer or pointer stored - // in the map infos corresponding MapInfoData, which is later accessed - // by genMapInfos and createTarget to help generate the kernel and - // kernel arg structure. It primarily becomes relevant in cases like - // bycopy, or byref range'd arrays. In the default case, we simply - // pass thee pointer byref as both basePointer and pointer. - if (!moduleTranslation.getOpenMPBuilder()->Config.isTargetDevice()) - createAlteredByCaptureMap(mapData, moduleTranslation, builder); - llvm::OpenMPIRBuilder::MapInfosTy combinedInfos; auto genMapInfoCB = [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & { diff --git a/mlir/test/Target/LLVMIR/omptarget-fortran-allocatable-types-host.mlir b/mlir/test/Target/LLVMIR/omptarget-fortran-allocatable-types-host.mlir index e8c388627a0a..7cb22dbb10b1 100644 --- a/mlir/test/Target/LLVMIR/omptarget-fortran-allocatable-types-host.mlir +++ b/mlir/test/Target/LLVMIR/omptarget-fortran-allocatable-types-host.mlir @@ -26,7 +26,7 @@ module attributes {omp.is_target_device = false} { %14 = llvm.sub %11, %2 : i64 %15 = omp.map.bounds lower_bound(%7 : i64) upper_bound(%14 : i64) extent(%11 : i64) stride(%13 : i64) start_idx(%9 : i64) {stride_in_bytes = true} %16 = llvm.getelementptr %3[0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> - %17 = omp.map.info var_ptr(%16 : !llvm.ptr, f32) map_clauses(tofrom) capture(ByRef) bounds(%15) -> !llvm.ptr {name = "full_arr"} + %17 = omp.map.info var_ptr(%3 : !llvm.ptr, f32) var_ptr_ptr(%16 : !llvm.ptr) map_clauses(tofrom) capture(ByRef) bounds(%15) -> !llvm.ptr {name = "full_arr"} %18 = omp.map.info var_ptr(%3 : !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>) map_clauses(tofrom) capture(ByRef) members(%17 : !llvm.ptr) -> !llvm.ptr {name = "full_arr"} %19 = llvm.getelementptr %6[0, 7, %7, 0] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> %20 = llvm.load %19 : !llvm.ptr -> i64 @@ -81,20 +81,19 @@ module attributes {omp.is_target_device = false} { // CHECK: %[[ARR_SECT_SIZE2:.*]] = add i64 %[[ARR_SECT_SIZE3]], 1 // CHECK: %[[ARR_SECT_SIZE1:.*]] = mul i64 1, %[[ARR_SECT_SIZE2]] // CHECK: %[[ARR_SECT_SIZE:.*]] = mul i64 %[[ARR_SECT_SIZE1]], 4 -// CHECK: %[[FULL_ARR_DESC_SIZE:.*]] = sdiv exact i64 sub (i64 ptrtoint (ptr getelementptr inbounds ({ ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr @_QFEfull_arr, i32 1) to i64), i64 ptrtoint (ptr @_QFEfull_arr to i64)), ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64) // CHECK: %[[LFULL_ARR:.*]] = load ptr, ptr @_QFEfull_arr, align 8 // CHECK: %[[FULL_ARR_PTR:.*]] = getelementptr inbounds float, ptr %[[LFULL_ARR]], i64 0 -// CHECK: %[[ARR_SECT_DESC_SIZE:.*]] = sdiv exact i64 sub (i64 ptrtoint (ptr getelementptr inbounds ({ ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr @_QFEsect_arr, i32 1) to i64), i64 ptrtoint (ptr @_QFEsect_arr to i64)), ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64) // CHECK: %[[ARR_SECT_OFFSET1:.*]] = mul i64 %[[ARR_SECT_OFFSET2]], 1 // CHECK: %[[LARR_SECT:.*]] = load ptr, ptr @_QFEsect_arr, align 8 // CHECK: %[[ARR_SECT_PTR:.*]] = getelementptr inbounds i32, ptr %[[LARR_SECT]], i64 %[[ARR_SECT_OFFSET1]] +// CHECK: %[[SCALAR_PTR_LOAD:.*]] = load ptr, ptr %[[SCALAR_BASE]], align 8 +// CHECK: %[[FULL_ARR_DESC_SIZE:.*]] = sdiv exact i64 sub (i64 ptrtoint (ptr getelementptr inbounds ({ ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr @_QFEfull_arr, i32 1) to i64), i64 ptrtoint (ptr @_QFEfull_arr to i64)), ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64) +// CHECK: %[[ARR_SECT_DESC_SIZE:.*]] = sdiv exact i64 sub (i64 ptrtoint (ptr getelementptr inbounds ({ ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr @_QFEsect_arr, i32 1) to i64), i64 ptrtoint (ptr @_QFEsect_arr to i64)), ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64) // CHECK: %[[SCALAR_DESC_SZ4:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8 }, ptr %[[SCALAR_ALLOCA]], i32 1 // CHECK: %[[SCALAR_DESC_SZ3:.*]] = ptrtoint ptr %[[SCALAR_DESC_SZ4]] to i64 // CHECK: %[[SCALAR_DESC_SZ2:.*]] = ptrtoint ptr %[[SCALAR_ALLOCA]] to i64 // CHECK: %[[SCALAR_DESC_SZ1:.*]] = sub i64 %[[SCALAR_DESC_SZ3]], %[[SCALAR_DESC_SZ2]] // CHECK: %[[SCALAR_DESC_SZ:.*]] = sdiv exact i64 %[[SCALAR_DESC_SZ1]], ptrtoint (ptr getelementptr (i8, ptr null, i32 1) to i64) -// CHECK: %[[SCALAR_PTR_LOAD:.*]] = load ptr, ptr %[[SCALAR_BASE]], align 8 -// CHECK: %[[SCALAR_PTR:.*]] = getelementptr inbounds float, ptr %[[SCALAR_PTR_LOAD]], i64 0 // CHECK: %[[OFFLOADBASEPTRS:.*]] = getelementptr inbounds [9 x ptr], ptr %.offload_baseptrs, i32 0, i32 0 // CHECK: store ptr @_QFEfull_arr, ptr %[[OFFLOADBASEPTRS]], align 8 @@ -145,4 +144,4 @@ module attributes {omp.is_target_device = false} { // CHECK: %[[OFFLOADBASEPTRS:.*]] = getelementptr inbounds [9 x ptr], ptr %.offload_baseptrs, i32 0, i32 8 // CHECK: store ptr %[[SCALAR_BASE]], ptr %[[OFFLOADBASEPTRS]], align 8 // CHECK: %[[OFFLOADPTRS:.*]] = getelementptr inbounds [9 x ptr], ptr %.offload_ptrs, i32 0, i32 8 -// CHECK: store ptr %[[SCALAR_PTR]], ptr %[[OFFLOADPTRS]], align 8 +// CHECK: store ptr %[[SCALAR_PTR_LOAD]], ptr %[[OFFLOADPTRS]], align 8 diff --git a/mlir/test/Target/LLVMIR/omptarget-llvm.mlir b/mlir/test/Target/LLVMIR/omptarget-llvm.mlir index 4b1d5d58f14e..2f629675442d 100644 --- a/mlir/test/Target/LLVMIR/omptarget-llvm.mlir +++ b/mlir/test/Target/LLVMIR/omptarget-llvm.mlir @@ -66,16 +66,17 @@ llvm.func @_QPopenmp_target_data_region(%0 : !llvm.ptr) { // CHECK: %[[VAL_2:.*]] = alloca [1 x ptr], align 8 // CHECK: br label %[[VAL_3:.*]] // CHECK: entry: ; preds = %[[VAL_4:.*]] +// CHECK: %[[ARR_OFFSET:.*]] = getelementptr inbounds [1024 x i32], ptr %[[ARR_DATA:.*]], i64 0, i64 0 // CHECK: %[[VAL_5:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0 -// CHECK: store ptr %[[VAL_6:.*]], ptr %[[VAL_5]], align 8 +// CHECK: store ptr %[[ARR_DATA]], ptr %[[VAL_5]], align 8 // CHECK: %[[VAL_7:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0 -// CHECK: store ptr %[[VAL_6]], ptr %[[VAL_7]], align 8 +// CHECK: store ptr %[[ARR_OFFSET]], ptr %[[VAL_7]], align 8 // CHECK: %[[VAL_8:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_2]], i64 0, i64 0 // CHECK: store ptr null, ptr %[[VAL_8]], align 8 // CHECK: %[[VAL_9:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0 // CHECK: %[[VAL_10:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0 // CHECK: call void @__tgt_target_data_begin_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_9]], ptr %[[VAL_10]], ptr @.offload_sizes, ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null) -// CHECK: %[[VAL_11:.*]] = getelementptr [1024 x i32], ptr %[[VAL_6]], i32 0, i64 0 +// CHECK: %[[VAL_11:.*]] = getelementptr [1024 x i32], ptr %[[ARR_DATA]], i32 0, i64 0 // CHECK: store i32 99, ptr %[[VAL_11]], align 4 // CHECK: %[[VAL_12:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0 // CHECK: %[[VAL_13:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0 @@ -153,16 +154,18 @@ llvm.func @_QPomp_target_enter_exit(%1 : !llvm.ptr, %3 : !llvm.ptr) { // CHECK: entry: ; preds = %[[VAL_12:.*]] // CHECK: br i1 %[[VAL_9]], label %[[VAL_13:.*]], label %[[VAL_14:.*]] // CHECK: omp_if.then: ; preds = %[[VAL_11]] +// CHECK: %[[ARR_OFFSET1:.*]] = getelementptr inbounds [1024 x i32], ptr %[[VAL_16:.*]], i64 0, i64 0 +// CHECK: %[[ARR_OFFSET2:.*]] = getelementptr inbounds [512 x i32], ptr %[[VAL_20:.*]], i64 0, i64 0 // CHECK: %[[VAL_15:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_3]], i32 0, i32 0 // CHECK: store ptr %[[VAL_16:.*]], ptr %[[VAL_15]], align 8 // CHECK: %[[VAL_17:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_4]], i32 0, i32 0 -// CHECK: store ptr %[[VAL_16]], ptr %[[VAL_17]], align 8 +// CHECK: store ptr %[[ARR_OFFSET1]], ptr %[[VAL_17]], align 8 // CHECK: %[[VAL_18:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_5]], i64 0, i64 0 // CHECK: store ptr null, ptr %[[VAL_18]], align 8 // CHECK: %[[VAL_19:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_3]], i32 0, i32 1 // CHECK: store ptr %[[VAL_20:.*]], ptr %[[VAL_19]], align 8 // CHECK: %[[VAL_21:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_4]], i32 0, i32 1 -// CHECK: store ptr %[[VAL_20]], ptr %[[VAL_21]], align 8 +// CHECK: store ptr %[[ARR_OFFSET2]], ptr %[[VAL_21]], align 8 // CHECK: %[[VAL_22:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_5]], i64 0, i64 1 // CHECK: store ptr null, ptr %[[VAL_22]], align 8 // CHECK: %[[VAL_23:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_3]], i32 0, i32 0 @@ -176,26 +179,28 @@ llvm.func @_QPomp_target_enter_exit(%1 : !llvm.ptr, %3 : !llvm.ptr) { // CHECK: %[[VAL_27:.*]] = icmp sgt i32 %[[VAL_26]], 10 // CHECK: %[[VAL_28:.*]] = load i32, ptr %[[VAL_6]], align 4 // CHECK: br i1 %[[VAL_27]], label %[[VAL_29:.*]], label %[[VAL_30:.*]] -// CHECK: omp_if.then1: ; preds = %[[VAL_25]] +// CHECK: omp_if.then2: ; preds = %[[VAL_25]] +// CHECK: %[[ARR_OFFSET3:.*]] = getelementptr inbounds [1024 x i32], ptr %[[VAL_16]], i64 0, i64 0 +// CHECK: %[[ARR_OFFSET4:.*]] = getelementptr inbounds [512 x i32], ptr %[[VAL_20]], i64 0, i64 0 // CHECK: %[[VAL_31:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_0]], i32 0, i32 0 // CHECK: store ptr %[[VAL_16]], ptr %[[VAL_31]], align 8 // CHECK: %[[VAL_32:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_1]], i32 0, i32 0 -// CHECK: store ptr %[[VAL_16]], ptr %[[VAL_32]], align 8 +// CHECK: store ptr %[[ARR_OFFSET3]], ptr %[[VAL_32]], align 8 // CHECK: %[[VAL_33:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_2]], i64 0, i64 0 // CHECK: store ptr null, ptr %[[VAL_33]], align 8 // CHECK: %[[VAL_34:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_0]], i32 0, i32 1 // CHECK: store ptr %[[VAL_20]], ptr %[[VAL_34]], align 8 // CHECK: %[[VAL_35:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_1]], i32 0, i32 1 -// CHECK: store ptr %[[VAL_20]], ptr %[[VAL_35]], align 8 +// CHECK: store ptr %[[ARR_OFFSET4]], ptr %[[VAL_35]], align 8 // CHECK: %[[VAL_36:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_2]], i64 0, i64 1 // CHECK: store ptr null, ptr %[[VAL_36]], align 8 // CHECK: %[[VAL_37:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_0]], i32 0, i32 0 // CHECK: %[[VAL_38:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_1]], i32 0, i32 0 // CHECK: call void @__tgt_target_data_end_mapper(ptr @3, i64 -1, i32 2, ptr %[[VAL_37]], ptr %[[VAL_38]], ptr @.offload_sizes.1, ptr @.offload_maptypes.2, ptr @.offload_mapnames.3, ptr null) // CHECK: br label %[[VAL_39:.*]] -// CHECK: omp_if.else5: ; preds = %[[VAL_25]] +// CHECK: omp_if.else8: ; preds = %[[VAL_25]] // CHECK: br label %[[VAL_39]] -// CHECK: omp_if.end6: ; preds = %[[VAL_30]], %[[VAL_29]] +// CHECK: omp_if.end9: ; preds = %[[VAL_30]], %[[VAL_29]] // CHECK: ret void // ----- diff --git a/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-2.f90 b/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-2.f90 new file mode 100644 index 000000000000..489c2532a762 --- /dev/null +++ b/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-2.f90 @@ -0,0 +1,39 @@ +! Offloading test checking interaction of an +! enter and exit map of an array of scalars +! REQUIRES: flang, amdgcn-amd-amdhsa +! UNSUPPORTED: nvptx64-nvidia-cuda +! UNSUPPORTED: nvptx64-nvidia-cuda-LTO +! UNSUPPORTED: aarch64-unknown-linux-gnu +! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO +! UNSUPPORTED: x86_64-pc-linux-gnu +! UNSUPPORTED: x86_64-pc-linux-gnu-LTO + +! RUN: %libomptarget-compile-fortran-run-and-check-generic +program main + integer :: array(10) + + do I = 1, 10 + array(I) = I + I + end do + + !$omp target enter data map(to: array) + + ! Shouldn't overwrite data already locked in + ! on target via enter, this will then be + ! overwritten by our exit + do I = 1, 10 + array(I) = 10 + end do + + !$omp target + do i=1,10 + array(i) = array(i) + i + end do + !$omp end target + + !$omp target exit data map(from: array) + + print*, array +end program + +!CHECK: 3 6 9 12 15 18 21 24 27 30 diff --git a/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-bounds.f90 b/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-bounds.f90 new file mode 100644 index 000000000000..3c8c3507ed72 --- /dev/null +++ b/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-array-bounds.f90 @@ -0,0 +1,44 @@ +! Offloading test checking interaction of an +! enter and exit map of an array of scalars +! with specified bounds +! REQUIRES: flang, amdgcn-amd-amdhsa +! UNSUPPORTED: nvptx64-nvidia-cuda +! UNSUPPORTED: nvptx64-nvidia-cuda-LTO +! UNSUPPORTED: aarch64-unknown-linux-gnu +! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO +! UNSUPPORTED: x86_64-pc-linux-gnu +! UNSUPPORTED: x86_64-pc-linux-gnu-LTO + +! RUN: %libomptarget-compile-fortran-run-and-check-generic + +program main + integer :: array(10) + + do I = 1, 10 + array(I) = I + I + end do + + !$omp target enter data map(to: array(3:6)) + + ! Shouldn't overwrite data already locked in + ! on target via enter, which will then be + ! overwritten by our exit + do I = 1, 10 + array(I) = 10 + end do + + ! The compiler/runtime is less lenient about read/write out of + ! bounds when using enter and exit, we have to specifically loop + ! over the correctly mapped range + !$omp target + do i=3,6 + array(i) = array(i) + i + end do + !$omp end target + + !$omp target exit data map(from: array(3:6)) + + print *, array +end program + +!CHECK: 10 10 9 12 15 18 10 10 10 10 diff --git a/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-scalar.f90 b/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-scalar.f90 new file mode 100644 index 000000000000..29a0b5ee3e62 --- /dev/null +++ b/openmp/libomptarget/test/offloading/fortran/target-map-enter-exit-scalar.f90 @@ -0,0 +1,33 @@ +! Offloading test checking interaction of an +! enter and exit map of an scalar +! REQUIRES: flang, amdgcn-amd-amdhsa +! UNSUPPORTED: nvptx64-nvidia-cuda +! UNSUPPORTED: nvptx64-nvidia-cuda-LTO +! UNSUPPORTED: aarch64-unknown-linux-gnu +! UNSUPPORTED: aarch64-unknown-linux-gnu-LTO +! UNSUPPORTED: x86_64-pc-linux-gnu +! UNSUPPORTED: x86_64-pc-linux-gnu-LTO + +! RUN: %libomptarget-compile-fortran-run-and-check-generic +program main + integer :: scalar + scalar = 10 + + !$omp target enter data map(to: scalar) + + !ignored, as we've already attached + scalar = 20 + + !$omp target + scalar = scalar + 50 + !$omp end target + + !$omp target exit data map(from: scalar) + + ! not the answer one may expect, but it is the same + ! answer Clang gives so we are correctly on par with + ! Clang for the moment. + print *, scalar +end program + +!CHECK: 10 -- GitLab From 71db97152173a524a3e16e02b7fdc50f405c8695 Mon Sep 17 00:00:00 2001 From: Matthias Gehre Date: Fri, 22 Mar 2024 15:39:52 +0100 Subject: [PATCH 261/296] [mlir][emitc] Arith to EmitC: Handle addi, subi and muli (#86120) Important to consider that `arith` has wrap around semantics, and in C++ signed overflow is UB. Unless the operation guarantees that no signed overflow happens, we will perform the arithmetic in an equivalent unsigned type. `bool` also doesn't wrap around in C++, and is not addressed here. --- .../Conversion/ArithToEmitC/ArithToEmitC.cpp | 52 +++++++++++++++++++ .../ArithToEmitC/arith-to-emitc-failed.mlir | 15 ++++++ .../ArithToEmitC/arith-to-emitc.mlir | 51 ++++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 mlir/test/Conversion/ArithToEmitC/arith-to-emitc-failed.mlir diff --git a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp index 3532785c31b9..db493c1294ba 100644 --- a/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp +++ b/mlir/lib/Conversion/ArithToEmitC/ArithToEmitC.cpp @@ -55,6 +55,55 @@ public: } }; +template +class IntegerOpConversion final : public OpConversionPattern { +public: + using OpConversionPattern::OpConversionPattern; + + LogicalResult + matchAndRewrite(ArithOp op, typename ArithOp::Adaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + + Type type = this->getTypeConverter()->convertType(op.getType()); + if (!isa_and_nonnull(type)) { + return rewriter.notifyMatchFailure(op, "expected integer type"); + } + + if (type.isInteger(1)) { + // arith expects wrap-around arithmethic, which doesn't happen on `bool`. + return rewriter.notifyMatchFailure(op, "i1 type is not implemented"); + } + + Value lhs = adaptor.getLhs(); + Value rhs = adaptor.getRhs(); + Type arithmeticType = type; + if ((type.isSignlessInteger() || type.isSignedInteger()) && + !bitEnumContainsAll(op.getOverflowFlags(), + arith::IntegerOverflowFlags::nsw)) { + // If the C type is signed and the op doesn't guarantee "No Signed Wrap", + // we compute in unsigned integers to avoid UB. + arithmeticType = rewriter.getIntegerType(type.getIntOrFloatBitWidth(), + /*isSigned=*/false); + } + if (arithmeticType != type) { + lhs = rewriter.template create(op.getLoc(), arithmeticType, + lhs); + rhs = rewriter.template create(op.getLoc(), arithmeticType, + rhs); + } + + Value result = rewriter.template create(op.getLoc(), + arithmeticType, lhs, rhs); + + if (arithmeticType != type) { + result = + rewriter.template create(op.getLoc(), type, result); + } + rewriter.replaceOp(op, result); + return success(); + } +}; + class SelectOpConversion : public OpConversionPattern { public: using OpConversionPattern::OpConversionPattern; @@ -96,6 +145,9 @@ void mlir::populateArithToEmitCPatterns(TypeConverter &typeConverter, ArithOpConversion, ArithOpConversion, ArithOpConversion, + IntegerOpConversion, + IntegerOpConversion, + IntegerOpConversion, SelectOpConversion >(typeConverter, ctx); // clang-format on diff --git a/mlir/test/Conversion/ArithToEmitC/arith-to-emitc-failed.mlir b/mlir/test/Conversion/ArithToEmitC/arith-to-emitc-failed.mlir new file mode 100644 index 000000000000..30abd81f3d44 --- /dev/null +++ b/mlir/test/Conversion/ArithToEmitC/arith-to-emitc-failed.mlir @@ -0,0 +1,15 @@ +// RUN: mlir-opt -convert-arith-to-emitc %s -split-input-file -verify-diagnostics + +func.func @bool(%arg0: i1, %arg1: i1) { + // expected-error@+1 {{failed to legalize operation 'arith.addi'}} + %0 = arith.addi %arg0, %arg1 : i1 + return +} + +// ----- + +func.func @vector(%arg0: vector<4xi32>, %arg1: vector<4xi32>) { + // expected-error@+1 {{failed to legalize operation 'arith.addi'}} + %0 = arith.addi %arg0, %arg1 : vector<4xi32> + return +} diff --git a/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir b/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir index 022530ef4db8..76ba518577ab 100644 --- a/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir +++ b/mlir/test/Conversion/ArithToEmitC/arith-to-emitc.mlir @@ -37,6 +37,57 @@ func.func @arith_ops(%arg0: f32, %arg1: f32) { // ----- +// CHECK-LABEL: arith_integer_ops +func.func @arith_integer_ops(%arg0: i32, %arg1: i32) { + // CHECK: %[[C1:[^ ]*]] = emitc.cast %arg0 : i32 to ui32 + // CHECK: %[[C2:[^ ]*]] = emitc.cast %arg1 : i32 to ui32 + // CHECK: %[[ADD:[^ ]*]] = emitc.add %[[C1]], %[[C2]] : (ui32, ui32) -> ui32 + // CHECK: %[[C3:[^ ]*]] = emitc.cast %[[ADD]] : ui32 to i32 + %0 = arith.addi %arg0, %arg1 : i32 + // CHECK: %[[C1:[^ ]*]] = emitc.cast %arg0 : i32 to ui32 + // CHECK: %[[C2:[^ ]*]] = emitc.cast %arg1 : i32 to ui32 + // CHECK: %[[SUB:[^ ]*]] = emitc.sub %[[C1]], %[[C2]] : (ui32, ui32) -> ui32 + // CHECK: %[[C3:[^ ]*]] = emitc.cast %[[SUB]] : ui32 to i32 + %1 = arith.subi %arg0, %arg1 : i32 + // CHECK: %[[C1:[^ ]*]] = emitc.cast %arg0 : i32 to ui32 + // CHECK: %[[C2:[^ ]*]] = emitc.cast %arg1 : i32 to ui32 + // CHECK: %[[MUL:[^ ]*]] = emitc.mul %[[C1]], %[[C2]] : (ui32, ui32) -> ui32 + // CHECK: %[[C3:[^ ]*]] = emitc.cast %[[MUL]] : ui32 to i32 + %2 = arith.muli %arg0, %arg1 : i32 + + return +} + +// ----- + +// CHECK-LABEL: arith_integer_ops_signed_nsw +func.func @arith_integer_ops_signed_nsw(%arg0: i32, %arg1: i32) { + // CHECK: emitc.add %arg0, %arg1 : (i32, i32) -> i32 + %0 = arith.addi %arg0, %arg1 overflow : i32 + // CHECK: emitc.sub %arg0, %arg1 : (i32, i32) -> i32 + %1 = arith.subi %arg0, %arg1 overflow : i32 + // CHECK: emitc.mul %arg0, %arg1 : (i32, i32) -> i32 + %2 = arith.muli %arg0, %arg1 overflow : i32 + + return +} + +// ----- + +// CHECK-LABEL: arith_index +func.func @arith_index(%arg0: index, %arg1: index) { + // CHECK: emitc.add %arg0, %arg1 : (index, index) -> index + %0 = arith.addi %arg0, %arg1 : index + // CHECK: emitc.sub %arg0, %arg1 : (index, index) -> index + %1 = arith.subi %arg0, %arg1 : index + // CHECK: emitc.mul %arg0, %arg1 : (index, index) -> index + %2 = arith.muli %arg0, %arg1 : index + + return +} + +// ----- + func.func @arith_select(%arg0: i1, %arg1: tensor<8xi32>, %arg2: tensor<8xi32>) -> () { // CHECK: [[V0:[^ ]*]] = emitc.conditional %arg0, %arg1, %arg2 : tensor<8xi32> %0 = arith.select %arg0, %arg1, %arg2 : i1, tensor<8xi32> -- GitLab From 9c0a0659d40f613e873e416833d2293365b48e06 Mon Sep 17 00:00:00 2001 From: Alexey Bataev Date: Fri, 22 Mar 2024 07:31:35 -0700 Subject: [PATCH 262/296] [SLP]Fix a crash for non-profitable non-schedulable single buildvector node tree, if the threshold allows its vectorization. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 2 + .../small-tree-not-schedulable-bv-node.ll | 263 ++++++++++++++++++ 2 files changed, 265 insertions(+) create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/small-tree-not-schedulable-bv-node.ll diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 7295ae0ba90b..0f7afa2fc25c 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -9278,6 +9278,8 @@ bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const { bool IsAllowedSingleBVNode = VectorizableTree.size() > 1 || (VectorizableTree.size() == 1 && VectorizableTree.front()->getOpcode() && + VectorizableTree.front()->getOpcode() != Instruction::PHI && + VectorizableTree.front()->getOpcode() != Instruction::GetElementPtr && allSameBlock(VectorizableTree.front()->Scalars)); if (any_of(VectorizableTree, [&](const std::unique_ptr &TE) { return TE->State == TreeEntry::NeedToGather && diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/small-tree-not-schedulable-bv-node.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/small-tree-not-schedulable-bv-node.ll new file mode 100644 index 000000000000..26f3fcae3a33 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/small-tree-not-schedulable-bv-node.ll @@ -0,0 +1,263 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -mtriple=riscv64-unknown-linux-gnu -slp-threshold=-100 -mattr=+v < %s | FileCheck %s + +define void @test1() personality ptr null { +; CHECK-LABEL: define void @test1( +; CHECK-SAME: ) #[[ATTR0:[0-9]+]] personality ptr null { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[CALL33:%.*]] = invoke ptr null(i64 0, ptr null) +; CHECK-NEXT: to label [[INVOKE_CONT32:%.*]] unwind label [[LPAD31_LOOPEXIT:%.*]] +; CHECK: invoke.cont32: +; CHECK-NEXT: invoke void null(ptr null, ptr null) +; CHECK-NEXT: to label [[INVOKE_CONT37:%.*]] unwind label [[LPAD34_LOOPEXIT:%.*]] +; CHECK: invoke.cont37: +; CHECK-NEXT: unreachable +; CHECK: lpad31.loopexit: +; CHECK-NEXT: [[LPAD_LOOPEXIT:%.*]] = landingpad { ptr, i32 } +; CHECK-NEXT: cleanup +; CHECK-NEXT: br label [[EHCLEANUP47:%.*]] +; CHECK: lpad34.loopexit: +; CHECK-NEXT: [[DOTLCSSA101:%.*]] = phi ptr [ null, [[INVOKE_CONT32]] ] +; CHECK-NEXT: [[CALL33_LCSSA96:%.*]] = phi ptr [ [[CALL33]], [[INVOKE_CONT32]] ] +; CHECK-NEXT: [[LPAD_LOOPEXIT56:%.*]] = landingpad { ptr, i32 } +; CHECK-NEXT: cleanup +; CHECK-NEXT: br label [[LPAD34_BODY:%.*]] +; CHECK: lpad34.loopexit.split-lp: +; CHECK-NEXT: [[LPAD_LOOPEXIT_SPLIT_LP57:%.*]] = landingpad { ptr, i32 } +; CHECK-NEXT: cleanup +; CHECK-NEXT: br label [[LPAD34_BODY]] +; CHECK: lpad34.body: +; CHECK-NEXT: [[TMP0:%.*]] = phi ptr [ [[DOTLCSSA101]], [[LPAD34_LOOPEXIT]] ], [ null, [[LPAD34_LOOPEXIT_SPLIT_LP:%.*]] ] +; CHECK-NEXT: [[CALL3399:%.*]] = phi ptr [ [[CALL33_LCSSA96]], [[LPAD34_LOOPEXIT]] ], [ null, [[LPAD34_LOOPEXIT_SPLIT_LP]] ] +; CHECK-NEXT: br label [[EHCLEANUP47]] +; CHECK: ehcleanup47: +; CHECK-NEXT: resume { ptr, i32 } zeroinitializer +; +entry: + %call33 = invoke ptr null(i64 0, ptr null) + to label %invoke.cont32 unwind label %lpad31.loopexit + +invoke.cont32: + invoke void null(ptr null, ptr null) + to label %invoke.cont37 unwind label %lpad34.loopexit + +invoke.cont37: + unreachable + +lpad31.loopexit: + %lpad.loopexit = landingpad { ptr, i32 } + cleanup + br label %ehcleanup47 + +lpad34.loopexit: + %.lcssa101 = phi ptr [ null, %invoke.cont32 ] + %call33.lcssa96 = phi ptr [ %call33, %invoke.cont32 ] + %lpad.loopexit56 = landingpad { ptr, i32 } + cleanup + br label %lpad34.body + +lpad34.loopexit.split-lp: + %lpad.loopexit.split-lp57 = landingpad { ptr, i32 } + cleanup + br label %lpad34.body + +lpad34.body: + %0 = phi ptr [ %.lcssa101, %lpad34.loopexit ], [ null, %lpad34.loopexit.split-lp ] + %call3399 = phi ptr [ %call33.lcssa96, %lpad34.loopexit ], [ null, %lpad34.loopexit.split-lp ] + br label %ehcleanup47 + +ehcleanup47: + resume { ptr, i32 } zeroinitializer +} + +define i32 @test2(i64 %idx.ext.i48.pre-phi) { +; CHECK-LABEL: define i32 @test2( +; CHECK-SAME: i64 [[IDX_EXT_I48_PRE_PHI:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[DO_ACTION:%.*]] +; CHECK: do_action: +; CHECK-NEXT: switch i32 0, label [[SW_DEFAULT:%.*]] [ +; CHECK-NEXT: i32 1, label [[CLEANUP185:%.*]] +; CHECK-NEXT: i32 2, label [[CLEANUP185]] +; CHECK-NEXT: i32 0, label [[CLEANUP185]] +; CHECK-NEXT: i32 4, label [[CLEANUP185]] +; CHECK-NEXT: i32 5, label [[CLEANUP185]] +; CHECK-NEXT: i32 6, label [[CLEANUP185]] +; CHECK-NEXT: i32 7, label [[CLEANUP185]] +; CHECK-NEXT: i32 8, label [[CLEANUP185]] +; CHECK-NEXT: i32 9, label [[CLEANUP185]] +; CHECK-NEXT: i32 10, label [[CLEANUP185]] +; CHECK-NEXT: i32 11, label [[CLEANUP185]] +; CHECK-NEXT: i32 12, label [[CLEANUP185]] +; CHECK-NEXT: i32 13, label [[CLEANUP185]] +; CHECK-NEXT: i32 14, label [[CLEANUP185]] +; CHECK-NEXT: i32 15, label [[CLEANUP185]] +; CHECK-NEXT: i32 16, label [[CLEANUP185]] +; CHECK-NEXT: i32 17, label [[CLEANUP185]] +; CHECK-NEXT: i32 18, label [[CLEANUP185]] +; CHECK-NEXT: i32 19, label [[CLEANUP185]] +; CHECK-NEXT: i32 20, label [[CLEANUP185]] +; CHECK-NEXT: i32 21, label [[CLEANUP185]] +; CHECK-NEXT: i32 22, label [[CLEANUP185]] +; CHECK-NEXT: i32 23, label [[CLEANUP185]] +; CHECK-NEXT: i32 24, label [[CLEANUP185]] +; CHECK-NEXT: i32 25, label [[CLEANUP185]] +; CHECK-NEXT: i32 26, label [[CLEANUP185]] +; CHECK-NEXT: i32 27, label [[CLEANUP185]] +; CHECK-NEXT: i32 28, label [[CLEANUP185]] +; CHECK-NEXT: i32 29, label [[CLEANUP185]] +; CHECK-NEXT: i32 30, label [[CLEANUP185]] +; CHECK-NEXT: i32 31, label [[CLEANUP185]] +; CHECK-NEXT: i32 32, label [[CLEANUP185]] +; CHECK-NEXT: i32 33, label [[CLEANUP185]] +; CHECK-NEXT: i32 34, label [[CLEANUP185]] +; CHECK-NEXT: i32 35, label [[CLEANUP185]] +; CHECK-NEXT: i32 36, label [[CLEANUP185]] +; CHECK-NEXT: i32 37, label [[CLEANUP185]] +; CHECK-NEXT: i32 38, label [[CLEANUP185]] +; CHECK-NEXT: i32 39, label [[CLEANUP185]] +; CHECK-NEXT: i32 40, label [[CLEANUP185]] +; CHECK-NEXT: i32 41, label [[CLEANUP185]] +; CHECK-NEXT: i32 42, label [[CLEANUP185]] +; CHECK-NEXT: i32 43, label [[CLEANUP185]] +; CHECK-NEXT: i32 44, label [[CLEANUP185]] +; CHECK-NEXT: i32 45, label [[CLEANUP185]] +; CHECK-NEXT: i32 46, label [[CLEANUP185]] +; CHECK-NEXT: i32 47, label [[CLEANUP185]] +; CHECK-NEXT: i32 48, label [[CLEANUP185]] +; CHECK-NEXT: i32 49, label [[CLEANUP185]] +; CHECK-NEXT: i32 50, label [[CLEANUP185]] +; CHECK-NEXT: i32 51, label [[CLEANUP185]] +; CHECK-NEXT: i32 52, label [[CLEANUP185]] +; CHECK-NEXT: i32 53, label [[CLEANUP185]] +; CHECK-NEXT: i32 54, label [[CLEANUP185]] +; CHECK-NEXT: i32 55, label [[CLEANUP185]] +; CHECK-NEXT: i32 56, label [[CLEANUP185]] +; CHECK-NEXT: i32 57, label [[DO_ACTION]] +; CHECK-NEXT: i32 58, label [[CLEANUP185]] +; CHECK-NEXT: i32 59, label [[CLEANUP185]] +; CHECK-NEXT: i32 60, label [[DO_ACTION]] +; CHECK-NEXT: i32 61, label [[DO_ACTION]] +; CHECK-NEXT: i32 62, label [[CLEANUP185]] +; CHECK-NEXT: i32 70, label [[SW_BB175:%.*]] +; CHECK-NEXT: i32 64, label [[CLEANUP185]] +; CHECK-NEXT: i32 65, label [[DO_ACTION]] +; CHECK-NEXT: i32 66, label [[DO_ACTION]] +; CHECK-NEXT: i32 67, label [[CLEANUP185]] +; CHECK-NEXT: i32 72, label [[CLEANUP185]] +; CHECK-NEXT: i32 69, label [[DO_ACTION]] +; CHECK-NEXT: i32 71, label [[CLEANUP185]] +; CHECK-NEXT: ] +; CHECK: yy_get_previous_state.exit.loopexit: +; CHECK-NEXT: br label [[YY_FIND_ACTION_BACKEDGE:%.*]] +; CHECK: yy_find_action.backedge: +; CHECK-NEXT: [[YY_BP_1_BE:%.*]] = phi ptr [ [[ADD_PTR_I49:%.*]], [[SW_BB175]] ], [ null, [[YY_GET_PREVIOUS_STATE_EXIT_LOOPEXIT:%.*]] ] +; CHECK-NEXT: [[YY_CP_2_BE:%.*]] = phi ptr [ [[ARRAYIDX178:%.*]], [[SW_BB175]] ], [ null, [[YY_GET_PREVIOUS_STATE_EXIT_LOOPEXIT]] ] +; CHECK-NEXT: br label [[DO_ACTION]] +; CHECK: sw.bb175: +; CHECK-NEXT: [[ARRAYIDX178]] = getelementptr i8, ptr null, i64 0 +; CHECK-NEXT: [[ADD_PTR_I49]] = getelementptr i8, ptr null, i64 [[IDX_EXT_I48_PRE_PHI]] +; CHECK-NEXT: [[CMP5_I50:%.*]] = icmp ult ptr [[ADD_PTR_I49]], [[ARRAYIDX178]] +; CHECK-NEXT: br label [[YY_FIND_ACTION_BACKEDGE]] +; CHECK: sw.default: +; CHECK-NEXT: unreachable +; CHECK: cleanup185: +; CHECK-NEXT: ret i32 0 +; +entry: + br label %do_action + +do_action: + switch i32 0, label %sw.default [ + i32 1, label %cleanup185 + i32 2, label %cleanup185 + i32 0, label %cleanup185 + i32 4, label %cleanup185 + i32 5, label %cleanup185 + i32 6, label %cleanup185 + i32 7, label %cleanup185 + i32 8, label %cleanup185 + i32 9, label %cleanup185 + i32 10, label %cleanup185 + i32 11, label %cleanup185 + i32 12, label %cleanup185 + i32 13, label %cleanup185 + i32 14, label %cleanup185 + i32 15, label %cleanup185 + i32 16, label %cleanup185 + i32 17, label %cleanup185 + i32 18, label %cleanup185 + i32 19, label %cleanup185 + i32 20, label %cleanup185 + i32 21, label %cleanup185 + i32 22, label %cleanup185 + i32 23, label %cleanup185 + i32 24, label %cleanup185 + i32 25, label %cleanup185 + i32 26, label %cleanup185 + i32 27, label %cleanup185 + i32 28, label %cleanup185 + i32 29, label %cleanup185 + i32 30, label %cleanup185 + i32 31, label %cleanup185 + i32 32, label %cleanup185 + i32 33, label %cleanup185 + i32 34, label %cleanup185 + i32 35, label %cleanup185 + i32 36, label %cleanup185 + i32 37, label %cleanup185 + i32 38, label %cleanup185 + i32 39, label %cleanup185 + i32 40, label %cleanup185 + i32 41, label %cleanup185 + i32 42, label %cleanup185 + i32 43, label %cleanup185 + i32 44, label %cleanup185 + i32 45, label %cleanup185 + i32 46, label %cleanup185 + i32 47, label %cleanup185 + i32 48, label %cleanup185 + i32 49, label %cleanup185 + i32 50, label %cleanup185 + i32 51, label %cleanup185 + i32 52, label %cleanup185 + i32 53, label %cleanup185 + i32 54, label %cleanup185 + i32 55, label %cleanup185 + i32 56, label %cleanup185 + i32 57, label %do_action + i32 58, label %cleanup185 + i32 59, label %cleanup185 + i32 60, label %do_action + i32 61, label %do_action + i32 62, label %cleanup185 + i32 70, label %sw.bb175 + i32 64, label %cleanup185 + i32 65, label %do_action + i32 66, label %do_action + i32 67, label %cleanup185 + i32 72, label %cleanup185 + i32 69, label %do_action + i32 71, label %cleanup185 + ] + +yy_get_previous_state.exit.loopexit: + br label %yy_find_action.backedge + +yy_find_action.backedge: + %yy_bp.1.be = phi ptr [ %add.ptr.i49, %sw.bb175 ], [ null, %yy_get_previous_state.exit.loopexit ] + %yy_cp.2.be = phi ptr [ %arrayidx178, %sw.bb175 ], [ null, %yy_get_previous_state.exit.loopexit ] + br label %do_action + +sw.bb175: + %arrayidx178 = getelementptr i8, ptr null, i64 0 + %add.ptr.i49 = getelementptr i8, ptr null, i64 %idx.ext.i48.pre-phi + %cmp5.i50 = icmp ult ptr %add.ptr.i49, %arrayidx178 + br label %yy_find_action.backedge + +sw.default: + unreachable + +cleanup185: + ret i32 0 +} -- GitLab From 6f44bb7717897191be25aa01161831c67cdf5b84 Mon Sep 17 00:00:00 2001 From: Antonio Frighetto Date: Fri, 22 Mar 2024 15:40:15 +0100 Subject: [PATCH 263/296] [Object] Ensure header size not to underflow in `OffloadBinary::create` Prevent potential integer underflows when header size is not valid. Fixes: https://github.com/llvm/llvm-project/issues/86280. --- llvm/lib/Object/OffloadBinary.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Object/OffloadBinary.cpp b/llvm/lib/Object/OffloadBinary.cpp index 4ab6536dc90b..6e9f8bed513c 100644 --- a/llvm/lib/Object/OffloadBinary.cpp +++ b/llvm/lib/Object/OffloadBinary.cpp @@ -189,7 +189,10 @@ OffloadBinary::create(MemoryBufferRef Buf) { return errorCodeToError(object_error::parse_failed); if (TheHeader->Size > Buf.getBufferSize() || - TheHeader->EntryOffset > TheHeader->Size - sizeof(Entry) || + TheHeader->Size < sizeof(Entry) || TheHeader->Size < sizeof(Header)) + return errorCodeToError(object_error::unexpected_eof); + + if (TheHeader->EntryOffset > TheHeader->Size - sizeof(Entry) || TheHeader->EntrySize > TheHeader->Size - sizeof(Header)) return errorCodeToError(object_error::unexpected_eof); -- GitLab From b44771f480385fa93ba7719a57e759e19747e709 Mon Sep 17 00:00:00 2001 From: Wang Pengcheng Date: Fri, 22 Mar 2024 18:24:23 +0800 Subject: [PATCH 264/296] [RISCV] Support RISC-V Profiles in -march option (#76357) This PR implements the draft https://github.com/riscv-non-isa/riscv-toolchain-conventions/pull/36. Currently, we replace specified profile in `-march` with standard arch string. This is recommitted as 66f88de was reverted because of failures caused by lacking `--target` option. --- clang/docs/ReleaseNotes.rst | 1 + clang/test/Driver/riscv-profiles.c | 324 +++++++++++++++++++++++++++++ llvm/lib/Support/RISCVISAInfo.cpp | 64 ++++++ 3 files changed, 389 insertions(+) create mode 100644 clang/test/Driver/riscv-profiles.c diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 45b2e01af997..d6e179ca9d69 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -520,6 +520,7 @@ RISC-V Support ^^^^^^^^^^^^^^ - ``__attribute__((rvv_vector_bits(N)))`` is now supported for RVV vbool*_t types. +- Profile names in ``-march`` option are now supported. CUDA/HIP Language Changes ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/clang/test/Driver/riscv-profiles.c b/clang/test/Driver/riscv-profiles.c new file mode 100644 index 000000000000..0227487015ba --- /dev/null +++ b/clang/test/Driver/riscv-profiles.c @@ -0,0 +1,324 @@ +// RUN: %clang --target=riscv32 -### -c %s 2>&1 -march=rvi20u32 \ +// RUN: | FileCheck -check-prefix=RVI20U32 %s +// RVI20U32: "-target-feature" "-a" +// RVI20U32: "-target-feature" "-c" +// RVI20U32: "-target-feature" "-d" +// RVI20U32: "-target-feature" "-f" +// RVI20U32: "-target-feature" "-m" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rvi20u64 \ +// RUN: | FileCheck -check-prefix=RVI20U64 %s +// RVI20U64: "-target-feature" "-a" +// RVI20U64: "-target-feature" "-c" +// RVI20U64: "-target-feature" "-d" +// RVI20U64: "-target-feature" "-f" +// RVI20U64: "-target-feature" "-m" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva20u64 \ +// RUN: | FileCheck -check-prefix=RVA20U64 %s +// RVA20U64: "-target-feature" "+m" +// RVA20U64: "-target-feature" "+a" +// RVA20U64: "-target-feature" "+f" +// RVA20U64: "-target-feature" "+d" +// RVA20U64: "-target-feature" "+c" +// RVA20U64: "-target-feature" "+ziccamoa" +// RVA20U64: "-target-feature" "+ziccif" +// RVA20U64: "-target-feature" "+zicclsm" +// RVA20U64: "-target-feature" "+ziccrse" +// RVA20U64: "-target-feature" "+zicntr" +// RVA20U64: "-target-feature" "+zicsr" +// RVA20U64: "-target-feature" "+za128rs" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva20s64 \ +// RUN: | FileCheck -check-prefix=RVA20S64 %s +// RVA20S64: "-target-feature" "+m" +// RVA20S64: "-target-feature" "+a" +// RVA20S64: "-target-feature" "+f" +// RVA20S64: "-target-feature" "+d" +// RVA20S64: "-target-feature" "+c" +// RVA20S64: "-target-feature" "+ziccamoa" +// RVA20S64: "-target-feature" "+ziccif" +// RVA20S64: "-target-feature" "+zicclsm" +// RVA20S64: "-target-feature" "+ziccrse" +// RVA20S64: "-target-feature" "+zicntr" +// RVA20S64: "-target-feature" "+zicsr" +// RVA20S64: "-target-feature" "+zifencei" +// RVA20S64: "-target-feature" "+za128rs" +// RVA20S64: "-target-feature" "+ssccptr" +// RVA20S64: "-target-feature" "+sstvala" +// RVA20S64: "-target-feature" "+sstvecd" +// RVA20S64: "-target-feature" "+svade" +// RVA20S64: "-target-feature" "+svbare" + +// RUN: %clang --target=riscv64 --target=riscv64 -### -c %s 2>&1 -march=rva22u64 \ +// RUN: | FileCheck -check-prefix=RVA22U64 %s +// RVA22U64: "-target-feature" "+m" +// RVA22U64: "-target-feature" "+a" +// RVA22U64: "-target-feature" "+f" +// RVA22U64: "-target-feature" "+d" +// RVA22U64: "-target-feature" "+c" +// RVA22U64: "-target-feature" "+zic64b" +// RVA22U64: "-target-feature" "+zicbom" +// RVA22U64: "-target-feature" "+zicbop" +// RVA22U64: "-target-feature" "+zicboz" +// RVA22U64: "-target-feature" "+ziccamoa" +// RVA22U64: "-target-feature" "+ziccif" +// RVA22U64: "-target-feature" "+zicclsm" +// RVA22U64: "-target-feature" "+ziccrse" +// RVA22U64: "-target-feature" "+zicntr" +// RVA22U64: "-target-feature" "+zicsr" +// RVA22U64: "-target-feature" "+zihintpause" +// RVA22U64: "-target-feature" "+zihpm" +// RVA22U64: "-target-feature" "+za64rs" +// RVA22U64: "-target-feature" "+zfhmin" +// RVA22U64: "-target-feature" "+zba" +// RVA22U64: "-target-feature" "+zbb" +// RVA22U64: "-target-feature" "+zbs" +// RVA22U64: "-target-feature" "+zkt" + +// RUN: %clang --target=riscv64 --target=riscv64 -### -c %s 2>&1 -march=rva22s64 \ +// RUN: | FileCheck -check-prefix=RVA22S64 %s +// RVA22S64: "-target-feature" "+m" +// RVA22S64: "-target-feature" "+a" +// RVA22S64: "-target-feature" "+f" +// RVA22S64: "-target-feature" "+d" +// RVA22S64: "-target-feature" "+c" +// RVA22S64: "-target-feature" "+zic64b" +// RVA22S64: "-target-feature" "+zicbom" +// RVA22S64: "-target-feature" "+zicbop" +// RVA22S64: "-target-feature" "+zicboz" +// RVA22S64: "-target-feature" "+ziccamoa" +// RVA22S64: "-target-feature" "+ziccif" +// RVA22S64: "-target-feature" "+zicclsm" +// RVA22S64: "-target-feature" "+ziccrse" +// RVA22S64: "-target-feature" "+zicntr" +// RVA22S64: "-target-feature" "+zicsr" +// RVA22S64: "-target-feature" "+zifencei" +// RVA22S64: "-target-feature" "+zihintpause" +// RVA22S64: "-target-feature" "+zihpm" +// RVA22S64: "-target-feature" "+za64rs" +// RVA22S64: "-target-feature" "+zfhmin" +// RVA22S64: "-target-feature" "+zba" +// RVA22S64: "-target-feature" "+zbb" +// RVA22S64: "-target-feature" "+zbs" +// RVA22S64: "-target-feature" "+zkt" +// RVA22S64: "-target-feature" "+ssccptr" +// RVA22S64: "-target-feature" "+sscounterenw" +// RVA22S64: "-target-feature" "+sstvala" +// RVA22S64: "-target-feature" "+sstvecd" +// RVA22S64: "-target-feature" "+svade" +// RVA22S64: "-target-feature" "+svbare" +// RVA22S64: "-target-feature" "+svinval" +// RVA22S64: "-target-feature" "+svpbmt" + +// RUN: %clang --target=riscv64 --target=riscv64 -### -c %s 2>&1 -march=rva23u64 -menable-experimental-extensions \ +// RUN: | FileCheck -check-prefix=RVA23U64 %s +// RVA23U64: "-target-feature" "+m" +// RVA23U64: "-target-feature" "+a" +// RVA23U64: "-target-feature" "+f" +// RVA23U64: "-target-feature" "+d" +// RVA23U64: "-target-feature" "+c" +// RVA23U64: "-target-feature" "+v" +// RVA23U64: "-target-feature" "+zic64b" +// RVA23U64: "-target-feature" "+zicbom" +// RVA23U64: "-target-feature" "+zicbop" +// RVA23U64: "-target-feature" "+zicboz" +// RVA23U64: "-target-feature" "+ziccamoa" +// RVA23U64: "-target-feature" "+ziccif" +// RVA23U64: "-target-feature" "+zicclsm" +// RVA23U64: "-target-feature" "+ziccrse" +// RVA23U64: "-target-feature" "+zicntr" +// RVA23U64: "-target-feature" "+zicond" +// RVA23U64: "-target-feature" "+zicsr" +// RVA23U64: "-target-feature" "+zihintntl" +// RVA23U64: "-target-feature" "+zihintpause" +// RVA23U64: "-target-feature" "+zihpm" +// RVA23U64: "-target-feature" "+experimental-zimop" +// RVA23U64: "-target-feature" "+za64rs" +// RVA23U64: "-target-feature" "+zawrs" +// RVA23U64: "-target-feature" "+zfa" +// RVA23U64: "-target-feature" "+zfhmin" +// RVA23U64: "-target-feature" "+zcb" +// RVA23U64: "-target-feature" "+experimental-zcmop" +// RVA23U64: "-target-feature" "+zba" +// RVA23U64: "-target-feature" "+zbb" +// RVA23U64: "-target-feature" "+zbs" +// RVA23U64: "-target-feature" "+zkt" +// RVA23U64: "-target-feature" "+zvbb" +// RVA23U64: "-target-feature" "+zvfhmin" +// RVA23U64: "-target-feature" "+zvkt" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva23s64 -menable-experimental-extensions \ +// RUN: | FileCheck -check-prefix=RVA23S64 %s +// RVA23S64: "-target-feature" "+m" +// RVA23S64: "-target-feature" "+a" +// RVA23S64: "-target-feature" "+f" +// RVA23S64: "-target-feature" "+d" +// RVA23S64: "-target-feature" "+c" +// RVA23S64: "-target-feature" "+v" +// RVA23S64: "-target-feature" "+h" +// RVA23S64: "-target-feature" "+zic64b" +// RVA23S64: "-target-feature" "+zicbom" +// RVA23S64: "-target-feature" "+zicbop" +// RVA23S64: "-target-feature" "+zicboz" +// RVA23S64: "-target-feature" "+ziccamoa" +// RVA23S64: "-target-feature" "+ziccif" +// RVA23S64: "-target-feature" "+zicclsm" +// RVA23S64: "-target-feature" "+ziccrse" +// RVA23S64: "-target-feature" "+zicntr" +// RVA23S64: "-target-feature" "+zicond" +// RVA23S64: "-target-feature" "+zicsr" +// RVA23S64: "-target-feature" "+zifencei" +// RVA23S64: "-target-feature" "+zihintntl" +// RVA23S64: "-target-feature" "+zihintpause" +// RVA23S64: "-target-feature" "+zihpm" +// RVA23S64: "-target-feature" "+experimental-zimop" +// RVA23S64: "-target-feature" "+za64rs" +// RVA23S64: "-target-feature" "+zawrs" +// RVA23S64: "-target-feature" "+zfa" +// RVA23S64: "-target-feature" "+zfhmin" +// RVA23S64: "-target-feature" "+zcb" +// RVA23S64: "-target-feature" "+experimental-zcmop" +// RVA23S64: "-target-feature" "+zba" +// RVA23S64: "-target-feature" "+zbb" +// RVA23S64: "-target-feature" "+zbs" +// RVA23S64: "-target-feature" "+zkt" +// RVA23S64: "-target-feature" "+zvbb" +// RVA23S64: "-target-feature" "+zvfhmin" +// RVA23S64: "-target-feature" "+zvkt" +// RVA23S64: "-target-feature" "+shcounterenw" +// RVA23S64: "-target-feature" "+shgatpa" +// RVA23S64: "-target-feature" "+shtvala" +// RVA23S64: "-target-feature" "+shvsatpa" +// RVA23S64: "-target-feature" "+shvstvala" +// RVA23S64: "-target-feature" "+shvstvecd" +// RVA23S64: "-target-feature" "+ssccptr" +// RVA23S64: "-target-feature" "+sscofpmf" +// RVA23S64: "-target-feature" "+sscounterenw" +// RVA23S64: "-target-feature" "+experimental-ssnpm" +// RVA23S64: "-target-feature" "+ssstateen" +// RVA23S64: "-target-feature" "+sstc" +// RVA23S64: "-target-feature" "+sstvala" +// RVA23S64: "-target-feature" "+sstvecd" +// RVA23S64: "-target-feature" "+ssu64xl" +// RVA23S64: "-target-feature" "+svade" +// RVA23S64: "-target-feature" "+svbare" +// RVA23S64: "-target-feature" "+svinval" +// RVA23S64: "-target-feature" "+svnapot" +// RVA23S64: "-target-feature" "+svpbmt" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rvb23u64 -menable-experimental-extensions \ +// RUN: | FileCheck -check-prefix=RVB23U64 %s +// RVB23U64: "-target-feature" "+m" +// RVB23U64: "-target-feature" "+a" +// RVB23U64: "-target-feature" "+f" +// RVB23U64: "-target-feature" "+d" +// RVB23U64: "-target-feature" "+c" +// RVB23U64: "-target-feature" "+zic64b" +// RVB23U64: "-target-feature" "+zicbom" +// RVB23U64: "-target-feature" "+zicbop" +// RVB23U64: "-target-feature" "+zicboz" +// RVB23U64: "-target-feature" "+ziccamoa" +// RVB23U64: "-target-feature" "+ziccif" +// RVB23U64: "-target-feature" "+zicclsm" +// RVB23U64: "-target-feature" "+ziccrse" +// RVB23U64: "-target-feature" "+zicntr" +// RVB23U64: "-target-feature" "+zicond" +// RVB23U64: "-target-feature" "+zicsr" +// RVB23U64: "-target-feature" "+zihintntl" +// RVB23U64: "-target-feature" "+zihintpause" +// RVB23U64: "-target-feature" "+zihpm" +// RVB23U64: "-target-feature" "+experimental-zimop" +// RVB23U64: "-target-feature" "+za64rs" +// RVB23U64: "-target-feature" "+zawrs" +// RVB23U64: "-target-feature" "+zfa" +// RVB23U64: "-target-feature" "+zcb" +// RVB23U64: "-target-feature" "+experimental-zcmop" +// RVB23U64: "-target-feature" "+zba" +// RVB23U64: "-target-feature" "+zbb" +// RVB23U64: "-target-feature" "+zbs" +// RVB23U64: "-target-feature" "+zkt" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rvb23s64 -menable-experimental-extensions \ +// RUN: | FileCheck -check-prefix=RVB23S64 %s +// RVB23S64: "-target-feature" "+m" +// RVB23S64: "-target-feature" "+a" +// RVB23S64: "-target-feature" "+f" +// RVB23S64: "-target-feature" "+d" +// RVB23S64: "-target-feature" "+c" +// RVB23S64: "-target-feature" "+zic64b" +// RVB23S64: "-target-feature" "+zicbom" +// RVB23S64: "-target-feature" "+zicbop" +// RVB23S64: "-target-feature" "+zicboz" +// RVB23S64: "-target-feature" "+ziccamoa" +// RVB23S64: "-target-feature" "+ziccif" +// RVB23S64: "-target-feature" "+zicclsm" +// RVB23S64: "-target-feature" "+ziccrse" +// RVB23S64: "-target-feature" "+zicntr" +// RVB23S64: "-target-feature" "+zicond" +// RVB23S64: "-target-feature" "+zicsr" +// RVB23S64: "-target-feature" "+zifencei" +// RVB23S64: "-target-feature" "+zihintntl" +// RVB23S64: "-target-feature" "+zihintpause" +// RVB23S64: "-target-feature" "+zihpm" +// RVB23S64: "-target-feature" "+experimental-zimop" +// RVB23S64: "-target-feature" "+za64rs" +// RVB23S64: "-target-feature" "+zawrs" +// RVB23S64: "-target-feature" "+zfa" +// RVB23S64: "-target-feature" "+zcb" +// RVB23S64: "-target-feature" "+experimental-zcmop" +// RVB23S64: "-target-feature" "+zba" +// RVB23S64: "-target-feature" "+zbb" +// RVB23S64: "-target-feature" "+zbs" +// RVB23S64: "-target-feature" "+zkt" +// RVB23S64: "-target-feature" "+ssccptr" +// RVB23S64: "-target-feature" "+sscofpmf" +// RVB23S64: "-target-feature" "+sscounterenw" +// RVB23S64: "-target-feature" "+sstc" +// RVB23S64: "-target-feature" "+sstvala" +// RVB23S64: "-target-feature" "+sstvecd" +// RVB23S64: "-target-feature" "+ssu64xl" +// RVB23S64: "-target-feature" "+svade" +// RVB23S64: "-target-feature" "+svbare" +// RVB23S64: "-target-feature" "+svinval" +// RVB23S64: "-target-feature" "+svnapot" +// RVB23S64: "-target-feature" "+svpbmt" + +// RUN: %clang --target=riscv32 -### -c %s 2>&1 -march=rvm23u32 -menable-experimental-extensions \ +// RUN: | FileCheck -check-prefix=RVM23U32 %s +// RVM23U32: "-target-feature" "+m" +// RVM23U32: "-target-feature" "+zicbop" +// RVM23U32: "-target-feature" "+zicond" +// RVM23U32: "-target-feature" "+zicsr" +// RVM23U32: "-target-feature" "+zihintntl" +// RVM23U32: "-target-feature" "+zihintpause" +// RVM23U32: "-target-feature" "+experimental-zimop" +// RVM23U32: "-target-feature" "+zce" +// RVM23U32: "-target-feature" "+experimental-zcmop" +// RVM23U32: "-target-feature" "+zba" +// RVM23U32: "-target-feature" "+zbb" +// RVM23U32: "-target-feature" "+zbs" + +// RUN: %clang --target=riscv64 -### -c %s 2>&1 -march=rva22u64_zfa \ +// RUN: | FileCheck -check-prefix=PROFILE-WITH-ADDITIONAL %s +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+m" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+a" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+f" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+d" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+c" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicbom" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicbop" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zicboz" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zihintpause" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zfa" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zfhmin" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zba" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zbb" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zbs" +// PROFILE-WITH-ADDITIONAL: "-target-feature" "+zkt" + +// RUN: not %clang --target=riscv64 -### -c %s 2>&1 -march=rva19u64_zfa | FileCheck -check-prefix=INVALID-PROFILE %s +// INVALID-PROFILE: error: invalid arch name 'rva19u64_zfa', unsupported profile + +// RUN: not %clang --target=riscv64 -### -c %s 2>&1 -march=rva22u64zfa | FileCheck -check-prefix=INVALID-ADDITIONAL %s +// INVALID-ADDITIONAL: error: invalid arch name 'rva22u64zfa', additional extensions must be after separator '_' diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp index 39235ace4724..67e6e5b962b1 100644 --- a/llvm/lib/Support/RISCVISAInfo.cpp +++ b/llvm/lib/Support/RISCVISAInfo.cpp @@ -36,6 +36,11 @@ struct RISCVSupportedExtension { } }; +struct RISCVProfile { + StringLiteral Name; + StringLiteral MArch; +}; + } // end anonymous namespace static constexpr StringLiteral AllStdExts = "mafdqlcbkjtpvnh"; @@ -244,6 +249,42 @@ static const RISCVSupportedExtension SupportedExperimentalExtensions[] = { }; // clang-format on +static constexpr RISCVProfile SupportedProfiles[] = { + {"rvi20u32", "rv32i"}, + {"rvi20u64", "rv64i"}, + {"rva20u64", "rv64imafdc_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_za128rs"}, + {"rva20s64", "rv64imafdc_ziccamoa_ziccif_zicclsm_ziccrse_zicntr_zifencei_" + "za128rs_ssccptr_sstvala_sstvecd_svade_svbare"}, + {"rva22u64", + "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zihintpause_zihpm_za64rs_zfhmin_zba_zbb_zbs_zkt"}, + {"rva22s64", + "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zifencei_zihintpause_zihpm_za64rs_zfhmin_zba_zbb_zbs_zkt_ssccptr_" + "sscounterenw_sstvala_sstvecd_svade_svbare_svinval_svpbmt"}, + {"rva23u64", + "rv64imafdcv_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zicond_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_zfa_" + "zfhmin_zcb_zcmop0p2_zba_zbb_zbs_zkt_zvbb_zvfhmin_zvkt"}, + {"rva23s64", + "rv64imafdcvh_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zicond_zifencei_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_" + "zfa_zfhmin_zcb_zcmop0p2_zba_zbb_zbs_zkt_zvbb_zvfhmin_zvkt_shcounterenw_" + "shgatpa_shtvala_shvsatpa_shvstvala_shvstvecd_ssccptr_sscofpmf_" + "sscounterenw_ssnpm0p8_ssstateen_sstc_sstvala_sstvecd_ssu64xl_svade_" + "svbare_svinval_svnapot_svpbmt"}, + {"rvb23u64", "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_" + "zicclsm_ziccrse_zicntr_zicond_zihintntl_zihintpause_zihpm_" + "zimop0p1_za64rs_zawrs_zfa_zcb_zcmop0p2_zba_zbb_zbs_zkt"}, + {"rvb23s64", + "rv64imafdc_zic64b_zicbom_zicbop_zicboz_ziccamoa_ziccif_zicclsm_ziccrse_" + "zicntr_zicond_zifencei_zihintntl_zihintpause_zihpm_zimop0p1_za64rs_zawrs_" + "zfa_zcb_zcmop0p2_zba_zbb_zbs_zkt_ssccptr_sscofpmf_sscounterenw_sstc_" + "sstvala_sstvecd_ssu64xl_svade_svbare_svinval_svnapot_svpbmt"}, + {"rvm23u32", "rv32im_zicbop_zicond_zicsr_zihintntl_zihintpause_zimop0p1_" + "zca_zcb_zce_zcmop0p2_zcmp_zcmt_zba_zbb_zbs"}, +}; + static void verifyTables() { #ifndef NDEBUG static std::atomic TableChecked(false); @@ -857,6 +898,29 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension, "string must be lowercase"); } + if (Arch.starts_with("rvi") || Arch.starts_with("rva") || + Arch.starts_with("rvb") || Arch.starts_with("rvm")) { + const auto *FoundProfile = + llvm::find_if(SupportedProfiles, [Arch](const RISCVProfile &Profile) { + return Arch.starts_with(Profile.Name); + }); + + if (FoundProfile == std::end(SupportedProfiles)) + return createStringError(errc::invalid_argument, "unsupported profile"); + + std::string NewArch = FoundProfile->MArch.str(); + StringRef ArchWithoutProfile = Arch.substr(FoundProfile->Name.size()); + if (!ArchWithoutProfile.empty()) { + if (!ArchWithoutProfile.starts_with("_")) + return createStringError( + errc::invalid_argument, + "additional extensions must be after separator '_'"); + NewArch += ArchWithoutProfile.str(); + } + return parseArchString(NewArch, EnableExperimentalExtension, + ExperimentalExtensionVersionCheck, IgnoreUnknown); + } + bool HasRV64 = Arch.starts_with("rv64"); // ISA string must begin with rv32 or rv64. if (!(Arch.starts_with("rv32") || HasRV64) || (Arch.size() < 5)) { -- GitLab From cdbec7baf1bc31b59526442c9d4d5f53aac746eb Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Fri, 22 Mar 2024 08:24:08 -0700 Subject: [PATCH 265/296] [libc] fix up the use of angle includes in include/ (#86027) Performed en-masse via: $ grep -rn "#include /#include "ll$1"/' $ grep -rn "#include <__" libc/include -l | \ xargs perl -pi -e 's/#include <__(.*)>/#include "__$1"/' Link: #83463 Link: #83210 --- libc/include/arpa/inet.h.def | 2 +- libc/include/assert.h.def | 2 +- libc/include/ctype.h.def | 2 +- libc/include/dirent.h.def | 2 +- libc/include/errno.h.def | 4 ++-- libc/include/fcntl.h.def | 4 ++-- libc/include/features.h.def | 4 ++-- libc/include/fenv.h.def | 4 ++-- libc/include/float.h.def | 2 +- libc/include/gpu/rpc.h.def | 4 ++-- libc/include/inttypes.h.def | 4 ++-- libc/include/limits.h.def | 2 +- .../llvm-libc-macros/containerof-macro.h | 2 +- .../llvm-libc-macros/sys-queue-macros.h | 4 ++-- libc/include/llvm-libc-types/__mutex_type.h | 2 +- .../llvm-libc-types/cookie_io_functions_t.h | 6 +++--- libc/include/llvm-libc-types/fd_set.h | 2 +- libc/include/llvm-libc-types/mtx_t.h | 2 +- libc/include/llvm-libc-types/once_flag.h | 2 +- libc/include/llvm-libc-types/pthread_attr_t.h | 2 +- .../include/llvm-libc-types/pthread_mutex_t.h | 2 +- libc/include/llvm-libc-types/pthread_once_t.h | 2 +- libc/include/llvm-libc-types/pthread_t.h | 2 +- libc/include/llvm-libc-types/siginfo_t.h | 8 ++++---- libc/include/llvm-libc-types/sigset_t.h | 2 +- libc/include/llvm-libc-types/stack_t.h | 2 +- libc/include/llvm-libc-types/struct_dirent.h | 4 ++-- .../llvm-libc-types/struct_epoll_event.h | 2 +- libc/include/llvm-libc-types/struct_rlimit.h | 2 +- libc/include/llvm-libc-types/struct_rusage.h | 2 +- .../llvm-libc-types/struct_sched_param.h | 6 +++--- .../llvm-libc-types/struct_sigaction.h | 4 ++-- .../include/llvm-libc-types/struct_sockaddr.h | 2 +- .../llvm-libc-types/struct_sockaddr_un.h | 2 +- libc/include/llvm-libc-types/struct_stat.h | 20 +++++++++---------- libc/include/llvm-libc-types/struct_termios.h | 6 +++--- .../include/llvm-libc-types/struct_timespec.h | 2 +- libc/include/llvm-libc-types/struct_timeval.h | 4 ++-- libc/include/llvm-libc-types/thrd_t.h | 2 +- libc/include/math.h.def | 6 +++--- libc/include/pthread.h.def | 2 +- libc/include/sched.h.def | 4 ++-- libc/include/search.h.def | 2 +- libc/include/setjmp.h.def | 2 +- libc/include/signal.h.def | 4 ++-- libc/include/spawn.h.def | 2 +- libc/include/stdbit.h.def | 4 ++-- libc/include/stdckdint.h.def | 4 ++-- libc/include/stdfix.h.def | 4 ++-- libc/include/stdint.h.def | 2 +- libc/include/stdio.h.def | 6 +++--- libc/include/stdlib.h.def | 4 ++-- libc/include/string.h.def | 4 ++-- libc/include/strings.h.def | 2 +- libc/include/sys/auxv.h.def | 4 ++-- libc/include/sys/epoll.h.def | 2 +- libc/include/sys/ioctl.h.def | 4 ++-- libc/include/sys/mman.h.def | 4 ++-- libc/include/sys/prctl.h.def | 2 +- libc/include/sys/queue.h | 2 +- libc/include/sys/random.h.def | 4 ++-- libc/include/sys/resource.h.def | 4 ++-- libc/include/sys/select.h.def | 4 ++-- libc/include/sys/sendfile.h.def | 2 +- libc/include/sys/socket.h.def | 4 ++-- libc/include/sys/stat.h.def | 4 ++-- libc/include/sys/time.h.def | 6 +++--- libc/include/sys/types.h.def | 2 +- libc/include/sys/utsname.h.def | 2 +- libc/include/sys/wait.h.def | 4 ++-- libc/include/termios.h.def | 4 ++-- libc/include/threads.h.def | 2 +- libc/include/time.h.def | 4 ++-- libc/include/uchar.h.def | 2 +- libc/include/unistd.h.def | 6 +++--- libc/include/wchar.h.def | 4 ++-- 76 files changed, 131 insertions(+), 131 deletions(-) diff --git a/libc/include/arpa/inet.h.def b/libc/include/arpa/inet.h.def index fdd5ae3e3f85..6a62b2c7be81 100644 --- a/libc/include/arpa/inet.h.def +++ b/libc/include/arpa/inet.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_ARPA_INET_H #define LLVM_LIBC_ARPA_INET_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" #include diff --git a/libc/include/assert.h.def b/libc/include/assert.h.def index e5d7dfbffdbb..e006133a7654 100644 --- a/libc/include/assert.h.def +++ b/libc/include/assert.h.def @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" // This file may be usefully included multiple times to change assert()'s // definition based on NDEBUG. diff --git a/libc/include/ctype.h.def b/libc/include/ctype.h.def index ac52a36bf72f..a9bb786931ea 100644 --- a/libc/include/ctype.h.def +++ b/libc/include/ctype.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_CTYPE_H #define LLVM_LIBC_CTYPE_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/dirent.h.def b/libc/include/dirent.h.def index 3de8b1c6713f..6786578fbd06 100644 --- a/libc/include/dirent.h.def +++ b/libc/include/dirent.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_DIRENT_H #define LLVM_LIBC_DIRENT_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/errno.h.def b/libc/include/errno.h.def index 90bd8bfecf2f..d7ae90ad4524 100644 --- a/libc/include/errno.h.def +++ b/libc/include/errno.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_ERRNO_H #define LLVM_LIBC_ERRNO_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" #ifdef __linux__ @@ -40,7 +40,7 @@ #endif // ENOTRECOVERABLE #else // __linux__ -#include +#include "llvm-libc-macros/generic-error-number-macros.h" #endif #if !defined(__AMDGPU__) && !defined(__NVPTX__) diff --git a/libc/include/fcntl.h.def b/libc/include/fcntl.h.def index b11645d18c5b..4f608845ce1e 100644 --- a/libc/include/fcntl.h.def +++ b/libc/include/fcntl.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_FCNTL_H #define LLVM_LIBC_FCNTL_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/fcntl-macros.h" %%public_api() diff --git a/libc/include/features.h.def b/libc/include/features.h.def index 64205f57acb5..238b88d4b90f 100644 --- a/libc/include/features.h.def +++ b/libc/include/features.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_FEATURES_H #define LLVM_LIBC_FEATURES_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/features-macros.h" %%public_api() diff --git a/libc/include/fenv.h.def b/libc/include/fenv.h.def index f131a44914ab..c677b2a5930d 100644 --- a/libc/include/fenv.h.def +++ b/libc/include/fenv.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_FENV_H #define LLVM_LIBC_FENV_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/fenv-macros.h" %%public_api() diff --git a/libc/include/float.h.def b/libc/include/float.h.def index 6d3599d78c69..3bcd7f5e3f98 100644 --- a/libc/include/float.h.def +++ b/libc/include/float.h.def @@ -9,6 +9,6 @@ #ifndef LLVM_LIBC_FLOAT_H #define LLVM_LIBC_FLOAT_H -#include +#include "llvm-libc-macros/float-macros.h" #endif // LLVM_LIBC_FLOAT_H diff --git a/libc/include/gpu/rpc.h.def b/libc/include/gpu/rpc.h.def index 0438cd65e7be..72acf0c81422 100644 --- a/libc/include/gpu/rpc.h.def +++ b/libc/include/gpu/rpc.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_GPU_RPC_H #define LLVM_LIBC_GPU_RPC_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-types/rpc_opcodes_t.h" %%public_api() diff --git a/libc/include/inttypes.h.def b/libc/include/inttypes.h.def index a99d4e931f51..5879d2d8e041 100644 --- a/libc/include/inttypes.h.def +++ b/libc/include/inttypes.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_INTTYPES_H #define LLVM_LIBC_INTTYPES_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/inttypes-macros.h" #include %%public_api() diff --git a/libc/include/limits.h.def b/libc/include/limits.h.def index de5f3490459e..c37c97c69a84 100644 --- a/libc/include/limits.h.def +++ b/libc/include/limits.h.def @@ -9,6 +9,6 @@ #ifndef LLVM_LIBC_LIMITS_H #define LLVM_LIBC_LIMITS_H -#include +#include "llvm-libc-macros/limits-macros.h" #endif // LLVM_LIBC_LIMITS_H diff --git a/libc/include/llvm-libc-macros/containerof-macro.h b/libc/include/llvm-libc-macros/containerof-macro.h index 62724abd3b0f..592acd6e3aa9 100644 --- a/libc/include/llvm-libc-macros/containerof-macro.h +++ b/libc/include/llvm-libc-macros/containerof-macro.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_MACROS_CONTAINEROF_MACRO_H #define LLVM_LIBC_MACROS_CONTAINEROF_MACRO_H -#include +#include "llvm-libc-macros/offsetof-macro.h" #define __containerof(ptr, type, member) \ ({ \ diff --git a/libc/include/llvm-libc-macros/sys-queue-macros.h b/libc/include/llvm-libc-macros/sys-queue-macros.h index fcac265333fc..089b6abaa024 100644 --- a/libc/include/llvm-libc-macros/sys-queue-macros.h +++ b/libc/include/llvm-libc-macros/sys-queue-macros.h @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_MACROS_SYS_QUEUE_MACROS_H #define LLVM_LIBC_MACROS_SYS_QUEUE_MACROS_H -#include -#include +#include "llvm-libc-macros/containerof-macro.h" +#include "llvm-libc-macros/null-macro.h" #ifdef __cplusplus #define QUEUE_TYPEOF(type) type diff --git a/libc/include/llvm-libc-types/__mutex_type.h b/libc/include/llvm-libc-types/__mutex_type.h index d27bf5db8377..3779c78203ed 100644 --- a/libc/include/llvm-libc-types/__mutex_type.h +++ b/libc/include/llvm-libc-types/__mutex_type.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES___MUTEX_TYPE_H #define LLVM_LIBC_TYPES___MUTEX_TYPE_H -#include +#include "llvm-libc-types/__futex_word.h" typedef struct { unsigned char __timed; diff --git a/libc/include/llvm-libc-types/cookie_io_functions_t.h b/libc/include/llvm-libc-types/cookie_io_functions_t.h index f9fa1a2d50ed..a3e7c32a5096 100644 --- a/libc/include/llvm-libc-types/cookie_io_functions_t.h +++ b/libc/include/llvm-libc-types/cookie_io_functions_t.h @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_TYPES_COOKIE_IO_FUNCTIONS_T_H #define LLVM_LIBC_TYPES_COOKIE_IO_FUNCTIONS_T_H -#include -#include -#include +#include "llvm-libc-types/off64_t.h" +#include "llvm-libc-types/size_t.h" +#include "llvm-libc-types/ssize_t.h" typedef ssize_t cookie_read_function_t(void *, char *, size_t); typedef ssize_t cookie_write_function_t(void *, const char *, size_t); diff --git a/libc/include/llvm-libc-types/fd_set.h b/libc/include/llvm-libc-types/fd_set.h index 58fc438bbdd2..fd1bde24c90e 100644 --- a/libc/include/llvm-libc-types/fd_set.h +++ b/libc/include/llvm-libc-types/fd_set.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_FD_SET_H #define LLVM_LIBC_TYPES_FD_SET_H -#include // FD_SETSIZE +#include "llvm-libc-macros/sys-select-macros.h" // FD_SETSIZE typedef struct { __FD_SET_WORD_TYPE __set[__FD_SET_ARRAYSIZE]; diff --git a/libc/include/llvm-libc-types/mtx_t.h b/libc/include/llvm-libc-types/mtx_t.h index 0f3882c26b6b..ebf79871c935 100644 --- a/libc/include/llvm-libc-types/mtx_t.h +++ b/libc/include/llvm-libc-types/mtx_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_MTX_T_H #define LLVM_LIBC_TYPES_MTX_T_H -#include +#include "llvm-libc-types/__mutex_type.h" typedef __mutex_type mtx_t; diff --git a/libc/include/llvm-libc-types/once_flag.h b/libc/include/llvm-libc-types/once_flag.h index cb8011284610..f80d35e317e9 100644 --- a/libc/include/llvm-libc-types/once_flag.h +++ b/libc/include/llvm-libc-types/once_flag.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_ONCE_FLAG_H #define LLVM_LIBC_TYPES_ONCE_FLAG_H -#include +#include "llvm-libc-types/__futex_word.h" #ifdef __linux__ typedef __futex_word once_flag; diff --git a/libc/include/llvm-libc-types/pthread_attr_t.h b/libc/include/llvm-libc-types/pthread_attr_t.h index 66c04de04a99..7512193ef97b 100644 --- a/libc/include/llvm-libc-types/pthread_attr_t.h +++ b/libc/include/llvm-libc-types/pthread_attr_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_PTHREAD_ATTR_T_H #define LLVM_LIBC_TYPES_PTHREAD_ATTR_T_H -#include +#include "llvm-libc-types/size_t.h" typedef struct { int __detachstate; diff --git a/libc/include/llvm-libc-types/pthread_mutex_t.h b/libc/include/llvm-libc-types/pthread_mutex_t.h index b1eb21f24fac..cf2194d719f3 100644 --- a/libc/include/llvm-libc-types/pthread_mutex_t.h +++ b/libc/include/llvm-libc-types/pthread_mutex_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_PTHREAD_MUTEX_T_H #define LLVM_LIBC_TYPES_PTHREAD_MUTEX_T_H -#include +#include "llvm-libc-types/__mutex_type.h" typedef __mutex_type pthread_mutex_t; diff --git a/libc/include/llvm-libc-types/pthread_once_t.h b/libc/include/llvm-libc-types/pthread_once_t.h index 3fe78b7ddff6..8ea926f4ee7d 100644 --- a/libc/include/llvm-libc-types/pthread_once_t.h +++ b/libc/include/llvm-libc-types/pthread_once_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_PTHREAD_ONCE_T_H #define LLVM_LIBC_TYPES_PTHREAD_ONCE_T_H -#include +#include "llvm-libc-types/__futex_word.h" #ifdef __linux__ typedef __futex_word pthread_once_t; diff --git a/libc/include/llvm-libc-types/pthread_t.h b/libc/include/llvm-libc-types/pthread_t.h index 72c14e1c2eea..63cc0d7dd74c 100644 --- a/libc/include/llvm-libc-types/pthread_t.h +++ b/libc/include/llvm-libc-types/pthread_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_PTHREAD_T_H #define LLVM_LIBC_TYPES_PTHREAD_T_H -#include +#include "llvm-libc-types/__thread_type.h" typedef __thread_type pthread_t; diff --git a/libc/include/llvm-libc-types/siginfo_t.h b/libc/include/llvm-libc-types/siginfo_t.h index 935ef4bbcb72..dafe9c1b5f8e 100644 --- a/libc/include/llvm-libc-types/siginfo_t.h +++ b/libc/include/llvm-libc-types/siginfo_t.h @@ -9,10 +9,10 @@ #ifndef LLVM_LIBC_TYPES_SIGINFO_T_H #define LLVM_LIBC_TYPES_SIGINFO_T_H -#include -#include -#include -#include +#include "llvm-libc-types/clock_t.h" +#include "llvm-libc-types/pid_t.h" +#include "llvm-libc-types/uid_t.h" +#include "llvm-libc-types/union_sigval.h" #define SI_MAX_SIZE 128 diff --git a/libc/include/llvm-libc-types/sigset_t.h b/libc/include/llvm-libc-types/sigset_t.h index f159c6c6c643..311a92b823ff 100644 --- a/libc/include/llvm-libc-types/sigset_t.h +++ b/libc/include/llvm-libc-types/sigset_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_SIGSET_T_H #define LLVM_LIBC_TYPES_SIGSET_T_H -#include +#include "llvm-libc-macros/signal-macros.h" // This definition can be adjusted/specialized for different targets and // platforms as necessary. This definition works for Linux on most targets. diff --git a/libc/include/llvm-libc-types/stack_t.h b/libc/include/llvm-libc-types/stack_t.h index 5fa4d3a6d3dc..9156425436e9 100644 --- a/libc/include/llvm-libc-types/stack_t.h +++ b/libc/include/llvm-libc-types/stack_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STACK_T_H #define LLVM_LIBC_TYPES_STACK_T_H -#include +#include "llvm-libc-types/size_t.h" typedef struct { // The order of the fields declared here should match the kernel definition diff --git a/libc/include/llvm-libc-types/struct_dirent.h b/libc/include/llvm-libc-types/struct_dirent.h index 3c5b361c3cbc..0bb71b9f3b84 100644 --- a/libc/include/llvm-libc-types/struct_dirent.h +++ b/libc/include/llvm-libc-types/struct_dirent.h @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_DIRENT_H #define LLVM_LIBC_TYPES_STRUCT_DIRENT_H -#include -#include +#include "llvm-libc-types/ino_t.h" +#include "llvm-libc-types/off_t.h" struct dirent { ino_t d_ino; diff --git a/libc/include/llvm-libc-types/struct_epoll_event.h b/libc/include/llvm-libc-types/struct_epoll_event.h index 6fc5b410348a..66cf86c1e2a0 100644 --- a/libc/include/llvm-libc-types/struct_epoll_event.h +++ b/libc/include/llvm-libc-types/struct_epoll_event.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_EPOLL_EVENT_H #define LLVM_LIBC_TYPES_STRUCT_EPOLL_EVENT_H -#include +#include "llvm-libc-types/struct_epoll_data.h" typedef struct epoll_event { __UINT32_TYPE__ events; diff --git a/libc/include/llvm-libc-types/struct_rlimit.h b/libc/include/llvm-libc-types/struct_rlimit.h index e093d9f306c9..11e6bee15f9d 100644 --- a/libc/include/llvm-libc-types/struct_rlimit.h +++ b/libc/include/llvm-libc-types/struct_rlimit.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_RLIMIT_H #define LLVM_LIBC_TYPES_STRUCT_RLIMIT_H -#include +#include "llvm-libc-types/rlim_t.h" struct rlimit { rlim_t rlim_cur; diff --git a/libc/include/llvm-libc-types/struct_rusage.h b/libc/include/llvm-libc-types/struct_rusage.h index 21ea8b1061c2..ed838d30ede3 100644 --- a/libc/include/llvm-libc-types/struct_rusage.h +++ b/libc/include/llvm-libc-types/struct_rusage.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_RUSAGE_H #define LLVM_LIBC_TYPES_STRUCT_RUSAGE_H -#include +#include "llvm-libc-types/struct_timeval.h" struct rusage { struct timeval ru_utime; diff --git a/libc/include/llvm-libc-types/struct_sched_param.h b/libc/include/llvm-libc-types/struct_sched_param.h index 0521a4df652f..86209ac3a181 100644 --- a/libc/include/llvm-libc-types/struct_sched_param.h +++ b/libc/include/llvm-libc-types/struct_sched_param.h @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_SCHED_PARAM_H #define LLVM_LIBC_TYPES_STRUCT_SCHED_PARAM_H -#include -#include -#include +#include "llvm-libc-types/pid_t.h" +#include "llvm-libc-types/struct_timespec.h" +#include "llvm-libc-types/time_t.h" struct sched_param { // Process or thread execution scheduling priority. diff --git a/libc/include/llvm-libc-types/struct_sigaction.h b/libc/include/llvm-libc-types/struct_sigaction.h index 54d2995f4ecd..ffce04d0f7e8 100644 --- a/libc/include/llvm-libc-types/struct_sigaction.h +++ b/libc/include/llvm-libc-types/struct_sigaction.h @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_SIGACTION_H #define LLVM_LIBC_TYPES_STRUCT_SIGACTION_H -#include -#include +#include "llvm-libc-types/siginfo_t.h" +#include "llvm-libc-types/sigset_t.h" struct sigaction { union { diff --git a/libc/include/llvm-libc-types/struct_sockaddr.h b/libc/include/llvm-libc-types/struct_sockaddr.h index 074b1ae50ef0..a98606323c52 100644 --- a/libc/include/llvm-libc-types/struct_sockaddr.h +++ b/libc/include/llvm-libc-types/struct_sockaddr.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_SOCKADDR_H #define LLVM_LIBC_TYPES_STRUCT_SOCKADDR_H -#include +#include "llvm-libc-types/sa_family_t.h" struct sockaddr { sa_family_t sa_family; diff --git a/libc/include/llvm-libc-types/struct_sockaddr_un.h b/libc/include/llvm-libc-types/struct_sockaddr_un.h index 4332419a5b71..3c0362ce24fb 100644 --- a/libc/include/llvm-libc-types/struct_sockaddr_un.h +++ b/libc/include/llvm-libc-types/struct_sockaddr_un.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_SOCKADDR_UN_H #define LLVM_LIBC_TYPES_STRUCT_SOCKADDR_UN_H -#include +#include "llvm-libc-types/sa_family_t.h" // This is the sockaddr specialization for AF_UNIX or AF_LOCAL sockets, as // defined by posix. diff --git a/libc/include/llvm-libc-types/struct_stat.h b/libc/include/llvm-libc-types/struct_stat.h index 3539fb5b920e..d8ae9dd6ffdc 100644 --- a/libc/include/llvm-libc-types/struct_stat.h +++ b/libc/include/llvm-libc-types/struct_stat.h @@ -9,16 +9,16 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_STAT_H #define LLVM_LIBC_TYPES_STRUCT_STAT_H -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "llvm-libc-types/blkcnt_t.h" +#include "llvm-libc-types/blksize_t.h" +#include "llvm-libc-types/dev_t.h" +#include "llvm-libc-types/gid_t.h" +#include "llvm-libc-types/ino_t.h" +#include "llvm-libc-types/mode_t.h" +#include "llvm-libc-types/nlink_t.h" +#include "llvm-libc-types/off_t.h" +#include "llvm-libc-types/struct_timespec.h" +#include "llvm-libc-types/uid_t.h" struct stat { dev_t st_dev; diff --git a/libc/include/llvm-libc-types/struct_termios.h b/libc/include/llvm-libc-types/struct_termios.h index 72aefe4f6926..51241192f741 100644 --- a/libc/include/llvm-libc-types/struct_termios.h +++ b/libc/include/llvm-libc-types/struct_termios.h @@ -9,9 +9,9 @@ #ifndef __LLVM_LIBC_TYPES_STRUCT_TERMIOS_H__ #define __LLVM_LIBC_TYPES_STRUCT_TERMIOS_H__ -#include -#include -#include +#include "llvm-libc-types/cc_t.h" +#include "llvm-libc-types/speed_t.h" +#include "llvm-libc-types/tcflag_t.h" struct termios { tcflag_t c_iflag; // Input mode flags diff --git a/libc/include/llvm-libc-types/struct_timespec.h b/libc/include/llvm-libc-types/struct_timespec.h index 5d56d9c9468b..4baab07c10f8 100644 --- a/libc/include/llvm-libc-types/struct_timespec.h +++ b/libc/include/llvm-libc-types/struct_timespec.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_TIMESPEC_H #define LLVM_LIBC_TYPES_STRUCT_TIMESPEC_H -#include +#include "llvm-libc-types/time_t.h" struct timespec { time_t tv_sec; /* Seconds. */ diff --git a/libc/include/llvm-libc-types/struct_timeval.h b/libc/include/llvm-libc-types/struct_timeval.h index 6a0b7bbaf825..365b835d345d 100644 --- a/libc/include/llvm-libc-types/struct_timeval.h +++ b/libc/include/llvm-libc-types/struct_timeval.h @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_TYPES_STRUCT_TIMEVAL_H #define LLVM_LIBC_TYPES_STRUCT_TIMEVAL_H -#include -#include +#include "llvm-libc-types/suseconds_t.h" +#include "llvm-libc-types/time_t.h" struct timeval { time_t tv_sec; // Seconds diff --git a/libc/include/llvm-libc-types/thrd_t.h b/libc/include/llvm-libc-types/thrd_t.h index 2e0f9a0d75ad..751ea5b9e4c0 100644 --- a/libc/include/llvm-libc-types/thrd_t.h +++ b/libc/include/llvm-libc-types/thrd_t.h @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_TYPES_THRD_T_H #define LLVM_LIBC_TYPES_THRD_T_H -#include +#include "llvm-libc-types/__thread_type.h" typedef __thread_type thrd_t; diff --git a/libc/include/math.h.def b/libc/include/math.h.def index 927e2d6697c6..cd2fe76f40bf 100644 --- a/libc/include/math.h.def +++ b/libc/include/math.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_MATH_H #define LLVM_LIBC_MATH_H -#include <__llvm-libc-common.h> -#include -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/math-macros.h" +#include "llvm-libc-types/float128.h" %%public_api() diff --git a/libc/include/pthread.h.def b/libc/include/pthread.h.def index 391ecd3c124f..abeb839ee83d 100644 --- a/libc/include/pthread.h.def +++ b/libc/include/pthread.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_PTHREAD_H #define LLVM_LIBC_PTHREAD_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" #define PTHREAD_STACK_MIN (1 << 14) // 16KB diff --git a/libc/include/sched.h.def b/libc/include/sched.h.def index 3b2d5e330859..493028e8dcc4 100644 --- a/libc/include/sched.h.def +++ b/libc/include/sched.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_SCHED_H #define LLVM_LIBC_SCHED_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/sched-macros.h" %%public_api() diff --git a/libc/include/search.h.def b/libc/include/search.h.def index 3435c1f8ad04..6301ba7b656c 100644 --- a/libc/include/search.h.def +++ b/libc/include/search.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SEARCH_H #define LLVM_LIBC_SEARCH_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" #define __need_size_t #include diff --git a/libc/include/setjmp.h.def b/libc/include/setjmp.h.def index 7447be2415bd..670bc1ac0fe2 100644 --- a/libc/include/setjmp.h.def +++ b/libc/include/setjmp.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SETJMP_H #define LLVM_LIBC_SETJMP_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/signal.h.def b/libc/include/signal.h.def index 0e7033452715..50a5f44c7337 100644 --- a/libc/include/signal.h.def +++ b/libc/include/signal.h.def @@ -9,12 +9,12 @@ #ifndef LLVM_LIBC_SIGNAL_H #define LLVM_LIBC_SIGNAL_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" #define __need_size_t #include -#include +#include "llvm-libc-macros/signal-macros.h" %%public_api() diff --git a/libc/include/spawn.h.def b/libc/include/spawn.h.def index 368ebff17ca1..a8d701585286 100644 --- a/libc/include/spawn.h.def +++ b/libc/include/spawn.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SPAWN_H #define LLVM_LIBC_SPAWN_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/stdbit.h.def b/libc/include/stdbit.h.def index c5a77329fbfe..28c147b01e22 100644 --- a/libc/include/stdbit.h.def +++ b/libc/include/stdbit.h.def @@ -9,10 +9,10 @@ #ifndef LLVM_LIBC_STDBIT_H #define LLVM_LIBC_STDBIT_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() -#include +#include "llvm-libc-macros/stdbit-macros.h" #endif // LLVM_LIBC_STDBIT_H diff --git a/libc/include/stdckdint.h.def b/libc/include/stdckdint.h.def index c82470911c33..d4a9d829a3c9 100644 --- a/libc/include/stdckdint.h.def +++ b/libc/include/stdckdint.h.def @@ -9,10 +9,10 @@ #ifndef LLVM_LIBC_STDCKDINT_H #define LLVM_LIBC_STDCKDINT_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() -#include +#include "llvm-libc-macros/stdckdint-macros.h" #endif // LLVM_LIBC_STDCKDINT_H diff --git a/libc/include/stdfix.h.def b/libc/include/stdfix.h.def index 368eeb33f2f0..8ac49be45fba 100644 --- a/libc/include/stdfix.h.def +++ b/libc/include/stdfix.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_STDFIX_H #define LLVM_LIBC_STDFIX_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/stdfix-macros.h" // From ISO/IEC TR 18037:2008 standard: // https://www.iso.org/standard/51126.html diff --git a/libc/include/stdint.h.def b/libc/include/stdint.h.def index 9e269101acd2..d7660860c91f 100644 --- a/libc/include/stdint.h.def +++ b/libc/include/stdint.h.def @@ -9,6 +9,6 @@ #ifndef LLVM_LIBC_STDINT_H #define LLVM_LIBC_STDINT_H -#include +#include "llvm-libc-macros/stdint-macros.h" #endif // LLVM_LIBC_STDINT_H diff --git a/libc/include/stdio.h.def b/libc/include/stdio.h.def index a28d009288d5..78d800c83b51 100644 --- a/libc/include/stdio.h.def +++ b/libc/include/stdio.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_STDIO_H #define LLVM_LIBC_STDIO_H -#include <__llvm-libc-common.h> -#include -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/file-seek-macros.h" +#include "llvm-libc-macros/stdio-macros.h" #include diff --git a/libc/include/stdlib.h.def b/libc/include/stdlib.h.def index 18df71a49a9b..d523f7a53024 100644 --- a/libc/include/stdlib.h.def +++ b/libc/include/stdlib.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_STDLIB_H #define LLVM_LIBC_STDLIB_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/stdlib-macros.h" %%public_api() diff --git a/libc/include/string.h.def b/libc/include/string.h.def index 26e6ef93d314..1bd2687db2be 100644 --- a/libc/include/string.h.def +++ b/libc/include/string.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_STRING_H #define LLVM_LIBC_STRING_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/null-macro.h" %%public_api() diff --git a/libc/include/strings.h.def b/libc/include/strings.h.def index f07ca30d5dbd..9b016bf0bc50 100644 --- a/libc/include/strings.h.def +++ b/libc/include/strings.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_STRINGS_H #define LLVM_LIBC_STRINGS_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/sys/auxv.h.def b/libc/include/sys/auxv.h.def index 504c2f68cb1e..11ab25bcfe2c 100644 --- a/libc/include/sys/auxv.h.def +++ b/libc/include/sys/auxv.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_AUXV_H #define LLVM_LIBC_SYS_AUXV_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-auxv-macros.h" %%public_api() diff --git a/libc/include/sys/epoll.h.def b/libc/include/sys/epoll.h.def index 490fad91db3c..85f7d9ad6091 100644 --- a/libc/include/sys/epoll.h.def +++ b/libc/include/sys/epoll.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SYS_EPOLL_H #define LLVM_LIBC_SYS_EPOLL_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/sys/ioctl.h.def b/libc/include/sys/ioctl.h.def index 90d91cf38291..2f37a1190ac1 100644 --- a/libc/include/sys/ioctl.h.def +++ b/libc/include/sys/ioctl.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_IOCTL_H #define LLVM_LIBC_SYS_IOCTL_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-ioctl-macros.h" %%public_api() diff --git a/libc/include/sys/mman.h.def b/libc/include/sys/mman.h.def index ab9fde1bb920..2e2c2f1997b8 100644 --- a/libc/include/sys/mman.h.def +++ b/libc/include/sys/mman.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_MMAN_H #define LLVM_LIBC_SYS_MMAN_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-mman-macros.h" %%public_api() diff --git a/libc/include/sys/prctl.h.def b/libc/include/sys/prctl.h.def index 0a11543d0729..08648c9f4792 100644 --- a/libc/include/sys/prctl.h.def +++ b/libc/include/sys/prctl.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SYS_PRCTL_H #define LLVM_LIBC_SYS_PRCTL_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" // Process control is highly platform specific, so the platform usually defines // the macros itself. diff --git a/libc/include/sys/queue.h b/libc/include/sys/queue.h index 1cde35e77a04..cca53c16f0f3 100644 --- a/libc/include/sys/queue.h +++ b/libc/include/sys/queue.h @@ -9,6 +9,6 @@ #ifndef SYS_QUEUE_H #define SYS_QUEUE_H -#include +#include "llvm-libc-macros/sys-queue-macros.h" #endif // SYS_QUEUE_H diff --git a/libc/include/sys/random.h.def b/libc/include/sys/random.h.def index b767f2479fc2..d11431b2755d 100644 --- a/libc/include/sys/random.h.def +++ b/libc/include/sys/random.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_RANDOM_H #define LLVM_LIBC_SYS_RANDOM_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-random-macros.h" %%public_api() diff --git a/libc/include/sys/resource.h.def b/libc/include/sys/resource.h.def index 31132d3b2608..365d803cf715 100644 --- a/libc/include/sys/resource.h.def +++ b/libc/include/sys/resource.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_RESOURCE_H #define LLVM_LIBC_SYS_RESOURCE_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-resource-macros.h" %%public_api() diff --git a/libc/include/sys/select.h.def b/libc/include/sys/select.h.def index 4f3cebaecbb9..529be7158f26 100644 --- a/libc/include/sys/select.h.def +++ b/libc/include/sys/select.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_SELECT_H #define LLVM_LIBC_SYS_SELECT_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-select-macros.h" %%public_api() diff --git a/libc/include/sys/sendfile.h.def b/libc/include/sys/sendfile.h.def index 947edc28e1ec..d7f21f91f95e 100644 --- a/libc/include/sys/sendfile.h.def +++ b/libc/include/sys/sendfile.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SYS_SENDFILE_H #define LLVM_LIBC_SYS_SENDFILE_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/sys/socket.h.def b/libc/include/sys/socket.h.def index 71654c64b988..933ef1512e45 100644 --- a/libc/include/sys/socket.h.def +++ b/libc/include/sys/socket.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_SOCKET_H #define LLVM_LIBC_SYS_SOCKET_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-socket-macros.h" %%public_api() diff --git a/libc/include/sys/stat.h.def b/libc/include/sys/stat.h.def index ed37d010f497..06a98a4aa029 100644 --- a/libc/include/sys/stat.h.def +++ b/libc/include/sys/stat.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_STAT_H #define LLVM_LIBC_SYS_STAT_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-stat-macros.h" %%public_api() diff --git a/libc/include/sys/time.h.def b/libc/include/sys/time.h.def index 9a3bd7bb49f8..5a87139aefc9 100644 --- a/libc/include/sys/time.h.def +++ b/libc/include/sys/time.h.def @@ -9,11 +9,11 @@ #ifndef LLVM_LIBC_SYS_TIME_H #define LLVM_LIBC_SYS_TIME_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-types/struct_timeval.h" -#include +#include "llvm-libc-macros/sys-time-macros.h" %%public_api() diff --git a/libc/include/sys/types.h.def b/libc/include/sys/types.h.def index 689482973fc7..f5c3bb2c928b 100644 --- a/libc/include/sys/types.h.def +++ b/libc/include/sys/types.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SYS_TYPES_H #define LLVM_LIBC_SYS_TYPES_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/sys/utsname.h.def b/libc/include/sys/utsname.h.def index 6d7daeb45f01..08dbbfc06245 100644 --- a/libc/include/sys/utsname.h.def +++ b/libc/include/sys/utsname.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_SYS_UTSNAME_H #define LLVM_LIBC_SYS_UTSNAME_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/sys/wait.h.def b/libc/include/sys/wait.h.def index b4fcce4d1652..0a76da019fdc 100644 --- a/libc/include/sys/wait.h.def +++ b/libc/include/sys/wait.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_SYS_WAIT_H #define LLVM_LIBC_SYS_WAIT_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" -#include +#include "llvm-libc-macros/sys-wait-macros.h" %%public_api() diff --git a/libc/include/termios.h.def b/libc/include/termios.h.def index be1cd2bff526..7538944c0985 100644 --- a/libc/include/termios.h.def +++ b/libc/include/termios.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_TERMIOS_H #define LLVM_LIBC_TERMIOS_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/termios-macros.h" %%public_api() diff --git a/libc/include/threads.h.def b/libc/include/threads.h.def index 93541b8d3bac..b114bea0ace3 100644 --- a/libc/include/threads.h.def +++ b/libc/include/threads.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_THREADS_H #define LLVM_LIBC_THREADS_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/time.h.def b/libc/include/time.h.def index d8988329a372..2355e8822fad 100644 --- a/libc/include/time.h.def +++ b/libc/include/time.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_TIME_H #define LLVM_LIBC_TIME_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/time-macros.h" %%public_api() diff --git a/libc/include/uchar.h.def b/libc/include/uchar.h.def index 7e62d43e9cc4..31b7fcb73ded 100644 --- a/libc/include/uchar.h.def +++ b/libc/include/uchar.h.def @@ -9,7 +9,7 @@ #ifndef LLVM_LIBC_UCHAR_H #define LLVM_LIBC_UCHAR_H -#include <__llvm-libc-common.h> +#include "__llvm-libc-common.h" %%public_api() diff --git a/libc/include/unistd.h.def b/libc/include/unistd.h.def index fa10af653fae..6b9137e14623 100644 --- a/libc/include/unistd.h.def +++ b/libc/include/unistd.h.def @@ -9,9 +9,9 @@ #ifndef LLVM_LIBC_UNISTD_H #define LLVM_LIBC_UNISTD_H -#include <__llvm-libc-common.h> -#include -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/file-seek-macros.h" +#include "llvm-libc-macros/unistd-macros.h" %%public_api() diff --git a/libc/include/wchar.h.def b/libc/include/wchar.h.def index ac72f80aa083..4c25de700d60 100644 --- a/libc/include/wchar.h.def +++ b/libc/include/wchar.h.def @@ -9,8 +9,8 @@ #ifndef LLVM_LIBC_WCHAR_H #define LLVM_LIBC_WCHAR_H -#include <__llvm-libc-common.h> -#include +#include "__llvm-libc-common.h" +#include "llvm-libc-macros/wchar-macros.h" %%public_api() -- GitLab From 4318f7e5301fb737a7abaacb3b43b6a9289055f3 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Fri, 22 Mar 2024 08:28:03 -0700 Subject: [PATCH 266/296] [libc][stdlib] initial support for __cxa_finalize (#85865) I'm trying to break up the pieces of supporting __cxa_finalize into smaller commits. Provide this symbol first, and make use of it from exit. Next will be to store __dso_handle, then finally to only run callbacks that were registered from a specific dso. Link: #85651 Link: https://itanium-cxx-abi.github.io/cxx-abi/abi.html#dso-dtor --- libc/src/stdlib/atexit.cpp | 31 +++++++++++++++++++------------ libc/src/stdlib/exit.cpp | 8 +++----- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/libc/src/stdlib/atexit.cpp b/libc/src/stdlib/atexit.cpp index 741ea4f25103..fa072b2fdf8d 100644 --- a/libc/src/stdlib/atexit.cpp +++ b/libc/src/stdlib/atexit.cpp @@ -55,14 +55,10 @@ void stdc_at_exit_func(void *payload) { reinterpret_cast(payload)(); } -} // namespace - -namespace internal { - void call_exit_callbacks() { handler_list_mtx.lock(); while (!exit_callbacks.empty()) { - auto unit = exit_callbacks.back(); + AtExitUnit &unit = exit_callbacks.back(); exit_callbacks.pop_back(); handler_list_mtx.unlock(); unit.callback(unit.payload); @@ -71,20 +67,31 @@ void call_exit_callbacks() { ExitCallbackList::destroy(&exit_callbacks); } -} // namespace internal - -static int add_atexit_unit(const AtExitUnit &unit) { +int add_atexit_unit(const AtExitUnit &unit) { MutexLock lock(&handler_list_mtx); - if (!exit_callbacks.push_back(unit)) - return -1; - return 0; + if (exit_callbacks.push_back(unit)) + return 0; + return -1; } +} // namespace + +extern "C" { + // TODO: Handle the last dso handle argument. -extern "C" int __cxa_atexit(AtExitCallback *callback, void *payload, void *) { +int __cxa_atexit(AtExitCallback *callback, void *payload, void *) { return add_atexit_unit({callback, payload}); } +// TODO: Handle the dso handle argument. call_exit_callbacks should only invoke +// the callbacks from this DSO. Requires adding support for __dso_handle. +void __cxa_finalize(void *dso) { + if (!dso) + call_exit_callbacks(); +} + +} // extern "C" + LLVM_LIBC_FUNCTION(int, atexit, (StdCAtExitCallback * callback)) { return add_atexit_unit( {&stdc_at_exit_func, reinterpret_cast(callback)}); diff --git a/libc/src/stdlib/exit.cpp b/libc/src/stdlib/exit.cpp index cc5ae6648d11..e754b34e4698 100644 --- a/libc/src/stdlib/exit.cpp +++ b/libc/src/stdlib/exit.cpp @@ -10,14 +10,12 @@ #include "src/__support/OSUtil/quick_exit.h" #include "src/__support/common.h" -namespace LIBC_NAMESPACE { +extern "C" void __cxa_finalize(void *); -namespace internal { -void call_exit_callbacks(); -} +namespace LIBC_NAMESPACE { LLVM_LIBC_FUNCTION(void, exit, (int status)) { - internal::call_exit_callbacks(); + __cxa_finalize(nullptr); quick_exit(status); __builtin_unreachable(); } -- GitLab From 6e28ecd79995a72a8dbde8f16a1afc18309442a1 Mon Sep 17 00:00:00 2001 From: Antonio Frighetto Date: Fri, 22 Mar 2024 16:23:19 +0100 Subject: [PATCH 267/296] [Object][ELF] Ensure offset to locate dyn section does not go past size Validate `p_offset` in `dynamicEntries` before computing the entry offset. Fixes: https://github.com/llvm/llvm-project/issues/85568. --- llvm/lib/Object/ELF.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Object/ELF.cpp b/llvm/lib/Object/ELF.cpp index 55dd0c8e06c0..0ac4e7a57759 100644 --- a/llvm/lib/Object/ELF.cpp +++ b/llvm/lib/Object/ELF.cpp @@ -560,7 +560,11 @@ Expected ELFFile::dynamicEntries() const { for (const Elf_Phdr &Phdr : *ProgramHeadersOrError) { if (Phdr.p_type == ELF::PT_DYNAMIC) { - Dyn = ArrayRef(reinterpret_cast(base() + Phdr.p_offset), + const uint8_t *DynOffset = base() + Phdr.p_offset; + if (DynOffset > end()) + return createError( + "dynamic section offset past file size: corrupted ELF"); + Dyn = ArrayRef(reinterpret_cast(DynOffset), Phdr.p_filesz / sizeof(Elf_Dyn)); break; } -- GitLab From f66d631bf8dc0fe33c6ba88c3dc7f00ac5946065 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Fri, 22 Mar 2024 08:38:40 -0700 Subject: [PATCH 268/296] Revert "[BOLT] Add BB index to BAT (#86044)" This reverts commit 3b3de48fd84b8269d5f45ee0a9dc6b7448368424. --- bolt/docs/BAT.md | 11 +++-- .../bolt/Profile/BoltAddressTranslation.h | 7 +--- bolt/lib/Profile/BoltAddressTranslation.cpp | 39 +++++------------ .../X86/bolt-address-translation-yaml.test | 2 +- bolt/test/X86/bolt-address-translation.test | 2 +- clang/lib/Driver/ToolChains/Clang.cpp | 4 +- clang/test/Driver/unsupported-option-gpu.c | 1 + lld/MachO/Driver.cpp | 42 ++----------------- lld/MachO/InputSection.cpp | 38 +++++++++++++++++ lld/MachO/InputSection.h | 3 ++ lld/MachO/ObjC.cpp | 16 +++---- lld/MachO/SyntheticSections.cpp | 4 +- 12 files changed, 77 insertions(+), 92 deletions(-) diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md index 436593478a39..186b0e5ea89d 100644 --- a/bolt/docs/BAT.md +++ b/bolt/docs/BAT.md @@ -90,12 +90,11 @@ current function. ### Address translation table Delta encoding means that only the difference with the previous corresponding entry is encoded. Input offsets implicitly start at zero. -| Entry | Encoding | Description | Branch/BB | -| ------ | ------| ----------- | ------ | -| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | Both | -| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | Both | -| `BBHash` | Optional, 8b | Basic block hash in input binary | BB | -| `BBIdx` | Optional, Delta, ULEB128 | Basic block index in input binary | BB | +| Entry | Encoding | Description | +| ------ | ------| ----------- | +| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | +| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | +| `BBHash` | Optional, 8b | Basic block entries only: basic block hash in input binary | `BRANCHENTRY` bit denotes whether a given offset pair is a control flow source (branch or call instruction). If not set, it signifies a control flow target diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index eda2b318f0d0..1f53f6d344ad 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -122,10 +122,6 @@ public: /// Returns BF hash by function output address (after BOLT). size_t getBFHash(uint64_t OutputAddress) const; - /// Returns BB index by function output address (after BOLT) and basic block - /// input offset. - unsigned getBBIndex(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const; - /// True if a given \p Address is a function with translation table entry. bool isBATFunction(uint64_t Address) const { return Maps.count(Address); } @@ -158,8 +154,7 @@ private: std::map Maps; - /// Map basic block input offset to a basic block index and hash pair. - using BBHashMap = std::unordered_map>; + using BBHashMap = std::unordered_map; std::unordered_map> FuncHashes; /// Links outlined cold bocks to their original function diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index 8fe976cc00e5..1d61a1b735b4 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -45,8 +45,6 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", getBBHash(HotFuncAddress, BBInputOffset))); (void)HotFuncAddress; - LLVM_DEBUG(dbgs() << formatv(" Index: {0}\n", - getBBIndex(HotFuncAddress, BBInputOffset))); // In case of conflicts (same Key mapping to different Vals), the last // update takes precedence. Of course it is not ideal to have conflicts and // those happen when we have an empty BB that either contained only @@ -219,7 +217,6 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, } size_t Index = 0; uint64_t InOffset = 0; - size_t PrevBBIndex = 0; // Output and Input addresses and delta-encoded for (std::pair &KeyVal : Map) { const uint64_t OutputAddress = KeyVal.first + Address; @@ -229,15 +226,11 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, encodeSLEB128(KeyVal.second - InOffset, OS); InOffset = KeyVal.second; // Keeping InOffset as if BRANCHENTRY is encoded if ((InOffset & BRANCHENTRY) == 0) { - unsigned BBIndex; - size_t BBHash; - std::tie(BBIndex, BBHash) = FuncHashPair.second[InOffset >> 1]; + // Basic block hash + size_t BBHash = FuncHashPair.second[InOffset >> 1]; OS.write(reinterpret_cast(&BBHash), 8); - // Basic block index in the input binary - encodeULEB128(BBIndex - PrevBBIndex, OS); - PrevBBIndex = BBIndex; - LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x} {3}\n", KeyVal.first, - InOffset >> 1, BBHash, BBIndex)); + LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x}\n", KeyVal.first, + InOffset >> 1, BBHash)); } } } @@ -323,7 +316,6 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, LLVM_DEBUG(dbgs() << "Parsing " << NumEntries << " entries for 0x" << Twine::utohexstr(Address) << "\n"); uint64_t InputOffset = 0; - size_t BBIndex = 0; for (uint32_t J = 0; J < NumEntries; ++J) { const uint64_t OutputDelta = DE.getULEB128(&Offset, &Err); const uint64_t OutputAddress = PrevAddress + OutputDelta; @@ -338,25 +330,19 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, } Map.insert(std::pair(OutputOffset, InputOffset)); size_t BBHash = 0; - size_t BBIndexDelta = 0; const bool IsBranchEntry = InputOffset & BRANCHENTRY; if (!IsBranchEntry) { BBHash = DE.getU64(&Offset, &Err); - BBIndexDelta = DE.getULEB128(&Offset, &Err); - BBIndex += BBIndexDelta; // Map basic block hash to hot fragment by input offset - FuncHashes[HotAddress].second.emplace(InputOffset >> 1, - std::pair(BBIndex, BBHash)); + FuncHashes[HotAddress].second.emplace(InputOffset >> 1, BBHash); } LLVM_DEBUG({ dbgs() << formatv( "{0:x} -> {1:x} ({2}/{3}b -> {4}/{5}b), {6:x}", OutputOffset, InputOffset, OutputDelta, getULEB128Size(OutputDelta), InputDelta, (J < EqualElems) ? 0 : getSLEB128Size(InputDelta), OutputAddress); - if (!IsBranchEntry) { - dbgs() << formatv(" {0:x} {1}/{2}b", BBHash, BBIndex, - getULEB128Size(BBIndexDelta)); - } + if (BBHash) + dbgs() << formatv(" {0:x}", BBHash); dbgs() << '\n'; }); } @@ -508,19 +494,14 @@ void BoltAddressTranslation::saveMetadata(BinaryContext &BC) { FuncHashes[BF.getAddress()].first = BF.computeHash(); BF.computeBlockHashes(); for (const BinaryBasicBlock &BB : BF) - FuncHashes[BF.getAddress()].second.emplace( - BB.getInputOffset(), std::pair(BB.getIndex(), BB.getHash())); + FuncHashes[BF.getAddress()].second.emplace(BB.getInputOffset(), + BB.getHash()); } } -unsigned BoltAddressTranslation::getBBIndex(uint64_t FuncOutputAddress, - uint32_t BBInputOffset) const { - return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).first; -} - size_t BoltAddressTranslation::getBBHash(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const { - return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).second; + return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset); } size_t BoltAddressTranslation::getBFHash(uint64_t OutputAddress) const { diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test index 4516a662697a..25ff4e7fbfcc 100644 --- a/bolt/test/X86/bolt-address-translation-yaml.test +++ b/bolt/test/X86/bolt-address-translation-yaml.test @@ -18,7 +18,7 @@ RUN: | FileCheck --check-prefix CHECK-BOLT-YAML %s WRITE-BAT-CHECK: BOLT-INFO: Wrote 5 BAT maps WRITE-BAT-CHECK: BOLT-INFO: Wrote 4 function and 22 basic block hashes -WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 376 +WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 344 READ-BAT-CHECK-NOT: BOLT-ERROR: unable to save profile in YAML format for input file processed by BOLT READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries diff --git a/bolt/test/X86/bolt-address-translation.test b/bolt/test/X86/bolt-address-translation.test index 5c1db89e3c6b..4277b4e0d0fe 100644 --- a/bolt/test/X86/bolt-address-translation.test +++ b/bolt/test/X86/bolt-address-translation.test @@ -37,7 +37,7 @@ # CHECK: BOLT: 3 out of 7 functions were overwritten. # CHECK: BOLT-INFO: Wrote 6 BAT maps # CHECK: BOLT-INFO: Wrote 3 function and 58 basic block hashes -# CHECK: BOLT-INFO: BAT section size (bytes): 920 +# CHECK: BOLT-INFO: BAT section size (bytes): 816 # # usqrt mappings (hot part). We match against any key (left side containing # the bolted binary offsets) because BOLT may change where it puts instructions diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index bc9cc8ce6cf5..86a287db72a4 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -5863,8 +5863,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, } else if (Triple.getArch() == llvm::Triple::x86_64) { Ok = llvm::is_contained({"small", "kernel", "medium", "large", "tiny"}, CM); - } else if (Triple.isNVPTX() || Triple.isAMDGPU()) { - // NVPTX/AMDGPU does not care about the code model and will accept + } else if (Triple.isNVPTX() || Triple.isAMDGPU() || Triple.isSPIRV()) { + // NVPTX/AMDGPU/SPIRV does not care about the code model and will accept // whatever works for the host. Ok = true; } else if (Triple.isSPARC64()) { diff --git a/clang/test/Driver/unsupported-option-gpu.c b/clang/test/Driver/unsupported-option-gpu.c index f23cb71ebfb0..5618b2cba72e 100644 --- a/clang/test/Driver/unsupported-option-gpu.c +++ b/clang/test/Driver/unsupported-option-gpu.c @@ -2,4 +2,5 @@ // DEFINE: %{check} = %clang -### --target=x86_64-linux-gnu -c -mcmodel=medium // RUN: %{check} -x cuda %s --cuda-path=%S/Inputs/CUDA/usr/local/cuda --offload-arch=sm_60 --no-cuda-version-check -fbasic-block-sections=all +// RUN: %{check} -x hip %s --offload=spirv64 -nogpulib -nogpuinc // RUN: %{check} -x hip %s --rocm-path=%S/Inputs/rocm -nogpulib -nogpuinc diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp index 36248925d65a..919a14b8bcf0 100644 --- a/lld/MachO/Driver.cpp +++ b/lld/MachO/Driver.cpp @@ -612,7 +612,7 @@ static void replaceCommonSymbols() { if (!osec) osec = ConcatOutputSection::getOrCreateForInput(isec); isec->parent = osec; - inputSections.push_back(isec); + addInputSection(isec); // FIXME: CommonSymbol should store isReferencedDynamically, noDeadStrip // and pass them on here. @@ -1220,53 +1220,18 @@ static void createFiles(const InputArgList &args) { static void gatherInputSections() { TimeTraceScope timeScope("Gathering input sections"); - int inputOrder = 0; for (const InputFile *file : inputFiles) { for (const Section *section : file->sections) { // Compact unwind entries require special handling elsewhere. (In // contrast, EH frames are handled like regular ConcatInputSections.) if (section->name == section_names::compactUnwind) continue; - ConcatOutputSection *osec = nullptr; - for (const Subsection &subsection : section->subsections) { - if (auto *isec = dyn_cast(subsection.isec)) { - if (isec->isCoalescedWeak()) - continue; - if (config->emitInitOffsets && - sectionType(isec->getFlags()) == S_MOD_INIT_FUNC_POINTERS) { - in.initOffsets->addInput(isec); - continue; - } - isec->outSecOff = inputOrder++; - if (!osec) - osec = ConcatOutputSection::getOrCreateForInput(isec); - isec->parent = osec; - inputSections.push_back(isec); - } else if (auto *isec = - dyn_cast(subsection.isec)) { - if (isec->getName() == section_names::objcMethname) { - if (in.objcMethnameSection->inputOrder == UnspecifiedInputOrder) - in.objcMethnameSection->inputOrder = inputOrder++; - in.objcMethnameSection->addInput(isec); - } else { - if (in.cStringSection->inputOrder == UnspecifiedInputOrder) - in.cStringSection->inputOrder = inputOrder++; - in.cStringSection->addInput(isec); - } - } else if (auto *isec = - dyn_cast(subsection.isec)) { - if (in.wordLiteralSection->inputOrder == UnspecifiedInputOrder) - in.wordLiteralSection->inputOrder = inputOrder++; - in.wordLiteralSection->addInput(isec); - } else { - llvm_unreachable("unexpected input section kind"); - } - } + for (const Subsection &subsection : section->subsections) + addInputSection(subsection.isec); } if (!file->objCImageInfo.empty()) in.objCImageInfo->addFile(file); } - assert(inputOrder <= UnspecifiedInputOrder); } static void foldIdenticalLiterals() { @@ -1422,6 +1387,7 @@ bool link(ArrayRef argsArr, llvm::raw_ostream &stdoutOS, concatOutputSections.clear(); inputFiles.clear(); inputSections.clear(); + inputSectionsOrder = 0; loadedArchives.clear(); loadedObjectFrameworks.clear(); missingAutolinkWarnings.clear(); diff --git a/lld/MachO/InputSection.cpp b/lld/MachO/InputSection.cpp index 8f5affb1dc21..22930d52dd1d 100644 --- a/lld/MachO/InputSection.cpp +++ b/lld/MachO/InputSection.cpp @@ -37,6 +37,44 @@ static_assert(sizeof(void *) != 8 || "instances of it"); std::vector macho::inputSections; +int macho::inputSectionsOrder = 0; + +// Call this function to add a new InputSection and have it routed to the +// appropriate container. Depending on its type and current config, it will +// either be added to 'inputSections' vector or to a synthetic section. +void lld::macho::addInputSection(InputSection *inputSection) { + if (auto *isec = dyn_cast(inputSection)) { + if (isec->isCoalescedWeak()) + return; + if (config->emitInitOffsets && + sectionType(isec->getFlags()) == S_MOD_INIT_FUNC_POINTERS) { + in.initOffsets->addInput(isec); + return; + } + isec->outSecOff = inputSectionsOrder++; + auto *osec = ConcatOutputSection::getOrCreateForInput(isec); + isec->parent = osec; + inputSections.push_back(isec); + } else if (auto *isec = dyn_cast(inputSection)) { + if (isec->getName() == section_names::objcMethname) { + if (in.objcMethnameSection->inputOrder == UnspecifiedInputOrder) + in.objcMethnameSection->inputOrder = inputSectionsOrder++; + in.objcMethnameSection->addInput(isec); + } else { + if (in.cStringSection->inputOrder == UnspecifiedInputOrder) + in.cStringSection->inputOrder = inputSectionsOrder++; + in.cStringSection->addInput(isec); + } + } else if (auto *isec = dyn_cast(inputSection)) { + if (in.wordLiteralSection->inputOrder == UnspecifiedInputOrder) + in.wordLiteralSection->inputOrder = inputSectionsOrder++; + in.wordLiteralSection->addInput(isec); + } else { + llvm_unreachable("unexpected input section kind"); + } + + assert(inputSectionsOrder <= UnspecifiedInputOrder); +} uint64_t InputSection::getFileSize() const { return isZeroFill(getFlags()) ? 0 : getSize(); diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h index b25f0638f4c6..694bdf734907 100644 --- a/lld/MachO/InputSection.h +++ b/lld/MachO/InputSection.h @@ -302,6 +302,8 @@ bool isEhFrameSection(const InputSection *); bool isGccExceptTabSection(const InputSection *); extern std::vector inputSections; +// This is used as a counter for specyfing input order for input sections +extern int inputSectionsOrder; namespace section_names { @@ -369,6 +371,7 @@ constexpr const char addrSig[] = "__llvm_addrsig"; } // namespace section_names +void addInputSection(InputSection *inputSection); } // namespace macho std::string toString(const macho::InputSection *); diff --git a/lld/MachO/ObjC.cpp b/lld/MachO/ObjC.cpp index 40df2243b26f..5902b82d30f5 100644 --- a/lld/MachO/ObjC.cpp +++ b/lld/MachO/ObjC.cpp @@ -790,7 +790,7 @@ void ObjcCategoryMerger::emitAndLinkProtocolList( infoCategoryWriter.catPtrListInfo.align); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; listSec->live = true; - allInputSections.push_back(listSec); + addInputSection(listSec); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; @@ -848,7 +848,7 @@ void ObjcCategoryMerger::emitAndLinkPointerList( infoCategoryWriter.catPtrListInfo.align); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; listSec->live = true; - allInputSections.push_back(listSec); + addInputSection(listSec); listSec->parent = infoCategoryWriter.catPtrListInfo.outputSection; @@ -889,7 +889,7 @@ ObjcCategoryMerger::emitCatListEntrySec(const std::string &forCateogryName, bodyData, infoCategoryWriter.catListInfo.align); newCatList->parent = infoCategoryWriter.catListInfo.outputSection; newCatList->live = true; - allInputSections.push_back(newCatList); + addInputSection(newCatList); newCatList->parent = infoCategoryWriter.catListInfo.outputSection; @@ -927,7 +927,7 @@ Defined *ObjcCategoryMerger::emitCategoryBody(const std::string &name, bodyData, infoCategoryWriter.catBodyInfo.align); newBodySec->parent = infoCategoryWriter.catBodyInfo.outputSection; newBodySec->live = true; - allInputSections.push_back(newBodySec); + addInputSection(newBodySec); std::string symName = objc::symbol_names::category + baseClassName + "_$_(" + name + ")"; @@ -1132,7 +1132,7 @@ void ObjcCategoryMerger::generateCatListForNonErasedCategories( infoCategoryWriter.catListInfo.align); listSec->parent = infoCategoryWriter.catListInfo.outputSection; listSec->live = true; - allInputSections.push_back(listSec); + addInputSection(listSec); std::string slotSymName = "<__objc_catlist slot for category "; slotSymName += nonErasedCatBody->getName(); @@ -1221,9 +1221,11 @@ void ObjcCategoryMerger::doCleanup() { generatedSectionData.clear(); } StringRef ObjcCategoryMerger::newStringData(const char *str) { uint32_t len = strlen(str); - auto &data = newSectionData(len + 1); + uint32_t bufSize = len + 1; + auto &data = newSectionData(bufSize); char *strData = reinterpret_cast(data.data()); - strncpy(strData, str, len); + // Copy the string chars and null-terminator + memcpy(strData, str, bufSize); return StringRef(strData, len); } diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp index 7ee3261ce307..1b3694528de1 100644 --- a/lld/MachO/SyntheticSections.cpp +++ b/lld/MachO/SyntheticSections.cpp @@ -793,7 +793,7 @@ void StubHelperSection::setUp() { in.imageLoaderCache->parent = ConcatOutputSection::getOrCreateForInput(in.imageLoaderCache); - inputSections.push_back(in.imageLoaderCache); + addInputSection(in.imageLoaderCache); // Since this isn't in the symbol table or in any input file, the noDeadStrip // argument doesn't matter. dyldPrivate = @@ -855,7 +855,7 @@ ConcatInputSection *ObjCSelRefsSection::makeSelRef(StringRef methname) { /*addend=*/static_cast(methnameOffset), /*referent=*/in.objcMethnameSection->isec}); objcSelref->parent = ConcatOutputSection::getOrCreateForInput(objcSelref); - inputSections.push_back(objcSelref); + addInputSection(objcSelref); objcSelref->isFinal = true; methnameToSelref[CachedHashStringRef(methname)] = objcSelref; return objcSelref; -- GitLab From b0e23639c5b19030bee2b307173802914f64aad6 Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Tue, 20 Feb 2024 16:35:08 -0800 Subject: [PATCH 269/296] [BOLT] Add BB index to BAT Add input basic block index to BAT metadata. This addresses the case where some basic blocks are eliminated, and output index is not equal to the input block index. These indices are used in non-stale-matching mode. Increases BAT section size to: - large binary: 39521512 bytes (1.02x original), - medium binary: 3799988 bytes (0.64x), - small binary: 920 bytes (0.64x). Test Plan: Updated bolt-address-translation{,-yaml}.test Pull Request: https://github.com/llvm/llvm-project/pull/86044 --- bolt/docs/BAT.md | 11 +++--- .../bolt/Profile/BoltAddressTranslation.h | 7 +++- bolt/lib/Profile/BoltAddressTranslation.cpp | 39 ++++++++++++++----- .../X86/bolt-address-translation-yaml.test | 2 +- bolt/test/X86/bolt-address-translation.test | 2 +- 5 files changed, 43 insertions(+), 18 deletions(-) diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md index 186b0e5ea89d..436593478a39 100644 --- a/bolt/docs/BAT.md +++ b/bolt/docs/BAT.md @@ -90,11 +90,12 @@ current function. ### Address translation table Delta encoding means that only the difference with the previous corresponding entry is encoded. Input offsets implicitly start at zero. -| Entry | Encoding | Description | -| ------ | ------| ----------- | -| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | -| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | -| `BBHash` | Optional, 8b | Basic block entries only: basic block hash in input binary | +| Entry | Encoding | Description | Branch/BB | +| ------ | ------| ----------- | ------ | +| `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary | Both | +| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit | Both | +| `BBHash` | Optional, 8b | Basic block hash in input binary | BB | +| `BBIdx` | Optional, Delta, ULEB128 | Basic block index in input binary | BB | `BRANCHENTRY` bit denotes whether a given offset pair is a control flow source (branch or call instruction). If not set, it signifies a control flow target diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index 1f53f6d344ad..eda2b318f0d0 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -122,6 +122,10 @@ public: /// Returns BF hash by function output address (after BOLT). size_t getBFHash(uint64_t OutputAddress) const; + /// Returns BB index by function output address (after BOLT) and basic block + /// input offset. + unsigned getBBIndex(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const; + /// True if a given \p Address is a function with translation table entry. bool isBATFunction(uint64_t Address) const { return Maps.count(Address); } @@ -154,7 +158,8 @@ private: std::map Maps; - using BBHashMap = std::unordered_map; + /// Map basic block input offset to a basic block index and hash pair. + using BBHashMap = std::unordered_map>; std::unordered_map> FuncHashes; /// Links outlined cold bocks to their original function diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index 1d61a1b735b4..8fe976cc00e5 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -45,6 +45,8 @@ void BoltAddressTranslation::writeEntriesForBB(MapTy &Map, LLVM_DEBUG(dbgs() << formatv(" Hash: {0:x}\n", getBBHash(HotFuncAddress, BBInputOffset))); (void)HotFuncAddress; + LLVM_DEBUG(dbgs() << formatv(" Index: {0}\n", + getBBIndex(HotFuncAddress, BBInputOffset))); // In case of conflicts (same Key mapping to different Vals), the last // update takes precedence. Of course it is not ideal to have conflicts and // those happen when we have an empty BB that either contained only @@ -217,6 +219,7 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, } size_t Index = 0; uint64_t InOffset = 0; + size_t PrevBBIndex = 0; // Output and Input addresses and delta-encoded for (std::pair &KeyVal : Map) { const uint64_t OutputAddress = KeyVal.first + Address; @@ -226,11 +229,15 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, encodeSLEB128(KeyVal.second - InOffset, OS); InOffset = KeyVal.second; // Keeping InOffset as if BRANCHENTRY is encoded if ((InOffset & BRANCHENTRY) == 0) { - // Basic block hash - size_t BBHash = FuncHashPair.second[InOffset >> 1]; + unsigned BBIndex; + size_t BBHash; + std::tie(BBIndex, BBHash) = FuncHashPair.second[InOffset >> 1]; OS.write(reinterpret_cast(&BBHash), 8); - LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x}\n", KeyVal.first, - InOffset >> 1, BBHash)); + // Basic block index in the input binary + encodeULEB128(BBIndex - PrevBBIndex, OS); + PrevBBIndex = BBIndex; + LLVM_DEBUG(dbgs() << formatv("{0:x} -> {1:x} {2:x} {3}\n", KeyVal.first, + InOffset >> 1, BBHash, BBIndex)); } } } @@ -316,6 +323,7 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, LLVM_DEBUG(dbgs() << "Parsing " << NumEntries << " entries for 0x" << Twine::utohexstr(Address) << "\n"); uint64_t InputOffset = 0; + size_t BBIndex = 0; for (uint32_t J = 0; J < NumEntries; ++J) { const uint64_t OutputDelta = DE.getULEB128(&Offset, &Err); const uint64_t OutputAddress = PrevAddress + OutputDelta; @@ -330,19 +338,25 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, } Map.insert(std::pair(OutputOffset, InputOffset)); size_t BBHash = 0; + size_t BBIndexDelta = 0; const bool IsBranchEntry = InputOffset & BRANCHENTRY; if (!IsBranchEntry) { BBHash = DE.getU64(&Offset, &Err); + BBIndexDelta = DE.getULEB128(&Offset, &Err); + BBIndex += BBIndexDelta; // Map basic block hash to hot fragment by input offset - FuncHashes[HotAddress].second.emplace(InputOffset >> 1, BBHash); + FuncHashes[HotAddress].second.emplace(InputOffset >> 1, + std::pair(BBIndex, BBHash)); } LLVM_DEBUG({ dbgs() << formatv( "{0:x} -> {1:x} ({2}/{3}b -> {4}/{5}b), {6:x}", OutputOffset, InputOffset, OutputDelta, getULEB128Size(OutputDelta), InputDelta, (J < EqualElems) ? 0 : getSLEB128Size(InputDelta), OutputAddress); - if (BBHash) - dbgs() << formatv(" {0:x}", BBHash); + if (!IsBranchEntry) { + dbgs() << formatv(" {0:x} {1}/{2}b", BBHash, BBIndex, + getULEB128Size(BBIndexDelta)); + } dbgs() << '\n'; }); } @@ -494,14 +508,19 @@ void BoltAddressTranslation::saveMetadata(BinaryContext &BC) { FuncHashes[BF.getAddress()].first = BF.computeHash(); BF.computeBlockHashes(); for (const BinaryBasicBlock &BB : BF) - FuncHashes[BF.getAddress()].second.emplace(BB.getInputOffset(), - BB.getHash()); + FuncHashes[BF.getAddress()].second.emplace( + BB.getInputOffset(), std::pair(BB.getIndex(), BB.getHash())); } } +unsigned BoltAddressTranslation::getBBIndex(uint64_t FuncOutputAddress, + uint32_t BBInputOffset) const { + return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).first; +} + size_t BoltAddressTranslation::getBBHash(uint64_t FuncOutputAddress, uint32_t BBInputOffset) const { - return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset); + return FuncHashes.at(FuncOutputAddress).second.at(BBInputOffset).second; } size_t BoltAddressTranslation::getBFHash(uint64_t OutputAddress) const { diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test index 25ff4e7fbfcc..4516a662697a 100644 --- a/bolt/test/X86/bolt-address-translation-yaml.test +++ b/bolt/test/X86/bolt-address-translation-yaml.test @@ -18,7 +18,7 @@ RUN: | FileCheck --check-prefix CHECK-BOLT-YAML %s WRITE-BAT-CHECK: BOLT-INFO: Wrote 5 BAT maps WRITE-BAT-CHECK: BOLT-INFO: Wrote 4 function and 22 basic block hashes -WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 344 +WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 376 READ-BAT-CHECK-NOT: BOLT-ERROR: unable to save profile in YAML format for input file processed by BOLT READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries diff --git a/bolt/test/X86/bolt-address-translation.test b/bolt/test/X86/bolt-address-translation.test index 4277b4e0d0fe..5c1db89e3c6b 100644 --- a/bolt/test/X86/bolt-address-translation.test +++ b/bolt/test/X86/bolt-address-translation.test @@ -37,7 +37,7 @@ # CHECK: BOLT: 3 out of 7 functions were overwritten. # CHECK: BOLT-INFO: Wrote 6 BAT maps # CHECK: BOLT-INFO: Wrote 3 function and 58 basic block hashes -# CHECK: BOLT-INFO: BAT section size (bytes): 816 +# CHECK: BOLT-INFO: BAT section size (bytes): 920 # # usqrt mappings (hot part). We match against any key (left side containing # the bolted binary offsets) because BOLT may change where it puts instructions -- GitLab From cb300c33059c1d14f72392ce5dffcf050ad7567d Mon Sep 17 00:00:00 2001 From: Christian Ulmann Date: Fri, 22 Mar 2024 16:44:06 +0100 Subject: [PATCH 270/296] [MLIR][LLVM][SROA] Fix pointer escape through stores bug (#86291) This commit resolves a SROA bug caused by not properly checking if a llvm store operation writes the pointer to memory or not. Now, we do no longer consider stores that use a slot pointer as a value to store as fixable. --- mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp | 4 ++++ mlir/test/Dialect/LLVMIR/sroa.mlir | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp index 0ef1d105aca6..f171bf7cc4be 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp @@ -251,6 +251,10 @@ bool LLVM::StoreOp::canRewire(const DestructurableMemorySlot &slot, if (getVolatile_()) return false; + // Storing the pointer to memory cannot be dealt with. + if (getValue() == slot.ptr) + return false; + // A store always accesses the first element of the destructured slot. auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0); Type subslotType = getTypeAtIndex(slot, index); diff --git a/mlir/test/Dialect/LLVMIR/sroa.mlir b/mlir/test/Dialect/LLVMIR/sroa.mlir index ca49b1298b0e..3f4d17c6a43f 100644 --- a/mlir/test/Dialect/LLVMIR/sroa.mlir +++ b/mlir/test/Dialect/LLVMIR/sroa.mlir @@ -305,3 +305,16 @@ llvm.func @vector_store_type_mismatch(%arg: vector<4xi32>) { llvm.store %arg, %1 : vector<4xi32>, !llvm.ptr llvm.return } + +// ----- + +// CHECK-LABEL: llvm.func @store_to_memory +// CHECK-SAME: %[[ARG:.*]]: !llvm.ptr +llvm.func @store_to_memory(%arg: !llvm.ptr) { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct< + %1 = llvm.alloca %0 x !llvm.struct<"foo", (vector<4xf32>)> : (i32) -> !llvm.ptr + // CHECK-NEXT: llvm.store %[[ALLOCA]], %[[ARG]] + llvm.store %1, %arg : !llvm.ptr, !llvm.ptr + llvm.return +} -- GitLab From ceba3a38e8f7b378ad20641832d568460892af1d Mon Sep 17 00:00:00 2001 From: Amir Ayupov Date: Fri, 22 Mar 2024 08:46:48 -0700 Subject: [PATCH 271/296] [BOLT] Add number of basic blocks to BAT YAML profile reader checks the number of basic blocks in regular, no-stale-matching mode. Add it to BAT. This increases the size of BAT section to: - large binary: 39583080 bytes (1.02x of the original), - medium binary: 3816492 bytes (0.64x), - small binary: 920 bytes (0.64x, no change due to alignment). Test Plan: Updated bolt-address-translation-yaml.test Reviewers: rafaelauler, ayermolo, maksfb, dcci Reviewed By: rafaelauler Pull Request: https://github.com/llvm/llvm-project/pull/86045 --- bolt/docs/BAT.md | 1 + bolt/include/bolt/Profile/BoltAddressTranslation.h | 3 +++ bolt/lib/Profile/BoltAddressTranslation.cpp | 10 ++++++++++ bolt/test/X86/bolt-address-translation-yaml.test | 2 +- 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md index 436593478a39..2279a070263e 100644 --- a/bolt/docs/BAT.md +++ b/bolt/docs/BAT.md @@ -79,6 +79,7 @@ Hot indices are delta encoded, implicitly starting at zero. | `Address` | Continuous, Delta, ULEB128 | Function address in the output binary | | `HotIndex` | Delta, ULEB128 | Cold functions only: index of corresponding hot function in hot functions table | | `FuncHash` | 8b | Hot functions only: function hash for input function | +| `NumBlocks` | ULEB128 | Hot functions only: number of basic blocks in the original function | | `NumEntries` | ULEB128 | Number of address translation entries for a function | | `EqualElems` | ULEB128 | Hot functions only: number of equal offsets in the beginning of a function | | `BranchEntries` | Bitmask, `alignTo(EqualElems, 8)` bits | Hot functions only: if `EqualElems` is non-zero, bitmask denoting entries with `BRANCHENTRY` bit | diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h index eda2b318f0d0..d583ce0b76a2 100644 --- a/bolt/include/bolt/Profile/BoltAddressTranslation.h +++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h @@ -162,6 +162,9 @@ private: using BBHashMap = std::unordered_map>; std::unordered_map> FuncHashes; + /// Map a function to its basic blocks count + std::unordered_map NumBasicBlocksMap; + /// Links outlined cold bocks to their original function std::map ColdPartSource; diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp index 8fe976cc00e5..31886f4c8025 100644 --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -196,6 +196,10 @@ void BoltAddressTranslation::writeMaps(std::map &Maps, // Function hash LLVM_DEBUG(dbgs() << "Hash: " << formatv("{0:x}\n", FuncHashPair.first)); OS.write(reinterpret_cast(&FuncHashPair.first), 8); + // Number of basic blocks + size_t NumBasicBlocks = FuncHashPair.second.size(); + LLVM_DEBUG(dbgs() << "Basic blocks: " << NumBasicBlocks << '\n'); + encodeULEB128(NumBasicBlocks, OS); } encodeULEB128(NumEntries, OS); // For hot fragments only: encode the number of equal offsets @@ -293,6 +297,12 @@ void BoltAddressTranslation::parseMaps(std::vector &HotFuncs, const size_t FuncHash = DE.getU64(&Offset, &Err); FuncHashes[Address].first = FuncHash; LLVM_DEBUG(dbgs() << formatv("{0:x}: hash {1:x}\n", Address, FuncHash)); + // Number of basic blocks + const size_t NumBasicBlocks = DE.getULEB128(&Offset, &Err); + NumBasicBlocksMap.emplace(Address, NumBasicBlocks); + LLVM_DEBUG(dbgs() << formatv("{0:x}: #bbs {1}, {2} bytes\n", Address, + NumBasicBlocks, + getULEB128Size(NumBasicBlocks))); } const uint32_t NumEntries = DE.getULEB128(&Offset, &Err); // Equal offsets, hot fragments only. diff --git a/bolt/test/X86/bolt-address-translation-yaml.test b/bolt/test/X86/bolt-address-translation-yaml.test index 4516a662697a..ee54a90a9f2a 100644 --- a/bolt/test/X86/bolt-address-translation-yaml.test +++ b/bolt/test/X86/bolt-address-translation-yaml.test @@ -18,7 +18,7 @@ RUN: | FileCheck --check-prefix CHECK-BOLT-YAML %s WRITE-BAT-CHECK: BOLT-INFO: Wrote 5 BAT maps WRITE-BAT-CHECK: BOLT-INFO: Wrote 4 function and 22 basic block hashes -WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 376 +WRITE-BAT-CHECK: BOLT-INFO: BAT section size (bytes): 380 READ-BAT-CHECK-NOT: BOLT-ERROR: unable to save profile in YAML format for input file processed by BOLT READ-BAT-CHECK: BOLT-INFO: Parsed 5 BAT entries -- GitLab From 2091c74796b1dac68e622284c63a870b88b7554f Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 22 Mar 2024 15:47:40 +0000 Subject: [PATCH 272/296] [RemoveDIs] Update DIBuilder C API with DbgRecord functions [2/2] (#85657) Follow on from #84915 which adds the DbgRecord function variants. Update the LLVMDIBuilderInsert... functions to insert DbgRecords instead of debug intrinsics. LLVMDIBuilderInsertDeclareBefore LLVMDIBuilderInsertDeclareAtEnd LLVMDIBuilderInsertDbgValueBefore LLVMDIBuilderInsertDbgValueAtEnd Calling these functions will now cause an assertion if the module is in the wrong debug info format. They should only be used when the module is in "new debug format". Use LLVMIsNewDbgInfoFormat to query and LLVMSetIsNewDbgInfoFormat to change the debug info format of a module. Please see https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-change (RemoveDIsDebugInfo.md) for more info. --- llvm/docs/RemoveDIsDebugInfo.md | 11 ++- llvm/include/llvm-c/DebugInfo.h | 60 +++++++++----- llvm/lib/IR/DebugInfo.cpp | 121 +++++++++++++++++++---------- llvm/tools/llvm-c-test/debuginfo.c | 13 ++-- 4 files changed, 137 insertions(+), 68 deletions(-) diff --git a/llvm/docs/RemoveDIsDebugInfo.md b/llvm/docs/RemoveDIsDebugInfo.md index a2f1e173d9d9..9e50a2a604aa 100644 --- a/llvm/docs/RemoveDIsDebugInfo.md +++ b/llvm/docs/RemoveDIsDebugInfo.md @@ -40,15 +40,22 @@ New functions (all to be deprecated) LLVMIsNewDbgInfoFormat # Returns true if the module is in the new non-instruction mode. LLVMSetIsNewDbgInfoFormat # Convert to the requested debug info format. -LLVMDIBuilderInsertDeclareIntrinsicBefore # Insert a debug intrinsic (old debug info format). +LLVMDIBuilderInsertDeclareIntrinsicBefore # Insert a debug intrinsic (old debug info format). LLVMDIBuilderInsertDeclareIntrinsicAtEnd # Same as above. LLVMDIBuilderInsertDbgValueIntrinsicBefore # Same as above. LLVMDIBuilderInsertDbgValueIntrinsicAtEnd # Same as above. -LLVMDIBuilderInsertDeclareRecordBefore # Insert a debug record (new debug info format). +LLVMDIBuilderInsertDeclareRecordBefore # Insert a debug record (new debug info format). LLVMDIBuilderInsertDeclareRecordAtEnd # Same as above. LLVMDIBuilderInsertDbgValueRecordBefore # Same as above. LLVMDIBuilderInsertDbgValueRecordAtEnd # Same as above. + +Existing functions (behaviour change) +------------------------------------- +LLVMDIBuilderInsertDeclareBefore # Insert a debug record (new debug info format) instead of a debug intrinsic (old debug info format). +LLVMDIBuilderInsertDeclareAtEnd # Same as above. +LLVMDIBuilderInsertDbgValueBefore # Same as above. +LLVMDIBuilderInsertDbgValueAtEnd # Same as above. ``` # Anything else? diff --git a/llvm/include/llvm-c/DebugInfo.h b/llvm/include/llvm-c/DebugInfo.h index b23ff63c862f..dab1d697761b 100644 --- a/llvm/include/llvm-c/DebugInfo.h +++ b/llvm/include/llvm-c/DebugInfo.h @@ -1249,7 +1249,12 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( LLVMMetadataRef Decl, uint32_t AlignInBits); /* - * Insert a new llvm.dbg.declare intrinsic call before the given instruction. + * Insert a new Declare DbgRecord before the given instruction. + * + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. + * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes + * * \param Builder The DIBuilder. * \param Storage The storage of the variable to declare. * \param VarInfo The variable's debug info descriptor. @@ -1257,13 +1262,13 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( * \param DebugLoc Debug info location. * \param Instr Instruction acting as a location for the new intrinsic. */ -LLVMValueRef +LLVMDbgRecordRef LLVMDIBuilderInsertDeclareBefore(LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "old debug mode" (LLVMIsNewDbgFormat() is false). + * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.declare intrinsic call before the given instruction. @@ -1279,7 +1284,7 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "new debug mode" (LLVMIsNewDbgFormat() is true). + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a Declare DbgRecord before the given instruction. @@ -1295,9 +1300,14 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** - * Insert a new llvm.dbg.declare intrinsic call at the end of the given basic - * block. If the basic block has a terminator instruction, the intrinsic is - * inserted before that terminator instruction. + * Insert a new Declare DbgRecord at the end of the given basic block. If the + * basic block has a terminator instruction, the intrinsic is inserted before + * that terminator instruction. + * + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. + * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes + * * \param Builder The DIBuilder. * \param Storage The storage of the variable to declare. * \param VarInfo The variable's debug info descriptor. @@ -1305,12 +1315,12 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore( * \param DebugLoc Debug info location. * \param Block Basic block acting as a location for the new intrinsic. */ -LLVMValueRef LLVMDIBuilderInsertDeclareAtEnd( +LLVMDbgRecordRef LLVMDIBuilderInsertDeclareAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "old debug mode" (LLVMIsNewDbgFormat() is false). + * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.declare intrinsic call at the end of the given basic @@ -1328,7 +1338,7 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicAtEnd( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "new debug mode" (LLVMIsNewDbgFormat() is true). + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a Declare DbgRecord at the end of the given basic block. If the basic @@ -1346,7 +1356,12 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** - * Insert a new llvm.dbg.value intrinsic call before the given instruction. + * Insert a new Value DbgRecord before the given instruction. + * + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. + * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes + * * \param Builder The DIBuilder. * \param Val The value of the variable. * \param VarInfo The variable's debug info descriptor. @@ -1354,13 +1369,13 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd( * \param DebugLoc Debug info location. * \param Instr Instruction acting as a location for the new intrinsic. */ -LLVMValueRef +LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueBefore(LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "old debug mode" (Module::IsNewDbgInfoFormat is false). + * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call before the given instruction. @@ -1376,7 +1391,7 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "new debug mode" (Module::IsNewDbgInfoFormat is true). + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call before the given instruction. @@ -1392,9 +1407,14 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** - * Insert a new llvm.dbg.value intrinsic call at the end of the given basic - * block. If the basic block has a terminator instruction, the intrinsic is - * inserted before that terminator instruction. + * Insert a new Value DbgRecord at the end of the given basic block. If the + * basic block has a terminator instruction, the intrinsic is inserted before + * that terminator instruction. + * + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. + * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes + * * \param Builder The DIBuilder. * \param Val The value of the variable. * \param VarInfo The variable's debug info descriptor. @@ -1402,12 +1422,12 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore( * \param DebugLoc Debug info location. * \param Block Basic block acting as a location for the new intrinsic. */ -LLVMValueRef LLVMDIBuilderInsertDbgValueAtEnd( +LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "old debug mode" (Module::IsNewDbgInfoFormat is false). + * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call at the end of the given basic @@ -1425,7 +1445,7 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicAtEnd( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "new debug mode" (Module::IsNewDbgInfoFormat is true). + * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call at the end of the given basic diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp index 09bce9df1f33..4206162d1768 100644 --- a/llvm/lib/IR/DebugInfo.cpp +++ b/llvm/lib/IR/DebugInfo.cpp @@ -1665,12 +1665,12 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( unwrapDI(Decl), nullptr, AlignInBits)); } -LLVMValueRef +LLVMDbgRecordRef LLVMDIBuilderInsertDeclareBefore(LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMValueRef Instr) { - return LLVMDIBuilderInsertDeclareIntrinsicBefore(Builder, Storage, VarInfo, - Expr, DL, Instr); + return LLVMDIBuilderInsertDeclareRecordBefore(Builder, Storage, VarInfo, Expr, + DL, Instr); } LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicBefore( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, @@ -1679,27 +1679,38 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicBefore( unwrap(Storage), unwrap(VarInfo), unwrap(Expr), unwrap(DL), unwrap(Instr)); + // This assert will fail if the module is in the new debug info format. + // This function should only be called if the module is in the old + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Inserted a DbgRecord into function using old debug info mode"); + "Function unexpectedly in new debug info format"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMValueRef Instr) { - return wrap( - unwrap(Builder) - ->insertDeclare(unwrap(Storage), unwrap(VarInfo), - unwrap(Expr), unwrap(DL), - unwrap(Instr)) - .get()); + DbgInstPtr DbgInst = unwrap(Builder)->insertDeclare( + unwrap(Storage), unwrap(VarInfo), + unwrap(Expr), unwrap(DL), + unwrap(Instr)); + // This assert will fail if the module is in the old debug info format. + // This function should only be called if the module is in the new + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. + assert(isa(DbgInst) && + "Function unexpectedly in old debug info format"); + return wrap(cast(DbgInst)); } -LLVMValueRef +LLVMDbgRecordRef LLVMDIBuilderInsertDeclareAtEnd(LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMBasicBlockRef Block) { - return LLVMDIBuilderInsertDeclareIntrinsicAtEnd(Builder, Storage, VarInfo, - Expr, DL, Block); + return LLVMDIBuilderInsertDeclareRecordAtEnd(Builder, Storage, VarInfo, Expr, + DL, Block); } LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, @@ -1707,26 +1718,36 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicAtEnd( DbgInstPtr DbgInst = unwrap(Builder)->insertDeclare( unwrap(Storage), unwrap(VarInfo), unwrap(Expr), unwrap(DL), unwrap(Block)); + // This assert will fail if the module is in the new debug info format. + // This function should only be called if the module is in the old + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Inserted a DbgRecord into function using old debug info mode"); + "Function unexpectedly in new debug info format"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMBasicBlockRef Block) { - return wrap(unwrap(Builder) - ->insertDeclare(unwrap(Storage), - unwrap(VarInfo), - unwrap(Expr), - unwrap(DL), unwrap(Block)) - .get()); + DbgInstPtr DbgInst = unwrap(Builder)->insertDeclare( + unwrap(Storage), unwrap(VarInfo), + unwrap(Expr), unwrap(DL), unwrap(Block)); + // This assert will fail if the module is in the old debug info format. + // This function should only be called if the module is in the new + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. + assert(isa(DbgInst) && + "Function unexpectedly in old debug info format"); + return wrap(cast(DbgInst)); } -LLVMValueRef LLVMDIBuilderInsertDbgValueBefore( +LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueBefore( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr) { - return LLVMDIBuilderInsertDbgValueIntrinsicBefore(Builder, Val, VarInfo, Expr, - DebugLoc, Instr); + return LLVMDIBuilderInsertDbgValueRecordBefore(Builder, Val, VarInfo, Expr, + DebugLoc, Instr); } LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicBefore( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, @@ -1734,26 +1755,36 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicBefore( DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( unwrap(Val), unwrap(VarInfo), unwrap(Expr), unwrap(DebugLoc), unwrap(Instr)); + // This assert will fail if the module is in the new debug info format. + // This function should only be called if the module is in the old + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Inserted a DbgRecord into function using old debug info mode"); + "Function unexpectedly in new debug info format"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr) { - return wrap(unwrap(Builder) - ->insertDbgValueIntrinsic( - unwrap(Val), unwrap(VarInfo), - unwrap(Expr), unwrap(DebugLoc), - unwrap(Instr)) - .get()); + DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( + unwrap(Val), unwrap(VarInfo), unwrap(Expr), + unwrap(DebugLoc), unwrap(Instr)); + // This assert will fail if the module is in the old debug info format. + // This function should only be called if the module is in the new + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. + assert(isa(DbgInst) && + "Function unexpectedly in old debug info format"); + return wrap(cast(DbgInst)); } -LLVMValueRef LLVMDIBuilderInsertDbgValueAtEnd( +LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block) { - return LLVMDIBuilderInsertDbgValueIntrinsicAtEnd(Builder, Val, VarInfo, Expr, - DebugLoc, Block); + return LLVMDIBuilderInsertDbgValueRecordAtEnd(Builder, Val, VarInfo, Expr, + DebugLoc, Block); } LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, @@ -1761,19 +1792,29 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicAtEnd( DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( unwrap(Val), unwrap(VarInfo), unwrap(Expr), unwrap(DebugLoc), unwrap(Block)); + // This assert will fail if the module is in the new debug info format. + // This function should only be called if the module is in the old + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Inserted a DbgRecord into function using old debug info mode"); + "Function unexpectedly in new debug info format"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block) { - return wrap(unwrap(Builder) - ->insertDbgValueIntrinsic( - unwrap(Val), unwrap(VarInfo), - unwrap(Expr), unwrap(DebugLoc), - unwrap(Block)) - .get()); + DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( + unwrap(Val), unwrap(VarInfo), unwrap(Expr), + unwrap(DebugLoc), unwrap(Block)); + // This assert will fail if the module is in the old debug info format. + // This function should only be called if the module is in the new + // debug info format. + // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, + // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. + assert(isa(DbgInst) && + "Function unexpectedly in old debug info format"); + return wrap(cast(DbgInst)); } LLVMMetadataRef LLVMDIBuilderCreateAutoVariable( diff --git a/llvm/tools/llvm-c-test/debuginfo.c b/llvm/tools/llvm-c-test/debuginfo.c index 78ccaf12a380..9b5c37b05d90 100644 --- a/llvm/tools/llvm-c-test/debuginfo.c +++ b/llvm/tools/llvm-c-test/debuginfo.c @@ -136,12 +136,13 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) { LLVMMetadataRef FooParamVar1 = LLVMDIBuilderCreateParameterVariable(DIB, FunctionMetadata, "a", 1, 1, File, 42, Int64Ty, true, 0); + if (LLVMIsNewDbgInfoFormat(M)) - LLVMDIBuilderInsertDeclareRecordAtEnd( + LLVMDIBuilderInsertDeclareAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar1, FooParamExpression, FooParamLocation, FooEntryBlock); else - LLVMDIBuilderInsertDeclareAtEnd( + LLVMDIBuilderInsertDeclareIntrinsicAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar1, FooParamExpression, FooParamLocation, FooEntryBlock); LLVMMetadataRef FooParamVar2 = @@ -149,11 +150,11 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) { 42, Int64Ty, true, 0); if (LLVMIsNewDbgInfoFormat(M)) - LLVMDIBuilderInsertDeclareRecordAtEnd( + LLVMDIBuilderInsertDeclareAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar2, FooParamExpression, FooParamLocation, FooEntryBlock); else - LLVMDIBuilderInsertDeclareAtEnd( + LLVMDIBuilderInsertDeclareIntrinsicAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar2, FooParamExpression, FooParamLocation, FooEntryBlock); @@ -161,11 +162,11 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) { LLVMDIBuilderCreateParameterVariable(DIB, FunctionMetadata, "c", 1, 3, File, 42, VectorTy, true, 0); if (LLVMIsNewDbgInfoFormat(M)) - LLVMDIBuilderInsertDeclareRecordAtEnd( + LLVMDIBuilderInsertDeclareAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar3, FooParamExpression, FooParamLocation, FooEntryBlock); else - LLVMDIBuilderInsertDeclareAtEnd( + LLVMDIBuilderInsertDeclareIntrinsicAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar3, FooParamExpression, FooParamLocation, FooEntryBlock); -- GitLab From 3054d0dae7a813c493d2bb8e969aa2321145a83b Mon Sep 17 00:00:00 2001 From: Changpeng Fang Date: Fri, 22 Mar 2024 08:51:53 -0700 Subject: [PATCH 273/296] AMDGPU: Rename and add bf16 support for global_load_tr builtins (#86202) Make the name of a clang builtin as close to the mnemonic instruction name as possible. The data type suffix may not be enough to tell what instruction the builtin is going to produce. This patch also add the bf16 support for global_load_tr_b128 builtins. --- clang/include/clang/Basic/BuiltinsAMDGPU.def | 16 ++++---- clang/lib/CodeGen/CGBuiltin.cpp | 34 +++++++++++------ ...uiltins-amdgcn-global-load-tr-gfx11-err.cl | 25 ++++++------ ...ins-amdgcn-global-load-tr-gfx12-w32-err.cl | 11 +++--- ...ins-amdgcn-global-load-tr-gfx12-w64-err.cl | 11 +++--- .../builtins-amdgcn-global-load-tr-w32.cl | 38 +++++++++---------- .../builtins-amdgcn-global-load-tr-w64.cl | 38 +++++++++---------- 7 files changed, 94 insertions(+), 79 deletions(-) diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def index 61ec8b79bf05..4153b316c22b 100644 --- a/clang/include/clang/Basic/BuiltinsAMDGPU.def +++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def @@ -432,13 +432,15 @@ TARGET_BUILTIN(__builtin_amdgcn_s_wakeup_barrier, "vi", "n", "gfx12-insts") TARGET_BUILTIN(__builtin_amdgcn_s_barrier_leave, "b", "n", "gfx12-insts") TARGET_BUILTIN(__builtin_amdgcn_s_get_barrier_state, "Uii", "n", "gfx12-insts") -TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v2i32, "V2iV2i*1", "nc", "gfx12-insts,wavefrontsize32") -TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v8i16, "V8sV8s*1", "nc", "gfx12-insts,wavefrontsize32") -TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v8f16, "V8hV8h*1", "nc", "gfx12-insts,wavefrontsize32") - -TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_i32, "ii*1", "nc", "gfx12-insts,wavefrontsize64") -TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v4i16, "V4sV4s*1", "nc", "gfx12-insts,wavefrontsize64") -TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_v4f16, "V4hV4h*1", "nc", "gfx12-insts,wavefrontsize64") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b64_v2i32, "V2iV2i*1", "nc", "gfx12-insts,wavefrontsize32") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v8i16, "V8sV8s*1", "nc", "gfx12-insts,wavefrontsize32") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v8f16, "V8hV8h*1", "nc", "gfx12-insts,wavefrontsize32") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v8bf16, "V8yV8y*1", "nc", "gfx12-insts,wavefrontsize32") + +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b64_i32, "ii*1", "nc", "gfx12-insts,wavefrontsize64") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v4i16, "V4sV4s*1", "nc", "gfx12-insts,wavefrontsize64") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v4f16, "V4hV4h*1", "nc", "gfx12-insts,wavefrontsize64") +TARGET_BUILTIN(__builtin_amdgcn_global_load_tr_b128_v4bf16, "V4yV4y*1", "nc", "gfx12-insts,wavefrontsize64") //===----------------------------------------------------------------------===// // WMMA builtins. diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index e14e89088282..2eaceeba6177 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -18531,35 +18531,45 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID, llvm::Function *F = CGM.getIntrinsic(IID, {ArgTy}); return Builder.CreateCall(F, {Addr, Val, ZeroI32, ZeroI32, ZeroI1}); } - case AMDGPU::BI__builtin_amdgcn_global_load_tr_i32: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v2i32: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4f16: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4i16: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8f16: - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8i16: { + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16: { llvm::Type *ArgTy; switch (BuiltinID) { - case AMDGPU::BI__builtin_amdgcn_global_load_tr_i32: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32: ArgTy = llvm::Type::getInt32Ty(getLLVMContext()); break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v2i32: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32: ArgTy = llvm::FixedVectorType::get( llvm::Type::getInt32Ty(getLLVMContext()), 2); break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4f16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4bf16: + ArgTy = llvm::FixedVectorType::get( + llvm::Type::getBFloatTy(getLLVMContext()), 4); + break; + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4f16: ArgTy = llvm::FixedVectorType::get( llvm::Type::getHalfTy(getLLVMContext()), 4); break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v4i16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16: ArgTy = llvm::FixedVectorType::get( llvm::Type::getInt16Ty(getLLVMContext()), 4); break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8f16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16: + ArgTy = llvm::FixedVectorType::get( + llvm::Type::getBFloatTy(getLLVMContext()), 8); + break; + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16: ArgTy = llvm::FixedVectorType::get( llvm::Type::getHalfTy(getLLVMContext()), 8); break; - case AMDGPU::BI__builtin_amdgcn_global_load_tr_v8i16: + case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16: ArgTy = llvm::FixedVectorType::get( llvm::Type::getInt16Ty(getLLVMContext()), 8); break; diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl index f7afb7cb97ed..4363769b8645 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx11-err.cl @@ -6,21 +6,22 @@ typedef int v2i __attribute__((ext_vector_type(2))); typedef half v8h __attribute__((ext_vector_type(8))); typedef short v8s __attribute__((ext_vector_type(8))); +typedef __bf16 v8bf16 __attribute__((ext_vector_type(8))); typedef half v4h __attribute__((ext_vector_type(4))); typedef short v4s __attribute__((ext_vector_type(4))); +typedef __bf16 v4bf16 __attribute__((ext_vector_type(4))); - - -void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr, global v8h* v8h_inptr, - global int* int_inptr, global v4s* v4s_inptr, global v4h* v4h_inptr) +void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr, global v8h* v8h_inptr, global v8bf16* v8bf16_inptr, + global int* int_inptr, global v4s* v4s_inptr, global v4h* v4h_inptr, global v4bf16* v4bf16_inptr) { - v2i out_1 = __builtin_amdgcn_global_load_tr_v2i32(v2i_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v2i32' needs target feature gfx12-insts,wavefrontsize32}} - v8s out_2 = __builtin_amdgcn_global_load_tr_v8i16(v8s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v8i16' needs target feature gfx12-insts,wavefrontsize32}} - v8h out_3 = __builtin_amdgcn_global_load_tr_v8f16(v8h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v8f16' needs target feature gfx12-insts,wavefrontsize32}} - - int out_4 = __builtin_amdgcn_global_load_tr_i32(int_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_i32' needs target feature gfx12-insts,wavefrontsize64}} - v4s out_5 = __builtin_amdgcn_global_load_tr_v4i16(v4s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v4i16' needs target feature gfx12-insts,wavefrontsize64}} - v4h out_6 = __builtin_amdgcn_global_load_tr_v4f16(v4h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v4f16' needs target feature gfx12-insts,wavefrontsize64}} + v2i out_1 = __builtin_amdgcn_global_load_tr_b64_v2i32(v2i_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_v2i32' needs target feature gfx12-insts,wavefrontsize32}} + v8s out_2 = __builtin_amdgcn_global_load_tr_b128_v8i16(v8s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8i16' needs target feature gfx12-insts,wavefrontsize32}} + v8h out_3 = __builtin_amdgcn_global_load_tr_b128_v8f16(v8h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8f16' needs target feature gfx12-insts,wavefrontsize32}} + v8bf16 o4 = __builtin_amdgcn_global_load_tr_b128_v8bf16(v8bf16_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8bf16' needs target feature gfx12-insts,wavefrontsize32}} + + int out_5 = __builtin_amdgcn_global_load_tr_b64_i32(int_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_i32' needs target feature gfx12-insts,wavefrontsize64}} + v4s out_6 = __builtin_amdgcn_global_load_tr_b128_v4i16(v4s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4i16' needs target feature gfx12-insts,wavefrontsize64}} + v4h out_7 = __builtin_amdgcn_global_load_tr_b128_v4f16(v4h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4f16' needs target feature gfx12-insts,wavefrontsize64}} + v4bf16 o8 = __builtin_amdgcn_global_load_tr_b128_v4bf16(v4bf16_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4bf16' needs target feature gfx12-insts,wavefrontsize64}} } - diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl index 04ac0a66db7c..208f92fc5d44 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w32-err.cl @@ -5,11 +5,12 @@ typedef half v4h __attribute__((ext_vector_type(4))); typedef short v4s __attribute__((ext_vector_type(4))); +typedef __bf16 v4bf16 __attribute__((ext_vector_type(4))); -void amdgcn_global_load_tr(global int* int_inptr, global v4s* v4s_inptr, global v4h* v4h_inptr) +void amdgcn_global_load_tr(global int* int_inptr, global v4s* v4s_inptr, global v4h* v4h_inptr, global v4bf16* v4bf16_inptr) { - int out_4 = __builtin_amdgcn_global_load_tr_i32(int_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_i32' needs target feature gfx12-insts,wavefrontsize64}} - v4s out_5 = __builtin_amdgcn_global_load_tr_v4i16(v4s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v4i16' needs target feature gfx12-insts,wavefrontsize64}} - v4h out_6 = __builtin_amdgcn_global_load_tr_v4f16(v4h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v4f16' needs target feature gfx12-insts,wavefrontsize64}} + int out_1 = __builtin_amdgcn_global_load_tr_b64_i32(int_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_i32' needs target feature gfx12-insts,wavefrontsize64}} + v4s out_2 = __builtin_amdgcn_global_load_tr_b128_v4i16(v4s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4i16' needs target feature gfx12-insts,wavefrontsize64}} + v4h out_3 = __builtin_amdgcn_global_load_tr_b128_v4f16(v4h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4f16' needs target feature gfx12-insts,wavefrontsize64}} + v4bf16 o4 = __builtin_amdgcn_global_load_tr_b128_v4bf16(v4bf16_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v4bf16' needs target feature gfx12-insts,wavefrontsize64}} } - diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl index 113b54b853a9..199146a9715d 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-gfx12-w64-err.cl @@ -6,11 +6,12 @@ typedef int v2i __attribute__((ext_vector_type(2))); typedef half v8h __attribute__((ext_vector_type(8))); typedef short v8s __attribute__((ext_vector_type(8))); +typedef __bf16 v8bf16 __attribute__((ext_vector_type(8))); -void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr, global v8h* v8h_inptr) +void amdgcn_global_load_tr(global v2i* v2i_inptr, global v8s* v8s_inptr, global v8h* v8h_inptr, global v8bf16* v8bf16_inptr) { - v2i out_1 = __builtin_amdgcn_global_load_tr_v2i32(v2i_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v2i32' needs target feature gfx12-insts,wavefrontsize32}} - v8s out_2 = __builtin_amdgcn_global_load_tr_v8i16(v8s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v8i16' needs target feature gfx12-insts,wavefrontsize32}} - v8h out_3 = __builtin_amdgcn_global_load_tr_v8f16(v8h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_v8f16' needs target feature gfx12-insts,wavefrontsize32}} + v2i out_1 = __builtin_amdgcn_global_load_tr_b64_v2i32(v2i_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b64_v2i32' needs target feature gfx12-insts,wavefrontsize32}} + v8s out_2 = __builtin_amdgcn_global_load_tr_b128_v8i16(v8s_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8i16' needs target feature gfx12-insts,wavefrontsize32}} + v8h out_3 = __builtin_amdgcn_global_load_tr_b128_v8f16(v8h_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8f16' needs target feature gfx12-insts,wavefrontsize32}} + v8bf16 o4 = __builtin_amdgcn_global_load_tr_b128_v8bf16(v8bf16_inptr); // expected-error{{'__builtin_amdgcn_global_load_tr_b128_v8bf16' needs target feature gfx12-insts,wavefrontsize32}} } - diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl index b5fcad68a470..0035b16b902b 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w32.cl @@ -5,44 +5,44 @@ typedef int v2i __attribute__((ext_vector_type(2))); typedef half v8h __attribute__((ext_vector_type(8))); typedef short v8s __attribute__((ext_vector_type(8))); +typedef __bf16 v8bf16 __attribute__((ext_vector_type(8))); -// Wave32 - -// -// amdgcn_global_load_tr -// - -// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_v2i32( +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b64_v2i32( // CHECK-GFX1200-NEXT: entry: // CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.global.load.tr.v2i32(ptr addrspace(1) [[INPTR:%.*]]) // CHECK-GFX1200-NEXT: ret <2 x i32> [[TMP0]] // -v2i test_amdgcn_global_load_tr_v2i32(global v2i* inptr) +v2i test_amdgcn_global_load_tr_b64_v2i32(global v2i* inptr) { - return __builtin_amdgcn_global_load_tr_v2i32(inptr); + return __builtin_amdgcn_global_load_tr_b64_v2i32(inptr); } -// -// amdgcn_global_load_tr -// - -// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_v8i16( +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v8i16( // CHECK-GFX1200-NEXT: entry: // CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <8 x i16> @llvm.amdgcn.global.load.tr.v8i16(ptr addrspace(1) [[INPTR:%.*]]) // CHECK-GFX1200-NEXT: ret <8 x i16> [[TMP0]] // -v8s test_amdgcn_global_load_tr_v8i16(global v8s* inptr) +v8s test_amdgcn_global_load_tr_b128_v8i16(global v8s* inptr) { - return __builtin_amdgcn_global_load_tr_v8i16(inptr); + return __builtin_amdgcn_global_load_tr_b128_v8i16(inptr); } -// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_v8f16( +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v8f16( // CHECK-GFX1200-NEXT: entry: // CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <8 x half> @llvm.amdgcn.global.load.tr.v8f16(ptr addrspace(1) [[INPTR:%.*]]) // CHECK-GFX1200-NEXT: ret <8 x half> [[TMP0]] // -v8h test_amdgcn_global_load_tr_v8f16(global v8h* inptr) +v8h test_amdgcn_global_load_tr_b128_v8f16(global v8h* inptr) { - return __builtin_amdgcn_global_load_tr_v8f16(inptr); + return __builtin_amdgcn_global_load_tr_b128_v8f16(inptr); } +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v8bf16( +// CHECK-GFX1200-NEXT: entry: +// CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <8 x bfloat> @llvm.amdgcn.global.load.tr.v8bf16(ptr addrspace(1) [[INPTR:%.*]]) +// CHECK-GFX1200-NEXT: ret <8 x bfloat> [[TMP0]] +// +v8bf16 test_amdgcn_global_load_tr_b128_v8bf16(global v8bf16* inptr) +{ + return __builtin_amdgcn_global_load_tr_b128_v8bf16(inptr); +} diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl index 9c48ac071b4d..6c025bb5a55a 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-global-load-tr-w64.cl @@ -4,44 +4,44 @@ typedef half v4h __attribute__((ext_vector_type(4))); typedef short v4s __attribute__((ext_vector_type(4))); +typedef __bf16 v4bf16 __attribute__((ext_vector_type(4))); -// Wave64 - -// -// amdgcn_global_load_tr -// - -// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_i32( +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b64_i32( // CHECK-GFX1200-NEXT: entry: // CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.global.load.tr.i32(ptr addrspace(1) [[INPTR:%.*]]) // CHECK-GFX1200-NEXT: ret i32 [[TMP0]] // -int test_amdgcn_global_load_tr_i32(global int* inptr) +int test_amdgcn_global_load_tr_b64_i32(global int* inptr) { - return __builtin_amdgcn_global_load_tr_i32(inptr); + return __builtin_amdgcn_global_load_tr_b64_i32(inptr); } -// -// amdgcn_global_load_tr -// - -// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_v4i16( +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v4i16( // CHECK-GFX1200-NEXT: entry: // CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <4 x i16> @llvm.amdgcn.global.load.tr.v4i16(ptr addrspace(1) [[INPTR:%.*]]) // CHECK-GFX1200-NEXT: ret <4 x i16> [[TMP0]] // -v4s test_amdgcn_global_load_tr_v4i16(global v4s* inptr) +v4s test_amdgcn_global_load_tr_b128_v4i16(global v4s* inptr) { - return __builtin_amdgcn_global_load_tr_v4i16(inptr); + return __builtin_amdgcn_global_load_tr_b128_v4i16(inptr); } -// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_v4f16( +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v4f16( // CHECK-GFX1200-NEXT: entry: // CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <4 x half> @llvm.amdgcn.global.load.tr.v4f16(ptr addrspace(1) [[INPTR:%.*]]) // CHECK-GFX1200-NEXT: ret <4 x half> [[TMP0]] // -v4h test_amdgcn_global_load_tr_v4f16(global v4h* inptr) +v4h test_amdgcn_global_load_tr_b128_v4f16(global v4h* inptr) { - return __builtin_amdgcn_global_load_tr_v4f16(inptr); + return __builtin_amdgcn_global_load_tr_b128_v4f16(inptr); } +// CHECK-GFX1200-LABEL: @test_amdgcn_global_load_tr_b128_v4bf16( +// CHECK-GFX1200-NEXT: entry: +// CHECK-GFX1200-NEXT: [[TMP0:%.*]] = tail call <4 x bfloat> @llvm.amdgcn.global.load.tr.v4bf16(ptr addrspace(1) [[INPTR:%.*]]) +// CHECK-GFX1200-NEXT: ret <4 x bfloat> [[TMP0]] +// +v4bf16 test_amdgcn_global_load_tr_b128_v4bf16(global v4bf16* inptr) +{ + return __builtin_amdgcn_global_load_tr_b128_v4bf16(inptr); +} -- GitLab From 631e54aa1a0b7a79d0dec8dce7ec0f5e506acf6c Mon Sep 17 00:00:00 2001 From: "long.chen" Date: Fri, 22 Mar 2024 23:52:47 +0800 Subject: [PATCH 274/296] [mlir][arith] fix wrong floordivsi fold (#83248) Fixs https://github.com/llvm/llvm-project/issues/83079 --- mlir/lib/Dialect/Arith/IR/ArithOps.cpp | 36 ++------ .../Dialect/Arith/Transforms/ExpandOps.cpp | 58 ++++++------- mlir/test/Dialect/Arith/expand-ops.mlir | 84 ++++++++----------- .../Standard/CPU/test-ceil-floor-pos-neg.mlir | 30 +++++++ mlir/test/Transforms/canonicalize.mlir | 9 ++ 5 files changed, 102 insertions(+), 115 deletions(-) diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp index 9f64a07f31e3..2f32d9a26e77 100644 --- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp +++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp @@ -689,43 +689,17 @@ OpFoldResult arith::FloorDivSIOp::fold(FoldAdaptor adaptor) { return getLhs(); // Don't fold if it would overflow or if it requires a division by zero. - bool overflowOrDiv0 = false; + bool overflowOrDiv = false; auto result = constFoldBinaryOp( adaptor.getOperands(), [&](APInt a, const APInt &b) { - if (overflowOrDiv0 || !b) { - overflowOrDiv0 = true; + if (b.isZero()) { + overflowOrDiv = true; return a; } - if (!a) - return a; - // After this point we know that neither a or b are zero. - unsigned bits = a.getBitWidth(); - APInt zero = APInt::getZero(bits); - bool aGtZero = a.sgt(zero); - bool bGtZero = b.sgt(zero); - if (aGtZero && bGtZero) { - // Both positive, return a / b. - return a.sdiv_ov(b, overflowOrDiv0); - } - if (!aGtZero && !bGtZero) { - // Both negative, return -a / -b. - APInt posA = zero.ssub_ov(a, overflowOrDiv0); - APInt posB = zero.ssub_ov(b, overflowOrDiv0); - return posA.sdiv_ov(posB, overflowOrDiv0); - } - if (!aGtZero && bGtZero) { - // A is negative, b is positive, return - ceil(-a, b). - APInt posA = zero.ssub_ov(a, overflowOrDiv0); - APInt ceil = signedCeilNonnegInputs(posA, b, overflowOrDiv0); - return zero.ssub_ov(ceil, overflowOrDiv0); - } - // A is positive, b is negative, return - ceil(a, -b). - APInt posB = zero.ssub_ov(b, overflowOrDiv0); - APInt ceil = signedCeilNonnegInputs(a, posB, overflowOrDiv0); - return zero.ssub_ov(ceil, overflowOrDiv0); + return a.sfloordiv_ov(b, overflowOrDiv); }); - return overflowOrDiv0 ? Attribute() : result; + return overflowOrDiv ? Attribute() : result; } //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp index 7f246daf99ff..71e14a153cfd 100644 --- a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp +++ b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp @@ -110,9 +110,13 @@ struct CeilDivSIOpConverter : public OpRewritePattern { } }; -/// Expands FloorDivSIOp (n, m) into -/// 1) x = (m<0) ? 1 : -1 -/// 2) return (n*m<0) ? - ((-n+x) / m) -1 : n / m +/// Expands FloorDivSIOp (x, y) into +/// z = x / y +/// if (z * y != x && (x < 0) != (y < 0)) { +/// return z - 1; +/// } else { +/// return z; +/// } struct FloorDivSIOpConverter : public OpRewritePattern { using OpRewritePattern::OpRewritePattern; LogicalResult matchAndRewrite(arith::FloorDivSIOp op, @@ -121,41 +125,29 @@ struct FloorDivSIOpConverter : public OpRewritePattern { Type type = op.getType(); Value a = op.getLhs(); Value b = op.getRhs(); - Value plusOne = createConst(loc, type, 1, rewriter); + + Value quotient = rewriter.create(loc, a, b); + Value product = rewriter.create(loc, quotient, b); + Value notEqualDivisor = rewriter.create( + loc, arith::CmpIPredicate::ne, a, product); Value zero = createConst(loc, type, 0, rewriter); - Value minusOne = createConst(loc, type, -1, rewriter); - // Compute x = (b<0) ? 1 : -1. - Value compare = - rewriter.create(loc, arith::CmpIPredicate::slt, b, zero); - Value x = rewriter.create(loc, compare, plusOne, minusOne); - // Compute negative res: -1 - ((x-a)/b). - Value xMinusA = rewriter.create(loc, x, a); - Value xMinusADivB = rewriter.create(loc, xMinusA, b); - Value negRes = rewriter.create(loc, minusOne, xMinusADivB); - // Compute positive res: a/b. - Value posRes = rewriter.create(loc, a, b); - // Result is (a*b<0) ? negative result : positive result. - // Note, we want to avoid using a*b because of possible overflow. - // The case that matters are a>0, a==0, a<0, b>0 and b<0. We do - // not particuliarly care if a*b<0 is true or false when b is zero - // as this will result in an illegal divide. So `a*b<0` can be reformulated - // as `(a>0 && b<0) || (a>0 && b<0)' or `(a>0 && b<0) || (a>0 && b<=0)'. - // We pick the first expression here. + Value aNeg = rewriter.create(loc, arith::CmpIPredicate::slt, a, zero); - Value aPos = - rewriter.create(loc, arith::CmpIPredicate::sgt, a, zero); Value bNeg = rewriter.create(loc, arith::CmpIPredicate::slt, b, zero); - Value bPos = - rewriter.create(loc, arith::CmpIPredicate::sgt, b, zero); - Value firstTerm = rewriter.create(loc, aNeg, bPos); - Value secondTerm = rewriter.create(loc, aPos, bNeg); - Value compareRes = - rewriter.create(loc, firstTerm, secondTerm); - // Perform substitution and return success. - rewriter.replaceOpWithNewOp(op, compareRes, negRes, - posRes); + + Value signOpposite = rewriter.create( + loc, arith::CmpIPredicate::ne, aNeg, bNeg); + Value cond = + rewriter.create(loc, notEqualDivisor, signOpposite); + + Value minusOne = createConst(loc, type, -1, rewriter); + Value quotientMinusOne = + rewriter.create(loc, quotient, minusOne); + + rewriter.replaceOpWithNewOp(op, cond, quotientMinusOne, + quotient); return success(); } }; diff --git a/mlir/test/Dialect/Arith/expand-ops.mlir b/mlir/test/Dialect/Arith/expand-ops.mlir index 91f652e5a270..6bed93e4c969 100644 --- a/mlir/test/Dialect/Arith/expand-ops.mlir +++ b/mlir/test/Dialect/Arith/expand-ops.mlir @@ -66,23 +66,17 @@ func.func @ceildivi_index(%arg0: index, %arg1: index) -> (index) { func.func @floordivi(%arg0: i32, %arg1: i32) -> (i32) { %res = arith.floordivsi %arg0, %arg1 : i32 return %res : i32 -// CHECK: [[ONE:%.+]] = arith.constant 1 : i32 -// CHECK: [[ZERO:%.+]] = arith.constant 0 : i32 -// CHECK: [[MIN1:%.+]] = arith.constant -1 : i32 -// CHECK: [[CMP1:%.+]] = arith.cmpi slt, [[ARG1]], [[ZERO]] : i32 -// CHECK: [[X:%.+]] = arith.select [[CMP1]], [[ONE]], [[MIN1]] : i32 -// CHECK: [[TRUE1:%.+]] = arith.subi [[X]], [[ARG0]] : i32 -// CHECK: [[TRUE2:%.+]] = arith.divsi [[TRUE1]], [[ARG1]] : i32 -// CHECK: [[TRUE3:%.+]] = arith.subi [[MIN1]], [[TRUE2]] : i32 -// CHECK: [[FALSE:%.+]] = arith.divsi [[ARG0]], [[ARG1]] : i32 -// CHECK: [[NNEG:%.+]] = arith.cmpi slt, [[ARG0]], [[ZERO]] : i32 -// CHECK: [[NPOS:%.+]] = arith.cmpi sgt, [[ARG0]], [[ZERO]] : i32 -// CHECK: [[MNEG:%.+]] = arith.cmpi slt, [[ARG1]], [[ZERO]] : i32 -// CHECK: [[MPOS:%.+]] = arith.cmpi sgt, [[ARG1]], [[ZERO]] : i32 -// CHECK: [[TERM1:%.+]] = arith.andi [[NNEG]], [[MPOS]] : i1 -// CHECK: [[TERM2:%.+]] = arith.andi [[NPOS]], [[MNEG]] : i1 -// CHECK: [[CMP2:%.+]] = arith.ori [[TERM1]], [[TERM2]] : i1 -// CHECK: [[RES:%.+]] = arith.select [[CMP2]], [[TRUE3]], [[FALSE]] : i32 +// CHECK: %[[QUOTIENT:.*]] = arith.divsi %arg0, %arg1 : i32 +// CHECK: %[[PRODUCT:.*]] = arith.muli %[[QUOTIENT]], %arg1 : i32 +// CHECK: %[[NOT_EQ_PRODUCT:.*]] = arith.cmpi ne, %arg0, %[[PRODUCT]] : i32 +// CHECK-DAG: %[[ZERO:.*]] = arith.constant 0 : i32 +// CHECK: %[[NEG_DIVISOR:.*]] = arith.cmpi slt, %arg0, %[[ZERO]] : i32 +// CHECK: %[[NEG_DIVIDEND:.*]] = arith.cmpi slt, %arg1, %[[ZERO]] : i32 +// CHECK: %[[OPPOSITE_SIGN:.*]] = arith.cmpi ne, %[[NEG_DIVISOR]], %[[NEG_DIVIDEND]] : i1 +// CHECK: %[[CONDITION:.*]] = arith.andi %[[NOT_EQ_PRODUCT]], %[[OPPOSITE_SIGN]] : i1 +// CHECK-DAG: %[[NEG_ONE:.*]] = arith.constant -1 : i32 +// CHECK: %[[MINUS_ONE:.*]] = arith.addi %[[QUOTIENT]], %[[NEG_ONE]] : i32 +// CHECK: %[[RES:.*]] = arith.select %[[CONDITION]], %[[MINUS_ONE]], %[[QUOTIENT]] : i32 } // ----- @@ -93,23 +87,17 @@ func.func @floordivi(%arg0: i32, %arg1: i32) -> (i32) { func.func @floordivi_index(%arg0: index, %arg1: index) -> (index) { %res = arith.floordivsi %arg0, %arg1 : index return %res : index -// CHECK: [[ONE:%.+]] = arith.constant 1 : index -// CHECK: [[ZERO:%.+]] = arith.constant 0 : index -// CHECK: [[MIN1:%.+]] = arith.constant -1 : index -// CHECK: [[CMP1:%.+]] = arith.cmpi slt, [[ARG1]], [[ZERO]] : index -// CHECK: [[X:%.+]] = arith.select [[CMP1]], [[ONE]], [[MIN1]] : index -// CHECK: [[TRUE1:%.+]] = arith.subi [[X]], [[ARG0]] : index -// CHECK: [[TRUE2:%.+]] = arith.divsi [[TRUE1]], [[ARG1]] : index -// CHECK: [[TRUE3:%.+]] = arith.subi [[MIN1]], [[TRUE2]] : index -// CHECK: [[FALSE:%.+]] = arith.divsi [[ARG0]], [[ARG1]] : index -// CHECK: [[NNEG:%.+]] = arith.cmpi slt, [[ARG0]], [[ZERO]] : index -// CHECK: [[NPOS:%.+]] = arith.cmpi sgt, [[ARG0]], [[ZERO]] : index -// CHECK: [[MNEG:%.+]] = arith.cmpi slt, [[ARG1]], [[ZERO]] : index -// CHECK: [[MPOS:%.+]] = arith.cmpi sgt, [[ARG1]], [[ZERO]] : index -// CHECK: [[TERM1:%.+]] = arith.andi [[NNEG]], [[MPOS]] : i1 -// CHECK: [[TERM2:%.+]] = arith.andi [[NPOS]], [[MNEG]] : i1 -// CHECK: [[CMP2:%.+]] = arith.ori [[TERM1]], [[TERM2]] : i1 -// CHECK: [[RES:%.+]] = arith.select [[CMP2]], [[TRUE3]], [[FALSE]] : index +// CHECK: %[[QUOTIENT:.*]] = arith.divsi %arg0, %arg1 : index +// CHECK: %[[PRODUCT:.*]] = arith.muli %[[QUOTIENT]], %arg1 : index +// CHECK: %[[NOT_EQ_PRODUCT:.*]] = arith.cmpi ne, %arg0, %[[PRODUCT]] : index +// CHECK-DAG: %[[ZERO:.*]] = arith.constant 0 : index +// CHECK: %[[NEG_DIVISOR:.*]] = arith.cmpi slt, %arg0, %[[ZERO]] : index +// CHECK: %[[NEG_DIVIDEND:.*]] = arith.cmpi slt, %arg1, %[[ZERO]] : index +// CHECK: %[[OPPOSITE_SIGN:.*]] = arith.cmpi ne, %[[NEG_DIVISOR]], %[[NEG_DIVIDEND]] : i1 +// CHECK: %[[CONDITION:.*]] = arith.andi %[[NOT_EQ_PRODUCT]], %[[OPPOSITE_SIGN]] : i1 +// CHECK: %[[NEG_ONE:.*]] = arith.constant -1 : index +// CHECK-DAG: %[[MINUS_ONE:.*]] = arith.addi %[[QUOTIENT]], %[[NEG_ONE]] : index +// CHECK: %[[RES:.*]] = arith.select %[[CONDITION]], %[[MINUS_ONE]], %[[QUOTIENT]] : index } // ----- @@ -121,23 +109,17 @@ func.func @floordivi_index(%arg0: index, %arg1: index) -> (index) { func.func @floordivi_vec(%arg0: vector<4xi32>, %arg1: vector<4xi32>) -> (vector<4xi32>) { %res = arith.floordivsi %arg0, %arg1 : vector<4xi32> return %res : vector<4xi32> -// CHECK: %[[VAL_2:.*]] = arith.constant dense<1> : vector<4xi32> -// CHECK: %[[VAL_3:.*]] = arith.constant dense<0> : vector<4xi32> -// CHECK: %[[VAL_4:.*]] = arith.constant dense<-1> : vector<4xi32> -// CHECK: %[[VAL_5:.*]] = arith.cmpi slt, %[[VAL_1]], %[[VAL_3]] : vector<4xi32> -// CHECK: %[[VAL_6:.*]] = arith.select %[[VAL_5]], %[[VAL_2]], %[[VAL_4]] : vector<4xi1>, vector<4xi32> -// CHECK: %[[VAL_7:.*]] = arith.subi %[[VAL_6]], %[[VAL_0]] : vector<4xi32> -// CHECK: %[[VAL_8:.*]] = arith.divsi %[[VAL_7]], %[[VAL_1]] : vector<4xi32> -// CHECK: %[[VAL_9:.*]] = arith.subi %[[VAL_4]], %[[VAL_8]] : vector<4xi32> -// CHECK: %[[VAL_10:.*]] = arith.divsi %[[VAL_0]], %[[VAL_1]] : vector<4xi32> -// CHECK: %[[VAL_11:.*]] = arith.cmpi slt, %[[VAL_0]], %[[VAL_3]] : vector<4xi32> -// CHECK: %[[VAL_12:.*]] = arith.cmpi sgt, %[[VAL_0]], %[[VAL_3]] : vector<4xi32> -// CHECK: %[[VAL_13:.*]] = arith.cmpi slt, %[[VAL_1]], %[[VAL_3]] : vector<4xi32> -// CHECK: %[[VAL_14:.*]] = arith.cmpi sgt, %[[VAL_1]], %[[VAL_3]] : vector<4xi32> -// CHECK: %[[VAL_15:.*]] = arith.andi %[[VAL_11]], %[[VAL_14]] : vector<4xi1> -// CHECK: %[[VAL_16:.*]] = arith.andi %[[VAL_12]], %[[VAL_13]] : vector<4xi1> -// CHECK: %[[VAL_17:.*]] = arith.ori %[[VAL_15]], %[[VAL_16]] : vector<4xi1> -// CHECK: %[[VAL_18:.*]] = arith.select %[[VAL_17]], %[[VAL_9]], %[[VAL_10]] : vector<4xi1>, vector<4xi32> +// CHECK: %[[QUOTIENT:.*]] = arith.divsi %arg0, %arg1 : vector<4xi32> +// CHECK: %[[PRODUCT:.*]] = arith.muli %[[QUOTIENT]], %arg1 : vector<4xi32> +// CHECK: %[[NOT_EQ_PRODUCT:.*]] = arith.cmpi ne, %arg0, %[[PRODUCT]] : vector<4xi32> +// CHECK-DAG: %[[ZERO:.*]] = arith.constant dense<0> : vector<4xi32> +// CHECK: %[[NEG_DIVISOR:.*]] = arith.cmpi slt, %arg0, %[[ZERO]] : vector<4xi32> +// CHECK: %[[NEG_DIVIDEND:.*]] = arith.cmpi slt, %arg1, %[[ZERO]] : vector<4xi32> +// CHECK: %[[OPPOSITE_SIGN:.*]] = arith.cmpi ne, %[[NEG_DIVISOR]], %[[NEG_DIVIDEND]] : vector<4xi1> +// CHECK: %[[CONDITION:.*]] = arith.andi %[[NOT_EQ_PRODUCT]], %[[OPPOSITE_SIGN]] : vector<4xi1> +// CHECK-DAG: %[[NEG_ONE:.*]] = arith.constant dense<-1> : vector<4xi32> +// CHECK: %[[MINUS_ONE:.*]] = arith.addi %[[QUOTIENT]], %[[NEG_ONE]] : vector<4xi32> +// CHECK: %[[RES:.*]] = arith.select %[[CONDITION]], %[[MINUS_ONE]], %[[QUOTIENT]] : vector<4xi1>, vector<4xi32> } // ----- diff --git a/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir b/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir index 39fbb67512c6..a7013eacc984 100644 --- a/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir +++ b/mlir/test/Integration/Dialect/Standard/CPU/test-ceil-floor-pos-neg.mlir @@ -2,6 +2,10 @@ // RUN: mlir-cpu-runner -e entry -entry-point-result=void \ // RUN: -shared-libs=%mlir_c_runner_utils | \ // RUN: FileCheck %s +// RUN: mlir-opt %s -pass-pipeline="builtin.module(func.func(convert-vector-to-scf,lower-affine,convert-scf-to-cf,memref-expand,arith-expand),convert-vector-to-llvm,finalize-memref-to-llvm,convert-func-to-llvm,reconcile-unrealized-casts)" | \ +// RUN: mlir-cpu-runner -e main -entry-point-result=void \ +// RUN: -shared-libs=%mlir_c_runner_utils | \ +// RUN: FileCheck %s --check-prefix=SCHECK func.func @transfer_read_2d(%A : memref<40xi32>, %base1: index) { %i42 = arith.constant -42: i32 @@ -101,3 +105,29 @@ func.func @entry() { // CHECK:( 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -2, -2, -2, -2, -2, -2, -2, -2, -2 ) // CHECK:( 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4 ) // CHECK:( 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ) + +// ----- + +func.func @non_inline_function() -> (i64, i64, i64, i64, i64, i64) { + %MIN_INT_MINUS_ONE = arith.constant -9223372036854775807 : i64 + %NEG_ONE = arith.constant -1 : i64 + %MIN_INT = arith.constant -9223372036854775808 : i64 + %ONE = arith.constant 1 : i64 + %MAX_INT = arith.constant 9223372036854775807 : i64 + return %MIN_INT_MINUS_ONE, %NEG_ONE, %MIN_INT, %ONE, %MAX_INT, %NEG_ONE : i64, i64, i64, i64, i64, i64 +} + +func.func @main() { + %0:6 = call @non_inline_function() : () -> (i64, i64, i64, i64, i64, i64) + %1 = arith.floordivsi %0#0, %0#1 : i64 + %2 = arith.floordivsi %0#2, %0#3 : i64 + %3 = arith.floordivsi %0#4, %0#5 : i64 + vector.print %1 : i64 + vector.print %2 : i64 + vector.print %3 : i64 + return +} + +// SCHECK: 9223372036854775807 +// SCHECK: -9223372036854775808 +// SCHECK: -9223372036854775807 diff --git a/mlir/test/Transforms/canonicalize.mlir b/mlir/test/Transforms/canonicalize.mlir index 2cf86b50d432..d2c2c12d3238 100644 --- a/mlir/test/Transforms/canonicalize.mlir +++ b/mlir/test/Transforms/canonicalize.mlir @@ -989,6 +989,15 @@ func.func @tensor_arith.floordivsi_by_one(%arg0: tensor<4x5xi32>) -> tensor<4x5x return %res : tensor<4x5xi32> } +// CHECK-LABEL: func @arith.floordivsi_by_one_overflow +func.func @arith.floordivsi_by_one_overflow() -> i64 { + %neg_one = arith.constant -1 : i64 + %min_int = arith.constant -9223372036854775808 : i64 + // CHECK: arith.floordivsi + %poision = arith.floordivsi %min_int, %neg_one : i64 + return %poision : i64 +} + // ----- // CHECK-LABEL: func @arith.ceildivsi_by_one -- GitLab From 01b1b0c1f728e2c2639edc654424f50830295989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= Date: Fri, 22 Mar 2024 15:53:04 +0000 Subject: [PATCH 275/296] [mlir][SVE] Add e2e for 1D depthwise WC convolution (#85225) Follow-up for https://github.com/llvm/llvm-project/pull/81625 --- .../Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir new file mode 100644 index 000000000000..57d69383c2de --- /dev/null +++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir @@ -0,0 +1,60 @@ +// DEFINE: %{compile} = mlir-opt %s \ +// DEFINE: -transform-interpreter -test-transform-dialect-erase-schedule \ +// DEFINE: -one-shot-bufferize="bufferize-function-boundaries" -lower-vector-mask -cse -canonicalize -convert-vector-to-scf -arm-sve-legalize-vector-storage \ +// DEFINE: -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t +// DEFINE: %{entry_point} = conv +// DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void --march=aarch64 --mattr="+sve"\ +// DEFINE: -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils + +// RUN: %{compile} | %{run} | FileCheck %s + +func.func @conv() { + // Define input/output tensors + %input_init = tensor.empty() : tensor<1x8x6xi32> + %output_init = tensor.empty() : tensor<1x7x6xi32> + + %five = arith.constant 5 : i32 + %zero = arith.constant 0 : i32 + %input = linalg.fill ins(%five : i32) outs(%input_init : tensor<1x8x6xi32>) -> tensor<1x8x6xi32> + %output = linalg.fill ins(%zero : i32) outs(%output_init : tensor<1x7x6xi32>) -> tensor<1x7x6xi32> + + // Define the filter tensor + %filter = arith.constant dense<[ + [ 1, 2, 3, 4, 5, 6], + [ 11, 12, 13, 14, 15, 16] + ]> : tensor<2x6xi32> + + // static sizes -> dynamic sizes + %input_dyn = tensor.cast %input_init : tensor<1x8x6xi32> to tensor<1x8x?xi32> + %output_dyn = tensor.cast %output : tensor<1x7x6xi32> to tensor<1x7x?xi32> + %filter_dyn = tensor.cast %filter : tensor<2x6xi32> to tensor<2x?xi32> + + // Run the convolution + %res = linalg.depthwise_conv_1d_nwc_wc + ins(%input_dyn, %filter_dyn : tensor<1x8x?xi32>, tensor<2x?xi32>) + outs(%output_dyn : tensor<1x7x?xi32>) -> tensor<1x7x?xi32> + + // Print the results + // CHECK: SVE: START OF TEST OUTPUT + vector.print str "SVE: START OF TEST OUTPUT\n" + + // CHECK-NEXT: Unranked Memref base@ = {{.*}} rank = 3 offset = 0 sizes = [1, 7, 6] strides = [42, 6, 1] data = + // CHECK-COUNT-7: [60, 70, 80, 90, 100, 110] + %xf = tensor.cast %res : tensor<1x7x?xi32> to tensor<*xi32> + call @printMemrefI32(%xf) : (tensor<*xi32>) -> () + + // CHECK-NEXT: SVE: END OF TEST OUTPUT + vector.print str "SVE: END OF TEST OUTPUT\n" + + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["linalg.depthwise_conv_1d_nwc_wc"]} in %arg0 : (!transform.any_op) -> !transform.any_op + transform.structured.vectorize %0 vector_sizes [1, 7, [8], 2] : !transform.any_op + transform.yield + } +} + +func.func private @printMemrefI32(%ptr : tensor<*xi32>) attributes { llvm.emit_c_interface } -- GitLab From ab8ace3bfd5165a8532f710f9c2d8dd40c3fac39 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 22 Mar 2024 09:04:50 -0700 Subject: [PATCH 276/296] [bazel] Update to 7.x (#86297) --- utils/bazel/.bazelrc | 5 +++++ utils/bazel/.bazelversion | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/utils/bazel/.bazelrc b/utils/bazel/.bazelrc index c06e9b341626..46894decc7ad 100644 --- a/utils/bazel/.bazelrc +++ b/utils/bazel/.bazelrc @@ -6,6 +6,11 @@ # Common flags that apply to all configurations. # Use sparingly for things common to all compilers and platforms. ############################################################################### + +# Flip off to disable MODULE.bazel until we're ready. +# https://github.com/llvm/llvm-project/issues/55924 +common --enable_bzlmod=false + # Prevent invalid caching if input files are modified during a build. build --experimental_guard_against_concurrent_changes diff --git a/utils/bazel/.bazelversion b/utils/bazel/.bazelversion index 5e3254243a3b..21c8c7b46b89 100644 --- a/utils/bazel/.bazelversion +++ b/utils/bazel/.bazelversion @@ -1 +1 @@ -6.1.2 +7.1.1 -- GitLab From 26857582e5ee7980a71133ef8f8f579bcd90bdc8 Mon Sep 17 00:00:00 2001 From: Orlando Cazalet-Hyams Date: Fri, 22 Mar 2024 16:21:50 +0000 Subject: [PATCH 277/296] Revert "[RemoveDIs] Update DIBuilder C API with DbgRecord functions [2/2] (#85657)" This reverts commit 2091c74796b1dac68e622284c63a870b88b7554f. Builtbot failure: https://lab.llvm.org/buildbot/#/builders/16/builds/63080 --- llvm/docs/RemoveDIsDebugInfo.md | 11 +-- llvm/include/llvm-c/DebugInfo.h | 60 +++++--------- llvm/lib/IR/DebugInfo.cpp | 121 ++++++++++------------------- llvm/tools/llvm-c-test/debuginfo.c | 13 ++-- 4 files changed, 68 insertions(+), 137 deletions(-) diff --git a/llvm/docs/RemoveDIsDebugInfo.md b/llvm/docs/RemoveDIsDebugInfo.md index 9e50a2a604aa..a2f1e173d9d9 100644 --- a/llvm/docs/RemoveDIsDebugInfo.md +++ b/llvm/docs/RemoveDIsDebugInfo.md @@ -40,22 +40,15 @@ New functions (all to be deprecated) LLVMIsNewDbgInfoFormat # Returns true if the module is in the new non-instruction mode. LLVMSetIsNewDbgInfoFormat # Convert to the requested debug info format. -LLVMDIBuilderInsertDeclareIntrinsicBefore # Insert a debug intrinsic (old debug info format). +LLVMDIBuilderInsertDeclareIntrinsicBefore # Insert a debug intrinsic (old debug info format). LLVMDIBuilderInsertDeclareIntrinsicAtEnd # Same as above. LLVMDIBuilderInsertDbgValueIntrinsicBefore # Same as above. LLVMDIBuilderInsertDbgValueIntrinsicAtEnd # Same as above. -LLVMDIBuilderInsertDeclareRecordBefore # Insert a debug record (new debug info format). +LLVMDIBuilderInsertDeclareRecordBefore # Insert a debug record (new debug info format). LLVMDIBuilderInsertDeclareRecordAtEnd # Same as above. LLVMDIBuilderInsertDbgValueRecordBefore # Same as above. LLVMDIBuilderInsertDbgValueRecordAtEnd # Same as above. - -Existing functions (behaviour change) -------------------------------------- -LLVMDIBuilderInsertDeclareBefore # Insert a debug record (new debug info format) instead of a debug intrinsic (old debug info format). -LLVMDIBuilderInsertDeclareAtEnd # Same as above. -LLVMDIBuilderInsertDbgValueBefore # Same as above. -LLVMDIBuilderInsertDbgValueAtEnd # Same as above. ``` # Anything else? diff --git a/llvm/include/llvm-c/DebugInfo.h b/llvm/include/llvm-c/DebugInfo.h index dab1d697761b..b23ff63c862f 100644 --- a/llvm/include/llvm-c/DebugInfo.h +++ b/llvm/include/llvm-c/DebugInfo.h @@ -1249,12 +1249,7 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( LLVMMetadataRef Decl, uint32_t AlignInBits); /* - * Insert a new Declare DbgRecord before the given instruction. - * - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). - * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. - * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes - * + * Insert a new llvm.dbg.declare intrinsic call before the given instruction. * \param Builder The DIBuilder. * \param Storage The storage of the variable to declare. * \param VarInfo The variable's debug info descriptor. @@ -1262,13 +1257,13 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( * \param DebugLoc Debug info location. * \param Instr Instruction acting as a location for the new intrinsic. */ -LLVMDbgRecordRef +LLVMValueRef LLVMDIBuilderInsertDeclareBefore(LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). + * Only use in "old debug mode" (LLVMIsNewDbgFormat() is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.declare intrinsic call before the given instruction. @@ -1284,7 +1279,7 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Only use in "new debug mode" (LLVMIsNewDbgFormat() is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a Declare DbgRecord before the given instruction. @@ -1300,14 +1295,9 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** - * Insert a new Declare DbgRecord at the end of the given basic block. If the - * basic block has a terminator instruction, the intrinsic is inserted before - * that terminator instruction. - * - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). - * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. - * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes - * + * Insert a new llvm.dbg.declare intrinsic call at the end of the given basic + * block. If the basic block has a terminator instruction, the intrinsic is + * inserted before that terminator instruction. * \param Builder The DIBuilder. * \param Storage The storage of the variable to declare. * \param VarInfo The variable's debug info descriptor. @@ -1315,12 +1305,12 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore( * \param DebugLoc Debug info location. * \param Block Basic block acting as a location for the new intrinsic. */ -LLVMDbgRecordRef LLVMDIBuilderInsertDeclareAtEnd( +LLVMValueRef LLVMDIBuilderInsertDeclareAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). + * Only use in "old debug mode" (LLVMIsNewDbgFormat() is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.declare intrinsic call at the end of the given basic @@ -1338,7 +1328,7 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicAtEnd( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Only use in "new debug mode" (LLVMIsNewDbgFormat() is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a Declare DbgRecord at the end of the given basic block. If the basic @@ -1356,12 +1346,7 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** - * Insert a new Value DbgRecord before the given instruction. - * - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). - * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. - * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes - * + * Insert a new llvm.dbg.value intrinsic call before the given instruction. * \param Builder The DIBuilder. * \param Val The value of the variable. * \param VarInfo The variable's debug info descriptor. @@ -1369,13 +1354,13 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd( * \param DebugLoc Debug info location. * \param Instr Instruction acting as a location for the new intrinsic. */ -LLVMDbgRecordRef +LLVMValueRef LLVMDIBuilderInsertDbgValueBefore(LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). + * Only use in "old debug mode" (Module::IsNewDbgInfoFormat is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call before the given instruction. @@ -1391,7 +1376,7 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** * Soon to be deprecated. - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Only use in "new debug mode" (Module::IsNewDbgInfoFormat is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call before the given instruction. @@ -1407,14 +1392,9 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr); /** - * Insert a new Value DbgRecord at the end of the given basic block. If the - * basic block has a terminator instruction, the intrinsic is inserted before - * that terminator instruction. - * - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). - * Use LLVMSetIsNewDbgInfoFormat(LLVMBool) to convert between formats. - * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes - * + * Insert a new llvm.dbg.value intrinsic call at the end of the given basic + * block. If the basic block has a terminator instruction, the intrinsic is + * inserted before that terminator instruction. * \param Builder The DIBuilder. * \param Val The value of the variable. * \param VarInfo The variable's debug info descriptor. @@ -1422,12 +1402,12 @@ LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore( * \param DebugLoc Debug info location. * \param Block Basic block acting as a location for the new intrinsic. */ -LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueAtEnd( +LLVMValueRef LLVMDIBuilderInsertDbgValueAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "old debug mode" (LLVMIsNewDbgInfoFormat() is false). + * Only use in "old debug mode" (Module::IsNewDbgInfoFormat is false). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call at the end of the given basic @@ -1445,7 +1425,7 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicAtEnd( LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block); /** * Soon to be deprecated. - * Only use in "new debug mode" (LLVMIsNewDbgInfoFormat() is true). + * Only use in "new debug mode" (Module::IsNewDbgInfoFormat is true). * See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes * * Insert a new llvm.dbg.value intrinsic call at the end of the given basic diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp index 4206162d1768..09bce9df1f33 100644 --- a/llvm/lib/IR/DebugInfo.cpp +++ b/llvm/lib/IR/DebugInfo.cpp @@ -1665,12 +1665,12 @@ LLVMMetadataRef LLVMDIBuilderCreateTempGlobalVariableFwdDecl( unwrapDI(Decl), nullptr, AlignInBits)); } -LLVMDbgRecordRef +LLVMValueRef LLVMDIBuilderInsertDeclareBefore(LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMValueRef Instr) { - return LLVMDIBuilderInsertDeclareRecordBefore(Builder, Storage, VarInfo, Expr, - DL, Instr); + return LLVMDIBuilderInsertDeclareIntrinsicBefore(Builder, Storage, VarInfo, + Expr, DL, Instr); } LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicBefore( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, @@ -1679,38 +1679,27 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicBefore( unwrap(Storage), unwrap(VarInfo), unwrap(Expr), unwrap(DL), unwrap(Instr)); - // This assert will fail if the module is in the new debug info format. - // This function should only be called if the module is in the old - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Function unexpectedly in new debug info format"); + "Inserted a DbgRecord into function using old debug info mode"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordBefore( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMValueRef Instr) { - DbgInstPtr DbgInst = unwrap(Builder)->insertDeclare( - unwrap(Storage), unwrap(VarInfo), - unwrap(Expr), unwrap(DL), - unwrap(Instr)); - // This assert will fail if the module is in the old debug info format. - // This function should only be called if the module is in the new - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. - assert(isa(DbgInst) && - "Function unexpectedly in old debug info format"); - return wrap(cast(DbgInst)); + return wrap( + unwrap(Builder) + ->insertDeclare(unwrap(Storage), unwrap(VarInfo), + unwrap(Expr), unwrap(DL), + unwrap(Instr)) + .get()); } -LLVMDbgRecordRef +LLVMValueRef LLVMDIBuilderInsertDeclareAtEnd(LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMBasicBlockRef Block) { - return LLVMDIBuilderInsertDeclareRecordAtEnd(Builder, Storage, VarInfo, Expr, - DL, Block); + return LLVMDIBuilderInsertDeclareIntrinsicAtEnd(Builder, Storage, VarInfo, + Expr, DL, Block); } LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, @@ -1718,36 +1707,26 @@ LLVMValueRef LLVMDIBuilderInsertDeclareIntrinsicAtEnd( DbgInstPtr DbgInst = unwrap(Builder)->insertDeclare( unwrap(Storage), unwrap(VarInfo), unwrap(Expr), unwrap(DL), unwrap(Block)); - // This assert will fail if the module is in the new debug info format. - // This function should only be called if the module is in the old - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Function unexpectedly in new debug info format"); + "Inserted a DbgRecord into function using old debug info mode"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDeclareRecordAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Storage, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DL, LLVMBasicBlockRef Block) { - DbgInstPtr DbgInst = unwrap(Builder)->insertDeclare( - unwrap(Storage), unwrap(VarInfo), - unwrap(Expr), unwrap(DL), unwrap(Block)); - // This assert will fail if the module is in the old debug info format. - // This function should only be called if the module is in the new - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. - assert(isa(DbgInst) && - "Function unexpectedly in old debug info format"); - return wrap(cast(DbgInst)); + return wrap(unwrap(Builder) + ->insertDeclare(unwrap(Storage), + unwrap(VarInfo), + unwrap(Expr), + unwrap(DL), unwrap(Block)) + .get()); } -LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueBefore( +LLVMValueRef LLVMDIBuilderInsertDbgValueBefore( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr) { - return LLVMDIBuilderInsertDbgValueRecordBefore(Builder, Val, VarInfo, Expr, - DebugLoc, Instr); + return LLVMDIBuilderInsertDbgValueIntrinsicBefore(Builder, Val, VarInfo, Expr, + DebugLoc, Instr); } LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicBefore( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, @@ -1755,36 +1734,26 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicBefore( DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( unwrap(Val), unwrap(VarInfo), unwrap(Expr), unwrap(DebugLoc), unwrap(Instr)); - // This assert will fail if the module is in the new debug info format. - // This function should only be called if the module is in the old - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Function unexpectedly in new debug info format"); + "Inserted a DbgRecord into function using old debug info mode"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordBefore( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMValueRef Instr) { - DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( - unwrap(Val), unwrap(VarInfo), unwrap(Expr), - unwrap(DebugLoc), unwrap(Instr)); - // This assert will fail if the module is in the old debug info format. - // This function should only be called if the module is in the new - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. - assert(isa(DbgInst) && - "Function unexpectedly in old debug info format"); - return wrap(cast(DbgInst)); + return wrap(unwrap(Builder) + ->insertDbgValueIntrinsic( + unwrap(Val), unwrap(VarInfo), + unwrap(Expr), unwrap(DebugLoc), + unwrap(Instr)) + .get()); } -LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueAtEnd( +LLVMValueRef LLVMDIBuilderInsertDbgValueAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block) { - return LLVMDIBuilderInsertDbgValueRecordAtEnd(Builder, Val, VarInfo, Expr, - DebugLoc, Block); + return LLVMDIBuilderInsertDbgValueIntrinsicAtEnd(Builder, Val, VarInfo, Expr, + DebugLoc, Block); } LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, @@ -1792,29 +1761,19 @@ LLVMValueRef LLVMDIBuilderInsertDbgValueIntrinsicAtEnd( DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( unwrap(Val), unwrap(VarInfo), unwrap(Expr), unwrap(DebugLoc), unwrap(Block)); - // This assert will fail if the module is in the new debug info format. - // This function should only be called if the module is in the old - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. assert(isa(DbgInst) && - "Function unexpectedly in new debug info format"); + "Inserted a DbgRecord into function using old debug info mode"); return wrap(cast(DbgInst)); } LLVMDbgRecordRef LLVMDIBuilderInsertDbgValueRecordAtEnd( LLVMDIBuilderRef Builder, LLVMValueRef Val, LLVMMetadataRef VarInfo, LLVMMetadataRef Expr, LLVMMetadataRef DebugLoc, LLVMBasicBlockRef Block) { - DbgInstPtr DbgInst = unwrap(Builder)->insertDbgValueIntrinsic( - unwrap(Val), unwrap(VarInfo), unwrap(Expr), - unwrap(DebugLoc), unwrap(Block)); - // This assert will fail if the module is in the old debug info format. - // This function should only be called if the module is in the new - // debug info format. - // See https://llvm.org/docs/RemoveDIsDebugInfo.html#c-api-changes, - // LLVMIsNewDbgInfoFormat, and LLVMSetIsNewDbgInfoFormat for more info. - assert(isa(DbgInst) && - "Function unexpectedly in old debug info format"); - return wrap(cast(DbgInst)); + return wrap(unwrap(Builder) + ->insertDbgValueIntrinsic( + unwrap(Val), unwrap(VarInfo), + unwrap(Expr), unwrap(DebugLoc), + unwrap(Block)) + .get()); } LLVMMetadataRef LLVMDIBuilderCreateAutoVariable( diff --git a/llvm/tools/llvm-c-test/debuginfo.c b/llvm/tools/llvm-c-test/debuginfo.c index 9b5c37b05d90..78ccaf12a380 100644 --- a/llvm/tools/llvm-c-test/debuginfo.c +++ b/llvm/tools/llvm-c-test/debuginfo.c @@ -136,13 +136,12 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) { LLVMMetadataRef FooParamVar1 = LLVMDIBuilderCreateParameterVariable(DIB, FunctionMetadata, "a", 1, 1, File, 42, Int64Ty, true, 0); - if (LLVMIsNewDbgInfoFormat(M)) - LLVMDIBuilderInsertDeclareAtEnd( + LLVMDIBuilderInsertDeclareRecordAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar1, FooParamExpression, FooParamLocation, FooEntryBlock); else - LLVMDIBuilderInsertDeclareIntrinsicAtEnd( + LLVMDIBuilderInsertDeclareAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar1, FooParamExpression, FooParamLocation, FooEntryBlock); LLVMMetadataRef FooParamVar2 = @@ -150,11 +149,11 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) { 42, Int64Ty, true, 0); if (LLVMIsNewDbgInfoFormat(M)) - LLVMDIBuilderInsertDeclareAtEnd( + LLVMDIBuilderInsertDeclareRecordAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar2, FooParamExpression, FooParamLocation, FooEntryBlock); else - LLVMDIBuilderInsertDeclareIntrinsicAtEnd( + LLVMDIBuilderInsertDeclareAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar2, FooParamExpression, FooParamLocation, FooEntryBlock); @@ -162,11 +161,11 @@ int llvm_test_dibuilder(bool NewDebugInfoFormat) { LLVMDIBuilderCreateParameterVariable(DIB, FunctionMetadata, "c", 1, 3, File, 42, VectorTy, true, 0); if (LLVMIsNewDbgInfoFormat(M)) - LLVMDIBuilderInsertDeclareAtEnd( + LLVMDIBuilderInsertDeclareRecordAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar3, FooParamExpression, FooParamLocation, FooEntryBlock); else - LLVMDIBuilderInsertDeclareIntrinsicAtEnd( + LLVMDIBuilderInsertDeclareAtEnd( DIB, LLVMConstInt(LLVMInt64Type(), 0, false), FooParamVar3, FooParamExpression, FooParamLocation, FooEntryBlock); -- GitLab From 8155ec13968b6457c61b8507f2ae8ba3ac3b748b Mon Sep 17 00:00:00 2001 From: "Yaxun (Sam) Liu" Date: Fri, 22 Mar 2024 12:30:02 -0400 Subject: [PATCH 278/296] [HIP][NFC] Refactor managed var codegen (#85976) Refactor managed variable handling in codegen so that the transformation is done separately from registration. This will allow the new driver to register the managed var in the linker wrapper. --- clang/lib/CodeGen/CGCUDANV.cpp | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp index d3f2573fd5e3..b756318c46a9 100644 --- a/clang/lib/CodeGen/CGCUDANV.cpp +++ b/clang/lib/CodeGen/CGCUDANV.cpp @@ -605,20 +605,10 @@ llvm::Function *CGNVCUDARuntime::makeRegisterGlobalsFn() { uint64_t VarSize = CGM.getDataLayout().getTypeAllocSize(Var->getValueType()); if (Info.Flags.isManaged()) { - auto *ManagedVar = new llvm::GlobalVariable( - CGM.getModule(), Var->getType(), - /*isConstant=*/false, Var->getLinkage(), - /*Init=*/Var->isDeclaration() - ? nullptr - : llvm::ConstantPointerNull::get(Var->getType()), - /*Name=*/"", /*InsertBefore=*/nullptr, - llvm::GlobalVariable::NotThreadLocal); - ManagedVar->setDSOLocal(Var->isDSOLocal()); - ManagedVar->setVisibility(Var->getVisibility()); - ManagedVar->setExternallyInitialized(true); - ManagedVar->takeName(Var); - Var->setName(Twine(ManagedVar->getName() + ".managed")); - replaceManagedVar(Var, ManagedVar); + assert(Var->getName().ends_with(".managed") && + "HIP managed variables not transformed"); + auto *ManagedVar = CGM.getModule().getNamedGlobal( + Var->getName().drop_back(StringRef(".managed").size())); llvm::Value *Args[] = { &GpuBinaryHandlePtr, ManagedVar, @@ -1093,7 +1083,9 @@ void CGNVCUDARuntime::transformManagedVars() { : llvm::ConstantPointerNull::get(Var->getType()), /*Name=*/"", /*InsertBefore=*/nullptr, llvm::GlobalVariable::NotThreadLocal, - CGM.getContext().getTargetAddressSpace(LangAS::cuda_device)); + CGM.getContext().getTargetAddressSpace(CGM.getLangOpts().CUDAIsDevice + ? LangAS::cuda_device + : LangAS::Default)); ManagedVar->setDSOLocal(Var->isDSOLocal()); ManagedVar->setVisibility(Var->getVisibility()); ManagedVar->setExternallyInitialized(true); @@ -1102,7 +1094,7 @@ void CGNVCUDARuntime::transformManagedVars() { Var->setName(Twine(ManagedVar->getName()) + ".managed"); // Keep managed variables even if they are not used in device code since // they need to be allocated by the runtime. - if (!Var->isDeclaration()) { + if (CGM.getLangOpts().CUDAIsDevice && !Var->isDeclaration()) { assert(!ManagedVar->isDeclaration()); CGM.addCompilerUsedGlobal(Var); CGM.addCompilerUsedGlobal(ManagedVar); @@ -1160,9 +1152,8 @@ void CGNVCUDARuntime::createOffloadingEntries() { // Returns module constructor to be added. llvm::Function *CGNVCUDARuntime::finalizeModule() { + transformManagedVars(); if (CGM.getLangOpts().CUDAIsDevice) { - transformManagedVars(); - // Mark ODR-used device variables as compiler used to prevent it from being // eliminated by optimization. This is necessary for device variables // ODR-used by host functions. Sema correctly marks them as ODR-used no -- GitLab From 7269570e4b2a5197201c959652c3e86804ed1eeb Mon Sep 17 00:00:00 2001 From: Abhin P Jose Date: Fri, 22 Mar 2024 17:39:43 +0100 Subject: [PATCH 279/296] Fixed build breaking due to #77178 and #86131 (#86290) Fixed a small issue of matching pthread signature, which was causing the build to break for the compiler-rt project after adding -Wcast-function-type-mismatch to -Wextra dignostic group (https://github.com/llvm/llvm-project/pull/77178 & https://github.com/llvm/llvm-project/pull/86131). --- compiler-rt/lib/asan/tests/asan_noinst_test.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/compiler-rt/lib/asan/tests/asan_noinst_test.cpp b/compiler-rt/lib/asan/tests/asan_noinst_test.cpp index 4c103609c83b..df7de2d7d15e 100644 --- a/compiler-rt/lib/asan/tests/asan_noinst_test.cpp +++ b/compiler-rt/lib/asan/tests/asan_noinst_test.cpp @@ -45,7 +45,8 @@ TEST(AddressSanitizer, InternalSimpleDeathTest) { EXPECT_DEATH(exit(1), ""); } -static void MallocStress(size_t n) { +static void *MallocStress(void *NumOfItrPtr) { + size_t n = *((size_t *)NumOfItrPtr); u32 seed = my_rand(); BufferedStackTrace stack1; stack1.trace_buffer[0] = 0xa123; @@ -90,20 +91,21 @@ static void MallocStress(size_t n) { } for (size_t i = 0; i < vec.size(); i++) __asan::asan_free(vec[i], &stack3, __asan::FROM_MALLOC); + return nullptr; } - TEST(AddressSanitizer, NoInstMallocTest) { - MallocStress(ASAN_LOW_MEMORY ? 300000 : 1000000); + const size_t kNumIterations = (ASAN_LOW_MEMORY) ? 300000 : 1000000; + MallocStress((void *)&kNumIterations); } TEST(AddressSanitizer, ThreadedMallocStressTest) { const int kNumThreads = 4; - const int kNumIterations = (ASAN_LOW_MEMORY) ? 10000 : 100000; + const size_t kNumIterations = (ASAN_LOW_MEMORY) ? 10000 : 100000; pthread_t t[kNumThreads]; for (int i = 0; i < kNumThreads; i++) { - PTHREAD_CREATE(&t[i], 0, (void* (*)(void *x))MallocStress, - (void*)kNumIterations); + PTHREAD_CREATE(&t[i], 0, (void *(*)(void *x))MallocStress, + (void *)&kNumIterations); } for (int i = 0; i < kNumThreads; i++) { PTHREAD_JOIN(t[i], 0); -- GitLab From d2f684685afeffcffba7e889e7267bce1d905911 Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Fri, 22 Mar 2024 12:48:25 -0400 Subject: [PATCH 280/296] [C99] Update status of DR290, which we do not yet implement --- clang/test/C/drs/dr290.c | 20 ++++++++++++++++++++ clang/www/c_dr_status.html | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 clang/test/C/drs/dr290.c diff --git a/clang/test/C/drs/dr290.c b/clang/test/C/drs/dr290.c new file mode 100644 index 000000000000..3a6fd1d0dab6 --- /dev/null +++ b/clang/test/C/drs/dr290.c @@ -0,0 +1,20 @@ +/* RUN: %clang_cc1 -fsyntax-only -ast-dump %s | FileCheck %s + */ + +/* WG14 DR290: no + * FLT_EVAL_METHOD and extra precision and/or range + * + * We retain an implicit conversion based on the float eval method being used + * instead of dropping it due to the explicit cast. See GH86304 and C23 6.5.5p7. + */ + +#pragma clang fp eval_method(double) +_Static_assert((float)(123.0F * 2.0F) == (float)246.0F, ""); + +// CHECK: StaticAssertDecl +// CHECK-NEXT: ImplicitCastExpr {{.*}} '_Bool' +// CHECK-NEXT: BinaryOperator {{.*}} 'int' '==' +// NB: the following implicit cast is incorrect. +// CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' FPEvalMethod=1 +// CHECK-NEXT: CStyleCastExpr {{.*}} 'float' FPEvalMethod=1 + diff --git a/clang/www/c_dr_status.html b/clang/www/c_dr_status.html index ed45123ffd0e..a41c4f717067 100644 --- a/clang/www/c_dr_status.html +++ b/clang/www/c_dr_status.html @@ -1686,7 +1686,7 @@ conformance.

290 C99 FLT_EVAL_METHOD and extra precision and/or range - Unknown + No 291 -- GitLab From 527a624205748814dd9309eda7ee308b40b2359a Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Fri, 22 Mar 2024 12:49:12 -0400 Subject: [PATCH 281/296] [C11] Update the status of N1365 on constant expression handling This paper is about constant expression handling in the presence of FLT_EVAL_METHOD, which we handle via insertion of implicit cast nodes in the AST. --- clang/test/C/C11/n1365.c | 60 ++++++++++++++++++++++++++++++++++++++++ clang/www/c_status.html | 2 +- 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 clang/test/C/C11/n1365.c diff --git a/clang/test/C/C11/n1365.c b/clang/test/C/C11/n1365.c new file mode 100644 index 000000000000..3f769faa365d --- /dev/null +++ b/clang/test/C/C11/n1365.c @@ -0,0 +1,60 @@ +// RUN: %clang_cc1 -ast-dump %s | FileCheck %s + +/* WG14 N1365: Clang 16 + * Constant expressions + */ + +// Note: we don't allow you to expand __FLT_EVAL_METHOD__ in the presence of a +// pragma that changes its value. However, we can test that we have the correct +// constant expression behavior by testing that the AST has the correct implicit +// casts, which also specify that the cast was inserted due to an evaluation +// method requirement. +void func(void) { + { + #pragma clang fp eval_method(double) + _Static_assert(123.0F * 2.0F == 246.0F, ""); + // CHECK: StaticAssertDecl + // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Bool' + // CHECK-NEXT: BinaryOperator {{.*}} 'int' '==' + // CHECK-NEXT: BinaryOperator {{.*}} 'double' '*' FPEvalMethod=1 + // CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' FPEvalMethod=1 + // CHECK-NEXT: FloatingLiteral + // CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' FPEvalMethod=1 + // CHECK-NEXT: FloatingLiteral + + // Ensure that a cast removes the extra precision. + _Static_assert((float)(123.0F * 2.0F) == 246.0F, ""); + // CHECK: StaticAssertDecl + // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Bool' + // CHECK-NEXT: BinaryOperator {{.*}} 'int' '==' + // CHECK-NEXT: BinaryOperator {{.*}} 'double' '*' FPEvalMethod=1 + // CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' FPEvalMethod=1 + // CHECK-NEXT: FloatingLiteral + // CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' FPEvalMethod=1 + // CHECK-NEXT: FloatingLiteral + } + + { + #pragma clang fp eval_method(extended) + _Static_assert(123.0F * 2.0F == 246.0F, ""); + // CHECK: StaticAssertDecl + // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Bool' + // CHECK-NEXT: BinaryOperator {{.*}} 'int' '==' + // CHECK-NEXT: BinaryOperator {{.*}} 'long double' '*' FPEvalMethod=2 + // CHECK-NEXT: ImplicitCastExpr {{.*}} 'long double' FPEvalMethod=2 + // CHECK-NEXT: FloatingLiteral + // CHECK-NEXT: ImplicitCastExpr {{.*}} 'long double' FPEvalMethod=2 + // CHECK-NEXT: FloatingLiteral + } + + { + #pragma clang fp eval_method(source) + _Static_assert(123.0F * 2.0F == 246.0F, ""); + // CHECK: StaticAssertDecl + // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Bool' + // CHECK-NEXT: BinaryOperator {{.*}} 'int' '==' + // CHECK-NEXT: BinaryOperator {{.*}} 'float' '*' FPEvalMethod=0 + // CHECK-NEXT: FloatingLiteral + // CHECK-NEXT: FloatingLiteral + } +} diff --git a/clang/www/c_status.html b/clang/www/c_status.html index 0069da74cbd5..f00d5a6b7094 100644 --- a/clang/www/c_status.html +++ b/clang/www/c_status.html @@ -471,7 +471,7 @@ conformance.

Constant expressions N1365 - Unknown + Full Contractions and expression evaluation methods -- GitLab From 72c729f354d71697a1402720c90b57ff521b6739 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 22 Mar 2024 09:51:20 -0700 Subject: [PATCH 282/296] [bazel] Add support for --incompatible_disallow_empty_glob (#85999) This bazel flag, that should be flipped in an upcoming release https://github.com/bazelbuild/bazel/pull/15327, fails if globs have no matches. This helps find libraries where you are accidentally not including files because of typos. This change removes the various globs that were not matching anything, and uncovered some targets that were doing nothing because their source files were deleted. There are a few cases where globs were intentionally optional in the case of loops that expanded to different potential options, so those now use `allow_empty = True`. This allows downstream consumers to also flip this flags for their own builds, where previously this would fail in LLVM instead. The downside to this change is that if files are added in these relatively standard locations, manual work will have to be done to add this patterns back. If folks prefer we could instead add `allow_empty = True` to every glob. --- utils/bazel/.bazelrc | 4 + .../llvm-project-overlay/bolt/BUILD.bazel | 2 - .../clang-tools-extra/clang-tidy/defs.bzl | 4 +- .../llvm-project-overlay/clang/BUILD.bazel | 22 +- .../llvm-project-overlay/lld/BUILD.bazel | 2 - .../llvm-project-overlay/llvm/BUILD.bazel | 337 +++++++----------- .../llvm-project-overlay/mlir/BUILD.bazel | 144 -------- .../mlir/test/BUILD.bazel | 13 - .../mlir/unittests/BUILD.bazel | 38 -- 9 files changed, 138 insertions(+), 428 deletions(-) diff --git a/utils/bazel/.bazelrc b/utils/bazel/.bazelrc index 46894decc7ad..e8d055ec2322 100644 --- a/utils/bazel/.bazelrc +++ b/utils/bazel/.bazelrc @@ -35,6 +35,10 @@ build --features=layering_check # See: https://bazel.build/reference/be/functions#exports_files build --incompatible_no_implicit_file_export +# Enable so downstream users can flip this flag globally, this should +# eventually become the default +common --incompatible_disallow_empty_glob + ############################################################################### # Options to select different strategies for linking potential dependent # libraries. The default leaves it disabled. diff --git a/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel b/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel index 043a3b61a75f..1c12c8167ba4 100644 --- a/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/bolt/BUILD.bazel @@ -221,8 +221,6 @@ cc_library( srcs = glob([ "lib/Target/AArch64/*.cpp", ]), - hdrs = glob([ - ]), includes = ["include"], deps = [ ":Core", diff --git a/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/defs.bzl b/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/defs.bzl index 41c03aad871c..5abe4b08f5d9 100644 --- a/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/defs.bzl +++ b/utils/bazel/llvm-project-overlay/clang-tools-extra/clang-tidy/defs.bzl @@ -16,8 +16,8 @@ _common_library_deps = [ ] def clang_tidy_library(name, **kwargs): - kwargs["srcs"] = kwargs.get("srcs", native.glob([paths.join(name, "*.cpp")])) - kwargs["hdrs"] = kwargs.get("hdrs", native.glob([paths.join(name, "*.h")])) + kwargs["srcs"] = kwargs.get("srcs", native.glob([paths.join(name, "*.cpp")], allow_empty = True)) + kwargs["hdrs"] = kwargs.get("hdrs", native.glob([paths.join(name, "*.h")], allow_empty=True)) kwargs["deps"] = kwargs.get("deps", []) + _common_library_deps cc_library( name = name, diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel index 865cafbf50c6..c01986815afe 100644 --- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel @@ -614,7 +614,6 @@ cc_library( "include/clang/Basic/Version.inc", ] + glob([ "lib/Basic/*.cpp", - "lib/Basic/*.c", "lib/Basic/*.h", "lib/Basic/Targets/*.cpp", "lib/Basic/Targets/*.h", @@ -1042,7 +1041,6 @@ cc_library( "lib/Analysis/FlowSensitive/Models/*.cpp", "lib/Analysis/FlowSensitive/*.cpp", "lib/Analysis/*.cpp", - "lib/Analysis/*.h", ]) + [ ":analysis_htmllogger_gen", ], @@ -1180,10 +1178,8 @@ gentbl( cc_library( name = "parse", - srcs = [ - ] + glob([ + srcs = glob([ "lib/Parse/*.cpp", - "lib/Parse/*.h", ]), hdrs = [ "include/clang/Parse/AttrParserStringSwitches.inc", @@ -1207,7 +1203,6 @@ cc_library( name = "ast_matchers", srcs = glob([ "lib/ASTMatchers/*.cpp", - "lib/ASTMatchers/*.h", ]), hdrs = glob(["include/clang/ASTMatchers/*.h"]), includes = ["include"], @@ -1241,7 +1236,6 @@ cc_library( name = "rewrite", srcs = glob([ "lib/Rewrite/*.cpp", - "lib/Rewrite/*.h", ]), hdrs = glob(["include/clang/Rewrite/Core/*.h"]), includes = ["include"], @@ -1275,7 +1269,6 @@ cc_library( name = "tooling_core", srcs = glob([ "lib/Tooling/Core/*.cpp", - "lib/Tooling/Core/*.h", ]), hdrs = glob(["include/clang/Tooling/Core/*.h"]), includes = ["include"], @@ -1340,11 +1333,9 @@ cc_library( name = "tooling_refactoring", srcs = glob([ "lib/Tooling/Refactoring/**/*.cpp", - "lib/Tooling/Refactoring/**/*.h", ]), hdrs = glob([ "include/clang/Tooling/Refactoring/**/*.h", - "include/clang/Tooling/Refactoring/**/*.def", ]), deps = [ ":ast", @@ -1593,9 +1584,6 @@ cc_library( srcs = glob( [ "lib/Driver/*.cpp", - "lib/Driver/*.h", - "lib/Driver/Arch/*.cpp", - "lib/Driver/Arch/*.h", "lib/Driver/ToolChains/*.cpp", "lib/Driver/ToolChains/*.h", "lib/Driver/ToolChains/Arch/*.cpp", @@ -1833,9 +1821,6 @@ cc_library( copts = ["$(STACK_FRAME_UNLIMITED)"], data = [":builtin_headers_gen"], includes = ["include"], - textual_hdrs = glob([ - "include/clang/Frontend/*.def", - ]), deps = [ ":apinotes", ":ast", @@ -1872,7 +1857,6 @@ cc_library( name = "frontend_rewrite", srcs = glob([ "lib/Frontend/Rewrite/*.cpp", - "lib/Frontend/Rewrite/*.h", ]), hdrs = glob(["include/clang/Rewrite/Frontend/*.h"]), includes = ["include"], @@ -2116,7 +2100,6 @@ cc_library( name = "frontend_tool", srcs = glob([ "lib/FrontendTool/*.cpp", - "lib/FrontendTool/*.h", ]), hdrs = glob(["include/clang/FrontendTool/*.h"]), includes = ["include"], @@ -2320,7 +2303,6 @@ cc_binary( testonly = 1, srcs = glob([ "tools/clang-import-test/*.cpp", - "tools/clang-import-test/*.h", ]), stamp = 0, deps = [ @@ -2350,7 +2332,6 @@ cc_library( name = "clang-driver", srcs = glob([ "tools/driver/*.cpp", - "tools/driver/*.h", ]) + ["clang-driver.cpp"], copts = [ # Disable stack frame size checks in the driver because @@ -2668,7 +2649,6 @@ cc_library( name = "extract_api", srcs = glob([ "lib/ExtractAPI/**/*.cpp", - "lib/ExtractAPI/**/*.h", ]), hdrs = glob(["include/clang/ExtractAPI/**/*.h"]), includes = ["include"], diff --git a/utils/bazel/llvm-project-overlay/lld/BUILD.bazel b/utils/bazel/llvm-project-overlay/lld/BUILD.bazel index 8fb71fc1f971..5a494a13acea 100644 --- a/utils/bazel/llvm-project-overlay/lld/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/lld/BUILD.bazel @@ -187,7 +187,6 @@ cc_library( name = "MinGW", srcs = glob([ "MinGW/*.cpp", - "MinGW/*.h", ]), includes = ["MinGW"], deps = [ @@ -296,7 +295,6 @@ cc_binary( name = "lld", srcs = glob([ "tools/lld/*.cpp", - "tools/lld/*.h", ]) + ["lld-driver.cpp"], deps = [ ":COFF", diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel index 07c5a00c07d7..f5a2d264b690 100644 --- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel @@ -66,7 +66,10 @@ enum_targets_gen( llvm_target_asm_parsers = [ t for t in llvm_targets - if glob(["lib/Target/{}/AsmParser/CMakeLists.txt".format(t)]) + if glob( + ["lib/Target/{}/AsmParser/CMakeLists.txt".format(t)], + allow_empty = True, + ) ] enum_targets_gen( @@ -81,7 +84,10 @@ enum_targets_gen( llvm_target_disassemblers = [ t for t in llvm_targets - if glob(["lib/Target/{}/Disassembler/CMakeLists.txt".format(t)]) + if glob( + ["lib/Target/{}/Disassembler/CMakeLists.txt".format(t)], + allow_empty = True, + ) ] enum_targets_gen( @@ -96,7 +102,10 @@ enum_targets_gen( llvm_target_mcas = [ t for t in llvm_targets - if glob(["lib/Target/{}/MCA/CMakeLists.txt".format(t)]) + if glob( + ["lib/Target/{}/MCA/CMakeLists.txt".format(t)], + allow_empty = True, + ) ] enum_targets_gen( @@ -111,7 +120,10 @@ enum_targets_gen( llvm_target_exegesis = [ t for t in llvm_targets - if glob(["tools/llvm-exegesis/lib/{}/CMakeLists.txt".format(t)]) + if glob( + ["tools/llvm-exegesis/lib/{}/CMakeLists.txt".format(t)], + allow_empty = True, + ) ] enum_targets_gen( @@ -168,7 +180,6 @@ cc_library( name = "Demangle", srcs = glob([ "lib/Demangle/*.cpp", - "lib/Demangle/*.h", ]), hdrs = glob([ "include/llvm/Demangle/*.h", @@ -203,7 +214,6 @@ cc_library( "include/llvm/Option/*.h", ]) + select({ "@platforms//os:windows": glob([ - "lib/Support/Windows/*.h", "lib/Support/Windows/*.inc", ]), "//conditions:default": glob([ @@ -315,7 +325,6 @@ cc_library( name = "LineEditor", srcs = glob([ "lib/LineEditor/*.cpp", - "lib/LineEditor/*.h", ]), hdrs = glob(["include/llvm/LineEditor/*.h"]), copts = llvm_copts, @@ -329,7 +338,6 @@ cc_library( name = "Option", srcs = glob([ "lib/Option/*.cpp", - "lib/Option/*.h", ]), hdrs = glob(["include/llvm/Option/*.h"]), copts = llvm_copts, @@ -376,8 +384,6 @@ cc_library( name = "BinaryFormat", srcs = glob([ "lib/BinaryFormat/*.cpp", - "lib/BinaryFormat/*.def", - "lib/BinaryFormat/*.h", ]), hdrs = glob([ "include/llvm/BinaryFormat/*.h", @@ -409,7 +415,6 @@ cc_library( name = "DebugInfoMSF", srcs = glob([ "lib/DebugInfo/MSF/*.cpp", - "lib/DebugInfo/MSF/*.h", ]), hdrs = glob(["include/llvm/DebugInfo/MSF/*.h"]), copts = llvm_copts, @@ -420,7 +425,6 @@ cc_library( name = "DebugInfoBTF", srcs = glob([ "lib/DebugInfo/BTF/*.cpp", - "lib/DebugInfo/BTF/*.h", ]), hdrs = glob(["include/llvm/DebugInfo/BTF/*.h"]) + [ "include/llvm/DebugInfo/BTF/BTF.def", @@ -437,7 +441,6 @@ cc_library( name = "DebugInfoCodeView", srcs = glob([ "lib/DebugInfo/CodeView/*.cpp", - "lib/DebugInfo/CodeView/*.h", ]), hdrs = glob([ "include/llvm/DebugInfo/CodeView/*.h", @@ -480,9 +483,7 @@ cc_library( name = "DebugInfoPDB", srcs = glob([ "lib/DebugInfo/PDB/*.cpp", - "lib/DebugInfo/PDB/*.h", "lib/DebugInfo/PDB/Native/*.cpp", - "lib/DebugInfo/PDB/Native/*.h", ]), hdrs = glob([ "include/llvm/DebugInfo/PDB/*.h", @@ -523,12 +524,9 @@ cc_library( name = "MC", srcs = glob([ "lib/MC/*.cpp", - "lib/MC/*.h", ]), hdrs = glob([ "include/llvm/MC/*.h", - "include/llvm/MC/*.def", - "include/llvm/MC/*.inc", ]), copts = llvm_copts, deps = [ @@ -545,7 +543,6 @@ cc_library( name = "DebugInfoDWARF", srcs = glob([ "lib/DebugInfo/DWARF/*.cpp", - "lib/DebugInfo/DWARF/*.h", ]), hdrs = glob(["include/llvm/DebugInfo/DWARF/*.h"]), copts = llvm_copts, @@ -563,7 +560,6 @@ cc_library( name = "DebugInfoGSYM", srcs = glob([ "lib/DebugInfo/GSYM/*.cpp", - "lib/DebugInfo/GSYM/*.h", ]), hdrs = glob(["include/llvm/DebugInfo/GSYM/*.h"]), copts = llvm_copts, @@ -580,7 +576,6 @@ cc_library( name = "Symbolize", srcs = glob([ "lib/DebugInfo/Symbolize/*.cpp", - "lib/DebugInfo/Symbolize/*.h", ]), hdrs = glob([ "include/llvm/DebugInfo/Symbolize/*.h", @@ -658,7 +653,6 @@ cc_binary( srcs = glob( [ "utils/TableGen/*.cpp", - "utils/TableGen/*.inc", "utils/TableGen/*.h", "utils/TableGen/GlobalISel/*.cpp", "utils/TableGen/GlobalISel/*.h", @@ -821,7 +815,6 @@ cc_library( name = "BitstreamReader", srcs = glob([ "lib/Bitstream/Reader/*.cpp", - "lib/Bitstream/Reader/*.h", ]), hdrs = [ "include/llvm/Bitstream/BitCodeEnums.h", @@ -836,9 +829,6 @@ cc_library( cc_library( name = "BitstreamWriter", - srcs = glob([ - "lib/Bitstream/Writer/*.h", - ]), hdrs = [ "include/llvm/Bitstream/BitCodeEnums.h", "include/llvm/Bitstream/BitCodes.h", @@ -956,7 +946,6 @@ cc_library( name = "MCParser", srcs = glob([ "lib/MC/MCParser/*.cpp", - "lib/MC/MCParser/*.h", ]), hdrs = glob(["include/llvm/MC/MCParser/*.h"]), copts = llvm_copts, @@ -1002,9 +991,7 @@ cc_library( srcs = glob([ "lib/TextAPI/BinaryReader/**/*.cpp", ]), - hdrs = ["include/llvm/TextAPI/DylibReader.h"] + glob( - ["lib/TextAPI/BinaryReader/**/*.h"], - ), + hdrs = ["include/llvm/TextAPI/DylibReader.h"], copts = llvm_copts, deps = [ ":Object", @@ -1067,7 +1054,6 @@ cc_library( name = "ObjectYAML", srcs = glob([ "lib/ObjectYAML/*.cpp", - "lib/ObjectYAML/*.h", ]), hdrs = glob(["include/llvm/ObjectYAML/*.h"]), copts = llvm_copts, @@ -1085,7 +1071,6 @@ cc_library( name = "ProfileData", srcs = glob([ "lib/ProfileData/*.cpp", - "lib/ProfileData/*.h", ]), hdrs = glob([ "include/llvm/ProfileData/*.h", @@ -1109,7 +1094,6 @@ cc_library( name = "Coverage", srcs = glob([ "lib/ProfileData/Coverage/*.cpp", - "lib/ProfileData/Coverage/*.h", ]), hdrs = glob(["include/llvm/ProfileData/Coverage/*.h"]), copts = llvm_copts, @@ -1126,8 +1110,6 @@ cc_library( srcs = glob( [ "lib/Analysis/*.cpp", - "lib/Analysis/*.h", - "lib/Analysis/*.def", ], ), hdrs = glob( @@ -1185,7 +1167,6 @@ cc_library( name = "Target", srcs = glob([ "lib/Target/*.cpp", - "lib/Target/*.h", ]), hdrs = glob([ "include/llvm/Target/*.h", @@ -1221,14 +1202,11 @@ cc_library( name = "TargetParser", srcs = glob([ "lib/TargetParser/*.cpp", - "lib/TargetParser/*.h", ]) + select({ "@platforms//os:windows": glob([ - "lib/TargetParser/Windows/*.h", "lib/TargetParser/Windows/*.inc", ]), "//conditions:default": glob([ - "lib/TargetParser/Unix/*.h", "lib/TargetParser/Unix/*.inc", ]), }), @@ -1252,7 +1230,6 @@ cc_library( name = "DWP", srcs = glob([ "lib/DWP/*.cpp", - "lib/DWP/*.h", ]), hdrs = glob(["include/llvm/DWP/*.h"]), copts = llvm_copts, @@ -1269,7 +1246,6 @@ cc_library( name = "TransformUtils", srcs = glob([ "lib/Transforms/Utils/*.cpp", - "lib/Transforms/Utils/*.h", ]), hdrs = glob(["include/llvm/Transforms/Utils/*.h"]) + [ "include/llvm/Transforms/Utils.h", @@ -1390,7 +1366,6 @@ cc_library( name = "Scalar", srcs = glob([ "lib/Transforms/Scalar/*.cpp", - "lib/Transforms/Scalar/*.h", ]), hdrs = glob(["include/llvm/Transforms/Scalar/*.h"]) + [ "include/llvm/Transforms/Scalar.h", @@ -1432,9 +1407,6 @@ cc_library( cc_library( name = "FrontendDebug", - srcs = glob([ - "lib/Frontend/Debug/*.cpp", - ]), hdrs = glob([ "include/llvm/Frontend/Debug/*.h", ]), @@ -1530,8 +1502,6 @@ cc_library( ]), hdrs = glob([ "include/llvm/Frontend/OpenMP/*.h", - "include/llvm/Frontend/OpenMP/OMP/*.h", - "include/llvm/Frontend/*.h", ]) + [ "include/llvm/Frontend/OpenMP/OMP.h.inc", "include/llvm/Frontend/OpenMP/OMP.inc", @@ -1591,9 +1561,7 @@ cc_library( ]) + [ "include/llvm/Frontend/OpenACC/ACC.inc", ], - hdrs = glob([ - "include/llvm/Frontend/OpenACC/*.h", - ]) + ["include/llvm/Frontend/OpenACC/ACC.h.inc"], + hdrs = ["include/llvm/Frontend/OpenACC/ACC.h.inc"], copts = llvm_copts, deps = [ ":Analysis", @@ -1607,7 +1575,6 @@ cc_library( name = "AsmParser", srcs = glob([ "lib/AsmParser/*.cpp", - "lib/AsmParser/*.h", ]), hdrs = glob(["include/llvm/AsmParser/*.h"]), copts = llvm_copts, @@ -1623,7 +1590,6 @@ cc_library( name = "IRPrinter", srcs = glob([ "lib/IRPrinter/*.cpp", - "lib/IRPrinter/*.h", ]), hdrs = glob([ "include/llvm/IRPrinter/*.h", @@ -1640,7 +1606,6 @@ cc_library( name = "IRReader", srcs = glob([ "lib/IRReader/*.cpp", - "lib/IRReader/*.h", ]), hdrs = glob([ "include/llvm/IRReader/*.h", @@ -1683,7 +1648,6 @@ cc_library( name = "IPO", srcs = glob([ "lib/Transforms/IPO/*.cpp", - "lib/Transforms/IPO/*.h", ]), hdrs = glob([ "include/llvm/Transforms/IPO/*.h", @@ -1721,7 +1685,6 @@ cc_library( name = "CFGuard", srcs = glob([ "lib/Transforms/CFGuard/*.cpp", - "lib/Transforms/CFGuard/*.h", ]), hdrs = ["include/llvm/Transforms/CFGuard.h"], copts = llvm_copts, @@ -1736,7 +1699,6 @@ cc_library( name = "HipStdPar", srcs = glob([ "lib/Transforms/HipStdPar/*.cpp", - "lib/Transforms/HipStdPar/*.h", ]), hdrs = ["include/llvm/Transforms/HipStdPar/HipStdPar.h"], copts = llvm_copts, @@ -1826,7 +1788,6 @@ cc_library( copts = llvm_copts, textual_hdrs = glob([ "include/llvm/CodeGen/**/*.def", - "include/llvm/CodeGen/**/*.inc", ]), deps = [ ":AggressiveInstCombine", @@ -2305,10 +2266,13 @@ gentbl( td_file = "lib/Target/" + target["name"] + "/" + target["short_name"] + ".td", td_srcs = [ ":common_target_td_sources", - ] + glob([ - "lib/Target/" + target["name"] + "/*.td", - "lib/Target/" + target["name"] + "/GISel/*.td", - ]), + ] + glob( + [ + "lib/Target/" + target["name"] + "/*.td", + "lib/Target/" + target["name"] + "/GISel/*.td", + ], + allow_empty = True, + ), deps = target.get("tbl_deps", []), )], [cc_library( @@ -2332,43 +2296,49 @@ gentbl( # a number of targets due to crisscrossing inclusion of headers. [cc_library( name = target["name"] + "UtilsAndDesc", - srcs = glob([ - "lib/Target/" + target["name"] + "/MCTargetDesc/*.cpp", - "lib/Target/" + target["name"] + "/Utils/*.cpp", - - # We have to include these headers here as well as in the `hdrs` - # below to allow the `.cpp` files to use file-relative-inclusion to - # find them, even though consumers of this library use inclusion - # relative to the target with the `strip_includes_prefix` of this - # library. This mixture is likely incompatible with header modules. - "lib/Target/" + target["name"] + "/MCTargetDesc/*.h", - "lib/Target/" + target["name"] + "/Utils/*.h", - ]), - hdrs = glob([ - "lib/Target/" + target["name"] + "/MCTargetDesc/*.h", - "lib/Target/" + target["name"] + "/Utils/*.h", - - # This a bit of a hack to allow us to expose common, internal - # target header files to other libraries within the target via - # target-relative includes. This usage of headers is inherently - # non-modular as there is a mixture of target-relative inclusion - # using this rule and file-relative inclusion using the repeated - # listing of these headers in the `srcs` of subsequent rules. - "lib/Target/" + target["name"] + "/*.h", - - # FIXME: The entries below should be `textual_hdrs` instead of - # `hdrs`, but unfortunately that doesn't work with - # `strip_include_prefix`: - # https://github.com/bazelbuild/bazel/issues/12424 - # - # Once that issue is fixed and released, we can switch this to - # `textual_hdrs` and remove the feature disabling the various Bazel - # features (both current and under-development) that motivated the - # distinction between these two. - "lib/Target/" + target["name"] + "/*.def", - "lib/Target/" + target["name"] + "/*.inc", - "lib/Target/" + target["name"] + "/MCTargetDesc/*.def", - ]), + srcs = glob( + [ + "lib/Target/" + target["name"] + "/MCTargetDesc/*.cpp", + "lib/Target/" + target["name"] + "/Utils/*.cpp", + + # We have to include these headers here as well as in the `hdrs` + # below to allow the `.cpp` files to use file-relative-inclusion to + # find them, even though consumers of this library use inclusion + # relative to the target with the `strip_includes_prefix` of this + # library. This mixture is likely incompatible with header modules. + "lib/Target/" + target["name"] + "/MCTargetDesc/*.h", + "lib/Target/" + target["name"] + "/Utils/*.h", + ], + allow_empty = True, + ), + hdrs = glob( + [ + "lib/Target/" + target["name"] + "/MCTargetDesc/*.h", + "lib/Target/" + target["name"] + "/Utils/*.h", + + # This a bit of a hack to allow us to expose common, internal + # target header files to other libraries within the target via + # target-relative includes. This usage of headers is inherently + # non-modular as there is a mixture of target-relative inclusion + # using this rule and file-relative inclusion using the repeated + # listing of these headers in the `srcs` of subsequent rules. + "lib/Target/" + target["name"] + "/*.h", + + # FIXME: The entries below should be `textual_hdrs` instead of + # `hdrs`, but unfortunately that doesn't work with + # `strip_include_prefix`: + # https://github.com/bazelbuild/bazel/issues/12424 + # + # Once that issue is fixed and released, we can switch this to + # `textual_hdrs` and remove the feature disabling the various Bazel + # features (both current and under-development) that motivated the + # distinction between these two. + "lib/Target/" + target["name"] + "/*.def", + "lib/Target/" + target["name"] + "/*.inc", + "lib/Target/" + target["name"] + "/MCTargetDesc/*.def", + ], + allow_empty = True, + ), copts = llvm_copts, features = [ "-parse_headers", @@ -2392,20 +2362,26 @@ gentbl( )], [cc_library( name = target["name"] + "CodeGen", - srcs = glob([ - "lib/Target/" + target["name"] + "/GISel/*.cpp", - "lib/Target/" + target["name"] + "/GISel/*.h", - "lib/Target/" + target["name"] + "/*.cpp", - "lib/Target/" + target["name"] + "/*.h", - ]), + srcs = glob( + [ + "lib/Target/" + target["name"] + "/GISel/*.cpp", + "lib/Target/" + target["name"] + "/GISel/*.h", + "lib/Target/" + target["name"] + "/*.cpp", + "lib/Target/" + target["name"] + "/*.h", + ], + allow_empty = True, + ), hdrs = ["lib/Target/" + target["name"] + "/" + target["short_name"] + ".h"], copts = llvm_copts, features = ["-layering_check"], strip_include_prefix = "lib/Target/" + target["name"], - textual_hdrs = glob([ - "lib/Target/" + target["name"] + "/*.def", - "lib/Target/" + target["name"] + "/*.inc", - ]), + textual_hdrs = glob( + [ + "lib/Target/" + target["name"] + "/*.def", + "lib/Target/" + target["name"] + "/*.inc", + ], + allow_empty = True, + ), deps = [ ":Analysis", ":BinaryFormat", @@ -2430,10 +2406,13 @@ gentbl( )], [cc_library( name = target["name"] + "AsmParser", - srcs = glob([ - "lib/Target/" + target["name"] + "/AsmParser/*.cpp", - "lib/Target/" + target["name"] + "/AsmParser/*.h", - ]), + srcs = glob( + [ + "lib/Target/" + target["name"] + "/AsmParser/*.cpp", + "lib/Target/" + target["name"] + "/AsmParser/*.h", + ], + allow_empty = True, + ), copts = llvm_copts, deps = [ ":BinaryFormat", @@ -2464,9 +2443,12 @@ gentbl( # `textual_hdrs` and remove the feature disabling the various Bazel # features (both current and under-development) that motivated the # distinction between these two. - hdrs = glob([ - "lib/Target/" + target["name"] + "/Disassembler/*.h", - ]), + hdrs = glob( + [ + "lib/Target/" + target["name"] + "/Disassembler/*.h", + ], + allow_empty = True, + ), features = [ "-parse_headers", "-header_modules", @@ -2475,11 +2457,14 @@ gentbl( )], [cc_library( name = target["name"] + "Disassembler", - srcs = glob([ - "lib/Target/" + target["name"] + "/Disassembler/*.cpp", - "lib/Target/" + target["name"] + "/Disassembler/*.c", - "lib/Target/" + target["name"] + "/Disassembler/*.h", - ]), + srcs = glob( + [ + "lib/Target/" + target["name"] + "/Disassembler/*.cpp", + "lib/Target/" + target["name"] + "/Disassembler/*.c", + "lib/Target/" + target["name"] + "/Disassembler/*.h", + ], + allow_empty = True, + ), copts = llvm_copts, features = ["-layering_check"], deps = [ @@ -2497,11 +2482,14 @@ gentbl( )], [cc_library( name = target["name"] + "TargetMCA", - srcs = glob([ - "lib/Target/" + target["name"] + "/MCA/*.cpp", - "lib/Target/" + target["name"] + "/MCA/*.c", - "lib/Target/" + target["name"] + "/MCA/*.h", - ]), + srcs = glob( + [ + "lib/Target/" + target["name"] + "/MCA/*.cpp", + "lib/Target/" + target["name"] + "/MCA/*.c", + "lib/Target/" + target["name"] + "/MCA/*.h", + ], + allow_empty = True, + ), copts = llvm_copts, features = ["-layering_check"], deps = [ @@ -2559,28 +2547,10 @@ cc_library( textual_hdrs = ["lib/Passes/PassRegistry.def"], ) -cc_library( - name = "MLPolicies", - srcs = glob([ - "lib/Analysis/ML/*.cpp", - "lib/Analysis/ML/*.h", - ]), - hdrs = glob([ - "include/llvm/Analysis/ML/*.h", - ]), - copts = llvm_copts, - deps = [ - ":Analysis", - ":Core", - ":Support", - ], -) - cc_library( name = "Passes", srcs = glob([ "lib/Passes/*.cpp", - "lib/Passes/*.h", ]), hdrs = glob([ "include/llvm/Passes/*.h", @@ -2601,7 +2571,6 @@ cc_library( ":InstCombine", ":Instrumentation", ":MC", - ":MLPolicies", ":ObjCARC", ":Scalar", ":Support", @@ -2618,7 +2587,6 @@ cc_library( name = "LTO", srcs = glob([ "lib/LTO/*.cpp", - "lib/LTO/*.h", ]), hdrs = glob([ "include/llvm/LTO/*.h", @@ -2658,7 +2626,6 @@ cc_library( name = "ExecutionEngine", srcs = glob([ "lib/ExecutionEngine/*.cpp", - "lib/ExecutionEngine/*.h", "lib/ExecutionEngine/RuntimeDyld/*.cpp", "lib/ExecutionEngine/RuntimeDyld/*.h", "lib/ExecutionEngine/RuntimeDyld/Targets/*.cpp", @@ -2772,11 +2739,9 @@ cc_library( name = "OrcJIT", srcs = glob([ "lib/ExecutionEngine/Orc/*.cpp", - "lib/ExecutionEngine/Orc/*.h", ]), hdrs = glob([ "include/llvm/ExecutionEngine/Orc/*.h", - "include/llvm/ExecutionEngine/Orc/RPC/*.h", ]) + [ "include/llvm-c/LLJIT.h", "include/llvm-c/Orc.h", @@ -2900,7 +2865,6 @@ cc_library( name = "DWARFLinker", srcs = glob([ "lib/DWARFLinker/Classic/*.cpp", - "lib/DWARFLinker/Classic/*.h", ]), hdrs = glob(["include/llvm/DWARFLinker/Classic/*.h"]), copts = llvm_copts, @@ -2921,7 +2885,6 @@ cc_library( name = "DWARFLinkerBase", srcs = glob([ "lib/DWARFLinker/*.cpp", - "lib/DWARFLinker/*.h", ]), hdrs = glob(["include/llvm/DWARFLinker/*.h"]), copts = llvm_copts, @@ -3012,7 +2975,6 @@ cc_library( name = "InterfaceStub", srcs = glob([ "lib/InterfaceStub/*.cpp", - "lib/InterfaceStub/*.h", ]), hdrs = glob([ "include/llvm/InterfaceStub/*.h", @@ -3063,7 +3025,6 @@ cc_library( name = "MCA", srcs = glob([ "lib/MCA/**/*.cpp", - "lib/MCA/**/*.h", ]), hdrs = glob([ "include/llvm/MCA/**/*.h", @@ -3090,7 +3051,6 @@ cc_library( name = "XRay", srcs = glob([ "lib/XRay/*.cpp", - "lib/XRay/*.h", ]), hdrs = glob(["include/llvm/XRay/*.h"]), copts = llvm_copts, @@ -3147,21 +3107,24 @@ cc_library( cc_library( name = "Exegesis", - srcs = glob([ - "tools/llvm-exegesis/lib/*.cpp", - # We have to include these headers here as well as in the `hdrs` below - # to allow the `.cpp` files to use file-relative-inclusion to find - # them, even though consumers of this library use inclusion relative to - # `tools/llvm-exegesis/lib` with the `strip_includes_prefix` of this - # library. This mixture appears to be incompatible with header modules. - "tools/llvm-exegesis/lib/*.h", - ] + [ - "tools/llvm-exegesis/lib/{}/*.cpp".format(t) - for t in llvm_target_exegesis - ] + [ - "tools/llvm-exegesis/lib/{}/*.h".format(t) - for t in llvm_target_exegesis - ]), + srcs = glob( + [ + "tools/llvm-exegesis/lib/*.cpp", + # We have to include these headers here as well as in the `hdrs` below + # to allow the `.cpp` files to use file-relative-inclusion to find + # them, even though consumers of this library use inclusion relative to + # `tools/llvm-exegesis/lib` with the `strip_includes_prefix` of this + # library. This mixture appears to be incompatible with header modules. + "tools/llvm-exegesis/lib/*.h", + ] + [ + "tools/llvm-exegesis/lib/{}/*.cpp".format(t) + for t in llvm_target_exegesis + ] + [ + "tools/llvm-exegesis/lib/{}/*.h".format(t) + for t in llvm_target_exegesis + ], + allow_empty = True, + ), hdrs = glob(["tools/llvm-exegesis/lib/*.h"]), copts = llvm_copts, features = [ @@ -3335,7 +3298,6 @@ cc_binary( name = "llvm-ar", srcs = glob([ "tools/llvm-ar/*.cpp", - "tools/llvm-ar/*.h", ]) + ["llvm-ar-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -3373,7 +3335,6 @@ cc_binary( name = "llvm-as", srcs = glob([ "tools/llvm-as/*.cpp", - "tools/llvm-as/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3390,7 +3351,6 @@ cc_binary( name = "llvm-bcanalyzer", srcs = glob([ "tools/llvm-bcanalyzer/*.cpp", - "tools/llvm-bcanalyzer/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3477,7 +3437,6 @@ cc_binary( name = "llvm-cvtres", srcs = glob([ "tools/llvm-cvtres/*.cpp", - "tools/llvm-cvtres/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3511,7 +3470,6 @@ cc_binary( name = "llvm-cxxmap", srcs = glob([ "tools/llvm-cxxmap/*.cpp", - "tools/llvm-cxxmap/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3546,7 +3504,6 @@ cc_binary( name = "llvm-cxxfilt", srcs = glob([ "tools/llvm-cxxfilt/*.cpp", - "tools/llvm-cxxfilt/*.h", ]) + ["llvm-cxxfilt-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -3579,7 +3536,6 @@ cc_binary( name = "llvm-debuginfod-find", srcs = glob([ "tools/llvm-debuginfod-find/*.cpp", - "tools/llvm-debuginfod-find/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3596,7 +3552,6 @@ cc_binary( name = "llvm-dis", srcs = glob([ "tools/llvm-dis/*.cpp", - "tools/llvm-dis/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3691,7 +3646,6 @@ cc_binary( name = "llvm-dwp", srcs = glob([ "tools/llvm-dwp/*.cpp", - "tools/llvm-dwp/*.h", ]) + ["llvm-dwp-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -3731,7 +3685,6 @@ cc_binary( name = "llvm-extract", srcs = glob([ "tools/llvm-extract/*.cpp", - "tools/llvm-extract/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3773,7 +3726,6 @@ cc_binary( name = "llvm-gsymutil", srcs = glob([ "tools/llvm-gsymutil/*.cpp", - "tools/llvm-gsymutil/*.h", ]) + ["llvm-gsymutil-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -3925,7 +3877,6 @@ cc_binary( name = "llvm-link", srcs = glob([ "tools/llvm-link/*.cpp", - "tools/llvm-link/*.h", ]), copts = llvm_copts, stamp = 0, @@ -3990,7 +3941,6 @@ cc_binary( name = "llvm-lto", srcs = glob([ "tools/llvm-lto/*.cpp", - "tools/llvm-lto/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4013,7 +3963,6 @@ cc_binary( name = "llvm-lto2", srcs = glob([ "tools/llvm-lto2/*.cpp", - "tools/llvm-lto2/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4159,7 +4108,6 @@ cc_binary( name = "llvm-mt", srcs = glob([ "tools/llvm-mt/*.cpp", - "tools/llvm-mt/*.h", ]) + ["llvm-mt-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -4196,7 +4144,6 @@ cc_binary( name = "llvm-nm", srcs = glob([ "tools/llvm-nm/*.cpp", - "tools/llvm-nm/*.h", ]) + ["llvm-nm-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -4280,7 +4227,6 @@ cc_binary( name = "llvm-stress", srcs = glob([ "tools/llvm-stress/*.cpp", - "tools/llvm-stress/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4457,7 +4403,6 @@ cc_binary( name = "llvm-profdata", srcs = glob([ "tools/llvm-profdata/*.cpp", - "tools/llvm-profdata/*.h", ]) + ["llvm-profdata-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -4641,7 +4586,6 @@ cc_binary( name = "llvm-rtdyld", srcs = glob([ "tools/llvm-rtdyld/*.cpp", - "tools/llvm-rtdyld/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4683,7 +4627,6 @@ cc_binary( name = "llvm-size", srcs = glob([ "tools/llvm-size/*.cpp", - "tools/llvm-size/*.h", ]) + ["llvm-size-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -4699,7 +4642,6 @@ cc_binary( name = "llvm-split", srcs = glob([ "tools/llvm-split/*.cpp", - "tools/llvm-split/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4729,7 +4671,6 @@ cc_binary( name = "llvm-strings", srcs = glob([ "tools/llvm-strings/*.cpp", - "tools/llvm-strings/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4766,7 +4707,6 @@ cc_binary( name = "llvm-symbolizer", srcs = glob([ "tools/llvm-symbolizer/*.cpp", - "tools/llvm-symbolizer/*.h", ]) + ["llvm-symbolizer-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -4792,7 +4732,6 @@ cc_binary( name = "llvm-undname", srcs = glob([ "tools/llvm-undname/*.cpp", - "tools/llvm-undname/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4806,7 +4745,6 @@ cc_binary( name = "llvm-xray", srcs = glob([ "tools/llvm-xray/*.cpp", - "tools/llvm-xray/*.cc", "tools/llvm-xray/*.h", ]), copts = llvm_copts, @@ -4889,7 +4827,6 @@ cc_binary( name = "sancov", srcs = glob([ "tools/sancov/*.cpp", - "tools/sancov/*.h", ]) + ["sancov-driver.cpp"], copts = llvm_copts, stamp = 0, @@ -4912,7 +4849,6 @@ cc_binary( name = "sanstats", srcs = glob([ "tools/sanstats/*.cpp", - "tools/sanstats/*.h", ]), copts = llvm_copts, stamp = 0, @@ -4927,7 +4863,6 @@ cc_binary( name = "split-file", srcs = glob([ "utils/split-file/*.cpp", - "utils/split-file/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5023,7 +4958,6 @@ cc_library( testonly = True, srcs = glob([ "lib/Testing/Support/*.cpp", - "lib/Testing/Support/*.h", ]), hdrs = glob(["include/llvm/Testing/Support/*.h"]), copts = llvm_copts, @@ -5052,7 +4986,6 @@ cc_binary( testonly = True, srcs = glob([ "utils/FileCheck/*.cpp", - "utils/FileCheck/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5098,7 +5031,6 @@ cc_binary( testonly = True, srcs = glob([ "utils/count/*.c", - "utils/count/*.h", ]), stamp = 0, deps = [":Support"], @@ -5109,7 +5041,6 @@ cc_binary( testonly = True, srcs = glob([ "tools/lli/ChildTarget/*.cpp", - "tools/lli/ChildTarget/*.h", ]), copts = llvm_copts, # The tests load code into this binary that expect to see symbols @@ -5176,7 +5107,6 @@ cc_binary( testonly = True, srcs = glob([ "tools/llvm-diff/*.cpp", - "tools/llvm-diff/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5194,7 +5124,6 @@ cc_binary( testonly = True, srcs = glob([ "tools/llvm-isel-fuzzer/*.cpp", - "tools/llvm-isel-fuzzer/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5240,7 +5169,6 @@ cc_binary( testonly = True, srcs = glob([ "utils/not/*.cpp", - "utils/not/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5320,7 +5248,6 @@ cc_binary( testonly = True, srcs = glob([ "tools/llvm-tli-checker/*.cpp", - "tools/llvm-tli-checker/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5367,7 +5294,6 @@ cc_binary( name = "verify-uselistorder", srcs = glob([ "tools/verify-uselistorder/*.cpp", - "tools/verify-uselistorder/*.h", ]), copts = llvm_copts, stamp = 0, @@ -5387,7 +5313,6 @@ cc_binary( testonly = True, srcs = glob([ "tools/yaml2obj/*.cpp", - "tools/yaml2obj/*.h", ]), copts = llvm_copts, stamp = 0, diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 3b575d4a413c..5b6e4678a05e 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -325,9 +325,7 @@ cc_library( "lib/IR/*.cpp", "lib/IR/*.h", "lib/IR/PDL/*.cpp", - "lib/Bytecode/Reader/*.h", "lib/Bytecode/Writer/*.h", - "lib/Bytecode/*.h", ]) + [ "include/mlir/IR/PDLPatternMatch.h.inc", "include/mlir/Interfaces/CallInterfaces.h", @@ -1630,7 +1628,6 @@ cc_library( srcs = glob( [ "lib/Dialect/AMDGPU/Transforms/*.cpp", - "lib/Dialect/AMDGPU/Transforms/*.h", ], ), hdrs = glob(["include/mlir/Dialect/AMDGPU/Transforms/*.h"]), @@ -1769,7 +1766,6 @@ cc_library( name = "TargetCpp", srcs = glob([ "lib/Target/Cpp/*.cpp", - "lib/Target/Cpp/*.h", ]), hdrs = glob(["include/mlir/Target/Cpp/*.h"]), deps = [ @@ -1968,7 +1964,6 @@ cc_library( name = "ArmNeon2dToIntr", srcs = glob([ "lib/Conversion/ArmNeon2dToIntr/*.cpp", - "lib/Conversion/ArmNeon2dToIntr/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ArmNeon2dToIntr/*.h", @@ -2840,7 +2835,6 @@ cc_library( name = "SCFTransforms", srcs = glob([ "lib/Dialect/SCF/Transforms/*.cpp", - "lib/Dialect/SCF/Transforms/*.h", ]), hdrs = glob([ "include/mlir/Dialect/SCF/Transforms/*.h", @@ -3189,7 +3183,6 @@ cc_library( name = "SparseTensorTransforms", srcs = glob([ "lib/Dialect/SparseTensor/Transforms/*.cpp", - "lib/Dialect/SparseTensor/Transforms/*.h", "lib/Dialect/SparseTensor/Transforms/Utils/*.cpp", "lib/Dialect/SparseTensor/Transforms/Utils/*.h", ]), @@ -3829,7 +3822,6 @@ cc_library( name = "Dialect", srcs = glob([ "lib/Dialect/*.cpp", - "lib/Dialect/*.h", ]), hdrs = glob([ "include/mlir/Dialect/*.h", @@ -3871,7 +3863,6 @@ cc_library( name = "DialectUtils", srcs = glob([ "lib/Dialect/Utils/*.cpp", - "lib/Dialect/Utils/*.h", ]), hdrs = glob([ "include/mlir/Dialect/Utils/*.h", @@ -3890,7 +3881,6 @@ cc_library( name = "AffineDialect", srcs = glob([ "lib/Dialect/Affine/IR/*.cpp", - "lib/Dialect/Affine/IR/*.h", ]), hdrs = glob([ "include/mlir/Dialect/Affine/IR/*.h", @@ -4012,7 +4002,6 @@ cc_library( name = "AffineAnalysis", srcs = glob([ "lib/Dialect/Affine/Analysis/*.cpp", - "lib/Dialect/Affine/Analysis/*.h", ]), hdrs = glob(["include/mlir/Dialect/Affine/Analysis/*.h"]), includes = ["include"], @@ -4036,7 +4025,6 @@ cc_library( srcs = glob( [ "lib/Dialect/Affine/Utils/*.cpp", - "lib/Dialect/Affine/Utils/*.h", ], ), hdrs = [ @@ -4083,7 +4071,6 @@ cc_library( name = "AffineTransforms", srcs = glob([ "lib/Dialect/Affine/Transforms/*.cpp", - "lib/Dialect/Affine/Transforms/*.h", ]), hdrs = [ "include/mlir/Dialect/Affine/Passes.h", @@ -4191,7 +4178,6 @@ cc_library( ":MemRefToSPIRV", ":NVGPUToNVVM", ":NVVMToLLVM", - ":OpenACCToLLVM", ":OpenACCToSCF", ":OpenMPToLLVM", ":PDLToPDLInterp", @@ -4224,7 +4210,6 @@ cc_library( name = "AsyncToLLVM", srcs = glob([ "lib/Conversion/AsyncToLLVM/*.cpp", - "lib/Conversion/AsyncToLLVM/*.h", ]), hdrs = glob(["include/mlir/Conversion/AsyncToLLVM/*.h"]), includes = ["include"], @@ -4250,7 +4235,6 @@ cc_library( name = "AffineToStandard", srcs = glob([ "lib/Conversion/AffineToStandard/*.cpp", - "lib/Conversion/AffineToStandard/*.h", ]), hdrs = glob(["include/mlir/Conversion/AffineToStandard/*.h"]), includes = ["include"], @@ -4271,32 +4255,11 @@ cc_library( ], ) -# SDBM dialect only contains attribute components that can be constructed given -# a dialect object, so whenever it is used it must also be registered. Therefore -# we don't split out the registration library for it. -cc_library( - name = "SDBM", - srcs = glob([ - "lib/Dialect/SDBM/*.cpp", - "lib/Dialect/SDBM/*.h", - ]), - hdrs = glob([ - "include/mlir/Dialect/SDBM/*.h", - ]), - includes = ["include"], - deps = [ - ":IR", - ":Support", - "//llvm:Support", - ], -) - cc_library( name = "SCFDialect", srcs = glob( [ "lib/Dialect/SCF/IR/*.cpp", - "lib/Dialect/SCF/IR/*.h", ], ), hdrs = glob( @@ -4605,7 +4568,6 @@ cc_library( name = "ShapeToStandard", srcs = glob([ "lib/Conversion/ShapeToStandard/*.cpp", - "lib/Conversion/ShapeToStandard/*.h", ]), hdrs = ["include/mlir/Conversion/ShapeToStandard/ShapeToStandard.h"], includes = ["include"], @@ -4645,7 +4607,6 @@ cc_library( name = "ShapeTransforms", srcs = glob([ "lib/Dialect/Shape/Transforms/*.cpp", - "lib/Dialect/Shape/Transforms/*.h", ]), hdrs = [ "include/mlir/Dialect/Shape/Analysis/ShapeMappingAnalysis.h", @@ -4728,7 +4689,6 @@ cc_library( srcs = glob( [ "lib/Dialect/ControlFlow/IR/*.cpp", - "lib/Dialect/ControlFlow/IR/*.h", ], ), hdrs = glob([ @@ -4774,13 +4734,10 @@ cc_library( srcs = glob( [ "lib/Dialect/Func/IR/*.cpp", - "lib/Dialect/Func/IR/*.h", - "lib/Dialect/Func/Utils/*.cpp", ], ), hdrs = glob([ "include/mlir/Dialect/Func/IR/*.h", - "include/mlir/Dialect/Func/Utils/*.h", ]), includes = ["include"], deps = [ @@ -4931,7 +4888,6 @@ cc_library( name = "FuncTransforms", srcs = glob([ "lib/Dialect/Func/Transforms/*.cpp", - "lib/Dialect/Func/Transforms/*.h", ]), hdrs = glob(["include/mlir/Dialect/Func/Transforms/*.h"]), includes = ["include"], @@ -5068,7 +5024,6 @@ cc_library( srcs = glob( [ "lib/Dialect/Vector/Transforms/*.cpp", - "lib/Dialect/Vector/Transforms/*.h", ], ), hdrs = glob([ @@ -5135,7 +5090,6 @@ cc_library( name = "Support", srcs = glob([ "lib/Support/*.cpp", - "lib/Support/*.h", ]), hdrs = glob(["include/mlir/Support/*.h"]), includes = ["include"], @@ -5149,11 +5103,8 @@ cc_library( name = "Debug", srcs = glob([ "lib/Debug/*.cpp", - "lib/Debug/*.h", "lib/Debug/BreakpointManagers/*.cpp", - "lib/Debug/BreakpointManagers/*.h", "lib/Debug/Observers/*.cpp", - "lib/Debug/Observers/*.h", ]), hdrs = glob([ "include/mlir/Debug/*.h", @@ -5195,8 +5146,6 @@ cc_library( [ "lib/Tools/mlir-lsp-server/*.cpp", "lib/Tools/mlir-lsp-server/*.h", - "lib/Tools/mlir-lsp-server/lsp/*.cpp", - "lib/Tools/mlir-lsp-server/lsp/*.h", ], ), hdrs = glob( @@ -5270,8 +5219,6 @@ cc_library( name = "BytecodeReader", srcs = glob([ "lib/Bytecode/Reader/*.cpp", - "lib/Bytecode/Reader/*.h", - "lib/Bytecode/*.h", ]), hdrs = glob([ "include/mlir/Bytecode/*.h", @@ -5291,7 +5238,6 @@ cc_library( srcs = glob([ "lib/Bytecode/Writer/*.cpp", "lib/Bytecode/Writer/*.h", - "lib/Bytecode/*.h", ]), hdrs = glob([ "include/mlir/Bytecode/*.h", @@ -5309,7 +5255,6 @@ cc_library( name = "Parser", srcs = glob([ "lib/Parser/*.cpp", - "lib/Parser/*.h", ]), hdrs = glob([ "include/mlir/Parser/*.h", @@ -5440,7 +5385,6 @@ cc_library( name = "LLVMIRTransforms", srcs = glob([ "lib/Dialect/LLVMIR/Transforms/*.cpp", - "lib/Dialect/LLVMIR/Transforms/*.h", ]), hdrs = glob(["include/mlir/Dialect/LLVMIR/Transforms/*.h"]), includes = ["include"], @@ -5593,7 +5537,6 @@ cc_library( srcs = glob( [ "lib/Dialect/GPU/IR/*.cpp", - "lib/Dialect/GPU/IR/*.h", ], ), hdrs = glob(["include/mlir/Dialect/GPU/IR/*.h"]), @@ -5687,7 +5630,6 @@ cc_library( srcs = glob( [ "lib/Dialect/GPU/Transforms/*.cpp", - "lib/Dialect/GPU/Transforms/*.h", ], ), hdrs = glob(["include/mlir/Dialect/GPU/Transforms/*.h"]), @@ -5877,7 +5819,6 @@ cc_library( name = "GPUToNVVMTransforms", srcs = glob([ "lib/Conversion/GPUToNVVM/*.cpp", - "lib/Conversion/GPUToNVVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/GPUToNVVM/*.h", @@ -5913,7 +5854,6 @@ cc_library( name = "AMDGPUToROCDL", srcs = glob([ "lib/Conversion/AMDGPUToROCDL/*.cpp", - "lib/Conversion/AMDGPUToROCDL/*.h", ]) + ["include/mlir/Conversion/GPUToROCDL/Runtimes.h"], hdrs = glob([ "include/mlir/Conversion/AMDGPUToROCDL/*.h", @@ -5938,7 +5878,6 @@ cc_library( name = "NVGPUToNVVM", srcs = glob([ "lib/Conversion/NVGPUToNVVM/*.cpp", - "lib/Conversion/NVGPUToNVVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/NVGPUToNVVM/*.h", @@ -5966,7 +5905,6 @@ cc_library( name = "VectorToSPIRV", srcs = glob([ "lib/Conversion/VectorToSPIRV/*.cpp", - "lib/Conversion/VectorToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/VectorToSPIRV/*.h", @@ -6095,7 +6033,6 @@ cc_library( name = "GPUToSPIRV", srcs = glob([ "lib/Conversion/GPUToSPIRV/*.cpp", - "lib/Conversion/GPUToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/GPUToSPIRV/*.h", @@ -6665,7 +6602,6 @@ cc_library( name = "PDLDialect", srcs = glob([ "lib/Dialect/PDL/IR/*.cpp", - "lib/Dialect/PDL/IR/*.h", ]), hdrs = glob([ "include/mlir/Dialect/PDL/IR/*.h", @@ -6740,7 +6676,6 @@ cc_library( name = "PDLInterpDialect", srcs = glob([ "lib/Dialect/PDLInterp/IR/*.cpp", - "lib/Dialect/PDLInterp/IR/*.h", ]), hdrs = glob([ "include/mlir/Dialect/PDLInterp/IR/*.h", @@ -7038,7 +6973,6 @@ cc_library( srcs = glob( [ "lib/Dialect/SPIRV/Transforms/*.cpp", - "lib/Dialect/SPIRV/Transforms/*.h", ], exclude = ["lib/Dialect/SPIRV/Transforms/SPIRVConversion.cpp"], ), @@ -7077,7 +7011,6 @@ cc_library( name = "MathToSPIRV", srcs = glob([ "lib/Conversion/MathToSPIRV/*.cpp", - "lib/Conversion/MathToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/MathToSPIRV/*.h", @@ -7104,7 +7037,6 @@ cc_library( name = "FuncToEmitC", srcs = glob([ "lib/Conversion/FuncToEmitC/*.cpp", - "lib/Conversion/FuncToEmitC/*.h", ]), hdrs = glob([ "include/mlir/Conversion/FuncToEmitC/*.h", @@ -7130,7 +7062,6 @@ cc_library( name = "FuncToSPIRV", srcs = glob([ "lib/Conversion/FuncToSPIRV/*.cpp", - "lib/Conversion/FuncToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/FuncToSPIRV/*.h", @@ -7162,7 +7093,6 @@ cc_library( name = "TensorToLinalg", srcs = glob([ "lib/Conversion/TensorToLinalg/*.cpp", - "lib/Conversion/TensorToLinalg/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TensorToLinalg/*.h", @@ -7191,7 +7121,6 @@ cc_library( name = "TensorToSPIRV", srcs = glob([ "lib/Conversion/TensorToSPIRV/*.cpp", - "lib/Conversion/TensorToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TensorToSPIRV/*.h", @@ -7477,7 +7406,6 @@ cc_library( srcs = glob( [ "lib/Dialect/Tensor/Transforms/*.cpp", - "lib/Dialect/Tensor/Transforms/*.h", ], ), hdrs = glob(["include/mlir/Dialect/Tensor/Transforms/*.h"]), @@ -7584,7 +7512,6 @@ cc_library( srcs = glob( include = [ "lib/Transforms/Utils/*.cpp", - "lib/Transforms/Utils/*.h", ], exclude = ["lib/Transforms/Utils/InliningUtils.cpp"], ), @@ -7923,7 +7850,6 @@ cc_library( name = "Transforms", srcs = glob([ "lib/Transforms/*.cpp", - "lib/Transforms/*.h", ]), hdrs = glob(["include/mlir/Transforms/*.h"]), includes = ["include"], @@ -8011,7 +7937,6 @@ cc_library( name = "SCFToSPIRV", srcs = glob([ "lib/Conversion/SCFToSPIRV/*.cpp", - "lib/Conversion/SCFToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/SCFToSPIRV/*.h", @@ -8261,7 +8186,6 @@ cc_library( name = "MemRefToEmitC", srcs = glob([ "lib/Conversion/MemRefToEmitC/*.cpp", - "lib/Conversion/MemRefToEmitC/*.h", ]), hdrs = glob([ "include/mlir/Conversion/MemRefToEmitC/*.h", @@ -8311,7 +8235,6 @@ cc_library( name = "MemRefToSPIRV", srcs = glob([ "lib/Conversion/MemRefToSPIRV/*.cpp", - "lib/Conversion/MemRefToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/MemRefToSPIRV/*.h", @@ -8371,7 +8294,6 @@ cc_library( name = "ArithToArmSME", srcs = glob([ "lib/Conversion/ArithToArmSME/*.cpp", - "lib/Conversion/ArithToArmSME/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ArithToArmSME/*.h", @@ -8392,7 +8314,6 @@ cc_library( name = "ArithToEmitC", srcs = glob([ "lib/Conversion/ArithToEmitC/*.cpp", - "lib/Conversion/ArithToEmitC/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ArithToEmitC/*.h", @@ -8683,9 +8604,7 @@ cc_library( srcs = glob( [ "lib/Analysis/*.cpp", - "lib/Analysis/*.h", "lib/Analysis/*/*.cpp", - "lib/Analysis/*/*.h", ], ), hdrs = glob( @@ -8991,7 +8910,6 @@ cc_library( ":IR", ":LLVMDialect", ":OpenACCDialect", - ":OpenACCToLLVM", ":OpenMPCommon", ":Support", ":ToLLVMIRTranslation", @@ -9357,7 +9275,6 @@ cc_library( ":SCFToGPU", ":SCFTransformOps", ":SCFTransforms", - ":SDBM", ":SPIRVDialect", ":SPIRVPassIncGen", ":SPIRVTarget", @@ -9468,7 +9385,6 @@ cc_binary( "//mlir/test:TestTosaDialect", "//mlir/test:TestTransformDialect", "//mlir/test:TestTransforms", - "//mlir/test:TestTypeDialect", "//mlir/test:TestVector", "//mlir/test:TestVectorToSPIRV", ], @@ -10086,7 +10002,6 @@ cc_library( srcs = glob( [ "lib/Dialect/OpenACC/IR/*.cpp", - "lib/Dialect/OpenACC/IR/*.h", ], ), hdrs = glob( @@ -10140,7 +10055,6 @@ cc_library( srcs = glob( [ "lib/Dialect/OpenACC/Transforms/*.cpp", - "lib/Dialect/OpenACC/Transforms/*.h", ], ), hdrs = glob(["include/mlir/Dialect/OpenACC/Transforms/*.h"]), @@ -10304,7 +10218,6 @@ cc_library( srcs = glob( [ "lib/Dialect/OpenMP/IR/*.cpp", - "lib/Dialect/OpenMP/IR/*.h", ], ), hdrs = glob( @@ -10340,7 +10253,6 @@ cc_library( name = "OpenACCToSCF", srcs = glob([ "lib/Conversion/OpenACCToSCF/*.cpp", - "lib/Conversion/OpenACCToSCF/*.h", ]), hdrs = glob([ "include/mlir/Conversion/OpenACCToSCF/*.h", @@ -10360,35 +10272,10 @@ cc_library( ], ) -cc_library( - name = "OpenACCToLLVM", - srcs = glob([ - "lib/Conversion/OpenACCToLLVM/*.cpp", - "lib/Conversion/OpenACCToLLVM/*.h", - ]), - hdrs = glob([ - "include/mlir/Conversion/OpenACCToLLVM/*.h", - ]), - includes = ["include"], - deps = [ - ":ConversionPassIncGen", - ":FuncDialect", - ":IR", - ":LLVMCommonConversion", - ":LLVMDialect", - ":OpenACCDialect", - ":Pass", - ":Transforms", - "//llvm:Core", - "//llvm:Support", - ], -) - cc_library( name = "OpenMPToLLVM", srcs = glob([ "lib/Conversion/OpenMPToLLVM/*.cpp", - "lib/Conversion/OpenMPToLLVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/OpenMPToLLVM/*.h", @@ -10598,7 +10485,6 @@ cc_library( name = "IndexToLLVM", srcs = glob([ "lib/Conversion/IndexToLLVM/*.cpp", - "lib/Conversion/IndexToLLVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/IndexToLLVM/*.h", @@ -10624,7 +10510,6 @@ cc_library( name = "IndexToSPIRV", srcs = glob([ "lib/Conversion/IndexToSPIRV/*.cpp", - "lib/Conversion/IndexToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/IndexToSPIRV/*.h", @@ -11015,7 +10900,6 @@ cc_library( name = "LinalgToStandard", srcs = glob([ "lib/Conversion/LinalgToStandard/*.cpp", - "lib/Conversion/LinalgToStandard/*.h", ]), hdrs = glob([ "include/mlir/Conversion/LinalgToStandard/*.h", @@ -11161,7 +11045,6 @@ cc_library( name = "LinalgUtils", srcs = glob([ "lib/Dialect/Linalg/Utils/*.cpp", - "lib/Dialect/Linalg/Utils/*.h", ]), hdrs = glob([ "include/mlir/Dialect/Linalg/Utils/*.h", @@ -11190,7 +11073,6 @@ cc_library( name = "LinalgTransforms", srcs = glob([ "lib/Dialect/Linalg/Transforms/*.cpp", - "lib/Dialect/Linalg/Transforms/*.h", ]), hdrs = [ "include/mlir/Dialect/Linalg/Passes.h", @@ -11542,7 +11424,6 @@ cc_library( name = "VectorToLLVM", srcs = glob([ "lib/Conversion/VectorToLLVM/*.cpp", - "lib/Conversion/VectorToLLVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/VectorToLLVM/*.h", @@ -11584,7 +11465,6 @@ cc_library( name = "VectorToArmSME", srcs = glob([ "lib/Conversion/VectorToArmSME/*.cpp", - "lib/Conversion/VectorToArmSME/*.h", ]), hdrs = glob([ "include/mlir/Conversion/VectorToArmSME/*.h", @@ -11605,7 +11485,6 @@ cc_library( name = "VectorToGPU", srcs = glob([ "lib/Conversion/VectorToGPU/*.cpp", - "lib/Conversion/VectorToGPU/*.h", ]), hdrs = glob([ "include/mlir/Conversion/VectorToGPU/*.h", @@ -11642,7 +11521,6 @@ cc_library( name = "VectorToSCF", srcs = glob([ "lib/Conversion/VectorToSCF/*.cpp", - "lib/Conversion/VectorToSCF/*.h", ]), hdrs = glob([ "include/mlir/Conversion/VectorToSCF/*.h", @@ -11792,7 +11670,6 @@ cc_library( name = "TosaDialect", srcs = glob([ "lib/Dialect/Tosa/IR/*.cpp", - "lib/Dialect/Tosa/IR/*.h", "lib/Dialect/Tosa/Utils/*.cpp", "lib/Dialect/Tosa/Transforms/*.cpp", ]), @@ -11832,7 +11709,6 @@ cc_library( name = "TosaToArith", srcs = glob([ "lib/Conversion/TosaToArith/*.cpp", - "lib/Conversion/TosaToArith/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TosaToArith/*.h", @@ -11856,7 +11732,6 @@ cc_library( name = "TosaToLinalg", srcs = glob([ "lib/Conversion/TosaToLinalg/*.cpp", - "lib/Conversion/TosaToLinalg/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TosaToLinalg/*.h", @@ -11889,7 +11764,6 @@ cc_library( name = "TosaToMLProgram", srcs = glob([ "lib/Conversion/TosaToMLProgram/*.cpp", - "lib/Conversion/TosaToMLProgram/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TosaToMLProgram/*.h", @@ -11913,7 +11787,6 @@ cc_library( name = "TosaToSCF", srcs = glob([ "lib/Conversion/TosaToSCF/*.cpp", - "lib/Conversion/TosaToSCF/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TosaToSCF/*.h", @@ -11938,7 +11811,6 @@ cc_library( name = "TosaToTensor", srcs = glob([ "lib/Conversion/TosaToTensor/*.cpp", - "lib/Conversion/TosaToTensor/*.h", ]), hdrs = glob([ "include/mlir/Conversion/TosaToTensor/*.h", @@ -12443,7 +12315,6 @@ cc_library( srcs = glob( [ "lib/Dialect/Complex/IR/*.cpp", - "lib/Dialect/Complex/IR/*.h", ], ), hdrs = ["include/mlir/Dialect/Complex/IR/Complex.h"], @@ -12466,7 +12337,6 @@ cc_library( name = "ComplexToLLVM", srcs = glob([ "lib/Conversion/ComplexToLLVM/*.cpp", - "lib/Conversion/ComplexToLLVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ComplexToLLVM/*.h", @@ -12494,7 +12364,6 @@ cc_library( name = "ComplexToLibm", srcs = glob([ "lib/Conversion/ComplexToLibm/*.cpp", - "lib/Conversion/ComplexToLibm/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ComplexToLibm/*.h", @@ -12518,7 +12387,6 @@ cc_library( name = "ComplexToSPIRV", srcs = glob([ "lib/Conversion/ComplexToSPIRV/*.cpp", - "lib/Conversion/ComplexToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ComplexToSPIRV/*.h", @@ -12543,7 +12411,6 @@ cc_library( name = "ComplexToStandard", srcs = glob([ "lib/Conversion/ComplexToStandard/*.cpp", - "lib/Conversion/ComplexToStandard/*.h", ]), hdrs = glob([ "include/mlir/Conversion/ComplexToStandard/*.h", @@ -12760,7 +12627,6 @@ cc_library( name = "ArithTransforms", srcs = glob([ "lib/Dialect/Arith/Transforms/*.cpp", - "lib/Dialect/Arith/Transforms/*.h", ]), hdrs = glob([ "include/mlir/Dialect/Arith/Transforms/*.h", @@ -12882,7 +12748,6 @@ cc_library( srcs = glob( [ "lib/Dialect/Math/IR/*.cpp", - "lib/Dialect/Math/IR/*.h", ], ), hdrs = [ @@ -12909,7 +12774,6 @@ cc_library( name = "MathTransforms", srcs = glob([ "lib/Dialect/Math/Transforms/*.cpp", - "lib/Dialect/Math/Transforms/*.h", ]), hdrs = glob(["include/mlir/Dialect/Math/Transforms/*.h"]), includes = ["include"], @@ -12933,7 +12797,6 @@ cc_library( name = "MathToLibm", srcs = glob([ "lib/Conversion/MathToLibm/*.cpp", - "lib/Conversion/MathToLibm/*.h", ]), hdrs = glob([ "include/mlir/Conversion/MathToLibm/*.h", @@ -13024,7 +12887,6 @@ cc_library( srcs = glob( [ "lib/Dialect/MemRef/IR/*.cpp", - "lib/Dialect/MemRef/IR/*.h", ], ), hdrs = [ @@ -13102,7 +12964,6 @@ cc_library( srcs = glob( [ "lib/Dialect/MemRef/Transforms/*.cpp", - "lib/Dialect/MemRef/Transforms/*.h", ], ), hdrs = glob(["include/mlir/Dialect/MemRef/Transforms/*.h"]), @@ -13730,7 +13591,6 @@ cc_library( srcs = glob( [ "lib/Dialect/Bufferization/Transforms/*.cpp", - "lib/Dialect/Bufferization/Transforms/*.h", ], ), hdrs = glob(["include/mlir/Dialect/Bufferization/Transforms/*.h"]), @@ -13922,7 +13782,6 @@ cc_library( srcs = glob( [ "lib/Tools/PDLL/ODS/*.cpp", - "lib/Tools/PDLL/ODS/*.h", ], ), hdrs = glob(["include/mlir/Tools/PDLL/ODS/*.h"]), @@ -13956,7 +13815,6 @@ cc_library( srcs = glob( [ "lib/Tools/PDLL/CodeGen/*.cpp", - "lib/Tools/PDLL/CodeGen/*.h", ], ), hdrs = glob(["include/mlir/Tools/PDLL/CodeGen/*.h"]), @@ -14117,7 +13975,6 @@ cc_library( name = "UBToLLVM", srcs = glob([ "lib/Conversion/UBToLLVM/*.cpp", - "lib/Conversion/UBToLLVM/*.h", ]), hdrs = glob([ "include/mlir/Conversion/UBToLLVM/*.h", @@ -14138,7 +13995,6 @@ cc_library( name = "UBToSPIRV", srcs = glob([ "lib/Conversion/UBToSPIRV/*.cpp", - "lib/Conversion/UBToSPIRV/*.h", ]), hdrs = glob([ "include/mlir/Conversion/UBToSPIRV/*.h", diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel index df2392fd8c6e..57b29eb46e08 100644 --- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel @@ -818,7 +818,6 @@ cc_library( cc_library( name = "TestMesh", srcs = glob(["lib/Dialect/Mesh/**/*.cpp"]), - hdrs = glob(["lib/Dialect/Mesh/**/*.h"]), includes = ["lib/Dialect/Test"], deps = [ ":TestDialect", @@ -1046,18 +1045,6 @@ cc_library( ], ) -cc_library( - name = "TestTypeDialect", - srcs = glob([ - "lib/Dialect/LLVMIR/*.cpp", - ]), - deps = [ - ":TestDialect", - "//mlir:IR", - "//mlir:LLVMDialect", - ], -) - cc_library( name = "TestTosaDialect", srcs = glob([ diff --git a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel index c6b630230bb3..7172beb4de9a 100644 --- a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel @@ -16,7 +16,6 @@ cc_test( size = "small", srcs = glob([ "Debug/*.cpp", - "Debug/*.h", ]), deps = [ "//llvm:Support", @@ -35,7 +34,6 @@ cc_test( size = "small", srcs = glob([ "IR/*.cpp", - "IR/*.h", ]), deps = [ "//llvm:Support", @@ -57,7 +55,6 @@ cc_test( size = "small", srcs = glob([ "Interfaces/*.cpp", - "Interfaces/*.h", ]), deps = [ "//llvm:Support", @@ -82,7 +79,6 @@ cc_test( size = "small", srcs = glob([ "Support/*.cpp", - "Support/*.h", ]), deps = [ "//llvm:Support", @@ -98,7 +94,6 @@ cc_test( size = "small", srcs = glob([ "Pass/*.cpp", - "Pass/*.h", ]), deps = [ "//llvm:Support", @@ -118,7 +113,6 @@ cc_test( size = "small", srcs = glob([ "Rewrite/*.cpp", - "Rewrite/*.h", ]), deps = [ "//mlir:IR", @@ -134,7 +128,6 @@ cc_test( size = "small", srcs = glob([ "Dialect/*.cpp", - "Dialect/*.h", ]), deps = [ "//llvm:Support", @@ -149,7 +142,6 @@ cc_test( size = "small", srcs = glob([ "Dialect/MemRef/*.cpp", - "Dialect/MemRef/*.h", ]), deps = [ "//llvm:TestingSupport", @@ -161,27 +153,11 @@ cc_test( ], ) -cc_test( - name = "quantops_tests", - size = "small", - srcs = glob([ - "Dialect/Quant/*.cpp", - "Dialect/Quant/*.h", - ]), - deps = [ - "//llvm:TestingSupport", - "//mlir:QuantOps", - "//mlir:Transforms", - "//third-party/unittest:gtest_main", - ], -) - cc_test( name = "scf_tests", size = "small", srcs = glob([ "Dialect/SCF/*.cpp", - "Dialect/SCF/*.h", ]), deps = [ "//mlir:ArithDialect", @@ -200,7 +176,6 @@ cc_test( size = "small", srcs = glob([ "Dialect/SparseTensor/*.cpp", - "Dialect/SparseTensor/*.h", ]), deps = [ "//llvm:Support", @@ -218,7 +193,6 @@ cc_test( size = "small", srcs = glob([ "Dialect/SPIRV/*.cpp", - "Dialect/SPIRV/*.h", ]), deps = [ "//llvm:Support", @@ -238,7 +212,6 @@ cc_test( size = "small", srcs = glob([ "Dialect/Transform/*.cpp", - "Dialect/Transform/*.h", ]), deps = [ "//llvm:Support", @@ -264,7 +237,6 @@ cc_test( size = "small", srcs = glob([ "Dialect/Utils/*.cpp", - "Dialect/Utils/*.h", ]), deps = [ "//llvm:Support", @@ -317,7 +289,6 @@ cc_test( size = "small", srcs = glob([ "TableGen/*.cpp", - "TableGen/*.h", ]) + [ "TableGen/EnumsGenTest.cpp.inc", "TableGen/EnumsGenTest.h.inc", @@ -343,7 +314,6 @@ cc_test( size = "small", srcs = glob([ "Transforms/*.cpp", - "Transforms/*.h", ]), deps = [ "//mlir:AffineAnalysis", @@ -363,8 +333,6 @@ cc_test( name = "analysis_tests", size = "small", srcs = glob([ - "Analysis/*.cpp", - "Analysis/*.h", "Analysis/*/*.cpp", "Analysis/*/*.h", ]), @@ -387,9 +355,6 @@ cc_test( size = "small", srcs = glob([ "Bytecode/*.cpp", - "Bytecode/*.h", - "Bytecode/*/*.cpp", - "Bytecode/*/*.h", ]), deps = [ "//llvm:Support", @@ -408,10 +373,7 @@ cc_test( name = "conversion_tests", size = "small", srcs = glob([ - "Conversion/*.cpp", - "Conversion/*.h", "Conversion/*/*.cpp", - "Conversion/*/*.h", ]), deps = [ "//mlir:ArithDialect", -- GitLab From 5184e6ad69b0ca69dfba6fb0982a675c595f49a2 Mon Sep 17 00:00:00 2001 From: Aaron Ballman Date: Fri, 22 Mar 2024 13:00:32 -0400 Subject: [PATCH 283/296] Removing accidental code from 527a624205748814dd9309eda7ee308b40b2359a --- clang/test/C/C11/n1365.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clang/test/C/C11/n1365.c b/clang/test/C/C11/n1365.c index 3f769faa365d..d60bb546b29a 100644 --- a/clang/test/C/C11/n1365.c +++ b/clang/test/C/C11/n1365.c @@ -23,7 +23,7 @@ void func(void) { // CHECK-NEXT: FloatingLiteral // Ensure that a cast removes the extra precision. - _Static_assert((float)(123.0F * 2.0F) == 246.0F, ""); + _Static_assert(123.0F * 2.0F == 246.0F, ""); // CHECK: StaticAssertDecl // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Bool' // CHECK-NEXT: BinaryOperator {{.*}} 'int' '==' -- GitLab From bd493756fa51e538575fc320aae50d75394f0567 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Fri, 22 Mar 2024 17:01:39 +0000 Subject: [PATCH 284/296] [llvm-exegesis] Refactor parent code to separate function (#86232) This patch refactors the parent code to a separate function in the subprocess executor to make the code more clear and easy to follow. --- .../llvm-exegesis/lib/BenchmarkRunner.cpp | 116 ++++++++++-------- 1 file changed, 63 insertions(+), 53 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index f0452605eb24..6d8da7bd86e3 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -278,59 +278,20 @@ private: return FD; } - Error createSubProcessAndRunBenchmark( - StringRef CounterName, SmallVectorImpl &CounterValues, - ArrayRef ValidationCounters, - SmallVectorImpl &ValidationCounterValues) const { - int PipeFiles[2]; - int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles); - if (PipeSuccessOrErr != 0) { - return make_error( - "Failed to create a pipe for interprocess communication between " - "llvm-exegesis and the benchmarking subprocess: " + - Twine(strerror(errno))); - } - - SubprocessMemory SPMemory; - Error MemoryInitError = SPMemory.initializeSubprocessMemory(getpid()); - if (MemoryInitError) - return MemoryInitError; - - Error AddMemDefError = - SPMemory.addMemoryDefinition(Key.MemoryValues, getpid()); - if (AddMemDefError) - return AddMemDefError; - - pid_t ParentOrChildPID = fork(); - - if (ParentOrChildPID == -1) { - return make_error("Failed to create child process: " + - Twine(strerror(errno))); - } - - if (ParentOrChildPID == 0) { - // We are in the child process, close the write end of the pipe. - close(PipeFiles[1]); - // Unregister handlers, signal handling is now handled through ptrace in - // the host process. - sys::unregisterHandlers(); - prepareAndRunBenchmark(PipeFiles[0], Key); - // The child process terminates in the above function, so we should never - // get to this point. - llvm_unreachable("Child process didn't exit when expected."); - } - + Error + runParentProcess(pid_t ChildPID, int WriteFD, StringRef CounterName, + SmallVectorImpl &CounterValues, + ArrayRef ValidationCounters, + SmallVectorImpl &ValidationCounterValues) const { const ExegesisTarget &ET = State.getExegesisTarget(); - auto CounterOrError = ET.createCounter( - CounterName, State, ValidationCounters, ParentOrChildPID); + auto CounterOrError = + ET.createCounter(CounterName, State, ValidationCounters, ChildPID); if (!CounterOrError) return CounterOrError.takeError(); pfm::CounterGroup *Counter = CounterOrError.get().get(); - close(PipeFiles[0]); - // Make sure to attach to the process (and wait for the sigstop to be // delivered and for the process to continue) before we write to the counter // file descriptor. Attaching to the process before writing to the socket @@ -338,7 +299,7 @@ private: // attach afterwards, the subprocess might exit before we get to the attach // call due to effects like scheduler contention, introducing transient // failures. - if (ptrace(PTRACE_ATTACH, ParentOrChildPID, NULL, NULL) != 0) + if (ptrace(PTRACE_ATTACH, ChildPID, NULL, NULL) != 0) return make_error("Failed to attach to the child process: " + Twine(strerror(errno))); @@ -348,14 +309,14 @@ private: Twine(strerror(errno))); } - if (ptrace(PTRACE_CONT, ParentOrChildPID, NULL, NULL) != 0) + if (ptrace(PTRACE_CONT, ChildPID, NULL, NULL) != 0) return make_error( "Failed to continue execution of the child process: " + Twine(strerror(errno))); int CounterFileDescriptor = Counter->getFileDescriptor(); Error SendError = - sendFileDescriptorThroughSocket(PipeFiles[1], CounterFileDescriptor); + sendFileDescriptorThroughSocket(WriteFD, CounterFileDescriptor); if (SendError) return SendError; @@ -395,8 +356,7 @@ private: // An error was encountered running the snippet, process it siginfo_t ChildSignalInfo; - if (ptrace(PTRACE_GETSIGINFO, ParentOrChildPID, NULL, &ChildSignalInfo) == - -1) { + if (ptrace(PTRACE_GETSIGINFO, ChildPID, NULL, &ChildSignalInfo) == -1) { return make_error("Getting signal info from the child failed: " + Twine(strerror(errno))); } @@ -422,6 +382,56 @@ private: return make_error(ChildSignalInfo.si_signo); } + Error createSubProcessAndRunBenchmark( + StringRef CounterName, SmallVectorImpl &CounterValues, + ArrayRef ValidationCounters, + SmallVectorImpl &ValidationCounterValues) const { + int PipeFiles[2]; + int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles); + if (PipeSuccessOrErr != 0) { + return make_error( + "Failed to create a pipe for interprocess communication between " + "llvm-exegesis and the benchmarking subprocess: " + + Twine(strerror(errno))); + } + + SubprocessMemory SPMemory; + Error MemoryInitError = SPMemory.initializeSubprocessMemory(getpid()); + if (MemoryInitError) + return MemoryInitError; + + Error AddMemDefError = + SPMemory.addMemoryDefinition(Key.MemoryValues, getpid()); + if (AddMemDefError) + return AddMemDefError; + + pid_t ParentOrChildPID = fork(); + + if (ParentOrChildPID == -1) { + return make_error("Failed to create child process: " + + Twine(strerror(errno))); + } + + if (ParentOrChildPID == 0) { + // We are in the child process, close the write end of the pipe. + close(PipeFiles[1]); + // Unregister handlers, signal handling is now handled through ptrace in + // the host process. + sys::unregisterHandlers(); + runChildSubprocess(PipeFiles[0], Key); + // The child process terminates in the above function, so we should never + // get to this point. + llvm_unreachable("Child process didn't exit when expected."); + } + + // Close the read end of the pipe as we only need to write to the subprocess + // from the parent process. + close(PipeFiles[0]); + return runParentProcess(ParentOrChildPID, PipeFiles[1], CounterName, + CounterValues, ValidationCounters, + ValidationCounterValues); + } + void disableCoreDumps() const { struct rlimit rlim; @@ -429,8 +439,8 @@ private: setrlimit(RLIMIT_CORE, &rlim); } - [[noreturn]] void prepareAndRunBenchmark(int Pipe, - const BenchmarkKey &Key) const { + [[noreturn]] void runChildSubprocess(int Pipe, + const BenchmarkKey &Key) const { // Disable core dumps in the child process as otherwise everytime we // encounter an execution failure like a segmentation fault, we will create // a core dump. We report the information directly rather than require the -- GitLab From d2f8ba7d6dc7251815f1431cf8715053576615f4 Mon Sep 17 00:00:00 2001 From: Sacha Coppey Date: Fri, 22 Mar 2024 18:08:13 +0100 Subject: [PATCH 285/296] [RISCV][NFC] Add generateMCInstSeq in RISCVMatInt (#84462) This allows to avoid duplicating the code handling the instructions outputted by `generateInstSeq` when emitting `MCInst`s. --- .../Target/RISCV/AsmParser/RISCVAsmParser.cpp | 31 ++------------- .../Target/RISCV/MCTargetDesc/RISCVMatInt.cpp | 38 +++++++++++++++++++ .../Target/RISCV/MCTargetDesc/RISCVMatInt.h | 5 +++ 3 files changed, 47 insertions(+), 27 deletions(-) diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index 1779959324da..cb2ba52390e2 100644 --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -3081,34 +3081,11 @@ void RISCVAsmParser::emitToStreamer(MCStreamer &S, const MCInst &Inst) { void RISCVAsmParser::emitLoadImm(MCRegister DestReg, int64_t Value, MCStreamer &Out) { - RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Value, getSTI()); - - MCRegister SrcReg = RISCV::X0; - for (const RISCVMatInt::Inst &Inst : Seq) { - switch (Inst.getOpndKind()) { - case RISCVMatInt::Imm: - emitToStreamer(Out, - MCInstBuilder(Inst.getOpcode()).addReg(DestReg).addImm(Inst.getImm())); - break; - case RISCVMatInt::RegX0: - emitToStreamer( - Out, MCInstBuilder(Inst.getOpcode()).addReg(DestReg).addReg(SrcReg).addReg( - RISCV::X0)); - break; - case RISCVMatInt::RegReg: - emitToStreamer( - Out, MCInstBuilder(Inst.getOpcode()).addReg(DestReg).addReg(SrcReg).addReg( - SrcReg)); - break; - case RISCVMatInt::RegImm: - emitToStreamer( - Out, MCInstBuilder(Inst.getOpcode()).addReg(DestReg).addReg(SrcReg).addImm( - Inst.getImm())); - break; - } + SmallVector Seq; + RISCVMatInt::generateMCInstSeq(Value, getSTI(), DestReg, Seq); - // Only the first instruction has X0 as its source. - SrcReg = DestReg; + for (MCInst &Inst : Seq) { + emitToStreamer(Out, Inst); } } diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp index 4358a5b878e6..c3bae152993e 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp @@ -9,6 +9,7 @@ #include "RISCVMatInt.h" #include "MCTargetDesc/RISCVMCTargetDesc.h" #include "llvm/ADT/APInt.h" +#include "llvm/MC/MCInstBuilder.h" #include "llvm/Support/MathExtras.h" using namespace llvm; @@ -436,6 +437,43 @@ InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI) { return Res; } +void generateMCInstSeq(int64_t Val, const MCSubtargetInfo &STI, + MCRegister DestReg, SmallVectorImpl &Insts) { + RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, STI); + + MCRegister SrcReg = RISCV::X0; + for (RISCVMatInt::Inst &Inst : Seq) { + switch (Inst.getOpndKind()) { + case RISCVMatInt::Imm: + Insts.push_back(MCInstBuilder(Inst.getOpcode()) + .addReg(DestReg) + .addImm(Inst.getImm())); + break; + case RISCVMatInt::RegX0: + Insts.push_back(MCInstBuilder(Inst.getOpcode()) + .addReg(DestReg) + .addReg(SrcReg) + .addReg(RISCV::X0)); + break; + case RISCVMatInt::RegReg: + Insts.push_back(MCInstBuilder(Inst.getOpcode()) + .addReg(DestReg) + .addReg(SrcReg) + .addReg(SrcReg)); + break; + case RISCVMatInt::RegImm: + Insts.push_back(MCInstBuilder(Inst.getOpcode()) + .addReg(DestReg) + .addReg(SrcReg) + .addImm(Inst.getImm())); + break; + } + + // Only the first instruction has X0 as its source. + SrcReg = DestReg; + } +} + InstSeq generateTwoRegInstSeq(int64_t Val, const MCSubtargetInfo &STI, unsigned &ShiftAmt, unsigned &AddOpc) { int64_t LoVal = SignExtend64<32>(Val); diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.h index 780f685463f3..e87e0f325647 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.h +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.h @@ -10,6 +10,7 @@ #define LLVM_LIB_TARGET_RISCV_MCTARGETDESC_MATINT_H #include "llvm/ADT/SmallVector.h" +#include "llvm/MC/MCRegister.h" #include "llvm/MC/MCSubtargetInfo.h" #include @@ -48,6 +49,10 @@ using InstSeq = SmallVector; // instruction selection. InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI); +// Helper to generate the generateInstSeq instruction sequence using MCInsts +void generateMCInstSeq(int64_t Val, const MCSubtargetInfo &STI, + MCRegister DestReg, SmallVectorImpl &Insts); + // Helper to generate an instruction sequence that can materialize the given // immediate value into a register using an additional temporary register. This // handles cases where the constant can be generated by (ADD (SLLI X, C), X) or -- GitLab From c3a41aac5f32475b9a0499e6e888e713763566dc Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Fri, 22 Mar 2024 10:25:23 -0700 Subject: [PATCH 286/296] Revert "[llvm-exegesis] Refactor parent code to separate function (#86232)" This reverts commit bd493756fa51e538575fc320aae50d75394f0567. Causes build failures on non-X86 platforms. https://lab.llvm.org/buildbot/#/changes/128363 --- .../llvm-exegesis/lib/BenchmarkRunner.cpp | 116 ++++++++---------- 1 file changed, 53 insertions(+), 63 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index 6d8da7bd86e3..f0452605eb24 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -278,20 +278,59 @@ private: return FD; } - Error - runParentProcess(pid_t ChildPID, int WriteFD, StringRef CounterName, - SmallVectorImpl &CounterValues, - ArrayRef ValidationCounters, - SmallVectorImpl &ValidationCounterValues) const { + Error createSubProcessAndRunBenchmark( + StringRef CounterName, SmallVectorImpl &CounterValues, + ArrayRef ValidationCounters, + SmallVectorImpl &ValidationCounterValues) const { + int PipeFiles[2]; + int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles); + if (PipeSuccessOrErr != 0) { + return make_error( + "Failed to create a pipe for interprocess communication between " + "llvm-exegesis and the benchmarking subprocess: " + + Twine(strerror(errno))); + } + + SubprocessMemory SPMemory; + Error MemoryInitError = SPMemory.initializeSubprocessMemory(getpid()); + if (MemoryInitError) + return MemoryInitError; + + Error AddMemDefError = + SPMemory.addMemoryDefinition(Key.MemoryValues, getpid()); + if (AddMemDefError) + return AddMemDefError; + + pid_t ParentOrChildPID = fork(); + + if (ParentOrChildPID == -1) { + return make_error("Failed to create child process: " + + Twine(strerror(errno))); + } + + if (ParentOrChildPID == 0) { + // We are in the child process, close the write end of the pipe. + close(PipeFiles[1]); + // Unregister handlers, signal handling is now handled through ptrace in + // the host process. + sys::unregisterHandlers(); + prepareAndRunBenchmark(PipeFiles[0], Key); + // The child process terminates in the above function, so we should never + // get to this point. + llvm_unreachable("Child process didn't exit when expected."); + } + const ExegesisTarget &ET = State.getExegesisTarget(); - auto CounterOrError = - ET.createCounter(CounterName, State, ValidationCounters, ChildPID); + auto CounterOrError = ET.createCounter( + CounterName, State, ValidationCounters, ParentOrChildPID); if (!CounterOrError) return CounterOrError.takeError(); pfm::CounterGroup *Counter = CounterOrError.get().get(); + close(PipeFiles[0]); + // Make sure to attach to the process (and wait for the sigstop to be // delivered and for the process to continue) before we write to the counter // file descriptor. Attaching to the process before writing to the socket @@ -299,7 +338,7 @@ private: // attach afterwards, the subprocess might exit before we get to the attach // call due to effects like scheduler contention, introducing transient // failures. - if (ptrace(PTRACE_ATTACH, ChildPID, NULL, NULL) != 0) + if (ptrace(PTRACE_ATTACH, ParentOrChildPID, NULL, NULL) != 0) return make_error("Failed to attach to the child process: " + Twine(strerror(errno))); @@ -309,14 +348,14 @@ private: Twine(strerror(errno))); } - if (ptrace(PTRACE_CONT, ChildPID, NULL, NULL) != 0) + if (ptrace(PTRACE_CONT, ParentOrChildPID, NULL, NULL) != 0) return make_error( "Failed to continue execution of the child process: " + Twine(strerror(errno))); int CounterFileDescriptor = Counter->getFileDescriptor(); Error SendError = - sendFileDescriptorThroughSocket(WriteFD, CounterFileDescriptor); + sendFileDescriptorThroughSocket(PipeFiles[1], CounterFileDescriptor); if (SendError) return SendError; @@ -356,7 +395,8 @@ private: // An error was encountered running the snippet, process it siginfo_t ChildSignalInfo; - if (ptrace(PTRACE_GETSIGINFO, ChildPID, NULL, &ChildSignalInfo) == -1) { + if (ptrace(PTRACE_GETSIGINFO, ParentOrChildPID, NULL, &ChildSignalInfo) == + -1) { return make_error("Getting signal info from the child failed: " + Twine(strerror(errno))); } @@ -382,56 +422,6 @@ private: return make_error(ChildSignalInfo.si_signo); } - Error createSubProcessAndRunBenchmark( - StringRef CounterName, SmallVectorImpl &CounterValues, - ArrayRef ValidationCounters, - SmallVectorImpl &ValidationCounterValues) const { - int PipeFiles[2]; - int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles); - if (PipeSuccessOrErr != 0) { - return make_error( - "Failed to create a pipe for interprocess communication between " - "llvm-exegesis and the benchmarking subprocess: " + - Twine(strerror(errno))); - } - - SubprocessMemory SPMemory; - Error MemoryInitError = SPMemory.initializeSubprocessMemory(getpid()); - if (MemoryInitError) - return MemoryInitError; - - Error AddMemDefError = - SPMemory.addMemoryDefinition(Key.MemoryValues, getpid()); - if (AddMemDefError) - return AddMemDefError; - - pid_t ParentOrChildPID = fork(); - - if (ParentOrChildPID == -1) { - return make_error("Failed to create child process: " + - Twine(strerror(errno))); - } - - if (ParentOrChildPID == 0) { - // We are in the child process, close the write end of the pipe. - close(PipeFiles[1]); - // Unregister handlers, signal handling is now handled through ptrace in - // the host process. - sys::unregisterHandlers(); - runChildSubprocess(PipeFiles[0], Key); - // The child process terminates in the above function, so we should never - // get to this point. - llvm_unreachable("Child process didn't exit when expected."); - } - - // Close the read end of the pipe as we only need to write to the subprocess - // from the parent process. - close(PipeFiles[0]); - return runParentProcess(ParentOrChildPID, PipeFiles[1], CounterName, - CounterValues, ValidationCounters, - ValidationCounterValues); - } - void disableCoreDumps() const { struct rlimit rlim; @@ -439,8 +429,8 @@ private: setrlimit(RLIMIT_CORE, &rlim); } - [[noreturn]] void runChildSubprocess(int Pipe, - const BenchmarkKey &Key) const { + [[noreturn]] void prepareAndRunBenchmark(int Pipe, + const BenchmarkKey &Key) const { // Disable core dumps in the child process as otherwise everytime we // encounter an execution failure like a segmentation fault, we will create // a core dump. We report the information directly rather than require the -- GitLab From 36a6afdd2c7fa02548260ebe4c993b705c6e6e38 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Fri, 22 Mar 2024 10:26:24 -0700 Subject: [PATCH 287/296] Reland "[llvm-exegesis] Refactor parent code to separate function (#86232)" This reverts commit c3a41aac5f32475b9a0499e6e888e713763566dc. This relands commit bd493756fa51e538575fc320aae50d75394f0567. Apparently I forgot to update a couple values, so this change failed on every builder that builds those sections (should be every Linux platform) rather than something architecture specific like originally thought. I swore I updated the values and ran check-llvm before merging. Wondering If I forgot to push those changes... --- .../llvm-exegesis/lib/BenchmarkRunner.cpp | 124 ++++++++++-------- 1 file changed, 67 insertions(+), 57 deletions(-) diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index f0452605eb24..6d5f0286fb9c 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -278,59 +278,20 @@ private: return FD; } - Error createSubProcessAndRunBenchmark( - StringRef CounterName, SmallVectorImpl &CounterValues, - ArrayRef ValidationCounters, - SmallVectorImpl &ValidationCounterValues) const { - int PipeFiles[2]; - int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles); - if (PipeSuccessOrErr != 0) { - return make_error( - "Failed to create a pipe for interprocess communication between " - "llvm-exegesis and the benchmarking subprocess: " + - Twine(strerror(errno))); - } - - SubprocessMemory SPMemory; - Error MemoryInitError = SPMemory.initializeSubprocessMemory(getpid()); - if (MemoryInitError) - return MemoryInitError; - - Error AddMemDefError = - SPMemory.addMemoryDefinition(Key.MemoryValues, getpid()); - if (AddMemDefError) - return AddMemDefError; - - pid_t ParentOrChildPID = fork(); - - if (ParentOrChildPID == -1) { - return make_error("Failed to create child process: " + - Twine(strerror(errno))); - } - - if (ParentOrChildPID == 0) { - // We are in the child process, close the write end of the pipe. - close(PipeFiles[1]); - // Unregister handlers, signal handling is now handled through ptrace in - // the host process. - sys::unregisterHandlers(); - prepareAndRunBenchmark(PipeFiles[0], Key); - // The child process terminates in the above function, so we should never - // get to this point. - llvm_unreachable("Child process didn't exit when expected."); - } - + Error + runParentProcess(pid_t ChildPID, int WriteFD, StringRef CounterName, + SmallVectorImpl &CounterValues, + ArrayRef ValidationCounters, + SmallVectorImpl &ValidationCounterValues) const { const ExegesisTarget &ET = State.getExegesisTarget(); - auto CounterOrError = ET.createCounter( - CounterName, State, ValidationCounters, ParentOrChildPID); + auto CounterOrError = + ET.createCounter(CounterName, State, ValidationCounters, ChildPID); if (!CounterOrError) return CounterOrError.takeError(); pfm::CounterGroup *Counter = CounterOrError.get().get(); - close(PipeFiles[0]); - // Make sure to attach to the process (and wait for the sigstop to be // delivered and for the process to continue) before we write to the counter // file descriptor. Attaching to the process before writing to the socket @@ -338,30 +299,30 @@ private: // attach afterwards, the subprocess might exit before we get to the attach // call due to effects like scheduler contention, introducing transient // failures. - if (ptrace(PTRACE_ATTACH, ParentOrChildPID, NULL, NULL) != 0) + if (ptrace(PTRACE_ATTACH, ChildPID, NULL, NULL) != 0) return make_error("Failed to attach to the child process: " + Twine(strerror(errno))); - if (waitpid(ParentOrChildPID, NULL, 0) == -1) { + if (waitpid(ChildPID, NULL, 0) == -1) { return make_error( "Failed to wait for child process to stop after attaching: " + Twine(strerror(errno))); } - if (ptrace(PTRACE_CONT, ParentOrChildPID, NULL, NULL) != 0) + if (ptrace(PTRACE_CONT, ChildPID, NULL, NULL) != 0) return make_error( "Failed to continue execution of the child process: " + Twine(strerror(errno))); int CounterFileDescriptor = Counter->getFileDescriptor(); Error SendError = - sendFileDescriptorThroughSocket(PipeFiles[1], CounterFileDescriptor); + sendFileDescriptorThroughSocket(WriteFD, CounterFileDescriptor); if (SendError) return SendError; int ChildStatus; - if (waitpid(ParentOrChildPID, &ChildStatus, 0) == -1) { + if (waitpid(ChildPID, &ChildStatus, 0) == -1) { return make_error( "Waiting for the child process to complete failed: " + Twine(strerror(errno))); @@ -395,8 +356,7 @@ private: // An error was encountered running the snippet, process it siginfo_t ChildSignalInfo; - if (ptrace(PTRACE_GETSIGINFO, ParentOrChildPID, NULL, &ChildSignalInfo) == - -1) { + if (ptrace(PTRACE_GETSIGINFO, ChildPID, NULL, &ChildSignalInfo) == -1) { return make_error("Getting signal info from the child failed: " + Twine(strerror(errno))); } @@ -405,13 +365,13 @@ private: // handlers to run, and calling SIGTERM would mean that ptrace will force // it to block in the signal-delivery-stop for the SIGSEGV/other signals, // and upon exit. - if (kill(ParentOrChildPID, SIGKILL) == -1) + if (kill(ChildPID, SIGKILL) == -1) return make_error("Failed to kill child benchmarking proces: " + Twine(strerror(errno))); // Wait for the process to exit so that there are no zombie processes left // around. - if (waitpid(ParentOrChildPID, NULL, 0) == -1) + if (waitpid(ChildPID, NULL, 0) == -1) return make_error("Failed to wait for process to die: " + Twine(strerror(errno))); @@ -422,6 +382,56 @@ private: return make_error(ChildSignalInfo.si_signo); } + Error createSubProcessAndRunBenchmark( + StringRef CounterName, SmallVectorImpl &CounterValues, + ArrayRef ValidationCounters, + SmallVectorImpl &ValidationCounterValues) const { + int PipeFiles[2]; + int PipeSuccessOrErr = socketpair(AF_UNIX, SOCK_DGRAM, 0, PipeFiles); + if (PipeSuccessOrErr != 0) { + return make_error( + "Failed to create a pipe for interprocess communication between " + "llvm-exegesis and the benchmarking subprocess: " + + Twine(strerror(errno))); + } + + SubprocessMemory SPMemory; + Error MemoryInitError = SPMemory.initializeSubprocessMemory(getpid()); + if (MemoryInitError) + return MemoryInitError; + + Error AddMemDefError = + SPMemory.addMemoryDefinition(Key.MemoryValues, getpid()); + if (AddMemDefError) + return AddMemDefError; + + pid_t ParentOrChildPID = fork(); + + if (ParentOrChildPID == -1) { + return make_error("Failed to create child process: " + + Twine(strerror(errno))); + } + + if (ParentOrChildPID == 0) { + // We are in the child process, close the write end of the pipe. + close(PipeFiles[1]); + // Unregister handlers, signal handling is now handled through ptrace in + // the host process. + sys::unregisterHandlers(); + runChildSubprocess(PipeFiles[0], Key); + // The child process terminates in the above function, so we should never + // get to this point. + llvm_unreachable("Child process didn't exit when expected."); + } + + // Close the read end of the pipe as we only need to write to the subprocess + // from the parent process. + close(PipeFiles[0]); + return runParentProcess(ParentOrChildPID, PipeFiles[1], CounterName, + CounterValues, ValidationCounters, + ValidationCounterValues); + } + void disableCoreDumps() const { struct rlimit rlim; @@ -429,8 +439,8 @@ private: setrlimit(RLIMIT_CORE, &rlim); } - [[noreturn]] void prepareAndRunBenchmark(int Pipe, - const BenchmarkKey &Key) const { + [[noreturn]] void runChildSubprocess(int Pipe, + const BenchmarkKey &Key) const { // Disable core dumps in the child process as otherwise everytime we // encounter an execution failure like a segmentation fault, we will create // a core dump. We report the information directly rather than require the -- GitLab From bbcfe6f4311af8cf6095a5bc5937fa68a87b4289 Mon Sep 17 00:00:00 2001 From: srcarroll <50210727+srcarroll@users.noreply.github.com> Date: Fri, 22 Mar 2024 12:37:39 -0500 Subject: [PATCH 288/296] [mlir][transform] Emit error message with `emitSilenceableFailure` (#86146) The previous implementation used a `notifyMatchFailure` to emit failure message inappropriately and then used the `emitDefaultSilenceableFailure`. This patch changes this to use the more appropriate `emitSilenceableFailure` with error message. Additionally a failure test has been added. --- .../TransformOps/LinalgTransformOps.cpp | 14 ++++---- .../Dialect/Linalg/flatten-unsupported.mlir | 33 +++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 mlir/test/Dialect/Linalg/flatten-unsupported.mlir diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp index ecf998312482..88819cd96435 100644 --- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp +++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp @@ -3269,22 +3269,24 @@ DiagnosedSilenceableFailure transform::FlattenElementwiseLinalgOp::applyToOne( transform::ApplyToEachResultList &results, transform::TransformState &state) { rewriter.setInsertionPoint(target); - if (!isElementwise(target)) { - failed(rewriter.notifyMatchFailure( - target, "only elementwise flattening is supported")); - return emitDefaultSilenceableFailure(target); - } + if (!isElementwise(target)) + return mlir::emitSilenceableFailure(target->getLoc()) + << "only elementwise flattening is supported"; + // If rank <= 1, do nothing if (target.getNumLoops() <= 1) { results.push_back(target); return DiagnosedSilenceableFailure::success(); } + + // Attempt to flatten all dims to one. ReassociationIndices reassociation(target.getNumLoops()); std::iota(reassociation.begin(), reassociation.end(), 0); auto maybeFlattened = collapseOpIterationDims(target, reassociation, rewriter); if (failed(maybeFlattened)) - return emitDefaultSilenceableFailure(target); + return mlir::emitSilenceableFailure(target->getLoc()) + << "attempted to flatten, but failed"; results.push_back(maybeFlattened->collapsedOp); rewriter.replaceOp(target, maybeFlattened->results); return DiagnosedSilenceableFailure::success(); diff --git a/mlir/test/Dialect/Linalg/flatten-unsupported.mlir b/mlir/test/Dialect/Linalg/flatten-unsupported.mlir new file mode 100644 index 000000000000..499db4cfb329 --- /dev/null +++ b/mlir/test/Dialect/Linalg/flatten-unsupported.mlir @@ -0,0 +1,33 @@ +// RUN: mlir-opt %s -transform-interpreter -split-input-file -verify-diagnostics + +func.func @non_elementwise(%arg0: memref<2x3xf32>, %arg1: memref<3x4xf32>, %arg2: memref<2x4xf32>) { + // expected-error @below {{only elementwise flattening is supported}} + linalg.matmul ins(%arg0, %arg1 : memref<2x3xf32>, memref<3x4xf32>) outs(%arg2: memref<2x4xf32>) + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match interface{LinalgOp} in %arg1 : (!transform.any_op) -> !transform.any_op + %flattened = transform.structured.flatten_elementwise %0 + : (!transform.any_op) -> !transform.any_op + transform.yield + } +} + +// ----- + +func.func @unsupported_memref(%arg0: memref<32x7xf32, strided<[7, 2]>>, %arg1: memref<32x7xf32, strided<[7, 2]>>, %arg2: memref<32x7xf32, strided<[7, 2]>>) { + // expected-error @below {{attempted to flatten, but failed}} + linalg.map {arith.addf} ins(%arg0, %arg1: memref<32x7xf32, strided<[7, 2]>>, memref<32x7xf32, strided<[7, 2]>>) outs(%arg2: memref<32x7xf32, strided<[7, 2]>>) + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match interface{LinalgOp} in %arg1 : (!transform.any_op) -> !transform.any_op + %flattened = transform.structured.flatten_elementwise %0 + : (!transform.any_op) -> !transform.any_op + transform.yield + } +} -- GitLab From cd8286a667d568c4319b09baa63ba899e3101a19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Danny=20M=C3=B6sch?= Date: Fri, 22 Mar 2024 18:51:44 +0100 Subject: [PATCH 289/296] [GitHub] Allow shortcut for "introductory issue" and request linking to issue in PR (#84635) The answer to many requests in issues to be assigned to users is often "just create a pull request". That's in contradiction to the "introductory issue" instructions posted by the GitHub bot. This change updates the instructions, mentioning the shortcut of "just creating a PR". Moreover, it now explains linking PRs to issues in order to close them automatically upon merge. --- llvm/utils/git/github-automation.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/llvm/utils/git/github-automation.py b/llvm/utils/git/github-automation.py index b21f14eca445..42a658cefac3 100755 --- a/llvm/utils/git/github-automation.py +++ b/llvm/utils/git/github-automation.py @@ -24,12 +24,13 @@ Hi! This issue may be a good introductory issue for people new to working on LLVM. If you would like to work on this issue, your first steps are: -1. In the comments of the issue, request for it to be assigned to you. -2. Fix the issue locally. -3. [Run the test suite](https://llvm.org/docs/TestingGuide.html#unit-and-regression-tests) locally. Remember that the subdirectories under `test/` create fine-grained testing targets, so you can e.g. use `make check-clang-ast` to only run Clang's AST tests. -4. Create a Git commit. -5. Run [`git clang-format HEAD~1`](https://clang.llvm.org/docs/ClangFormat.html#git-integration) to format your changes. -6. Open a [pull request](https://github.com/llvm/llvm-project/pulls) to the [upstream repository](https://github.com/llvm/llvm-project) on GitHub. Detailed instructions can be found [in GitHub's documentation](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request). +1. Check that no other contributor has already been assigned to this issue. If you believe that no one is actually working on it despite an assignment, ping the person. After one week without a response, the assignee may be changed. +1. In the comments of this issue, request for it to be assigned to you, or just create a [pull request](https://github.com/llvm/llvm-project/pulls) after following the steps below. [Mention](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue) this issue in the description of the pull request. +1. Fix the issue locally. +1. [Run the test suite](https://llvm.org/docs/TestingGuide.html#unit-and-regression-tests) locally. Remember that the subdirectories under `test/` create fine-grained testing targets, so you can e.g. use `make check-clang-ast` to only run Clang's AST tests. +1. Create a Git commit. +1. Run [`git clang-format HEAD~1`](https://clang.llvm.org/docs/ClangFormat.html#git-integration) to format your changes. +1. Open a [pull request](https://github.com/llvm/llvm-project/pulls) to the [upstream repository](https://github.com/llvm/llvm-project) on GitHub. Detailed instructions can be found [in GitHub's documentation](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request). [Mention](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue) this issue in the description of the pull request. If you have any further questions about this issue, don't hesitate to ask via a comment in the thread below. """ -- GitLab From fb329f18445cb33d242cc500ca618d03674b22ad Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Fri, 22 Mar 2024 11:15:45 -0700 Subject: [PATCH 290/296] [Target] Move SubRegIdxRanges from MCSubtargetInfo to TargetInfo. (#86245) I'm planning to add HwMode support to SubRegIdxRanges for RISC-V GPR pairs. The MC layer is currently unaware of the HwMode for registers and I'd like to keep it that way. This information is not used by the MC layer so I think it is safe to move it. --- .../include/llvm/CodeGen/TargetRegisterInfo.h | 31 ++++++++++++---- llvm/include/llvm/MC/MCRegisterInfo.h | 21 ----------- llvm/lib/CodeGen/TargetRegisterInfo.cpp | 36 +++++++++++-------- llvm/lib/MC/MCRegisterInfo.cpp | 12 ------- llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 4 +-- llvm/unittests/CodeGen/MFCommon.inc | 5 +-- llvm/utils/TableGen/RegisterInfoEmitter.cpp | 29 +++++++-------- 7 files changed, 65 insertions(+), 73 deletions(-) diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h index 117d3f718297..33c4c745c341 100644 --- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h +++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h @@ -243,9 +243,20 @@ public: unsigned RegSize, SpillSize, SpillAlignment; unsigned VTListOffset; }; + + /// SubRegCoveredBits - Emitted by tablegen: bit range covered by a subreg + /// index, -1 in any being invalid. + struct SubRegCoveredBits { + uint16_t Offset; + uint16_t Size; + }; + private: const TargetRegisterInfoDesc *InfoDesc; // Extra desc array for codegen const char *const *SubRegIndexNames; // Names of subreg indexes. + const SubRegCoveredBits *SubRegIdxRanges; // Pointer to the subreg covered + // bit ranges array. + // Pointer to array of lane masks, one per sub-reg index. const LaneBitmask *SubRegIndexLaneMasks; @@ -256,12 +267,10 @@ private: unsigned HwMode; protected: - TargetRegisterInfo(const TargetRegisterInfoDesc *ID, - regclass_iterator RCB, - regclass_iterator RCE, - const char *const *SRINames, - const LaneBitmask *SRILaneMasks, - LaneBitmask CoveringLanes, + TargetRegisterInfo(const TargetRegisterInfoDesc *ID, regclass_iterator RCB, + regclass_iterator RCE, const char *const *SRINames, + const SubRegCoveredBits *SubIdxRanges, + const LaneBitmask *SRILaneMasks, LaneBitmask CoveringLanes, const RegClassInfo *const RCIs, const MVT::SimpleValueType *const RCVTLists, unsigned Mode = 0); @@ -382,6 +391,16 @@ public: return SubRegIndexNames[SubIdx-1]; } + /// Get the size of the bit range covered by a sub-register index. + /// If the index isn't continuous, return the sum of the sizes of its parts. + /// If the index is used to access subregisters of different sizes, return -1. + unsigned getSubRegIdxSize(unsigned Idx) const; + + /// Get the offset of the bit range covered by a sub-register index. + /// If an Offset doesn't make sense (the index isn't continuous, or is used to + /// access sub-registers at different offsets), return -1. + unsigned getSubRegIdxOffset(unsigned Idx) const; + /// Return a bitmask representing the parts of a register that are covered by /// SubIdx \see LaneBitmask. /// diff --git a/llvm/include/llvm/MC/MCRegisterInfo.h b/llvm/include/llvm/MC/MCRegisterInfo.h index fb4d11ec1d4d..c648ef20fa84 100644 --- a/llvm/include/llvm/MC/MCRegisterInfo.h +++ b/llvm/include/llvm/MC/MCRegisterInfo.h @@ -153,13 +153,6 @@ public: bool operator<(DwarfLLVMRegPair RHS) const { return FromReg < RHS.FromReg; } }; - /// SubRegCoveredBits - Emitted by tablegen: bit range covered by a subreg - /// index, -1 in any being invalid. - struct SubRegCoveredBits { - uint16_t Offset; - uint16_t Size; - }; - private: const MCRegisterDesc *Desc; // Pointer to the descriptor array unsigned NumRegs; // Number of entries in the array @@ -176,8 +169,6 @@ private: const char *RegClassStrings; // Pointer to the class strings. const uint16_t *SubRegIndices; // Pointer to the subreg lookup // array. - const SubRegCoveredBits *SubRegIdxRanges; // Pointer to the subreg covered - // bit ranges array. unsigned NumSubRegIndices; // Number of subreg indices. const uint16_t *RegEncodingTable; // Pointer to array of register // encodings. @@ -278,7 +269,6 @@ public: const int16_t *DL, const LaneBitmask *RUMS, const char *Strings, const char *ClassStrings, const uint16_t *SubIndices, unsigned NumIndices, - const SubRegCoveredBits *SubIdxRanges, const uint16_t *RET) { Desc = D; NumRegs = NR; @@ -294,7 +284,6 @@ public: NumRegUnits = NRU; SubRegIndices = SubIndices; NumSubRegIndices = NumIndices; - SubRegIdxRanges = SubIdxRanges; RegEncodingTable = RET; // Initialize DWARF register mapping variables @@ -387,16 +376,6 @@ public: /// otherwise. unsigned getSubRegIndex(MCRegister RegNo, MCRegister SubRegNo) const; - /// Get the size of the bit range covered by a sub-register index. - /// If the index isn't continuous, return the sum of the sizes of its parts. - /// If the index is used to access subregisters of different sizes, return -1. - unsigned getSubRegIdxSize(unsigned Idx) const; - - /// Get the offset of the bit range covered by a sub-register index. - /// If an Offset doesn't make sense (the index isn't continuous, or is used to - /// access sub-registers at different offsets), return -1. - unsigned getSubRegIdxOffset(unsigned Idx) const; - /// Return the human-readable symbolic target-specific name for the /// specified physical register. const char *getName(MCRegister RegNo) const { diff --git a/llvm/lib/CodeGen/TargetRegisterInfo.cpp b/llvm/lib/CodeGen/TargetRegisterInfo.cpp index c9503fcb77bb..4120c74c23b1 100644 --- a/llvm/lib/CodeGen/TargetRegisterInfo.cpp +++ b/llvm/lib/CodeGen/TargetRegisterInfo.cpp @@ -50,20 +50,16 @@ static cl::opt "high compile time cost in global splitting."), cl::init(5000)); -TargetRegisterInfo::TargetRegisterInfo(const TargetRegisterInfoDesc *ID, - regclass_iterator RCB, regclass_iterator RCE, - const char *const *SRINames, - const LaneBitmask *SRILaneMasks, - LaneBitmask SRICoveringLanes, - const RegClassInfo *const RCIs, - const MVT::SimpleValueType *const RCVTLists, - unsigned Mode) - : InfoDesc(ID), SubRegIndexNames(SRINames), - SubRegIndexLaneMasks(SRILaneMasks), - RegClassBegin(RCB), RegClassEnd(RCE), - CoveringLanes(SRICoveringLanes), - RCInfos(RCIs), RCVTLists(RCVTLists), HwMode(Mode) { -} +TargetRegisterInfo::TargetRegisterInfo( + const TargetRegisterInfoDesc *ID, regclass_iterator RCB, + regclass_iterator RCE, const char *const *SRINames, + const SubRegCoveredBits *SubIdxRanges, const LaneBitmask *SRILaneMasks, + LaneBitmask SRICoveringLanes, const RegClassInfo *const RCIs, + const MVT::SimpleValueType *const RCVTLists, unsigned Mode) + : InfoDesc(ID), SubRegIndexNames(SRINames), SubRegIdxRanges(SubIdxRanges), + SubRegIndexLaneMasks(SRILaneMasks), RegClassBegin(RCB), RegClassEnd(RCE), + CoveringLanes(SRICoveringLanes), RCInfos(RCIs), RCVTLists(RCVTLists), + HwMode(Mode) {} TargetRegisterInfo::~TargetRegisterInfo() = default; @@ -596,6 +592,18 @@ bool TargetRegisterInfo::getCoveringSubRegIndexes( return BestIdx; } +unsigned TargetRegisterInfo::getSubRegIdxSize(unsigned Idx) const { + assert(Idx && Idx < getNumSubRegIndices() && + "This is not a subregister index"); + return SubRegIdxRanges[Idx].Size; +} + +unsigned TargetRegisterInfo::getSubRegIdxOffset(unsigned Idx) const { + assert(Idx && Idx < getNumSubRegIndices() && + "This is not a subregister index"); + return SubRegIdxRanges[Idx].Offset; +} + Register TargetRegisterInfo::lookThruCopyLike(Register SrcReg, const MachineRegisterInfo *MRI) const { diff --git a/llvm/lib/MC/MCRegisterInfo.cpp b/llvm/lib/MC/MCRegisterInfo.cpp index a2c1737e2964..334655616d8d 100644 --- a/llvm/lib/MC/MCRegisterInfo.cpp +++ b/llvm/lib/MC/MCRegisterInfo.cpp @@ -57,18 +57,6 @@ unsigned MCRegisterInfo::getSubRegIndex(MCRegister Reg, return 0; } -unsigned MCRegisterInfo::getSubRegIdxSize(unsigned Idx) const { - assert(Idx && Idx < getNumSubRegIndices() && - "This is not a subregister index"); - return SubRegIdxRanges[Idx].Size; -} - -unsigned MCRegisterInfo::getSubRegIdxOffset(unsigned Idx) const { - assert(Idx && Idx < getNumSubRegIndices() && - "This is not a subregister index"); - return SubRegIdxRanges[Idx].Offset; -} - int MCRegisterInfo::getDwarfRegNum(MCRegister RegNum, bool isEH) const { const DwarfLLVMRegPair *M = isEH ? EHL2DwarfRegs : L2DwarfRegs; unsigned Size = isEH ? EHL2DwarfRegsSize : L2DwarfRegsSize; diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp index 79a7d1cf66c4..245731ad5fc7 100644 --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp @@ -363,8 +363,8 @@ SIRegisterInfo::SIRegisterInfo(const GCNSubtarget &ST) for (auto &Row : SubRegFromChannelTable) Row.fill(AMDGPU::NoSubRegister); for (unsigned Idx = 1; Idx < getNumSubRegIndices(); ++Idx) { - unsigned Width = AMDGPUSubRegIdxRanges[Idx].Size / 32; - unsigned Offset = AMDGPUSubRegIdxRanges[Idx].Offset / 32; + unsigned Width = getSubRegIdxSize(Idx) / 32; + unsigned Offset = getSubRegIdxOffset(Idx) / 32; assert(Width < SubRegFromChannelTableWidthMap.size()); Width = SubRegFromChannelTableWidthMap[Width]; if (Width == 0) diff --git a/llvm/unittests/CodeGen/MFCommon.inc b/llvm/unittests/CodeGen/MFCommon.inc index 7de7eabdd1f6..1997e8052297 100644 --- a/llvm/unittests/CodeGen/MFCommon.inc +++ b/llvm/unittests/CodeGen/MFCommon.inc @@ -23,9 +23,10 @@ class BogusRegisterInfo : public TargetRegisterInfo { public: BogusRegisterInfo() : TargetRegisterInfo(nullptr, BogusRegisterClasses, BogusRegisterClasses, - nullptr, nullptr, LaneBitmask(~0u), nullptr, nullptr) { + nullptr, nullptr, nullptr, LaneBitmask(~0u), nullptr, + nullptr) { InitMCRegisterInfo(nullptr, 0, 0, 0, nullptr, 0, nullptr, 0, nullptr, - nullptr, nullptr, nullptr, nullptr, 0, nullptr, nullptr); + nullptr, nullptr, nullptr, nullptr, 0, nullptr); } const MCPhysReg * diff --git a/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/llvm/utils/TableGen/RegisterInfoEmitter.cpp index d074e31c6245..c4fc1930488c 100644 --- a/llvm/utils/TableGen/RegisterInfoEmitter.cpp +++ b/llvm/utils/TableGen/RegisterInfoEmitter.cpp @@ -955,16 +955,6 @@ void RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target, SubRegIdxSeqs.emit(OS, printSubRegIndex); OS << "};\n\n"; - // Emit the table of sub-register index sizes. - OS << "extern const MCRegisterInfo::SubRegCoveredBits " << TargetName - << "SubRegIdxRanges[] = {\n"; - OS << " { " << (uint16_t)-1 << ", " << (uint16_t)-1 << " },\n"; - for (const auto &Idx : SubRegIndices) { - OS << " { " << Idx.Offset << ", " << Idx.Size << " },\t// " - << Idx.getName() << "\n"; - } - OS << "};\n\n"; - // Emit the string table. RegStrings.layout(); RegStrings.emitStringLiteralDef(OS, Twine("extern const char ") + TargetName + @@ -1101,8 +1091,7 @@ void RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target, << TargetName << "LaneMaskLists, " << TargetName << "RegStrings, " << TargetName << "RegClassStrings, " << TargetName << "SubRegIdxLists, " << (std::distance(SubRegIndices.begin(), SubRegIndices.end()) + 1) << ",\n" - << TargetName << "SubRegIdxRanges, " << TargetName - << "RegEncodingTable);\n\n"; + << TargetName << "RegEncodingTable);\n\n"; EmitRegMapping(OS, Regs, false); @@ -1253,6 +1242,16 @@ void RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, } OS << "\" };\n\n"; + // Emit the table of sub-register index sizes. + OS << "static const TargetRegisterInfo::SubRegCoveredBits " + "SubRegIdxRangeTable[] = {\n"; + OS << " { " << (uint16_t)-1 << ", " << (uint16_t)-1 << " },\n"; + for (const auto &Idx : SubRegIndices) { + OS << " { " << Idx.Offset << ", " << Idx.Size << " },\t// " + << Idx.getName() << "\n"; + } + OS << "};\n\n"; + // Emit SubRegIndex lane masks, including 0. OS << "\nstatic const LaneBitmask SubRegIndexLaneMaskTable[] = {\n " "LaneBitmask::getAll(),\n"; @@ -1634,8 +1633,6 @@ void RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, OS << "extern const char " << TargetName << "RegClassStrings[];\n"; OS << "extern const MCPhysReg " << TargetName << "RegUnitRoots[][2];\n"; OS << "extern const uint16_t " << TargetName << "SubRegIdxLists[];\n"; - OS << "extern const MCRegisterInfo::SubRegCoveredBits " << TargetName - << "SubRegIdxRanges[];\n"; OS << "extern const uint16_t " << TargetName << "RegEncodingTable[];\n"; EmitRegMappingTables(OS, Regs, true); @@ -1646,7 +1643,8 @@ void RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, " unsigned PC, unsigned HwMode)\n" << " : TargetRegisterInfo(&" << TargetName << "RegInfoDesc" << ", RegisterClasses, RegisterClasses+" << RegisterClasses.size() << ",\n" - << " SubRegIndexNameTable, SubRegIndexLaneMaskTable,\n" + << " SubRegIndexNameTable, SubRegIdxRangeTable, " + "SubRegIndexLaneMaskTable,\n" << " "; printMask(OS, RegBank.CoveringLanes); OS << ", RegClassInfos, VTLists, HwMode) {\n" @@ -1661,7 +1659,6 @@ void RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, << " " << TargetName << "RegClassStrings,\n" << " " << TargetName << "SubRegIdxLists,\n" << " " << SubRegIndicesSize + 1 << ",\n" - << " " << TargetName << "SubRegIdxRanges,\n" << " " << TargetName << "RegEncodingTable);\n\n"; EmitRegMapping(OS, Regs, true); -- GitLab From b1575f9082071702bd6aaa2600ce9fe011a091e9 Mon Sep 17 00:00:00 2001 From: Kevin Frei Date: Fri, 22 Mar 2024 11:22:09 -0700 Subject: [PATCH 291/296] Missed a null-ptr check in previous PR for Debuginfod testing (#86292) @GeorgeHuyubo noticed an unchecked shared pointer result in https://github.com/llvm/llvm-project/pull/85693/. This is the fix for that issue. Co-authored-by: Kevin Frei --- lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp index 91b8b4a979e0..a9956aa9075f 100644 --- a/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp +++ b/lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp @@ -53,7 +53,7 @@ static bool IsDwpSymbolFile(const lldb::ModuleSP &module_sp, ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin( module_sp, &file_spec, 0, FileSystem::Instance().GetByteSize(file_spec), dwp_file_data_sp, dwp_file_data_offset); - if (!ObjectFileELF::classof(dwp_obj_file.get())) + if (!dwp_obj_file || !ObjectFileELF::classof(dwp_obj_file.get())) return false; // The presence of a debug_cu_index section is the key identifying feature of // a DWP file. Make sure we don't fill in the section list on dwp_obj_file -- GitLab From 721f149596f27f3d4c5c28ec2a2fac33340fb876 Mon Sep 17 00:00:00 2001 From: Job Henandez Lara Date: Fri, 22 Mar 2024 11:38:51 -0700 Subject: [PATCH 292/296] Fix typo (#86319) Im working on the floating point fmaximum and fminimum functions right now in a different pr and I ran the individual tests by doing ``` ninja libc.test.src.math.smoke._test.__unit__ ``` --- libc/src/math/docs/add_math_function.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libc/src/math/docs/add_math_function.md b/libc/src/math/docs/add_math_function.md index f8bc8a3bdd8b..d1bca222b428 100644 --- a/libc/src/math/docs/add_math_function.md +++ b/libc/src/math/docs/add_math_function.md @@ -177,7 +177,7 @@ implementation (which is very often glibc). - Build and Run a specific unit test: ``` - $ ninja libc.test.src.math._test + $ ninja libc.test.src.math._test.__unit__ $ projects/libc/test/src/math/libc.test.src.math._test ``` -- GitLab From 83e96977cdb6041196366fc01e8abdca52cadb2e Mon Sep 17 00:00:00 2001 From: Vinayak Dev <104419489+vinayakdsci@users.noreply.github.com> Date: Sat, 23 Mar 2024 00:09:53 +0530 Subject: [PATCH 293/296] [libc] Implement strfromd() and strfroml() (#86113) Follow up to #85438. Implements the functions `strfromd()` and `strfroml()` introduced in C23, and unifies the testing framework for `strfrom*()` functions. --- libc/config/linux/x86_64/entrypoints.txt | 2 + libc/spec/stdc.td | 2 + libc/src/stdlib/CMakeLists.txt | 20 ++ libc/src/stdlib/strfromd.cpp | 42 +++ libc/src/stdlib/strfromd.h | 21 ++ libc/src/stdlib/strfromf.h | 2 +- libc/src/stdlib/strfroml.cpp | 47 +++ libc/src/stdlib/strfroml.h | 21 ++ libc/test/src/stdlib/CMakeLists.txt | 31 ++ libc/test/src/stdlib/StrfromTest.h | 435 +++++++++++++++++++++++ libc/test/src/stdlib/strfromd_test.cpp | 13 + libc/test/src/stdlib/strfromf_test.cpp | 98 +---- libc/test/src/stdlib/strfroml_test.cpp | 13 + 13 files changed, 650 insertions(+), 97 deletions(-) create mode 100644 libc/src/stdlib/strfromd.cpp create mode 100644 libc/src/stdlib/strfromd.h create mode 100644 libc/src/stdlib/strfroml.cpp create mode 100644 libc/src/stdlib/strfroml.h create mode 100644 libc/test/src/stdlib/StrfromTest.h create mode 100644 libc/test/src/stdlib/strfromd_test.cpp create mode 100644 libc/test/src/stdlib/strfroml_test.cpp diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt index 8e1ab5cd65f0..8b1cd3fb1052 100644 --- a/libc/config/linux/x86_64/entrypoints.txt +++ b/libc/config/linux/x86_64/entrypoints.txt @@ -180,7 +180,9 @@ set(TARGET_LIBC_ENTRYPOINTS libc.src.stdlib.qsort_r libc.src.stdlib.rand libc.src.stdlib.srand + libc.src.stdlib.strfromd libc.src.stdlib.strfromf + libc.src.stdlib.strfroml libc.src.stdlib.strtod libc.src.stdlib.strtof libc.src.stdlib.strtol diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td index 76010a4b4533..3e58e3b88645 100644 --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -962,6 +962,8 @@ def StdC : StandardSpec<"stdc"> { FunctionSpec<"srand", RetValSpec, [ArgSpec]>, FunctionSpec<"strfromf", RetValSpec, [ArgSpec, ArgSpec, ArgSpec, ArgSpec]>, + FunctionSpec<"strfromd", RetValSpec, [ArgSpec, ArgSpec, ArgSpec, ArgSpec]>, + FunctionSpec<"strfroml", RetValSpec, [ArgSpec, ArgSpec, ArgSpec, ArgSpec]>, FunctionSpec<"strtof", RetValSpec, [ArgSpec, ArgSpec]>, FunctionSpec<"strtod", RetValSpec, [ArgSpec, ArgSpec]>, diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt index 22f7f990fb08..a2b16e73f353 100644 --- a/libc/src/stdlib/CMakeLists.txt +++ b/libc/src/stdlib/CMakeLists.txt @@ -62,6 +62,26 @@ add_entrypoint_object( .str_from_util ) +add_entrypoint_object( + strfromd + SRCS + strfromd.cpp + HDRS + strfromd.h + DEPENDS + .str_from_util +) + +add_entrypoint_object( + strfroml + SRCS + strfroml.cpp + HDRS + strfroml.h + DEPENDS + .str_from_util +) + add_header_library( str_from_util HDRS diff --git a/libc/src/stdlib/strfromd.cpp b/libc/src/stdlib/strfromd.cpp new file mode 100644 index 000000000000..1d02a7ad1124 --- /dev/null +++ b/libc/src/stdlib/strfromd.cpp @@ -0,0 +1,42 @@ +//===-- Implementation of strfromd ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdlib/strfromd.h" +#include "src/stdlib/str_from_util.h" + +#include +#include + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, strfromd, + (char *__restrict s, size_t n, const char *__restrict format, + double fp)) { + LIBC_ASSERT(s != nullptr); + + printf_core::FormatSection section = + internal::parse_format_string(format, fp); + printf_core::WriteBuffer wb(s, (n > 0 ? n - 1 : 0)); + printf_core::Writer writer(&wb); + + int result = 0; + if (section.has_conv) + result = internal::strfromfloat_convert(&writer, section); + else + result = writer.write(section.raw_string); + + if (result < 0) + return result; + + if (n > 0) + wb.buff[wb.buff_cur] = '\0'; + + return writer.get_chars_written(); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdlib/strfromd.h b/libc/src/stdlib/strfromd.h new file mode 100644 index 000000000000..d2c3fefb6300 --- /dev/null +++ b/libc/src/stdlib/strfromd.h @@ -0,0 +1,21 @@ +//===-- Implementation header for strfromd ------------------------*- C++--===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDLIB_STRFROMD_H +#define LLVM_LIBC_SRC_STDLIB_STRFROMD_H + +#include + +namespace LIBC_NAMESPACE { + +int strfromd(char *__restrict s, size_t n, const char *__restrict format, + double fp); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDLIB_STRFROMD_H diff --git a/libc/src/stdlib/strfromf.h b/libc/src/stdlib/strfromf.h index b551a58af05a..492c2c33cf08 100644 --- a/libc/src/stdlib/strfromf.h +++ b/libc/src/stdlib/strfromf.h @@ -18,4 +18,4 @@ int strfromf(char *__restrict s, size_t n, const char *__restrict format, } // namespace LIBC_NAMESPACE -#endif // LLVM_LIBC_SRC_STDLIB_STRTOF_H +#endif // LLVM_LIBC_SRC_STDLIB_STRFROMF_H diff --git a/libc/src/stdlib/strfroml.cpp b/libc/src/stdlib/strfroml.cpp new file mode 100644 index 000000000000..6b8781e7c04e --- /dev/null +++ b/libc/src/stdlib/strfroml.cpp @@ -0,0 +1,47 @@ +//===-- Implementation of strfroml ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/stdlib/strfroml.h" +#include "src/stdlib/str_from_util.h" + +#include +#include + +namespace LIBC_NAMESPACE { + +LLVM_LIBC_FUNCTION(int, strfroml, + (char *__restrict s, size_t n, const char *__restrict format, + long double fp)) { + LIBC_ASSERT(s != nullptr); + + printf_core::FormatSection section = + internal::parse_format_string(format, fp); + + // To ensure that the conversion function actually uses long double, + // the length modifier has to be set to LenghtModifier::L + section.length_modifier = printf_core::LengthModifier::L; + + printf_core::WriteBuffer wb(s, (n > 0 ? n - 1 : 0)); + printf_core::Writer writer(&wb); + + int result = 0; + if (section.has_conv) + result = internal::strfromfloat_convert(&writer, section); + else + result = writer.write(section.raw_string); + + if (result < 0) + return result; + + if (n > 0) + wb.buff[wb.buff_cur] = '\0'; + + return writer.get_chars_written(); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/stdlib/strfroml.h b/libc/src/stdlib/strfroml.h new file mode 100644 index 000000000000..e99d035e4da6 --- /dev/null +++ b/libc/src/stdlib/strfroml.h @@ -0,0 +1,21 @@ +//===-- Implementation header for strfroml ------------------------*- C++--===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDLIB_STRFROML_H +#define LLVM_LIBC_SRC_STDLIB_STRFROML_H + +#include + +namespace LIBC_NAMESPACE { + +int strfroml(char *__restrict s, size_t n, const char *__restrict format, + long double fp); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STDLIB_STRFROML_H diff --git a/libc/test/src/stdlib/CMakeLists.txt b/libc/test/src/stdlib/CMakeLists.txt index cb42bc56f51c..3ccc1cde9193 100644 --- a/libc/test/src/stdlib/CMakeLists.txt +++ b/libc/test/src/stdlib/CMakeLists.txt @@ -168,6 +168,14 @@ add_libc_test( .strtol_test_support ) +add_header_library( + strfrom_test_support + HDRS + StrfromTest.h + DEPENDS + libc.src.__support.CPP.type_traits +) + add_libc_test( strfromf_test SUITE @@ -175,9 +183,32 @@ add_libc_test( SRCS strfromf_test.cpp DEPENDS + .strfrom_test_support libc.src.stdlib.strfromf ) +add_libc_test( + strfromd_test + SUITE + libc-stdlib-tests + SRCS + strfromd_test.cpp + DEPENDS + .strfrom_test_support + libc.src.stdlib.strfromd +) + +add_libc_test( + strfroml_test + SUITE + libc-stdlib-tests + SRCS + strfroml_test.cpp + DEPENDS + .strfrom_test_support + libc.src.stdlib.strfroml +) + add_libc_test( abs_test SUITE diff --git a/libc/test/src/stdlib/StrfromTest.h b/libc/test/src/stdlib/StrfromTest.h new file mode 100644 index 000000000000..f695bbb335bd --- /dev/null +++ b/libc/test/src/stdlib/StrfromTest.h @@ -0,0 +1,435 @@ +//===-- A template class for testing strfrom functions ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "src/__support/CPP/type_traits.h" +#include "test/UnitTest/Test.h" + +#define ASSERT_STREQ_LEN(actual_written, actual_str, expected_str) \ + EXPECT_EQ(actual_written, static_cast(sizeof(expected_str) - 1)); \ + EXPECT_STREQ(actual_str, expected_str); + +template +class StrfromTest : public LIBC_NAMESPACE::testing::Test { + + static const bool is_single_prec = + LIBC_NAMESPACE::cpp::is_same::value; + static const bool is_double_prec = + LIBC_NAMESPACE::cpp::is_same::value; + + using FunctionT = int (*)(char *, size_t, const char *, InputT fp); + +public: + void floatDecimalFormat(FunctionT func) { + if (is_single_prec) + floatDecimalSinglePrec(func); + else if (is_double_prec) + floatDecimalDoublePrec(func); + else + floatDecimalLongDoublePrec(func); + } + + void floatHexExpFormat(FunctionT func) { + if (is_single_prec) + floatHexExpSinglePrec(func); + else if (is_double_prec) + floatHexExpDoublePrec(func); + else + floatHexExpLongDoublePrec(func); + } + + void floatDecimalExpFormat(FunctionT func) { + if (is_single_prec) + floatDecimalExpSinglePrec(func); + else if (is_double_prec) + floatDecimalExpDoublePrec(func); + else + floatDecimalExpLongDoublePrec(func); + } + + void floatDecimalAutoFormat(FunctionT func) { + if (is_single_prec) + floatDecimalAutoSinglePrec(func); + else if (is_double_prec) + floatDecimalAutoDoublePrec(func); + else + floatDecimalAutoLongDoublePrec(func); + } + + void improperFormatString(FunctionT func) { + char buff[100]; + int written; + const bool is_long_double = !is_single_prec && !is_double_prec; + + written = func(buff, 37, "A simple string with no conversions.", 1.0); + ASSERT_STREQ_LEN(written, buff, "A simple string with no conversions."); + + written = + func(buff, 37, + "%A simple string with one conversion, should overwrite.", 1.0); + ASSERT_STREQ_LEN(written, buff, is_long_double ? "0X8P-3" : "0X1P+0"); + + written = func(buff, 74, + "A simple string with one conversion in %A " + "between, writes string as it is", + 1.0); + ASSERT_STREQ_LEN(written, buff, + "A simple string with one conversion in %A between, " + "writes string as it is"); + + written = func(buff, 36, "A simple string with one conversion", 1.0); + ASSERT_STREQ_LEN(written, buff, "A simple string with one conversion"); + + written = func(buff, 20, "%1f", 1234567890.0); + ASSERT_STREQ_LEN(written, buff, "%1f"); + } + + void insufficentBufsize(FunctionT func) { + char buff[20]; + int written; + + written = func(buff, 5, "%f", 1234567890.0); + EXPECT_EQ(written, 17); + ASSERT_STREQ(buff, "1234"); + + written = func(buff, 5, "%.5f", 1.05); + EXPECT_EQ(written, 7); + ASSERT_STREQ(buff, "1.05"); + + written = func(buff, 0, "%g", 1.0); + EXPECT_EQ(written, 1); + ASSERT_STREQ(buff, "1.05"); // Make sure that buff has not changed + } + + void floatDecimalSinglePrec(FunctionT func) { + char buff[70]; + int written; + + written = func(buff, 16, "%f", 1.0); + ASSERT_STREQ_LEN(written, buff, "1.000000"); + + written = func(buff, 20, "%f", 1234567890.0); + ASSERT_STREQ_LEN(written, buff, "1234567936.000000"); + + written = func(buff, 67, "%.3f", 1.0); + ASSERT_STREQ_LEN(written, buff, "1.000"); + } + + void floatDecimalDoublePrec(FunctionT func) { + char buff[500]; + int written; + + written = func(buff, 99, "%f", 1.0); + ASSERT_STREQ_LEN(written, buff, "1.000000"); + + written = func(buff, 99, "%F", -1.0); + ASSERT_STREQ_LEN(written, buff, "-1.000000"); + + written = func(buff, 99, "%f", -1.234567); + ASSERT_STREQ_LEN(written, buff, "-1.234567"); + + written = func(buff, 99, "%f", 0.0); + ASSERT_STREQ_LEN(written, buff, "0.000000"); + + written = func(buff, 99, "%f", 1.5); + ASSERT_STREQ_LEN(written, buff, "1.500000"); + + written = func(buff, 499, "%f", 1e300); + ASSERT_STREQ_LEN(written, buff, + "100000000000000005250476025520442024870446858110815915491" + "585411551180245" + "798890819578637137508044786404370444383288387817694252323" + "536043057564479" + "218478670698284838720092657580373783023379478809005936895" + "323497079994508" + "111903896764088007465274278014249457925878882005684283811" + "566947219638686" + "5459400540160.000000"); + + written = func(buff, 99, "%f", 0.1); + ASSERT_STREQ_LEN(written, buff, "0.100000"); + + written = func(buff, 99, "%f", 1234567890123456789.0); + ASSERT_STREQ_LEN(written, buff, "1234567890123456768.000000"); + + written = func(buff, 99, "%f", 9999999999999.99); + ASSERT_STREQ_LEN(written, buff, "9999999999999.990234"); + + written = func(buff, 99, "%f", 0.1); + ASSERT_STREQ_LEN(written, buff, "0.100000"); + + written = func(buff, 99, "%f", 1234567890123456789.0); + ASSERT_STREQ_LEN(written, buff, "1234567890123456768.000000"); + + written = func(buff, 99, "%f", 9999999999999.99); + ASSERT_STREQ_LEN(written, buff, "9999999999999.990234"); + + // Precision Tests + written = func(buff, 100, "%.2f", 9999999999999.99); + ASSERT_STREQ_LEN(written, buff, "9999999999999.99"); + + written = func(buff, 100, "%.1f", 9999999999999.99); + ASSERT_STREQ_LEN(written, buff, "10000000000000.0"); + + written = func(buff, 100, "%.5f", 1.25); + ASSERT_STREQ_LEN(written, buff, "1.25000"); + + written = func(buff, 100, "%.0f", 1.25); + ASSERT_STREQ_LEN(written, buff, "1"); + + written = func(buff, 100, "%.20f", 1.234e-10); + ASSERT_STREQ_LEN(written, buff, "0.00000000012340000000"); + } + + void floatDecimalLongDoublePrec(FunctionT func) { + char buff[45]; + int written; + + written = func(buff, 40, "%f", 1.0L); + ASSERT_STREQ_LEN(written, buff, "1.000000"); + + written = func(buff, 10, "%.f", -2.5L); + ASSERT_STREQ_LEN(written, buff, "-2"); + } + + void floatHexExpSinglePrec(FunctionT func) { + char buff[25]; + int written; + + written = func(buff, 0, "%a", 1234567890.0); + EXPECT_EQ(written, 14); + + written = func(buff, 20, "%a", 1234567890.0); + EXPECT_EQ(written, 14); + ASSERT_STREQ(buff, "0x1.26580cp+30"); + + written = func(buff, 20, "%A", 1234567890.0); + EXPECT_EQ(written, 14); + ASSERT_STREQ(buff, "0X1.26580CP+30"); + } + + void floatHexExpDoublePrec(FunctionT func) { + char buff[60]; + int written; + + written = func(buff, 10, "%a", 1.0); + ASSERT_STREQ_LEN(written, buff, "0x1p+0"); + + written = func(buff, 10, "%A", -1.0); + ASSERT_STREQ_LEN(written, buff, "-0X1P+0"); + + written = func(buff, 30, "%a", -0x1.abcdef12345p0); + ASSERT_STREQ_LEN(written, buff, "-0x1.abcdef12345p+0"); + + written = func(buff, 50, "%A", 0x1.abcdef12345p0); + ASSERT_STREQ_LEN(written, buff, "0X1.ABCDEF12345P+0"); + + written = func(buff, 10, "%a", 0.0); + ASSERT_STREQ_LEN(written, buff, "0x0p+0"); + + written = func(buff, 40, "%a", 1.0e100); + ASSERT_STREQ_LEN(written, buff, "0x1.249ad2594c37dp+332"); + + written = func(buff, 30, "%a", 0.1); + ASSERT_STREQ_LEN(written, buff, "0x1.999999999999ap-4"); + } + + void floatHexExpLongDoublePrec(FunctionT func) { + char buff[55]; + int written; + + written = func(buff, 50, "%a", 0.1L); +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + ASSERT_STREQ_LEN(written, buff, "0xc.ccccccccccccccdp-7"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64) + ASSERT_STREQ_LEN(written, buff, "0x1.999999999999ap-4"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128) + ASSERT_STREQ_LEN(written, buff, "0x1.999999999999999999999999999ap-4"); +#endif + + written = func(buff, 20, "%.1a", 0.1L); +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + ASSERT_STREQ_LEN(written, buff, "0xc.dp-7"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64) + ASSERT_STREQ_LEN(written, buff, "0x1.ap-4"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128) + ASSERT_STREQ_LEN(written, buff, "0x1.ap-4"); +#endif + + written = func(buff, 50, "%a", 1.0e1000L); +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + ASSERT_STREQ_LEN(written, buff, "0xf.38db1f9dd3dac05p+3318"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64) + ASSERT_STREQ_LEN(written, buff, "inf"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128) + ASSERT_STREQ_LEN(written, buff, "0x1.e71b63f3ba7b580af1a52d2a7379p+3321"); +#endif + + written = func(buff, 50, "%a", 1.0e-1000L); +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + ASSERT_STREQ_LEN(written, buff, "0x8.68a9188a89e1467p-3325"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64) + ASSERT_STREQ_LEN(written, buff, "0x0p+0"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128) + ASSERT_STREQ_LEN(written, buff, "0x1.0d152311513c28ce202627c06ec2p-3322"); +#endif + + written = func(buff, 50, "%.1a", 0xf.fffffffffffffffp16380L); +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + ASSERT_STREQ_LEN(written, buff, "0x1.0p+16384"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64) + ASSERT_STREQ_LEN(written, buff, "inf"); +#elif defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT128) + ASSERT_STREQ_LEN(written, buff, "0x2.0p+16383"); +#endif + } + + void floatDecimalExpSinglePrec(FunctionT func) { + char buff[25]; + int written; + + written = func(buff, 20, "%.9e", 1234567890.0); + ASSERT_STREQ_LEN(written, buff, "1.234567936e+09"); + + written = func(buff, 20, "%.9E", 1234567890.0); + ASSERT_STREQ_LEN(written, buff, "1.234567936E+09"); + } + + void floatDecimalExpDoublePrec(FunctionT func) { + char buff[101]; + int written; + + written = func(buff, 100, "%e", 1.0); + ASSERT_STREQ_LEN(written, buff, "1.000000e+00"); + + written = func(buff, 100, "%E", -1.0); + ASSERT_STREQ_LEN(written, buff, "-1.000000E+00"); + + written = func(buff, 100, "%e", -1.234567); + ASSERT_STREQ_LEN(written, buff, "-1.234567e+00"); + + written = func(buff, 100, "%e", 0.0); + ASSERT_STREQ_LEN(written, buff, "0.000000e+00"); + + written = func(buff, 100, "%e", 1.5); + ASSERT_STREQ_LEN(written, buff, "1.500000e+00"); + + written = func(buff, 100, "%e", 1e300); + ASSERT_STREQ_LEN(written, buff, "1.000000e+300"); + + written = func(buff, 100, "%e", 1234567890123456789.0); + ASSERT_STREQ_LEN(written, buff, "1.234568e+18"); + + // Precision Tests + written = func(buff, 100, "%.1e", 1.0); + ASSERT_STREQ_LEN(written, buff, "1.0e+00"); + + written = func(buff, 100, "%.1e", 1.99); + ASSERT_STREQ_LEN(written, buff, "2.0e+00"); + + written = func(buff, 100, "%.1e", 9.99); + ASSERT_STREQ_LEN(written, buff, "1.0e+01"); + } + + void floatDecimalExpLongDoublePrec(FunctionT func) { + char buff[100]; + int written; + +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + written = func(buff, 90, "%.9e", 1000000000500000000.1L); + ASSERT_STREQ_LEN(written, buff, "1.000000001e+18"); + + written = func(buff, 90, "%.9e", 1000000000500000000.0L); + ASSERT_STREQ_LEN(written, buff, "1.000000000e+18"); + + written = func(buff, 90, "%e", 0xf.fffffffffffffffp+16380L); + ASSERT_STREQ_LEN(written, buff, "1.189731e+4932"); +#endif // LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80 + } + + void floatDecimalAutoSinglePrec(FunctionT func) { + char buff[25]; + int written; + + written = func(buff, 20, "%.9g", 1234567890.0); + ASSERT_STREQ_LEN(written, buff, "1.23456794e+09"); + + written = func(buff, 20, "%.9G", 1234567890.0); + ASSERT_STREQ_LEN(written, buff, "1.23456794E+09"); + } + + void floatDecimalAutoDoublePrec(FunctionT func) { + char buff[120]; + int written; + + written = func(buff, 100, "%g", 1234567890123456789.0); + ASSERT_STREQ_LEN(written, buff, "1.23457e+18"); + + written = func(buff, 100, "%g", 9999990000000.00); + ASSERT_STREQ_LEN(written, buff, "9.99999e+12"); + + written = func(buff, 100, "%g", 9999999000000.00); + ASSERT_STREQ_LEN(written, buff, "1e+13"); + + written = func(buff, 100, "%g", 0xa.aaaaaaaaaaaaaabp-7); + ASSERT_STREQ_LEN(written, buff, "0.0833333"); + + written = func(buff, 100, "%g", 0.00001); + ASSERT_STREQ_LEN(written, buff, "1e-05"); + + // Precision Tests + written = func(buff, 100, "%.0g", 0.0); + ASSERT_STREQ_LEN(written, buff, "0"); + + written = func(buff, 100, "%.2g", 0.1); + ASSERT_STREQ_LEN(written, buff, "0.1"); + + written = func(buff, 100, "%.2g", 1.09); + ASSERT_STREQ_LEN(written, buff, "1.1"); + + written = func(buff, 100, "%.15g", 22.25); + ASSERT_STREQ_LEN(written, buff, "22.25"); + + written = func(buff, 100, "%.20g", 1.234e-10); + ASSERT_STREQ_LEN(written, buff, "1.2340000000000000814e-10"); + } + + void floatDecimalAutoLongDoublePrec(FunctionT func) { + char buff[100]; + int written; + +#if defined(LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80) + written = func(buff, 99, "%g", 0xf.fffffffffffffffp+16380L); + ASSERT_STREQ_LEN(written, buff, "1.18973e+4932"); + + written = func(buff, 99, "%g", 0xa.aaaaaaaaaaaaaabp-7L); + ASSERT_STREQ_LEN(written, buff, "0.0833333"); + + written = func(buff, 99, "%g", 9.99999999999e-100L); + ASSERT_STREQ_LEN(written, buff, "1e-99"); +#endif // LIBC_TYPES_LONG_DOUBLE_IS_X86_FLOAT80 + } +}; + +#define STRFROM_TEST(InputType, name, func) \ + using LlvmLibc##name##Test = StrfromTest; \ + TEST_F(LlvmLibc##name##Test, FloatDecimalFormat) { \ + floatDecimalFormat(func); \ + } \ + TEST_F(LlvmLibc##name##Test, FloatHexExpFormat) { floatHexExpFormat(func); } \ + TEST_F(LlvmLibc##name##Test, FloatDecimalAutoFormat) { \ + floatDecimalAutoFormat(func); \ + } \ + TEST_F(LlvmLibc##name##Test, FloatDecimalExpFormat) { \ + floatDecimalExpFormat(func); \ + } \ + TEST_F(LlvmLibc##name##Test, ImproperFormatString) { \ + improperFormatString(func); \ + } \ + TEST_F(LlvmLibc##name##Test, InsufficientBufferSize) { \ + insufficentBufsize(func); \ + } diff --git a/libc/test/src/stdlib/strfromd_test.cpp b/libc/test/src/stdlib/strfromd_test.cpp new file mode 100644 index 000000000000..55724d7e902b --- /dev/null +++ b/libc/test/src/stdlib/strfromd_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for strfromd --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "StrfromTest.h" +#include "src/stdlib/strfromd.h" +#include "test/UnitTest/Test.h" + +STRFROM_TEST(double, Strfromd, LIBC_NAMESPACE::strfromd) diff --git a/libc/test/src/stdlib/strfromf_test.cpp b/libc/test/src/stdlib/strfromf_test.cpp index c5489f5f3af2..8b987fd434ac 100644 --- a/libc/test/src/stdlib/strfromf_test.cpp +++ b/libc/test/src/stdlib/strfromf_test.cpp @@ -6,102 +6,8 @@ // //===----------------------------------------------------------------------===// +#include "StrfromTest.h" #include "src/stdlib/strfromf.h" #include "test/UnitTest/Test.h" -TEST(LlvmLibcStrfromfTest, DecimalFloatFormat) { - char buff[100]; - int written; - - written = LIBC_NAMESPACE::strfromf(buff, 16, "%f", 1.0); - EXPECT_EQ(written, 8); - ASSERT_STREQ(buff, "1.000000"); - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%f", 1234567890.0); - EXPECT_EQ(written, 17); - ASSERT_STREQ(buff, "1234567936.000000"); - - written = LIBC_NAMESPACE::strfromf(buff, 5, "%f", 1234567890.0); - EXPECT_EQ(written, 17); - ASSERT_STREQ(buff, "1234"); - - written = LIBC_NAMESPACE::strfromf(buff, 67, "%.3f", 1.0); - EXPECT_EQ(written, 5); - ASSERT_STREQ(buff, "1.000"); - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%1f", 1234567890.0); - EXPECT_EQ(written, 3); - ASSERT_STREQ(buff, "%1f"); -} - -TEST(LlvmLibcStrfromfTest, HexExpFloatFormat) { - char buff[100]; - int written; - - written = LIBC_NAMESPACE::strfromf(buff, 0, "%a", 1234567890.0); - EXPECT_EQ(written, 14); - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%a", 1234567890.0); - EXPECT_EQ(written, 14); - ASSERT_STREQ(buff, "0x1.26580cp+30"); - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%A", 1234567890.0); - EXPECT_EQ(written, 14); - ASSERT_STREQ(buff, "0X1.26580CP+30"); -} - -TEST(LlvmLibcStrfromfTest, DecimalExpFloatFormat) { - char buff[100]; - int written; - written = LIBC_NAMESPACE::strfromf(buff, 20, "%.9e", 1234567890.0); - EXPECT_EQ(written, 15); - ASSERT_STREQ(buff, "1.234567936e+09"); - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%.9E", 1234567890.0); - EXPECT_EQ(written, 15); - ASSERT_STREQ(buff, "1.234567936E+09"); -} - -TEST(LlvmLibcStrfromfTest, AutoDecimalFloatFormat) { - char buff[100]; - int written; - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%.9g", 1234567890.0); - EXPECT_EQ(written, 14); - ASSERT_STREQ(buff, "1.23456794e+09"); - - written = LIBC_NAMESPACE::strfromf(buff, 20, "%.9G", 1234567890.0); - EXPECT_EQ(written, 14); - ASSERT_STREQ(buff, "1.23456794E+09"); - - written = LIBC_NAMESPACE::strfromf(buff, 0, "%G", 1.0); - EXPECT_EQ(written, 1); -} - -TEST(LlvmLibcStrfromfTest, ImproperFormatString) { - - char buff[100]; - int retval; - retval = LIBC_NAMESPACE::strfromf( - buff, 37, "A simple string with no conversions.", 1.0); - EXPECT_EQ(retval, 36); - ASSERT_STREQ(buff, "A simple string with no conversions."); - - retval = LIBC_NAMESPACE::strfromf( - buff, 37, "%A simple string with one conversion, should overwrite.", 1.0); - EXPECT_EQ(retval, 6); - ASSERT_STREQ(buff, "0X1P+0"); - - retval = LIBC_NAMESPACE::strfromf(buff, 74, - "A simple string with one conversion in %A " - "between, writes string as it is", - 1.0); - EXPECT_EQ(retval, 73); - ASSERT_STREQ(buff, "A simple string with one conversion in %A between, " - "writes string as it is"); - - retval = LIBC_NAMESPACE::strfromf(buff, 36, - "A simple string with one conversion", 1.0); - EXPECT_EQ(retval, 35); - ASSERT_STREQ(buff, "A simple string with one conversion"); -} +STRFROM_TEST(float, StrFromf, LIBC_NAMESPACE::strfromf) diff --git a/libc/test/src/stdlib/strfroml_test.cpp b/libc/test/src/stdlib/strfroml_test.cpp new file mode 100644 index 000000000000..cf472a39a5bf --- /dev/null +++ b/libc/test/src/stdlib/strfroml_test.cpp @@ -0,0 +1,13 @@ +//===-- Unittests for strfroml --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "StrfromTest.h" +#include "src/stdlib/strfroml.h" +#include "test/UnitTest/Test.h" + +STRFROM_TEST(long double, Strfroml, LIBC_NAMESPACE::strfroml) -- GitLab From e64e15ee597370a9731fcba0b2b8a514f26125e7 Mon Sep 17 00:00:00 2001 From: Craig Topper Date: Fri, 22 Mar 2024 11:39:42 -0700 Subject: [PATCH 294/296] [RISCV] Move the RISCVSchedule.td include after RISCVRegisterInfo.td. NFC Registers shouldn't depend on the scheduler, but a scheduler predicate could depend on a register. This would make it possible to move VLDSX0Pred out of the SiFive7 scheduler model to RISCVSchedule.td if another model needed it. --- llvm/lib/Target/RISCV/RISCV.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/RISCV/RISCV.td b/llvm/lib/Target/RISCV/RISCV.td index 22736edc5f07..9fb84efd5b6f 100644 --- a/llvm/lib/Target/RISCV/RISCV.td +++ b/llvm/lib/Target/RISCV/RISCV.td @@ -24,8 +24,8 @@ include "RISCVSystemOperands.td" // Registers, calling conventions, instruction descriptions. //===----------------------------------------------------------------------===// -include "RISCVSchedule.td" include "RISCVRegisterInfo.td" +include "RISCVSchedule.td" include "RISCVCallingConv.td" include "RISCVInstrInfo.td" include "GISel/RISCVRegisterBanks.td" -- GitLab From 2120f574103c487787390263b3692c4b167f6bdf Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Fri, 22 Mar 2024 11:45:51 -0700 Subject: [PATCH 295/296] Reapply [workflows] Split pr-code-format into two parts to make it more secure (#78215) (#80495) Actions triggered by pull_request_target events have access to all repository secrets, so it is unsafe to use them when executing untrusted code. The pr-code-format workflow does not execute any untrusted code, but it passes untrused input into clang-format. An attacker could use this to exploit a flaw in clang-format and potentially gain access to the repository secrets. By splitting the workflow, we can use the pull_request target which is more secure and isolate the issue write permissions in a separate job. The pull_request target also makes it easier to test changes to the code-format-helepr.py script, because the version of the script from the pull request will be used rather than the version of the script from main. Fixes #77142 --- .github/workflows/issue-write.yml | 128 +++++++++++++++++++++++++++ .github/workflows/pr-code-format.yml | 20 +++-- llvm/utils/git/code-format-helper.py | 26 ++++++ 3 files changed, 167 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/issue-write.yml diff --git a/.github/workflows/issue-write.yml b/.github/workflows/issue-write.yml new file mode 100644 index 000000000000..02a5f7c213e8 --- /dev/null +++ b/.github/workflows/issue-write.yml @@ -0,0 +1,128 @@ +name: Comment on an issue + +on: + workflow_run: + workflows: ["Check code formatting"] + types: + - completed + +permissions: + contents: read + +jobs: + pr-comment: + runs-on: ubuntu-latest + permissions: + pull-requests: write + if: > + github.event.workflow_run.event == 'pull_request' + steps: + - name: 'Download artifact' + uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1 + with: + github-token: ${{ secrets.ISSUE_WRITE_DOWNLOAD_ARTIFACT }} + run-id: ${{ github.event.workflow_run.id }} + name: workflow-args + + - name: 'Comment on PR' + uses: actions/github-script@v3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + var fs = require('fs'); + const comments = JSON.parse(fs.readFileSync('./comments')); + if (!comments) { + return; + } + + let runInfo = await github.actions.getWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id + }); + + console.log(runInfo); + + + // Query to find the number of the pull request that triggered this job. + // The associated pull requests are based off of the branch name, so if + // you create a pull request for a branch, close it, and then create + // another pull request with the same branch, then this query will return + // two associated pull requests. This is why we have to fetch all the + // associated pull requests and then iterate through them to find the + // one that is open. + const gql_query = ` + query($repo_owner : String!, $repo_name : String!, $branch: String!) { + repository(owner: $repo_owner, name: $repo_name) { + ref (qualifiedName: $branch) { + associatedPullRequests(first: 100) { + nodes { + baseRepository { + owner { + login + } + } + number + state + } + } + } + } + } + ` + const gql_variables = { + repo_owner: runInfo.data.head_repository.owner.login, + repo_name: runInfo.data.head_repository.name, + branch: runInfo.data.head_branch + } + const gql_result = await github.graphql(gql_query, gql_variables); + console.log(gql_result); + console.log(gql_result.repository.ref.associatedPullRequests.nodes); + + var pr_number = 0; + gql_result.repository.ref.associatedPullRequests.nodes.forEach((pr) => { + if (pr.baseRepository.owner.login = context.repo.owner && pr.state == 'OPEN') { + pr_number = pr.number; + } + }); + if (pr_number == 0) { + console.log("Error retrieving pull request number"); + return; + } + + await comments.forEach(function (comment) { + if (comment.id) { + // Security check: Ensure that this comment was created by + // the github-actions bot, so a malicious input won't overwrite + // a user's comment. + github.issues.getComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: comment.id + }).then((old_comment) => { + console.log(old_comment); + if (old_comment.data.user.login != "github-actions[bot]") { + console.log("Invalid comment id: " + comment.id); + return; + } + github.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr_number, + comment_id: comment.id, + body: comment.body + }); + }); + } else { + github.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr_number, + body: comment.body + }); + } + }); + + - name: Dump comments file + if: always() + run: cat comments diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml index 1d1fa2483b65..2ed9b05cac12 100644 --- a/.github/workflows/pr-code-format.yml +++ b/.github/workflows/pr-code-format.yml @@ -1,12 +1,9 @@ name: "Check code formatting" on: - pull_request_target: + pull_request: branches: - main -permissions: - pull-requests: write - jobs: code_formatter: runs-on: ubuntu-latest @@ -31,12 +28,13 @@ jobs: separator: "," skip_initial_fetch: true - # We need to make sure that we aren't executing/using any code from the - # PR for security reasons as we're using pull_request_target. Checkout - # the target branch with the necessary files. + # We need to pull the script from the main branch, so that we ensure + # we get the latest version of this script. - name: Fetch code formatting utils uses: actions/checkout@v4 with: + reository: ${{ github.repository }} + ref: ${{ github.base_ref }} sparse-checkout: | llvm/utils/git/requirements_formatting.txt llvm/utils/git/code-format-helper.py @@ -77,8 +75,16 @@ jobs: # the merge base. run: | python ./code-format-tools/llvm/utils/git/code-format-helper.py \ + --write-comment-to-file \ --token ${{ secrets.GITHUB_TOKEN }} \ --issue-number $GITHUB_PR_NUMBER \ --start-rev $(git merge-base $START_REV $END_REV) \ --end-rev $END_REV \ --changed-files "$CHANGED_FILES" + + - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 + if: always() + with: + name: workflow-args + path: | + comments diff --git a/llvm/utils/git/code-format-helper.py b/llvm/utils/git/code-format-helper.py index 1113bf02570b..af1bb3b5aec5 100755 --- a/llvm/utils/git/code-format-helper.py +++ b/llvm/utils/git/code-format-helper.py @@ -44,6 +44,7 @@ class FormatArgs: token: str = None verbose: bool = True issue_number: int = 0 + write_comment_to_file: bool = False def __init__(self, args: argparse.Namespace = None) -> None: if not args is None: @@ -53,12 +54,14 @@ class FormatArgs: self.token = args.token self.changed_files = args.changed_files self.issue_number = args.issue_number + self.write_comment_to_file = args.write_comment_to_file class FormatHelper: COMMENT_TAG = "" name: str friendly_name: str + comment: dict = None @property def comment_tag(self) -> str: @@ -119,6 +122,13 @@ View the diff from {self.name} here. comment_text = self.comment_tag + "\n\n" + comment_text existing_comment = self.find_comment(pr) + + if args.write_comment_to_file: + self.comment = {"body": comment_text} + if existing_comment: + self.comment["id"] = existing_comment.id + return + if existing_comment: existing_comment.edit(comment_text) elif create_new: @@ -310,6 +320,8 @@ def hook_main(): if fmt.has_tool(): if not fmt.run(args.changed_files, args): failed_fmts.append(fmt.name) + if fmt.comment: + comments.append(fmt.comment) else: print(f"Couldn't find {fmt.name}, can't check " + fmt.friendly_name.lower()) @@ -350,6 +362,11 @@ if __name__ == "__main__": type=str, help="Comma separated list of files that has been changed", ) + parser.add_argument( + "--write-comment-to-file", + action="store_true", + help="Don't post comments on the PR, instead write the comments and metadata a file called 'comment'", + ) args = FormatArgs(parser.parse_args()) @@ -358,9 +375,18 @@ if __name__ == "__main__": changed_files = args.changed_files.split(",") failed_formatters = [] + comments = [] for fmt in ALL_FORMATTERS: if not fmt.run(changed_files, args): failed_formatters.append(fmt.name) + if fmt.comment: + comments.append(fmt.comment) + + if len(comments): + with open("comments", "w") as f: + import json + + json.dump(comments, f) if len(failed_formatters) > 0: print(f"error: some formatters failed: {' '.join(failed_formatters)}") -- GitLab From 80fc61270d6cae8d1bfbd9211727fe1d22fc0cd5 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Fri, 22 Mar 2024 11:46:06 -0700 Subject: [PATCH 296/296] Revert "[bazel] Update to 7.x (#86297)" (#86325) Reverting for https://github.com/llvm/llvm-project/pull/86297#issuecomment-2015660662 This reverts commit ab8ace3bfd5165a8532f710f9c2d8dd40c3fac39. --- utils/bazel/.bazelrc | 5 ----- utils/bazel/.bazelversion | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/utils/bazel/.bazelrc b/utils/bazel/.bazelrc index e8d055ec2322..1d7cf4a4df1b 100644 --- a/utils/bazel/.bazelrc +++ b/utils/bazel/.bazelrc @@ -6,11 +6,6 @@ # Common flags that apply to all configurations. # Use sparingly for things common to all compilers and platforms. ############################################################################### - -# Flip off to disable MODULE.bazel until we're ready. -# https://github.com/llvm/llvm-project/issues/55924 -common --enable_bzlmod=false - # Prevent invalid caching if input files are modified during a build. build --experimental_guard_against_concurrent_changes diff --git a/utils/bazel/.bazelversion b/utils/bazel/.bazelversion index 21c8c7b46b89..5e3254243a3b 100644 --- a/utils/bazel/.bazelversion +++ b/utils/bazel/.bazelversion @@ -1 +1 @@ -7.1.1 +6.1.2 -- GitLab