aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/CodeGen/SelectionDAG
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/CodeGen/SelectionDAG')
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp20
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp14
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp3
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp26
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp100
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp17
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h4
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp2
8 files changed, 159 insertions, 27 deletions
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index f144f17..4f2eb1e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -18863,6 +18863,26 @@ SDValue DAGCombiner::visitFCOPYSIGN(SDNode *N) {
if (SimplifyDemandedBits(SDValue(N, 0)))
return SDValue(N, 0);
+ if (VT != N1.getValueType())
+ return SDValue();
+
+ // If this is equivalent to a disjoint or, replace it with one. This can
+ // happen if the sign operand is a sign mask (i.e., x << sign_bit_position).
+ if (DAG.SignBitIsZeroFP(N0) &&
+ DAG.computeKnownBits(N1).Zero.isMaxSignedValue()) {
+ // TODO: Just directly match the shift pattern. computeKnownBits is heavy
+ // for a such a narrowly targeted case.
+ EVT IntVT = VT.changeTypeToInteger();
+ // TODO: It appears to be profitable in some situations to unconditionally
+ // emit a fabs(n0) to perform this combine.
+ SDValue CastSrc0 = DAG.getNode(ISD::BITCAST, DL, IntVT, N0);
+ SDValue CastSrc1 = DAG.getNode(ISD::BITCAST, DL, IntVT, N1);
+
+ SDValue SignOr = DAG.getNode(ISD::OR, DL, IntVT, CastSrc0, CastSrc1,
+ SDNodeFlags::Disjoint);
+ return DAG.getNode(ISD::BITCAST, DL, VT, SignOr);
+ }
+
return SDValue();
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 316aacd..a0baf82 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4842,9 +4842,15 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
? RTLIB::getSINCOS(VT)
: RTLIB::getSINCOSPI(VT);
- bool Expanded = DAG.expandMultipleResultFPLibCall(LC, Node, Results);
- if (!Expanded)
- llvm_unreachable("Expected scalar FSINCOS[PI] to expand to libcall!");
+ bool Expanded = DAG.expandMultipleResultFPLibCall(LC, Node, Results, VT);
+ if (!Expanded) {
+ DAG.getContext()->emitError(Twine("no libcall available for ") +
+ Node->getOperationName(&DAG));
+ SDValue Poison = DAG.getPOISON(VT);
+ Results.push_back(Poison);
+ Results.push_back(Poison);
+ }
+
break;
}
case ISD::FLOG:
@@ -4934,7 +4940,7 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
EVT VT = Node->getValueType(0);
RTLIB::Libcall LC = Node->getOpcode() == ISD::FMODF ? RTLIB::getMODF(VT)
: RTLIB::getFREXP(VT);
- bool Expanded = DAG.expandMultipleResultFPLibCall(LC, Node, Results,
+ bool Expanded = DAG.expandMultipleResultFPLibCall(LC, Node, Results, VT,
/*CallRetResNo=*/0);
if (!Expanded)
llvm_unreachable("Expected scalar FFREXP/FMODF to expand to libcall!");
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 58983cb..29c4dac 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -1726,7 +1726,8 @@ void DAGTypeLegalizer::ExpandFloatRes_UnaryWithTwoFPResults(
SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
assert(!N->isStrictFPOpcode() && "strictfp not implemented");
SmallVector<SDValue> Results;
- DAG.expandMultipleResultFPLibCall(LC, N, Results, CallRetResNo);
+ DAG.expandMultipleResultFPLibCall(LC, N, Results, N->getValueType(0),
+ CallRetResNo);
for (auto [ResNo, Res] : enumerate(Results)) {
SDValue Lo, Hi;
GetPairElements(Res, Lo, Hi);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 94751be5..f5a54497 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -1268,20 +1268,30 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
return;
break;
- case ISD::FSINCOS:
+
case ISD::FSINCOSPI: {
+ EVT VT = Node->getValueType(0);
+ RTLIB::Libcall LC = RTLIB::getSINCOSPI(VT);
+ if (LC != RTLIB::UNKNOWN_LIBCALL &&
+ DAG.expandMultipleResultFPLibCall(LC, Node, Results, VT))
+ return;
+
+ // TODO: Try to see if there's a narrower call available to use before
+ // scalarizing.
+ break;
+ }
+ case ISD::FSINCOS: {
+ // FIXME: Try to directly match vector case like fsincospi
EVT VT = Node->getValueType(0).getVectorElementType();
- RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
- ? RTLIB::getSINCOS(VT)
- : RTLIB::getSINCOSPI(VT);
- if (DAG.expandMultipleResultFPLibCall(LC, Node, Results))
+ RTLIB::Libcall LC = RTLIB::getSINCOS(VT);
+ if (DAG.expandMultipleResultFPLibCall(LC, Node, Results, VT))
return;
break;
}
case ISD::FMODF: {
- RTLIB::Libcall LC =
- RTLIB::getMODF(Node->getValueType(0).getVectorElementType());
- if (DAG.expandMultipleResultFPLibCall(LC, Node, Results,
+ EVT VT = Node->getValueType(0).getVectorElementType();
+ RTLIB::Libcall LC = RTLIB::getMODF(VT);
+ if (DAG.expandMultipleResultFPLibCall(LC, Node, Results, VT,
/*CallRetResNo=*/0))
return;
break;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 80bbfea..b5d502b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -2514,18 +2514,20 @@ static bool canFoldStoreIntoLibCallOutputPointers(StoreSDNode *StoreNode,
bool SelectionDAG::expandMultipleResultFPLibCall(
RTLIB::Libcall LC, SDNode *Node, SmallVectorImpl<SDValue> &Results,
- std::optional<unsigned> CallRetResNo) {
- LLVMContext &Ctx = *getContext();
- EVT VT = Node->getValueType(0);
- unsigned NumResults = Node->getNumValues();
-
+ EVT CallVT, std::optional<unsigned> CallRetResNo) {
if (LC == RTLIB::UNKNOWN_LIBCALL)
return false;
- const char *LCName = TLI->getLibcallName(LC);
- if (!LCName)
+ EVT VT = Node->getValueType(0);
+
+ RTLIB::LibcallImpl Impl = TLI->getLibcallImpl(LC);
+ if (Impl == RTLIB::Unsupported)
return false;
+ StringRef LCName = TLI->getLibcallImplName(Impl);
+
+ // FIXME: This should not use TargetLibraryInfo. There should be
+ // RTLIB::Libcall entries for each used vector type, and directly matched.
auto getVecDesc = [&]() -> VecDesc const * {
for (bool Masked : {false, true}) {
if (VecDesc const *VD = getLibInfo().getVectorMappingInfo(
@@ -2538,9 +2540,34 @@ bool SelectionDAG::expandMultipleResultFPLibCall(
// For vector types, we must find a vector mapping for the libcall.
VecDesc const *VD = nullptr;
- if (VT.isVector() && !(VD = getVecDesc()))
+ if (VT.isVector() && !CallVT.isVector() && !(VD = getVecDesc()))
return false;
+ bool IsMasked = (VD && VD->isMasked()) ||
+ RTLIB::RuntimeLibcallsInfo::hasVectorMaskArgument(Impl);
+
+ // This wrapper function exists because getVectorMappingInfo works in terms of
+ // function names instead of RTLIB enums.
+
+ // FIXME: If we used a vector mapping, this assumes the calling convention of
+ // the vector function is the same as the scalar.
+
+ StringRef Name = VD ? VD->getVectorFnName() : LCName;
+
+ return expandMultipleResultFPLibCall(Name,
+ TLI->getLibcallImplCallingConv(Impl),
+ Node, Results, CallRetResNo, IsMasked);
+}
+
+// FIXME: This belongs in TargetLowering
+bool SelectionDAG::expandMultipleResultFPLibCall(
+ StringRef Name, CallingConv::ID CC, SDNode *Node,
+ SmallVectorImpl<SDValue> &Results, std::optional<unsigned> CallRetResNo,
+ bool IsMasked) {
+ LLVMContext &Ctx = *getContext();
+ EVT VT = Node->getValueType(0);
+ unsigned NumResults = Node->getNumValues();
+
// Find users of the node that store the results (and share input chains). The
// destination pointers can be used instead of creating stack allocations.
SDValue StoresInChain;
@@ -2598,7 +2625,7 @@ bool SelectionDAG::expandMultipleResultFPLibCall(
SDLoc DL(Node);
// Pass the vector mask (if required).
- if (VD && VD->isMasked()) {
+ if (IsMasked) {
EVT MaskVT = TLI->getSetCCResultType(getDataLayout(), Ctx, VT);
SDValue Mask = getBoolConstant(true, DL, MaskVT, VT);
Args.emplace_back(Mask, MaskVT.getTypeForEVT(Ctx));
@@ -2608,11 +2635,11 @@ bool SelectionDAG::expandMultipleResultFPLibCall(
? Node->getValueType(*CallRetResNo).getTypeForEVT(Ctx)
: Type::getVoidTy(Ctx);
SDValue InChain = StoresInChain ? StoresInChain : getEntryNode();
- SDValue Callee = getExternalSymbol(VD ? VD->getVectorFnName().data() : LCName,
- TLI->getPointerTy(getDataLayout()));
+ SDValue Callee =
+ getExternalSymbol(Name.data(), TLI->getPointerTy(getDataLayout()));
TargetLowering::CallLoweringInfo CLI(*this);
- CLI.setDebugLoc(DL).setChain(InChain).setLibCallee(
- TLI->getLibcallCallingConv(LC), RetType, Callee, std::move(Args));
+ CLI.setDebugLoc(DL).setChain(InChain).setLibCallee(CC, RetType, Callee,
+ std::move(Args));
auto [Call, CallChain] = TLI->LowerCallTo(CLI);
@@ -2920,6 +2947,34 @@ bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth);
}
+bool SelectionDAG::SignBitIsZeroFP(SDValue Op, unsigned Depth) const {
+ if (Depth >= MaxRecursionDepth)
+ return false; // Limit search depth.
+
+ unsigned Opc = Op.getOpcode();
+ switch (Opc) {
+ case ISD::FABS:
+ return true;
+ case ISD::AssertNoFPClass: {
+ FPClassTest NoFPClass =
+ static_cast<FPClassTest>(Op.getConstantOperandVal(1));
+
+ const FPClassTest TestMask = fcNan | fcNegative;
+ return (NoFPClass & TestMask) == TestMask;
+ }
+ case ISD::ARITH_FENCE:
+ return SignBitIsZeroFP(Op, Depth + 1);
+ case ISD::FEXP:
+ case ISD::FEXP2:
+ case ISD::FEXP10:
+ return Op->getFlags().hasNoNaNs();
+ default:
+ return false;
+ }
+
+ llvm_unreachable("covered opcode switch");
+}
+
/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
/// this predicate to simplify operations downstream. Mask is known to be zero
/// for bits that V cannot have.
@@ -4121,6 +4176,25 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
Known.One.clearLowBits(LogOfAlign);
break;
}
+ case ISD::AssertNoFPClass: {
+ Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+
+ FPClassTest NoFPClass =
+ static_cast<FPClassTest>(Op.getConstantOperandVal(1));
+ const FPClassTest NegativeTestMask = fcNan | fcNegative;
+ if ((NoFPClass & NegativeTestMask) == NegativeTestMask) {
+ // Cannot be negative.
+ Known.makeNonNegative();
+ }
+
+ const FPClassTest PositiveTestMask = fcNan | fcPositive;
+ if ((NoFPClass & PositiveTestMask) == PositiveTestMask) {
+ // Cannot be positive.
+ Known.makeNegative();
+ }
+
+ break;
+ }
case ISD::FGETSIGN:
// All bits are zero except the low bit.
Known.Zero.setBitsFrom(1);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 88b0809..6a9022d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4638,6 +4638,12 @@ static std::optional<ConstantRange> getRange(const Instruction &I) {
return std::nullopt;
}
+static FPClassTest getNoFPClass(const Instruction &I) {
+ if (const auto *CB = dyn_cast<CallBase>(&I))
+ return CB->getRetNoFPClass();
+ return fcNone;
+}
+
void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
if (I.isAtomic())
return visitAtomicLoad(I);
@@ -9132,6 +9138,7 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
if (Result.first.getNode()) {
Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first);
+ Result.first = lowerNoFPClassToAssertNoFPClass(DAG, CB, Result.first);
setValue(&CB, Result.first);
}
@@ -10718,6 +10725,16 @@ SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
return DAG.getMergeValues(Ops, SL);
}
+SDValue SelectionDAGBuilder::lowerNoFPClassToAssertNoFPClass(
+ SelectionDAG &DAG, const Instruction &I, SDValue Op) {
+ FPClassTest Classes = getNoFPClass(I);
+ if (Classes == fcNone)
+ return Op;
+
+ return DAG.getNode(ISD::AssertNoFPClass, SDLoc(Op), Op.getValueType(), Op,
+ DAG.getTargetConstant(Classes, SDLoc(), MVT::i32));
+}
+
/// Populate a CallLowerinInfo (into \p CLI) based on the properties of
/// the call being lowered.
///
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index ed63bee..13e2daa 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -429,6 +429,10 @@ public:
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I,
SDValue Op);
+ // Lower nofpclass attributes to AssertNoFPClass
+ SDValue lowerNoFPClassToAssertNoFPClass(SelectionDAG &DAG,
+ const Instruction &I, SDValue Op);
+
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI,
const CallBase *Call, unsigned ArgIdx,
unsigned NumArgs, SDValue Callee,
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 8bc5d2f..e78dfb1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -2448,7 +2448,7 @@ bool SelectionDAGISel::IsLegalToFold(SDValue N, SDNode *U, SDNode *Root,
// a cycle in the scheduling graph.
// If the node has glue, walk down the graph to the "lowest" node in the
- // glueged set.
+ // glued set.
EVT VT = Root->getValueType(Root->getNumValues()-1);
while (VT == MVT::Glue) {
SDNode *GU = Root->getGluedUser();
generated by cgit v1.1 at 2025-11-11 06:49:27 +0000