[InstCombine] Handle isNanOrInf idioms (#80414)

This patch folds:
```
(icmp eq (and (bitcast X to int), ExponentMask), ExponentMask) --> llvm.is.fpclass(X, fcInf|fcNan)
(icmp ne (and (bitcast X to int), ExponentMask), ExponentMask) --> llvm.is.fpclass(X, ~(fcInf|fcNan))
```
Alive2: https://alive2.llvm.org/ce/z/_hXAAF

1 files changed, 27 insertions, 0 deletions

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index d295853..15ec3fd 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -1827,6 +1827,33 @@ Instruction *InstCombinerImpl::foldICmpAndConstConst(ICmpInst &Cmp,
     }
   }
 
+  // (icmp eq (and (bitcast X to int), ExponentMask), ExponentMask) -->
+  // llvm.is.fpclass(X, fcInf|fcNan)
+  // (icmp ne (and (bitcast X to int), ExponentMask), ExponentMask) -->
+  // llvm.is.fpclass(X, ~(fcInf|fcNan))
+  Value *V;
+  if (!Cmp.getParent()->getParent()->hasFnAttribute(
+          Attribute::NoImplicitFloat) &&
+      Cmp.isEquality() && match(X, m_OneUse(m_BitCast(m_Value(V))))) {
+    Type *SrcType = V->getType();
+    Type *DstType = X->getType();
+    Type *FPType = SrcType->getScalarType();
+    // Make sure the bitcast doesn't change between scalar and vector and
+    // doesn't change the number of vector elements.
+    if (SrcType->isVectorTy() == DstType->isVectorTy() &&
+        SrcType->getScalarSizeInBits() == DstType->getScalarSizeInBits() &&
+        FPType->isIEEELikeFPTy() && C1 == *C2) {
+      APInt ExponentMask =
+          APFloat::getInf(FPType->getFltSemantics()).bitcastToAPInt();
+      if (C1 == ExponentMask) {
+        unsigned Mask = FPClassTest::fcNan | FPClassTest::fcInf;
+        if (isICMP_NE)
+          Mask = ~Mask & fcAllFlags;
+        return replaceInstUsesWith(Cmp, Builder.createIsFPClass(V, Mask));
+      }
+    }
+  }
+
   return nullptr;
 }