[clang][CodeGen] Lower Likelihood attributes to @llvm.expect intrin instead of branch weights

08196e0b2e1f8aaa8a854585335c17ba479114df exposed LowerExpectIntrinsic's
internal implementation detail in the form of
LikelyBranchWeight/UnlikelyBranchWeight options to the outside.

While this isn't incorrect from the results viewpoint,
this is suboptimal from the layering viewpoint,
and causes confusion - should transforms also use those weights,
or should they use something else, D98898?

So go back to status quo by making LikelyBranchWeight/UnlikelyBranchWeight
internal again, and fixing all the code that used it directly,
which currently is only clang codegen, thankfully,
to emit proper @llvm.expect intrinsics instead.

7 files changed, 524 insertions, 157 deletions

diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 38f3aa9..fb719ef 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -821,8 +821,11 @@ void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
     llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
     if (ConditionScope.requiresCleanups())
       ExitBlock = createBasicBlock("while.exit");
-    llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop(
-        S.getCond(), getProfileCount(S.getBody()), S.getBody());
+    llvm::MDNode *Weights =
+        createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
+    if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
+      BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
+          BoolCondVal, Stmt::getLikelihood(S.getBody()));
     Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock, Weights);
 
     if (ExitBlock != LoopExit.getBlock()) {
@@ -1008,8 +1011,11 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S,
     // C99 6.8.5p2/p4: The first substatement is executed if the expression
     // compares unequal to 0.  The condition must be a scalar type.
     llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
-    llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop(
-        S.getCond(), getProfileCount(S.getBody()), S.getBody());
+    llvm::MDNode *Weights =
+        createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
+    if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
+      BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
+          BoolCondVal, Stmt::getLikelihood(S.getBody()));
 
     if (llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal))
       if (C->isOne())
@@ -1094,8 +1100,11 @@ CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
   // The body is executed if the expression, contextually converted
   // to bool, is true.
   llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
-  llvm::MDNode *Weights = createProfileOrBranchWeightsForLoop(
-      S.getCond(), getProfileCount(S.getBody()), S.getBody());
+  llvm::MDNode *Weights =
+      createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
+  if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
+    BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
+        BoolCondVal, Stmt::getLikelihood(S.getBody()));
   Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights);
 
   if (ExitBlock != LoopExit.getBlock()) {
@@ -1369,7 +1378,7 @@ void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S,
     // this case.
     (*SwitchWeights)[0] += ThisCount;
   } else if (SwitchLikelihood)
-    Weights = createBranchWeights(LH);
+    Cond = emitCondLikelihoodViaExpectIntrinsic(Cond, LH);
 
   Builder.CreateCondBr(Cond, CaseDest, FalseDest, Weights);
 
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index a00ae74..1c53826 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -1764,31 +1764,39 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
     return;
   }
 
-  // If the branch has a condition wrapped by __builtin_unpredictable,
-  // create metadata that specifies that the branch is unpredictable.
-  // Don't bother if not optimizing because that metadata would not be used.
-  llvm::MDNode *Unpredictable = nullptr;
-  auto *Call = dyn_cast<CallExpr>(Cond->IgnoreImpCasts());
-  if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
-    auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
-    if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
-      llvm::MDBuilder MDHelper(getLLVMContext());
-      Unpredictable = MDHelper.createUnpredictable();
-    }
-  }
-
-  llvm::MDNode *Weights = createBranchWeights(LH);
-  if (!Weights) {
-    uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount);
-    Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount);
-  }
-
   // Emit the code with the fully general case.
   llvm::Value *CondV;
   {
     ApplyDebugLocation DL(*this, Cond);
     CondV = EvaluateExprAsBool(Cond);
   }
+
+  llvm::MDNode *Weights = nullptr;
+  llvm::MDNode *Unpredictable = nullptr;
+
+  // If optimizing, lower unpredictability/probability knowledge about cond.
+  if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
+    // If the branch has a condition wrapped by __builtin_unpredictable,
+    // create metadata that specifies that the branch is unpredictable.
+    if (auto *Call = dyn_cast<CallExpr>(Cond->IgnoreImpCasts())) {
+      auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
+      if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
+        llvm::MDBuilder MDHelper(getLLVMContext());
+        Unpredictable = MDHelper.createUnpredictable();
+      }
+    }
+
+    // If there is a Likelihood knowledge for the cond, lower it.
+    llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(CondV, LH);
+    if (CondV != NewCondV)
+      CondV = NewCondV;
+    else {
+      // Otherwise, lower profile counts.
+      uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount);
+      Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount);
+    }
+  }
+
   Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable);
 }
 
@@ -2632,35 +2640,26 @@ llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
   return llvm::DebugLoc();
 }
 
-static Optional<std::pair<uint32_t, uint32_t>>
-getLikelihoodWeights(Stmt::Likelihood LH) {
+llvm::Value *
+CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
+                                                      Stmt::Likelihood LH) {
   switch (LH) {
-  case Stmt::LH_Unlikely:
-    return std::pair<uint32_t, uint32_t>(llvm::UnlikelyBranchWeight,
-                                         llvm::LikelyBranchWeight);
   case Stmt::LH_None:
-    return None;
+    return Cond;
   case Stmt::LH_Likely:
-    return std::pair<uint32_t, uint32_t>(llvm::LikelyBranchWeight,
-                                         llvm::UnlikelyBranchWeight);
+  case Stmt::LH_Unlikely:
+    // Don't generate llvm.expect on -O0 as the backend won't use it for
+    // anything.
+    if (CGM.getCodeGenOpts().OptimizationLevel == 0)
+      return Cond;
+    llvm::Type *CondTy = Cond->getType();
+    assert(CondTy->isIntegerTy(1) && "expecting condition to be a boolean");
+    llvm::Function *FnExpect =
+        CGM.getIntrinsic(llvm::Intrinsic::expect, CondTy);
+    llvm::Value *ExpectedValueOfCond =
+        llvm::ConstantInt::getBool(CondTy, LH == Stmt::LH_Likely);
+    return Builder.CreateCall(FnExpect, {Cond, ExpectedValueOfCond},
+                              Cond->getName() + ".expval");
   }
   llvm_unreachable("Unknown Likelihood");
 }
-
-llvm::MDNode *CodeGenFunction::createBranchWeights(Stmt::Likelihood LH) const {
-  Optional<std::pair<uint32_t, uint32_t>> LHW = getLikelihoodWeights(LH);
-  if (!LHW)
-    return nullptr;
-
-  llvm::MDBuilder MDHelper(CGM.getLLVMContext());
-  return MDHelper.createBranchWeights(LHW->first, LHW->second);
-}
-
-llvm::MDNode *CodeGenFunction::createProfileOrBranchWeightsForLoop(
-    const Stmt *Cond, uint64_t LoopCount, const Stmt *Body) const {
-  llvm::MDNode *Weights = createProfileWeightsForLoop(Cond, LoopCount);
-  if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
-    Weights = createBranchWeights(Stmt::getLikelihood(Body));
-
-  return Weights;
-}
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 4bca21a..11de916 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -1445,8 +1445,9 @@ private:
   };
   OpenMPCancelExitStack OMPCancelStack;
 
-  /// Calculate branch weights for the likelihood attribute
-  llvm::MDNode *createBranchWeights(Stmt::Likelihood LH) const;
+  /// Lower the Likelihood knowledge about the \p Cond via llvm.expect intrin.
+  llvm::Value *emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
+                                                    Stmt::Likelihood LH);
 
   CodeGenPGO PGO;
 
@@ -1457,13 +1458,6 @@ private:
   llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond,
                                             uint64_t LoopCount) const;
 
-  /// Calculate the branch weight for PGO data or the likelihood attribute.
-  /// The function tries to get the weight of \ref createProfileWeightsForLoop.
-  /// If that fails it gets the weight of \ref createBranchWeights.
-  llvm::MDNode *createProfileOrBranchWeightsForLoop(const Stmt *Cond,
-                                                    uint64_t LoopCount,
-                                                    const Stmt *Body) const;
-
 public:
   /// Increment the profiler's counter for the given statement by \p StepV.
   /// If \p StepV is null, the default increment is 1.
diff --git a/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp b/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp
index ec2ee37..27e8a49 100644
--- a/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp
+++ b/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp
@@ -1,60 +1,263 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // RUN: %clang_cc1 -O1 -disable-llvm-passes -emit-llvm %s -o - -triple=x86_64-linux-gnu -verify
 // RUN: %clang_cc1 -O1 -disable-llvm-passes -emit-llvm %s -o - -triple=x86_64-linux-gnu | FileCheck %s
 
+// CHECK-LABEL: @_Z2wli(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[E_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2:![0-9]+]]
+// CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+// CHECK:       while.cond:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[TMP0]], 0
+// CHECK-NEXT:    [[TOBOOL_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[TOBOOL]], i1 true)
+// CHECK-NEXT:    br i1 [[TOBOOL_EXPVAL]], label [[WHILE_BODY:%.*]], label [[WHILE_END:%.*]]
+// CHECK:       while.body:
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+// CHECK-NEXT:    store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[WHILE_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK:       while.end:
+// CHECK-NEXT:    ret void
+//
 void wl(int e){
-  // CHECK-LABEL: define{{.*}}wl
-  // CHECK: br {{.*}} !prof !6
   while(e) [[likely]] ++e;
 }
 
+// CHECK-LABEL: @_Z2wui(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[E_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+// CHECK:       while.cond:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[TMP0]], 0
+// CHECK-NEXT:    [[TOBOOL_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[TOBOOL]], i1 false)
+// CHECK-NEXT:    br i1 [[TOBOOL_EXPVAL]], label [[WHILE_BODY:%.*]], label [[WHILE_END:%.*]]
+// CHECK:       while.body:
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+// CHECK-NEXT:    store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[WHILE_COND]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK:       while.end:
+// CHECK-NEXT:    ret void
+//
 void wu(int e){
-  // CHECK-LABEL: define{{.*}}wu
-  // CHECK: br {{.*}} !prof !10
   while(e) [[unlikely]] ++e;
 }
 
+// CHECK-LABEL: @_Z15w_branch_elidedj(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[E_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
+// CHECK:       while.body:
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[INC:%.*]] = add i32 [[TMP0]], 1
+// CHECK-NEXT:    store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[WHILE_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+//
 void w_branch_elided(unsigned e){
-  // CHECK-LABEL: define{{.*}}w_branch_elided
-  // CHECK-NOT: br {{.*}} !prof
   // expected-warning@+2 {{attribute 'likely' has no effect when annotating an infinite loop}}
   // expected-note@+1 {{annotating the infinite loop here}}
   while(1) [[likely]] ++e;
 }
 
+// CHECK-LABEL: @_Z2flj(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[E_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP0]]) #[[ATTR4:[0-9]+]]
+// CHECK-NEXT:    store i32 0, i32* [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK:       for.cond:
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], [[TMP2]]
+// CHECK-NEXT:    [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 true)
+// CHECK-NEXT:    br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+// CHECK:       for.cond.cleanup:
+// CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP3]]) #[[ATTR4]]
+// CHECK-NEXT:    br label [[FOR_END:%.*]]
+// CHECK:       for.body:
+// CHECK-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK:       for.inc:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[INC:%.*]] = add i32 [[TMP4]], 1
+// CHECK-NEXT:    store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP11:![0-9]+]]
+// CHECK:       for.end:
+// CHECK-NEXT:    ret void
+//
 void fl(unsigned e)
 {
-  // CHECK-LABEL: define{{.*}}fl
-  // CHECK: br {{.*}} !prof !6
   for(int i = 0; i != e; ++e) [[likely]];
 }
 
+// CHECK-LABEL: @_Z2fui(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[E_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i32 [[E:%.*]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP0]]) #[[ATTR4]]
+// CHECK-NEXT:    store i32 0, i32* [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK:       for.cond:
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], [[TMP2]]
+// CHECK-NEXT:    [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 false)
+// CHECK-NEXT:    br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+// CHECK:       for.cond.cleanup:
+// CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP3]]) #[[ATTR4]]
+// CHECK-NEXT:    br label [[FOR_END:%.*]]
+// CHECK:       for.body:
+// CHECK-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK:       for.inc:
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP4]], 1
+// CHECK-NEXT:    store i32 [[INC]], i32* [[E_ADDR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
+// CHECK:       for.end:
+// CHECK-NEXT:    ret void
+//
 void fu(int e)
 {
-  // CHECK-LABEL: define{{.*}}fu
-  // CHECK: br {{.*}} !prof !10
   for(int i = 0; i != e; ++e) [[unlikely]];
 }
 
+// CHECK-LABEL: @_Z15f_branch_elidedv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK:       for.cond:
+// CHECK-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP13:![0-9]+]]
+//
 void f_branch_elided()
 {
-  // CHECK-LABEL: define{{.*}}f_branch_elided
-  // CHECK-NOT: br {{.*}} !prof
   for(;;) [[likely]];
 }
 
+// CHECK-LABEL: @_Z3frlOA4_i(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[E_ADDR:%.*]] = alloca [4 x i32]*, align 8
+// CHECK-NEXT:    [[__RANGE1:%.*]] = alloca [4 x i32]*, align 8
+// CHECK-NEXT:    [[__BEGIN1:%.*]] = alloca i32*, align 8
+// CHECK-NEXT:    [[__END1:%.*]] = alloca i32*, align 8
+// CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store [4 x i32]* [[E:%.*]], [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14:![0-9]+]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP0]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP1:%.*]] = load [4 x i32]*, [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    store [4 x i32]* [[TMP1]], [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32** [[__BEGIN1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP2]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP3]], i64 0, i64 0
+// CHECK-NEXT:    store i32* [[ARRAYDECAY]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32** [[__END1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP4]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP5:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP5]], i64 0, i64 0
+// CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAYDECAY1]], i64 4
+// CHECK-NEXT:    store i32* [[ADD_PTR]], i32** [[__END1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK:       for.cond:
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[TMP7:%.*]] = load i32*, i32** [[__END1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32* [[TMP6]], [[TMP7]]
+// CHECK-NEXT:    [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 true)
+// CHECK-NEXT:    br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+// CHECK:       for.cond.cleanup:
+// CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32** [[__END1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP8]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32** [[__BEGIN1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP9]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP10:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP10]]) #[[ATTR4]]
+// CHECK-NEXT:    br label [[FOR_END:%.*]]
+// CHECK:       for.body:
+// CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP11]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP12:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i32 [[TMP13]], i32* [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP14]]) #[[ATTR4]]
+// CHECK-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK:       for.inc:
+// CHECK-NEXT:    [[TMP15:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[TMP15]], i32 1
+// CHECK-NEXT:    store i32* [[INCDEC_PTR]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP16:![0-9]+]]
+// CHECK:       for.end:
+// CHECK-NEXT:    ret void
+//
 void frl(int (&&e) [4])
 {
-  // CHECK-LABEL: define{{.*}}frl
-  // CHECK: br {{.*}} !prof !6
   for(int i : e) [[likely]];
 }
 
+// CHECK-LABEL: @_Z3fruOA4_i(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[E_ADDR:%.*]] = alloca [4 x i32]*, align 8
+// CHECK-NEXT:    [[__RANGE1:%.*]] = alloca [4 x i32]*, align 8
+// CHECK-NEXT:    [[__BEGIN1:%.*]] = alloca i32*, align 8
+// CHECK-NEXT:    [[__END1:%.*]] = alloca i32*, align 8
+// CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store [4 x i32]* [[E:%.*]], [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP0]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP1:%.*]] = load [4 x i32]*, [4 x i32]** [[E_ADDR]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    store [4 x i32]* [[TMP1]], [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32** [[__BEGIN1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP2]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP3]], i64 0, i64 0
+// CHECK-NEXT:    store i32* [[ARRAYDECAY]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32** [[__END1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP4]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP5:%.*]] = load [4 x i32]*, [4 x i32]** [[__RANGE1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[TMP5]], i64 0, i64 0
+// CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAYDECAY1]], i64 4
+// CHECK-NEXT:    store i32* [[ADD_PTR]], i32** [[__END1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK:       for.cond:
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[TMP7:%.*]] = load i32*, i32** [[__END1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32* [[TMP6]], [[TMP7]]
+// CHECK-NEXT:    [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 false)
+// CHECK-NEXT:    br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+// CHECK:       for.cond.cleanup:
+// CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32** [[__END1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP8]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32** [[__BEGIN1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP9]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP10:%.*]] = bitcast [4 x i32]** [[__RANGE1]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP10]]) #[[ATTR4]]
+// CHECK-NEXT:    br label [[FOR_END:%.*]]
+// CHECK:       for.body:
+// CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP11]]) #[[ATTR4]]
+// CHECK-NEXT:    [[TMP12:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    store i32 [[TMP13]], i32* [[I]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32* [[I]] to i8*
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP14]]) #[[ATTR4]]
+// CHECK-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK:       for.inc:
+// CHECK-NEXT:    [[TMP15:%.*]] = load i32*, i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[TMP15]], i32 1
+// CHECK-NEXT:    store i32* [[INCDEC_PTR]], i32** [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
+// CHECK-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP17:![0-9]+]]
+// CHECK:       for.end:
+// CHECK-NEXT:    ret void
+//
 void fru(int (&&e) [4])
 {
-  // CHECK-LABEL: define{{.*}}fru
-  // CHECK: br {{.*}} !prof !10
   for(int i : e) [[unlikely]];
 }
-
-// CHECK: !6 = !{!"branch_weights", i32 2000, i32 1}
-// CHECK: !10 = !{!"branch_weights", i32 1, i32 2000}
diff --git a/clang/test/CodeGenCXX/attr-likelihood-switch-branch-weights.cpp b/clang/test/CodeGenCXX/attr-likelihood-switch-branch-weights.cpp
index 5fb7a67..05729eb 100644
--- a/clang/test/CodeGenCXX/attr-likelihood-switch-branch-weights.cpp
+++ b/clang/test/CodeGenCXX/attr-likelihood-switch-branch-weights.cpp
@@ -1,59 +1,111 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // RUN: %clang_cc1 -O1 -disable-llvm-passes -emit-llvm %s -o - -triple=x86_64-linux-gnu | FileCheck %s
 
 extern volatile int i;
 
+// CHECK-LABEL: @_Z8OneCaseLv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2:![0-9]+]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_EPILOG]]
+// CHECK-NEXT:    ], !prof !6
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void OneCaseL() {
-  // CHECK-LABEL: define{{.*}}OneCaseL
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !6
   switch (i) {
     [[likely]] case 1: break;
   }
 }
 
+// CHECK-LABEL: @_Z8OneCaseUv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_BB:%.*]]
+// CHECK-NEXT:    ], !prof !7
+// CHECK:       sw.bb:
+// CHECK-NEXT:    [[TMP1:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+// CHECK-NEXT:    store volatile i32 [[INC]], i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void OneCaseU() {
-  // CHECK-LABEL: define{{.*}}OneCaseU
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !7
   switch (i) {
     [[unlikely]] case 1: ++i; break;
   }
 }
 
+// CHECK-LABEL: @_Z10TwoCasesLNv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_EPILOG]]
+// CHECK-NEXT:    i32 2, label [[SW_EPILOG]]
+// CHECK-NEXT:    ], !prof !8
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void TwoCasesLN() {
-  // CHECK-LABEL: define{{.*}}TwoCasesLN
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !8
   switch (i) {
     [[likely]] case 1: break;
     case 2: break;
   }
 }
 
+// CHECK-LABEL: @_Z10TwoCasesUNv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_EPILOG]]
+// CHECK-NEXT:    i32 2, label [[SW_EPILOG]]
+// CHECK-NEXT:    ], !prof !9
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void TwoCasesUN() {
-  // CHECK-LABEL: define{{.*}}TwoCasesUN
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !9
   switch (i) {
     [[unlikely]] case 1: break;
     case 2: break;
   }
 }
 
+// CHECK-LABEL: @_Z10TwoCasesLUv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_EPILOG]]
+// CHECK-NEXT:    i32 2, label [[SW_EPILOG]]
+// CHECK-NEXT:    ], !prof !10
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void TwoCasesLU() {
-  // CHECK-LABEL: define{{.*}}TwoCasesLU
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !10
   switch (i) {
     [[likely]] case 1: break;
     [[unlikely]] case 2: break;
   }
 }
 
+// CHECK-LABEL: @_Z20CasesFallthroughNNLNv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_BB:%.*]]
+// CHECK-NEXT:    i32 2, label [[SW_BB]]
+// CHECK-NEXT:    i32 3, label [[SW_BB1:%.*]]
+// CHECK-NEXT:    i32 4, label [[SW_BB1]]
+// CHECK-NEXT:    ], !prof !11
+// CHECK:       sw.bb:
+// CHECK-NEXT:    br label [[SW_BB1]]
+// CHECK:       sw.bb1:
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void CasesFallthroughNNLN() {
-  // CHECK-LABEL: define{{.*}}CasesFallthroughNNLN
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !11
   switch (i) {
     case 1:
     case 2:
@@ -62,10 +114,23 @@ void CasesFallthroughNNLN() {
   }
 }
 
+// CHECK-LABEL: @_Z20CasesFallthroughNNUNv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_BB:%.*]]
+// CHECK-NEXT:    i32 2, label [[SW_BB]]
+// CHECK-NEXT:    i32 3, label [[SW_BB1:%.*]]
+// CHECK-NEXT:    i32 4, label [[SW_BB1]]
+// CHECK-NEXT:    ], !prof !12
+// CHECK:       sw.bb:
+// CHECK-NEXT:    br label [[SW_BB1]]
+// CHECK:       sw.bb1:
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void CasesFallthroughNNUN() {
-  // CHECK-LABEL: define{{.*}}CasesFallthroughNNUN
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !12
   switch (i) {
     case 1:
     case 2:
@@ -74,10 +139,32 @@ void CasesFallthroughNNUN() {
   }
 }
 
+// CHECK-LABEL: @_Z28CasesFallthroughRangeSmallLNv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_BB:%.*]]
+// CHECK-NEXT:    i32 2, label [[SW_BB]]
+// CHECK-NEXT:    i32 3, label [[SW_BB]]
+// CHECK-NEXT:    i32 4, label [[SW_BB]]
+// CHECK-NEXT:    i32 5, label [[SW_BB]]
+// CHECK-NEXT:    i32 102, label [[SW_BB1:%.*]]
+// CHECK-NEXT:    i32 103, label [[SW_BB2:%.*]]
+// CHECK-NEXT:    i32 104, label [[SW_BB2]]
+// CHECK-NEXT:    ], !prof !13
+// CHECK:       sw.bb:
+// CHECK-NEXT:    [[TMP1:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+// CHECK-NEXT:    store volatile i32 [[INC]], i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[SW_BB1]]
+// CHECK:       sw.bb1:
+// CHECK-NEXT:    br label [[SW_BB2]]
+// CHECK:       sw.bb2:
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void CasesFallthroughRangeSmallLN() {
-  // CHECK-LABEL: define{{.*}}CasesFallthroughRangeSmallLN
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !13
   switch (i) {
     case 1 ... 5: ++i;
     case 102:
@@ -86,10 +173,32 @@ void CasesFallthroughRangeSmallLN() {
   }
 }
 
+// CHECK-LABEL: @_Z28CasesFallthroughRangeSmallUNv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_EPILOG:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_BB:%.*]]
+// CHECK-NEXT:    i32 2, label [[SW_BB]]
+// CHECK-NEXT:    i32 3, label [[SW_BB]]
+// CHECK-NEXT:    i32 4, label [[SW_BB]]
+// CHECK-NEXT:    i32 5, label [[SW_BB]]
+// CHECK-NEXT:    i32 102, label [[SW_BB1:%.*]]
+// CHECK-NEXT:    i32 103, label [[SW_BB2:%.*]]
+// CHECK-NEXT:    i32 104, label [[SW_BB2]]
+// CHECK-NEXT:    ], !prof !14
+// CHECK:       sw.bb:
+// CHECK-NEXT:    [[TMP1:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+// CHECK-NEXT:    store volatile i32 [[INC]], i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    br label [[SW_BB1]]
+// CHECK:       sw.bb1:
+// CHECK-NEXT:    br label [[SW_BB2]]
+// CHECK:       sw.bb2:
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void CasesFallthroughRangeSmallUN() {
-  // CHECK-LABEL: define{{.*}}CasesFallthroughRangeSmallUN
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !14
   switch (i) {
     case 1 ... 5: ++i;
     case 102:
@@ -98,12 +207,26 @@ void CasesFallthroughRangeSmallUN() {
   }
 }
 
+// CHECK-LABEL: @_Z29CasesFallthroughRangeLargeLLNv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_CASERANGE:%.*]] [
+// CHECK-NEXT:    i32 1003, label [[SW_BB1:%.*]]
+// CHECK-NEXT:    i32 104, label [[SW_BB1]]
+// CHECK-NEXT:    ], !prof !8
+// CHECK:       sw.bb:
+// CHECK-NEXT:    br label [[SW_BB1]]
+// CHECK:       sw.bb1:
+// CHECK-NEXT:    br label [[SW_EPILOG:%.*]]
+// CHECK:       sw.caserange:
+// CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], 0
+// CHECK-NEXT:    [[INBOUNDS:%.*]] = icmp ule i32 [[TMP1]], 64
+// CHECK-NEXT:    [[INBOUNDS_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[INBOUNDS]], i1 true)
+// CHECK-NEXT:    br i1 [[INBOUNDS_EXPVAL]], label [[SW_BB:%.*]], label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void CasesFallthroughRangeLargeLLN() {
-  // CHECK-LABEL: define{{.*}}CasesFallthroughRangeLargeLLN
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !8
-  // CHECK: caserange
-  // CHECK: br{{.*}} !prof !15
   switch (i) {
     [[likely]] case 0 ... 64:
     [[likely]] case 1003:
@@ -111,12 +234,26 @@ void CasesFallthroughRangeLargeLLN() {
   }
 }
 
+// CHECK-LABEL: @_Z29CasesFallthroughRangeLargeUUNv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_CASERANGE:%.*]] [
+// CHECK-NEXT:    i32 1003, label [[SW_BB1:%.*]]
+// CHECK-NEXT:    i32 104, label [[SW_BB1]]
+// CHECK-NEXT:    ], !prof !9
+// CHECK:       sw.bb:
+// CHECK-NEXT:    br label [[SW_BB1]]
+// CHECK:       sw.bb1:
+// CHECK-NEXT:    br label [[SW_EPILOG:%.*]]
+// CHECK:       sw.caserange:
+// CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], 0
+// CHECK-NEXT:    [[INBOUNDS:%.*]] = icmp ule i32 [[TMP1]], 64
+// CHECK-NEXT:    [[INBOUNDS_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[INBOUNDS]], i1 false)
+// CHECK-NEXT:    br i1 [[INBOUNDS_EXPVAL]], label [[SW_BB:%.*]], label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void CasesFallthroughRangeLargeUUN() {
-  // CHECK-LABEL: define{{.*}}CasesFallthroughRangeLargeUUN
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !9
-  // CHECK: caserange
-  // CHECK: br{{.*}} !prof !16
   switch (i) {
     [[unlikely]] case 0 ... 64:
     [[unlikely]] case 1003:
@@ -124,30 +261,55 @@ void CasesFallthroughRangeLargeUUN() {
   }
 }
 
+// CHECK-LABEL: @_Z15OneCaseDefaultLv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_EPILOG:%.*]]
+// CHECK-NEXT:    ], !prof !15
+// CHECK:       sw.default:
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void OneCaseDefaultL() {
-  // CHECK-LABEL: define{{.*}}OneCaseDefaultL
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !17
   switch (i) {
     case 1: break;
     [[likely]] default: break;
   }
 }
 
+// CHECK-LABEL: @_Z15OneCaseDefaultUv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_EPILOG:%.*]]
+// CHECK-NEXT:    ], !prof !16
+// CHECK:       sw.default:
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void OneCaseDefaultU() {
-  // CHECK-LABEL: define{{.*}}OneCaseDefaultU
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !18
   switch (i) {
     case 1: break;
     [[unlikely]] default: break;
   }
 }
 
+// CHECK-LABEL: @_Z18TwoCasesDefaultLNLv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_EPILOG:%.*]]
+// CHECK-NEXT:    i32 2, label [[SW_EPILOG]]
+// CHECK-NEXT:    ], !prof !17
+// CHECK:       sw.default:
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void TwoCasesDefaultLNL() {
-  // CHECK-LABEL: define{{.*}}TwoCasesDefaultLNL
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !19
   switch (i) {
     [[likely]] case 1: break;
     case 2: break;
@@ -155,10 +317,19 @@ void TwoCasesDefaultLNL() {
   }
 }
 
+// CHECK-LABEL: @_Z18TwoCasesDefaultLNNv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_EPILOG:%.*]]
+// CHECK-NEXT:    i32 2, label [[SW_EPILOG]]
+// CHECK-NEXT:    ], !prof !8
+// CHECK:       sw.default:
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void TwoCasesDefaultLNN() {
-  // CHECK-LABEL: define{{.*}}TwoCasesDefaultLNN
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !8
   switch (i) {
     [[likely]] case 1: break;
     case 2: break;
@@ -166,29 +337,22 @@ void TwoCasesDefaultLNN() {
   }
 }
 
+// CHECK-LABEL: @_Z18TwoCasesDefaultLNUv(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load volatile i32, i32* @i, align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    switch i32 [[TMP0]], label [[SW_DEFAULT:%.*]] [
+// CHECK-NEXT:    i32 1, label [[SW_EPILOG:%.*]]
+// CHECK-NEXT:    i32 2, label [[SW_EPILOG]]
+// CHECK-NEXT:    ], !prof !18
+// CHECK:       sw.default:
+// CHECK-NEXT:    br label [[SW_EPILOG]]
+// CHECK:       sw.epilog:
+// CHECK-NEXT:    ret void
+//
 void TwoCasesDefaultLNU() {
-  // CHECK-LABEL: define{{.*}}TwoCasesDefaultLNU
-  // CHECK: switch
-  // CHECK: {{.*}} !prof !20
   switch (i) {
     [[likely]] case 1: break;
     case 2: break;
     [[unlikely]] default: break;
   }
 }
-
-// CHECK: !6 = !{!"branch_weights", i32 357913942, i32 715827883}
-// CHECK: !7 = !{!"branch_weights", i32 536870912, i32 1}
-// CHECK: !8 = !{!"branch_weights", i32 238609295, i32 715827883, i32 238609295}
-// CHECK: !9 = !{!"branch_weights", i32 357913942, i32 1, i32 357913942}
-// CHECK: !10 = !{!"branch_weights", i32 357913942, i32 715827883, i32 1}
-// CHECK: !11 = !{!"branch_weights", i32 143165577, i32 143165577, i32 143165577, i32 715827883, i32 143165577}
-// CHECK: !12 = !{!"branch_weights", i32 214748365, i32 214748365, i32 214748365, i32 1, i32 214748365}
-// CHECK: !13 = !{!"branch_weights", i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 79536432, i32 715827883, i32 79536432}
-// CHECK: !14 = !{!"branch_weights", i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 119304648, i32 1, i32 119304648}
-// CHECK: !15 = !{!"branch_weights", i32 2000, i32 1}
-// CHECK: !16 = !{!"branch_weights", i32 1, i32 2000}
-// CHECK: !17 = !{!"branch_weights", i32 715827883, i32 357913942}
-// CHECK: !18 = !{!"branch_weights", i32 1, i32 536870912}
-// CHECK: !19 = !{!"branch_weights", i32 536870912, i32 536870912, i32 268435456}
-// CHECK: !20 = !{!"branch_weights", i32 1, i32 715827883, i32 357913942}
diff --git a/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h b/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h
index 22b2e64..4e47ff7 100644
--- a/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h
+++ b/llvm/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h
@@ -17,7 +17,6 @@
 
 #include "llvm/IR/Function.h"
 #include "llvm/IR/PassManager.h"
-#include "llvm/Support/CommandLine.h"
 
 namespace llvm {
 
@@ -32,8 +31,6 @@ struct LowerExpectIntrinsicPass : PassInfoMixin<LowerExpectIntrinsicPass> {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &);
 };
 
-extern cl::opt<uint32_t> LikelyBranchWeight;
-extern cl::opt<uint32_t> UnlikelyBranchWeight;
 }
 
 #endif
diff --git a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
index da13075..7f3549a 100644
--- a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
@@ -24,6 +24,7 @@
 #include "llvm/IR/Metadata.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Transforms/Scalar.h"
 
@@ -41,15 +42,15 @@ STATISTIC(ExpectIntrinsicsHandled,
 // only be used in extreme cases, we could make this ratio higher. As it stands,
 // programmers may be using __builtin_expect() / llvm.expect to annotate that a
 // branch is likely or unlikely to be taken.
-//
-// There is a known dependency on this ratio in CodeGenPrepare when transforming
-// 'select' instructions. It may be worthwhile to hoist these values to some
-// shared space, so they can be used directly by other passes.
 
-cl::opt<uint32_t> llvm::LikelyBranchWeight(
+// WARNING: these values are internal implementation detail of the pass.
+// They should not be exposed to the outside of the pass, front-end codegen
+// should emit @llvm.expect intrinsics instead of using these weights directly.
+// Transforms should use TargetLowering getPredictableBranchThreshold() hook.
+static cl::opt<uint32_t> LikelyBranchWeight(
     "likely-branch-weight", cl::Hidden, cl::init(2000),
     cl::desc("Weight of the branch likely to be taken (default = 2000)"));
-cl::opt<uint32_t> llvm::UnlikelyBranchWeight(
+static cl::opt<uint32_t> UnlikelyBranchWeight(
     "unlikely-branch-weight", cl::Hidden, cl::init(1),
     cl::desc("Weight of the branch unlikely to be taken (default = 1)"));