diff options
author | Haohai Wen <haohai.wen@intel.com> | 2024-03-28 20:07:15 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-03-28 20:07:15 +0800 |
commit | 896037c75ace929327e5b0bf5832157f9d81e6e7 (patch) | |
tree | d6675288240ea4544ffdea1f20273cdc54e03002 | |
parent | daa755ba7b7fe11e078f1e6f43d446234023f859 (diff) | |
download | llvm-896037c75ace929327e5b0bf5832157f9d81e6e7.zip llvm-896037c75ace929327e5b0bf5832157f9d81e6e7.tar.gz llvm-896037c75ace929327e5b0bf5832157f9d81e6e7.tar.bz2 |
[LoopRotate] Set loop back edge weight to not less than exit weight (#86496)
Branch weight from sample-based PGO may be not inaccurate due to
sampling. If the loop body must be executed, then origin loop back
edge weight must be not less than exit weight.
-rw-r--r-- | llvm/lib/Transforms/Utils/LoopRotationUtils.cpp | 10 | ||||
-rw-r--r-- | llvm/test/Transforms/LoopRotate/update-branch-weights.ll | 4 |
2 files changed, 12 insertions, 2 deletions
diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp index bc67117..0f55af3 100644 --- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp @@ -347,9 +347,19 @@ static void updateBranchWeights(BranchInst &PreHeaderBI, BranchInst &LoopBI, // probabilities as if there are only 0-trip and 1-trip cases. ExitWeight0 = OrigLoopExitWeight - OrigLoopBackedgeWeight; } + } else { + // Theoretically, if the loop body must be executed at least once, the + // backedge count must be not less than exit count. However the branch + // weight collected by sampling-based PGO may be not very accurate due to + // sampling. Therefore this workaround is required here to avoid underflow + // of unsigned in following update of branch weight. + if (OrigLoopExitWeight > OrigLoopBackedgeWeight) + OrigLoopBackedgeWeight = OrigLoopExitWeight; } + assert(OrigLoopExitWeight >= ExitWeight0 && "Bad branch weight"); ExitWeight1 = OrigLoopExitWeight - ExitWeight0; EnterWeight = ExitWeight1; + assert(OrigLoopBackedgeWeight >= EnterWeight && "Bad branch weight"); LoopBackWeight = OrigLoopBackedgeWeight - EnterWeight; } else if (OrigLoopExitWeight == 0) { if (OrigLoopBackedgeWeight == 0) { diff --git a/llvm/test/Transforms/LoopRotate/update-branch-weights.ll b/llvm/test/Transforms/LoopRotate/update-branch-weights.ll index acb2038..9a1f36e 100644 --- a/llvm/test/Transforms/LoopRotate/update-branch-weights.ll +++ b/llvm/test/Transforms/LoopRotate/update-branch-weights.ll @@ -240,7 +240,7 @@ loop_exit: ; BFI_AFTER-LABEL: block-frequency-info: func6_inaccurate_branch_weight ; BFI_AFTER: - entry: {{.*}} count = 1024 -; BFI_AFTER: - loop_body: {{.*}} count = 4294967296 +; BFI_AFTER: - loop_body: {{.*}} count = 1024 ; BFI_AFTER: - loop_exit: {{.*}} count = 1024 ; IR-LABEL: define void @func6_inaccurate_branch_weight( @@ -292,4 +292,4 @@ loop_exit: ; IR: [[PROF_FUNC3_0]] = !{!"branch_weights", i32 0, i32 1} ; IR: [[PROF_FUNC4_0]] = !{!"branch_weights", i32 1, i32 0} ; IR: [[PROF_FUNC5_0]] = !{!"branch_weights", i32 0, i32 0} -; IR: [[PROF_FUNC6_0]] = !{!"branch_weights", i32 -1, i32 1024} +; IR: [[PROF_FUNC6_0]] = !{!"branch_weights", i32 0, i32 1024} |