; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
; RUN: opt -p loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -S %s | FileCheck %s


; FIXME: currently the live-outs are not handled correctly.
; Test for https://github.com/llvm/llvm-project/issues/154967.
define i8 @iv_used_in_exit_with_math(i8 noundef %g) {
; CHECK-LABEL: define i8 @iv_used_in_exit_with_math(
; CHECK-SAME: i8 noundef [[G:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i8
; CHECK-NEXT:    [[TMP0:%.*]] = add i8 [[OFFSET_IDX]], 1
; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i8 1, [[OFFSET_IDX]]
; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i8 1, [[TMP0]]
; CHECK-NEXT:    [[TMP3:%.*]] = and i8 [[TMP1]], [[G]]
; CHECK-NEXT:    [[TMP4:%.*]] = and i8 [[TMP2]], [[G]]
; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP3]], 0
; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i8 [[TMP4]], 0
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT:    [[TMP12:%.*]] = freeze i1 [[TMP7]]
; CHECK-NEXT:    [[TMP13:%.*]] = freeze i1 [[TMP8]]
; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP12]], [[TMP13]]
; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], 4
; CHECK-NEXT:    [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP10]]
; CHECK-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK:       [[MIDDLE_SPLIT]]:
; CHECK-NEXT:    br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[RETURN:.*]]
; CHECK:       [[VECTOR_EARLY_EXIT]]:
; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i1 [[TMP8]], false
; CHECK-NEXT:    [[TMP33:%.*]] = zext i1 [[TMP32]] to i64
; CHECK-NEXT:    [[TMP34:%.*]] = add i64 1, [[TMP33]]
; CHECK-NEXT:    [[TMP35:%.*]] = icmp eq i1 [[TMP7]], false
; CHECK-NEXT:    [[TMP14:%.*]] = zext i1 [[TMP35]] to i64
; CHECK-NEXT:    [[TMP15:%.*]] = add i64 0, [[TMP14]]
; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne i64 [[TMP14]], 1
; CHECK-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i64 [[TMP15]], i64 [[TMP34]]
; CHECK-NEXT:    [[TMP18:%.*]] = trunc i64 [[TMP17]] to i32
; CHECK-NEXT:    [[TMP19:%.*]] = add i32 [[INDEX]], [[TMP18]]
; CHECK-NEXT:    [[TMP20:%.*]] = trunc i32 [[TMP19]] to i8
; CHECK-NEXT:    [[TMP23:%.*]] = trunc i32 [[TMP19]] to i8
; CHECK-NEXT:    br label %[[RETURN]]
; CHECK:       [[SCALAR_PH:.*]]:
; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
; CHECK:       [[LOOP_HEADER]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT:    [[S:%.*]] = shl nuw i8 1, [[IV]]
; CHECK-NEXT:    [[A:%.*]] = and i8 [[S]], [[G]]
; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[A]], 0
; CHECK-NEXT:    br i1 [[C]], label %[[LOOP_LATCH]], label %[[RETURN]]
; CHECK:       [[LOOP_LATCH]]:
; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i8 [[IV]], 1
; CHECK-NEXT:    [[EC:%.*]] = icmp eq i8 [[IV_NEXT]], 4
; CHECK-NEXT:    br i1 [[EC]], label %[[RETURN]], label %[[LOOP_HEADER]]
; CHECK:       [[RETURN]]:
; CHECK-NEXT:    [[RES_IV1:%.*]] = phi i8 [ 32, %[[LOOP_LATCH]] ], [ [[IV]], %[[LOOP_HEADER]] ], [ 32, %[[MIDDLE_BLOCK]] ], [ [[TMP20]], %[[VECTOR_EARLY_EXIT]] ]
; CHECK-NEXT:    [[RES_IV2:%.*]] = phi i8 [ 0, %[[LOOP_LATCH]] ], [ [[IV]], %[[LOOP_HEADER]] ], [ 0, %[[MIDDLE_BLOCK]] ], [ [[TMP23]], %[[VECTOR_EARLY_EXIT]] ]
; CHECK-NEXT:    [[RES:%.*]] = add i8 [[RES_IV1]], [[RES_IV2]]
; CHECK-NEXT:    ret i8 [[RES]]
;
entry:
  br label %loop.header

loop.header:
  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %s = shl nuw i8 1, %iv
  %a = and i8 %s, %g
  %c = icmp eq i8 %a, 0
  br i1 %c, label %loop.latch, label %return

loop.latch:
  %iv.next = add nuw nsw i8 %iv, 1
  %ec = icmp eq i8 %iv.next, 4
  br i1 %ec, label %return, label %loop.header

return:
  %res.iv1 = phi i8 [ 32, %loop.latch ], [ %iv, %loop.header ]
  %res.iv2 = phi i8 [ 0, %loop.latch ], [ %iv, %loop.header ]
  %res = add i8 %res.iv1, %res.iv2
  ret i8 %res
}

define i32 @iv_used_in_exit_with_loads(ptr align 4 dereferenceable(128) %src) {
; CHECK-LABEL: define i32 @iv_used_in_exit_with_loads(
; CHECK-SAME: ptr align 4 dereferenceable(128) [[SRC:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*:]]
; CHECK-NEXT:    br label %[[VECTOR_PH:.*]]
; CHECK:       [[VECTOR_PH]]:
; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
; CHECK:       [[VECTOR_BODY]]:
; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 1
; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[INDEX]]
; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP0]]
; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP1]], align 4
; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP2]], align 4
; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i32 [[TMP3]], 0
; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i32 [[TMP4]], 0
; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
; CHECK-NEXT:    [[TMP12:%.*]] = freeze i1 [[TMP7]]
; CHECK-NEXT:    [[TMP13:%.*]] = freeze i1 [[TMP8]]
; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP12]], [[TMP13]]
; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], 32
; CHECK-NEXT:    [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP10]]
; CHECK-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK:       [[MIDDLE_SPLIT]]:
; CHECK-NEXT:    br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
; CHECK:       [[MIDDLE_BLOCK]]:
; CHECK-NEXT:    br label %[[RETURN:.*]]
; CHECK:       [[VECTOR_EARLY_EXIT]]:
; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i1 [[TMP8]], false
; CHECK-NEXT:    [[TMP21:%.*]] = zext i1 [[TMP20]] to i64
; CHECK-NEXT:    [[TMP22:%.*]] = add i64 1, [[TMP21]]
; CHECK-NEXT:    [[TMP23:%.*]] = icmp eq i1 [[TMP7]], false
; CHECK-NEXT:    [[TMP24:%.*]] = zext i1 [[TMP23]] to i64
; CHECK-NEXT:    [[TMP25:%.*]] = add i64 0, [[TMP24]]
; CHECK-NEXT:    [[TMP26:%.*]] = icmp ne i64 [[TMP24]], 1
; CHECK-NEXT:    [[TMP27:%.*]] = select i1 [[TMP26]], i64 [[TMP25]], i64 [[TMP22]]
; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i32
; CHECK-NEXT:    [[TMP29:%.*]] = add i32 [[INDEX]], [[TMP28]]
; CHECK-NEXT:    br label %[[RETURN]]
; CHECK:       [[SCALAR_PH:.*]]:
; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
; CHECK:       [[LOOP_HEADER]]:
; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[IV]]
; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP]], align 4
; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[L]], 0
; CHECK-NEXT:    br i1 [[C]], label %[[LOOP_LATCH]], label %[[RETURN]]
; CHECK:       [[LOOP_LATCH]]:
; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 32
; CHECK-NEXT:    br i1 [[EC]], label %[[RETURN]], label %[[LOOP_HEADER]]
; CHECK:       [[RETURN]]:
; CHECK-NEXT:    [[RES_IV1:%.*]] = phi i32 [ 32, %[[LOOP_LATCH]] ], [ [[IV]], %[[LOOP_HEADER]] ], [ 32, %[[MIDDLE_BLOCK]] ], [ [[TMP29]], %[[VECTOR_EARLY_EXIT]] ]
; CHECK-NEXT:    [[RES_IV2:%.*]] = phi i32 [ 0, %[[LOOP_LATCH]] ], [ [[IV]], %[[LOOP_HEADER]] ], [ 0, %[[MIDDLE_BLOCK]] ], [ [[TMP29]], %[[VECTOR_EARLY_EXIT]] ]
; CHECK-NEXT:    [[RES:%.*]] = add i32 [[RES_IV1]], [[RES_IV2]]
; CHECK-NEXT:    ret i32 [[RES]]
;
entry:
  br label %loop.header

loop.header:
  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
  %gep = getelementptr inbounds i32, ptr %src, i32 %iv
  %l = load i32, ptr %gep
  %c = icmp eq i32 %l, 0
  br i1 %c, label %loop.latch, label %return

loop.latch:
  %iv.next = add nuw nsw i32 %iv, 1
  %ec = icmp eq i32 %iv.next, 32
  br i1 %ec, label %return, label %loop.header

return:
  %res.iv1 = phi i32 [ 32, %loop.latch ], [ %iv, %loop.header ]
  %res.iv2 = phi i32 [ 0, %loop.latch ], [ %iv, %loop.header ]
  %res = add i32 %res.iv1, %res.iv2
  ret i32 %res
}