; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefixes=X86,X86-NOCMOV
; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+cmov | FileCheck %s --check-prefixes=X86,X86-CMOV
; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=X64
; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+bmi,+lzcnt | FileCheck %s --check-prefix=X86-CLZ
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi,+lzcnt | FileCheck %s --check-prefix=X64-CLZ
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi,+lzcnt,+fast-lzcnt | FileCheck %s --check-prefix=X64-FASTLZCNT
; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+bmi,+lzcnt,+fast-lzcnt | FileCheck %s --check-prefix=X86-FASTLZCNT

declare i8 @llvm.cttz.i8(i8, i1)
declare i16 @llvm.cttz.i16(i16, i1)
declare i32 @llvm.cttz.i32(i32, i1)
declare i64 @llvm.cttz.i64(i64, i1)

define i8 @cttz_i8(i8 %x)  {
; X86-LABEL: cttz_i8:
; X86:       # %bb.0:
; X86-NEXT:    rep bsfl {{[0-9]+}}(%esp), %eax
; X86-NEXT:    # kill: def $al killed $al killed $eax
; X86-NEXT:    retl
;
; X64-LABEL: cttz_i8:
; X64:       # %bb.0:
; X64-NEXT:    rep bsfl %edi, %eax
; X64-NEXT:    # kill: def $al killed $al killed $eax
; X64-NEXT:    retq
;
; X86-CLZ-LABEL: cttz_i8:
; X86-CLZ:       # %bb.0:
; X86-CLZ-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-CLZ-NEXT:    # kill: def $al killed $al killed $eax
; X86-CLZ-NEXT:    retl
;
; X64-CLZ-LABEL: cttz_i8:
; X64-CLZ:       # %bb.0:
; X64-CLZ-NEXT:    tzcntl %edi, %eax
; X64-CLZ-NEXT:    # kill: def $al killed $al killed $eax
; X64-CLZ-NEXT:    retq
;
; X64-FASTLZCNT-LABEL: cttz_i8:
; X64-FASTLZCNT:       # %bb.0:
; X64-FASTLZCNT-NEXT:    tzcntl %edi, %eax
; X64-FASTLZCNT-NEXT:    # kill: def $al killed $al killed $eax
; X64-FASTLZCNT-NEXT:    retq
;
; X86-FASTLZCNT-LABEL: cttz_i8:
; X86-FASTLZCNT:       # %bb.0:
; X86-FASTLZCNT-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-FASTLZCNT-NEXT:    # kill: def $al killed $al killed $eax
; X86-FASTLZCNT-NEXT:    retl
  %tmp = call i8 @llvm.cttz.i8( i8 %x, i1 true )
  ret i8 %tmp
}

define i16 @cttz_i16(i16 %x)  {
; X86-LABEL: cttz_i16:
; X86:       # %bb.0:
; X86-NEXT:    rep bsfl {{[0-9]+}}(%esp), %eax
; X86-NEXT:    # kill: def $ax killed $ax killed $eax
; X86-NEXT:    retl
;
; X64-LABEL: cttz_i16:
; X64:       # %bb.0:
; X64-NEXT:    rep bsfl %edi, %eax
; X64-NEXT:    # kill: def $ax killed $ax killed $eax
; X64-NEXT:    retq
;
; X86-CLZ-LABEL: cttz_i16:
; X86-CLZ:       # %bb.0:
; X86-CLZ-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-CLZ-NEXT:    # kill: def $ax killed $ax killed $eax
; X86-CLZ-NEXT:    retl
;
; X64-CLZ-LABEL: cttz_i16:
; X64-CLZ:       # %bb.0:
; X64-CLZ-NEXT:    tzcntl %edi, %eax
; X64-CLZ-NEXT:    # kill: def $ax killed $ax killed $eax
; X64-CLZ-NEXT:    retq
;
; X64-FASTLZCNT-LABEL: cttz_i16:
; X64-FASTLZCNT:       # %bb.0:
; X64-FASTLZCNT-NEXT:    tzcntl %edi, %eax
; X64-FASTLZCNT-NEXT:    # kill: def $ax killed $ax killed $eax
; X64-FASTLZCNT-NEXT:    retq
;
; X86-FASTLZCNT-LABEL: cttz_i16:
; X86-FASTLZCNT:       # %bb.0:
; X86-FASTLZCNT-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-FASTLZCNT-NEXT:    # kill: def $ax killed $ax killed $eax
; X86-FASTLZCNT-NEXT:    retl
  %tmp = call i16 @llvm.cttz.i16( i16 %x, i1 true )
  ret i16 %tmp
}

define i32 @cttz_i32(i32 %x)  {
; X86-LABEL: cttz_i32:
; X86:       # %bb.0:
; X86-NEXT:    rep bsfl {{[0-9]+}}(%esp), %eax
; X86-NEXT:    retl
;
; X64-LABEL: cttz_i32:
; X64:       # %bb.0:
; X64-NEXT:    rep bsfl %edi, %eax
; X64-NEXT:    retq
;
; X86-CLZ-LABEL: cttz_i32:
; X86-CLZ:       # %bb.0:
; X86-CLZ-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-CLZ-NEXT:    retl
;
; X64-CLZ-LABEL: cttz_i32:
; X64-CLZ:       # %bb.0:
; X64-CLZ-NEXT:    tzcntl %edi, %eax
; X64-CLZ-NEXT:    retq
;
; X64-FASTLZCNT-LABEL: cttz_i32:
; X64-FASTLZCNT:       # %bb.0:
; X64-FASTLZCNT-NEXT:    tzcntl %edi, %eax
; X64-FASTLZCNT-NEXT:    retq
;
; X86-FASTLZCNT-LABEL: cttz_i32:
; X86-FASTLZCNT:       # %bb.0:
; X86-FASTLZCNT-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-FASTLZCNT-NEXT:    retl
  %tmp = call i32 @llvm.cttz.i32( i32 %x, i1 true )
  ret i32 %tmp
}

define i64 @cttz_i64(i64 %x)  {
; X86-NOCMOV-LABEL: cttz_i64:
; X86-NOCMOV:       # %bb.0:
; X86-NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NOCMOV-NEXT:    testl %eax, %eax
; X86-NOCMOV-NEXT:    jne .LBB3_1
; X86-NOCMOV-NEXT:  # %bb.2:
; X86-NOCMOV-NEXT:    rep bsfl {{[0-9]+}}(%esp), %eax
; X86-NOCMOV-NEXT:    addl $32, %eax
; X86-NOCMOV-NEXT:    xorl %edx, %edx
; X86-NOCMOV-NEXT:    retl
; X86-NOCMOV-NEXT:  .LBB3_1:
; X86-NOCMOV-NEXT:    rep bsfl %eax, %eax
; X86-NOCMOV-NEXT:    xorl %edx, %edx
; X86-NOCMOV-NEXT:    retl
;
; X86-CMOV-LABEL: cttz_i64:
; X86-CMOV:       # %bb.0:
; X86-CMOV-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-CMOV-NEXT:    rep bsfl %ecx, %edx
; X86-CMOV-NEXT:    rep bsfl {{[0-9]+}}(%esp), %eax
; X86-CMOV-NEXT:    addl $32, %eax
; X86-CMOV-NEXT:    testl %ecx, %ecx
; X86-CMOV-NEXT:    cmovnel %edx, %eax
; X86-CMOV-NEXT:    xorl %edx, %edx
; X86-CMOV-NEXT:    retl
;
; X64-LABEL: cttz_i64:
; X64:       # %bb.0:
; X64-NEXT:    rep bsfq %rdi, %rax
; X64-NEXT:    retq
;
; X86-CLZ-LABEL: cttz_i64:
; X86-CLZ:       # %bb.0:
; X86-CLZ-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-CLZ-NEXT:    testl %eax, %eax
; X86-CLZ-NEXT:    jne .LBB3_1
; X86-CLZ-NEXT:  # %bb.2:
; X86-CLZ-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-CLZ-NEXT:    addl $32, %eax
; X86-CLZ-NEXT:    xorl %edx, %edx
; X86-CLZ-NEXT:    retl
; X86-CLZ-NEXT:  .LBB3_1:
; X86-CLZ-NEXT:    tzcntl %eax, %eax
; X86-CLZ-NEXT:    xorl %edx, %edx
; X86-CLZ-NEXT:    retl
;
; X64-CLZ-LABEL: cttz_i64:
; X64-CLZ:       # %bb.0:
; X64-CLZ-NEXT:    tzcntq %rdi, %rax
; X64-CLZ-NEXT:    retq
;
; X64-FASTLZCNT-LABEL: cttz_i64:
; X64-FASTLZCNT:       # %bb.0:
; X64-FASTLZCNT-NEXT:    tzcntq %rdi, %rax
; X64-FASTLZCNT-NEXT:    retq
;
; X86-FASTLZCNT-LABEL: cttz_i64:
; X86-FASTLZCNT:       # %bb.0:
; X86-FASTLZCNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-FASTLZCNT-NEXT:    testl %eax, %eax
; X86-FASTLZCNT-NEXT:    jne .LBB3_1
; X86-FASTLZCNT-NEXT:  # %bb.2:
; X86-FASTLZCNT-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-FASTLZCNT-NEXT:    addl $32, %eax
; X86-FASTLZCNT-NEXT:    xorl %edx, %edx
; X86-FASTLZCNT-NEXT:    retl
; X86-FASTLZCNT-NEXT:  .LBB3_1:
; X86-FASTLZCNT-NEXT:    tzcntl %eax, %eax
; X86-FASTLZCNT-NEXT:    xorl %edx, %edx
; X86-FASTLZCNT-NEXT:    retl
  %tmp = call i64 @llvm.cttz.i64( i64 %x, i1 true )
  ret i64 %tmp
}

; Promote i8 cttz to i32 and mask bit8 to prevent (slow) zero-src bsf case.
define i8 @cttz_i8_zero_test(i8 %n) {
; X86-LABEL: cttz_i8_zero_test:
; X86:       # %bb.0:
; X86-NEXT:    movl $256, %eax # imm = 0x100
; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
; X86-NEXT:    rep bsfl %eax, %eax
; X86-NEXT:    # kill: def $al killed $al killed $eax
; X86-NEXT:    retl
;
; X64-LABEL: cttz_i8_zero_test:
; X64:       # %bb.0:
; X64-NEXT:    orl $256, %edi # imm = 0x100
; X64-NEXT:    rep bsfl %edi, %eax
; X64-NEXT:    # kill: def $al killed $al killed $eax
; X64-NEXT:    retq
;
; X86-CLZ-LABEL: cttz_i8_zero_test:
; X86-CLZ:       # %bb.0:
; X86-CLZ-NEXT:    movl $256, %eax # imm = 0x100
; X86-CLZ-NEXT:    orl {{[0-9]+}}(%esp), %eax
; X86-CLZ-NEXT:    tzcntl %eax, %eax
; X86-CLZ-NEXT:    # kill: def $al killed $al killed $eax
; X86-CLZ-NEXT:    retl
;
; X64-CLZ-LABEL: cttz_i8_zero_test:
; X64-CLZ:       # %bb.0:
; X64-CLZ-NEXT:    orl $256, %edi # imm = 0x100
; X64-CLZ-NEXT:    tzcntl %edi, %eax
; X64-CLZ-NEXT:    # kill: def $al killed $al killed $eax
; X64-CLZ-NEXT:    retq
;
; X64-FASTLZCNT-LABEL: cttz_i8_zero_test:
; X64-FASTLZCNT:       # %bb.0:
; X64-FASTLZCNT-NEXT:    orl $256, %edi # imm = 0x100
; X64-FASTLZCNT-NEXT:    tzcntl %edi, %eax
; X64-FASTLZCNT-NEXT:    # kill: def $al killed $al killed $eax
; X64-FASTLZCNT-NEXT:    retq
;
; X86-FASTLZCNT-LABEL: cttz_i8_zero_test:
; X86-FASTLZCNT:       # %bb.0:
; X86-FASTLZCNT-NEXT:    movl $256, %eax # imm = 0x100
; X86-FASTLZCNT-NEXT:    orl {{[0-9]+}}(%esp), %eax
; X86-FASTLZCNT-NEXT:    tzcntl %eax, %eax
; X86-FASTLZCNT-NEXT:    # kill: def $al killed $al killed $eax
; X86-FASTLZCNT-NEXT:    retl
  %tmp1 = call i8 @llvm.cttz.i8(i8 %n, i1 false)
  ret i8 %tmp1
}

; Promote i16 cttz to i32 and mask bit16 to prevent (slow) zero-src bsf case.
define i16 @cttz_i16_zero_test(i16 %n) {
; X86-LABEL: cttz_i16_zero_test:
; X86:       # %bb.0:
; X86-NEXT:    movl $65536, %eax # imm = 0x10000
; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
; X86-NEXT:    rep bsfl %eax, %eax
; X86-NEXT:    # kill: def $ax killed $ax killed $eax
; X86-NEXT:    retl
;
; X64-LABEL: cttz_i16_zero_test:
; X64:       # %bb.0:
; X64-NEXT:    orl $65536, %edi # imm = 0x10000
; X64-NEXT:    rep bsfl %edi, %eax
; X64-NEXT:    # kill: def $ax killed $ax killed $eax
; X64-NEXT:    retq
;
; X86-CLZ-LABEL: cttz_i16_zero_test:
; X86-CLZ:       # %bb.0:
; X86-CLZ-NEXT:    movl $65536, %eax # imm = 0x10000
; X86-CLZ-NEXT:    orl {{[0-9]+}}(%esp), %eax
; X86-CLZ-NEXT:    tzcntl %eax, %eax
; X86-CLZ-NEXT:    # kill: def $ax killed $ax killed $eax
; X86-CLZ-NEXT:    retl
;
; X64-CLZ-LABEL: cttz_i16_zero_test:
; X64-CLZ:       # %bb.0:
; X64-CLZ-NEXT:    orl $65536, %edi # imm = 0x10000
; X64-CLZ-NEXT:    tzcntl %edi, %eax
; X64-CLZ-NEXT:    # kill: def $ax killed $ax killed $eax
; X64-CLZ-NEXT:    retq
;
; X64-FASTLZCNT-LABEL: cttz_i16_zero_test:
; X64-FASTLZCNT:       # %bb.0:
; X64-FASTLZCNT-NEXT:    orl $65536, %edi # imm = 0x10000
; X64-FASTLZCNT-NEXT:    tzcntl %edi, %eax
; X64-FASTLZCNT-NEXT:    # kill: def $ax killed $ax killed $eax
; X64-FASTLZCNT-NEXT:    retq
;
; X86-FASTLZCNT-LABEL: cttz_i16_zero_test:
; X86-FASTLZCNT:       # %bb.0:
; X86-FASTLZCNT-NEXT:    movl $65536, %eax # imm = 0x10000
; X86-FASTLZCNT-NEXT:    orl {{[0-9]+}}(%esp), %eax
; X86-FASTLZCNT-NEXT:    tzcntl %eax, %eax
; X86-FASTLZCNT-NEXT:    # kill: def $ax killed $ax killed $eax
; X86-FASTLZCNT-NEXT:    retl
  %tmp1 = call i16 @llvm.cttz.i16(i16 %n, i1 false)
  ret i16 %tmp1
}

; Generate a test and branch to handle zero inputs because bsr/bsf are very slow.
define i32 @cttz_i32_zero_test(i32 %n) {
; X86-NOCMOV-LABEL: cttz_i32_zero_test:
; X86-NOCMOV:       # %bb.0:
; X86-NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NOCMOV-NEXT:    testl %eax, %eax
; X86-NOCMOV-NEXT:    je .LBB6_1
; X86-NOCMOV-NEXT:  # %bb.2: # %cond.false
; X86-NOCMOV-NEXT:    rep bsfl %eax, %eax
; X86-NOCMOV-NEXT:    retl
; X86-NOCMOV-NEXT:  .LBB6_1:
; X86-NOCMOV-NEXT:    movl $32, %eax
; X86-NOCMOV-NEXT:    retl
;
; X86-CMOV-LABEL: cttz_i32_zero_test:
; X86-CMOV:       # %bb.0:
; X86-CMOV-NEXT:    bsfl {{[0-9]+}}(%esp), %ecx
; X86-CMOV-NEXT:    movl $32, %eax
; X86-CMOV-NEXT:    cmovnel %ecx, %eax
; X86-CMOV-NEXT:    retl
;
; X64-LABEL: cttz_i32_zero_test:
; X64:       # %bb.0:
; X64-NEXT:    movl $32, %eax
; X64-NEXT:    rep bsfl %edi, %eax
; X64-NEXT:    retq
;
; X86-CLZ-LABEL: cttz_i32_zero_test:
; X86-CLZ:       # %bb.0:
; X86-CLZ-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-CLZ-NEXT:    retl
;
; X64-CLZ-LABEL: cttz_i32_zero_test:
; X64-CLZ:       # %bb.0:
; X64-CLZ-NEXT:    tzcntl %edi, %eax
; X64-CLZ-NEXT:    retq
;
; X64-FASTLZCNT-LABEL: cttz_i32_zero_test:
; X64-FASTLZCNT:       # %bb.0:
; X64-FASTLZCNT-NEXT:    tzcntl %edi, %eax
; X64-FASTLZCNT-NEXT:    retq
;
; X86-FASTLZCNT-LABEL: cttz_i32_zero_test:
; X86-FASTLZCNT:       # %bb.0:
; X86-FASTLZCNT-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-FASTLZCNT-NEXT:    retl
  %tmp1 = call i32 @llvm.cttz.i32(i32 %n, i1 false)
  ret i32 %tmp1
}

; Generate a test and branch to handle zero inputs because bsr/bsf are very slow.
define i64 @cttz_i64_zero_test(i64 %n) nounwind {
; X86-NOCMOV-LABEL: cttz_i64_zero_test:
; X86-NOCMOV:       # %bb.0:
; X86-NOCMOV-NEXT:    pushl %esi
; X86-NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NOCMOV-NEXT:    xorl %edx, %edx
; X86-NOCMOV-NEXT:    movl %ecx, %esi
; X86-NOCMOV-NEXT:    orl %eax, %esi
; X86-NOCMOV-NEXT:    je .LBB7_1
; X86-NOCMOV-NEXT:  # %bb.2: # %cond.false
; X86-NOCMOV-NEXT:    testl %ecx, %ecx
; X86-NOCMOV-NEXT:    jne .LBB7_3
; X86-NOCMOV-NEXT:  # %bb.4: # %cond.false
; X86-NOCMOV-NEXT:    rep bsfl %eax, %eax
; X86-NOCMOV-NEXT:    addl $32, %eax
; X86-NOCMOV-NEXT:    popl %esi
; X86-NOCMOV-NEXT:    retl
; X86-NOCMOV-NEXT:  .LBB7_1:
; X86-NOCMOV-NEXT:    movl $64, %eax
; X86-NOCMOV-NEXT:    popl %esi
; X86-NOCMOV-NEXT:    retl
; X86-NOCMOV-NEXT:  .LBB7_3:
; X86-NOCMOV-NEXT:    rep bsfl %ecx, %eax
; X86-NOCMOV-NEXT:    popl %esi
; X86-NOCMOV-NEXT:    retl
;
; X86-CMOV-LABEL: cttz_i64_zero_test:
; X86-CMOV:       # %bb.0:
; X86-CMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-CMOV-NOT:     rep
; X86-CMOV-NEXT:    bsfl {{[0-9]+}}(%esp), %ecx
; X86-CMOV-NEXT:    movl $32, %edx
; X86-CMOV-NEXT:    cmovnel %ecx, %edx
; X86-CMOV-NEXT:    addl $32, %edx
; X86-CMOV-NOT:     rep
; X86-CMOV-NEXT:    bsfl %eax, %eax
; X86-CMOV-NEXT:    cmovel %edx, %eax
; X86-CMOV-NEXT:    xorl %edx, %edx
; X86-CMOV-NEXT:    retl
;
; X64-LABEL: cttz_i64_zero_test:
; X64:       # %bb.0:
; X64-NEXT:    movl $64, %eax
; X64-NEXT:    rep bsfq %rdi, %rax
; X64-NEXT:    retq
;
; X86-CLZ-LABEL: cttz_i64_zero_test:
; X86-CLZ:       # %bb.0:
; X86-CLZ-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-CLZ-NEXT:    testl %eax, %eax
; X86-CLZ-NEXT:    jne .LBB7_1
; X86-CLZ-NEXT:  # %bb.2:
; X86-CLZ-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-CLZ-NEXT:    addl $32, %eax
; X86-CLZ-NEXT:    xorl %edx, %edx
; X86-CLZ-NEXT:    retl
; X86-CLZ-NEXT:  .LBB7_1:
; X86-CLZ-NEXT:    tzcntl %eax, %eax
; X86-CLZ-NEXT:    xorl %edx, %edx
; X86-CLZ-NEXT:    retl
;
; X64-CLZ-LABEL: cttz_i64_zero_test:
; X64-CLZ:       # %bb.0:
; X64-CLZ-NEXT:    tzcntq %rdi, %rax
; X64-CLZ-NEXT:    retq
;
; X64-FASTLZCNT-LABEL: cttz_i64_zero_test:
; X64-FASTLZCNT:       # %bb.0:
; X64-FASTLZCNT-NEXT:    tzcntq %rdi, %rax
; X64-FASTLZCNT-NEXT:    retq
;
; X86-FASTLZCNT-LABEL: cttz_i64_zero_test:
; X86-FASTLZCNT:       # %bb.0:
; X86-FASTLZCNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-FASTLZCNT-NEXT:    testl %eax, %eax
; X86-FASTLZCNT-NEXT:    jne .LBB7_1
; X86-FASTLZCNT-NEXT:  # %bb.2:
; X86-FASTLZCNT-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-FASTLZCNT-NEXT:    addl $32, %eax
; X86-FASTLZCNT-NEXT:    xorl %edx, %edx
; X86-FASTLZCNT-NEXT:    retl
; X86-FASTLZCNT-NEXT:  .LBB7_1:
; X86-FASTLZCNT-NEXT:    tzcntl %eax, %eax
; X86-FASTLZCNT-NEXT:    xorl %edx, %edx
; X86-FASTLZCNT-NEXT:    retl
  %tmp1 = call i64 @llvm.cttz.i64(i64 %n, i1 false)
  ret i64 %tmp1
}

define i8 @cttz_i8_knownbits(i8 %x)  {
; X86-LABEL: cttz_i8_knownbits:
; X86:       # %bb.0:
; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
; X86-NEXT:    orb $2, %al
; X86-NEXT:    movzbl %al, %eax
; X86-NEXT:    rep bsfl %eax, %eax
; X86-NEXT:    # kill: def $al killed $al killed $eax
; X86-NEXT:    retl
;
; X64-LABEL: cttz_i8_knownbits:
; X64:       # %bb.0:
; X64-NEXT:    orb $2, %dil
; X64-NEXT:    movzbl %dil, %eax
; X64-NEXT:    rep bsfl %eax, %eax
; X64-NEXT:    # kill: def $al killed $al killed $eax
; X64-NEXT:    retq
;
; X86-CLZ-LABEL: cttz_i8_knownbits:
; X86-CLZ:       # %bb.0:
; X86-CLZ-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
; X86-CLZ-NEXT:    orb $2, %al
; X86-CLZ-NEXT:    movzbl %al, %eax
; X86-CLZ-NEXT:    tzcntl %eax, %eax
; X86-CLZ-NEXT:    # kill: def $al killed $al killed $eax
; X86-CLZ-NEXT:    retl
;
; X64-CLZ-LABEL: cttz_i8_knownbits:
; X64-CLZ:       # %bb.0:
; X64-CLZ-NEXT:    orb $2, %dil
; X64-CLZ-NEXT:    movzbl %dil, %eax
; X64-CLZ-NEXT:    tzcntl %eax, %eax
; X64-CLZ-NEXT:    # kill: def $al killed $al killed $eax
; X64-CLZ-NEXT:    retq
;
; X64-FASTLZCNT-LABEL: cttz_i8_knownbits:
; X64-FASTLZCNT:       # %bb.0:
; X64-FASTLZCNT-NEXT:    orb $2, %dil
; X64-FASTLZCNT-NEXT:    movzbl %dil, %eax
; X64-FASTLZCNT-NEXT:    tzcntl %eax, %eax
; X64-FASTLZCNT-NEXT:    # kill: def $al killed $al killed $eax
; X64-FASTLZCNT-NEXT:    retq
;
; X86-FASTLZCNT-LABEL: cttz_i8_knownbits:
; X86-FASTLZCNT:       # %bb.0:
; X86-FASTLZCNT-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
; X86-FASTLZCNT-NEXT:    orb $2, %al
; X86-FASTLZCNT-NEXT:    movzbl %al, %eax
; X86-FASTLZCNT-NEXT:    tzcntl %eax, %eax
; X86-FASTLZCNT-NEXT:    # kill: def $al killed $al killed $eax
; X86-FASTLZCNT-NEXT:    retl
  %x2 = or i8 %x, 2
  %tmp = call i8 @llvm.cttz.i8(i8 %x2, i1 true )
  %tmp2 = and i8 %tmp, 1
  ret i8 %tmp2
}

; Make sure we can detect that the input is non-zero and avoid cmov after BSF
; This is relevant for 32-bit mode without tzcnt
define i64 @cttz_i64_zero_test_knownneverzero(i64 %n) {
; X86-NOCMOV-LABEL: cttz_i64_zero_test_knownneverzero:
; X86-NOCMOV:       # %bb.0:
; X86-NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NOCMOV-NEXT:    testl %eax, %eax
; X86-NOCMOV-NEXT:    jne .LBB9_1
; X86-NOCMOV-NEXT:  # %bb.2:
; X86-NOCMOV-NEXT:    movl $-2147483648, %eax # imm = 0x80000000
; X86-NOCMOV-NEXT:    orl {{[0-9]+}}(%esp), %eax
; X86-NOCMOV-NEXT:    rep bsfl %eax, %eax
; X86-NOCMOV-NEXT:    orl $32, %eax
; X86-NOCMOV-NEXT:    xorl %edx, %edx
; X86-NOCMOV-NEXT:    retl
; X86-NOCMOV-NEXT:  .LBB9_1:
; X86-NOCMOV-NEXT:    rep bsfl %eax, %eax
; X86-NOCMOV-NEXT:    xorl %edx, %edx
; X86-NOCMOV-NEXT:    retl
;
; X86-CMOV-LABEL: cttz_i64_zero_test_knownneverzero:
; X86-CMOV:       # %bb.0:
; X86-CMOV-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-CMOV-NEXT:    movl $-2147483648, %eax # imm = 0x80000000
; X86-CMOV-NEXT:    orl {{[0-9]+}}(%esp), %eax
; X86-CMOV-NEXT:    rep bsfl %ecx, %edx
; X86-CMOV-NEXT:    rep bsfl %eax, %eax
; X86-CMOV-NEXT:    orl $32, %eax
; X86-CMOV-NEXT:    testl %ecx, %ecx
; X86-CMOV-NEXT:    cmovnel %edx, %eax
; X86-CMOV-NEXT:    xorl %edx, %edx
; X86-CMOV-NEXT:    retl
;
; X64-LABEL: cttz_i64_zero_test_knownneverzero:
; X64:       # %bb.0:
; X64-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
; X64-NEXT:    orq %rdi, %rax
; X64-NEXT:    rep bsfq %rax, %rax
; X64-NEXT:    retq
;
; X86-CLZ-LABEL: cttz_i64_zero_test_knownneverzero:
; X86-CLZ:       # %bb.0:
; X86-CLZ-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-CLZ-NEXT:    testl %eax, %eax
; X86-CLZ-NEXT:    jne .LBB9_1
; X86-CLZ-NEXT:  # %bb.2:
; X86-CLZ-NEXT:    movl $-2147483648, %eax # imm = 0x80000000
; X86-CLZ-NEXT:    orl {{[0-9]+}}(%esp), %eax
; X86-CLZ-NEXT:    tzcntl %eax, %eax
; X86-CLZ-NEXT:    orl $32, %eax
; X86-CLZ-NEXT:    xorl %edx, %edx
; X86-CLZ-NEXT:    retl
; X86-CLZ-NEXT:  .LBB9_1:
; X86-CLZ-NEXT:    tzcntl %eax, %eax
; X86-CLZ-NEXT:    xorl %edx, %edx
; X86-CLZ-NEXT:    retl
;
; X64-CLZ-LABEL: cttz_i64_zero_test_knownneverzero:
; X64-CLZ:       # %bb.0:
; X64-CLZ-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
; X64-CLZ-NEXT:    orq %rdi, %rax
; X64-CLZ-NEXT:    tzcntq %rax, %rax
; X64-CLZ-NEXT:    retq
;
; X64-FASTLZCNT-LABEL: cttz_i64_zero_test_knownneverzero:
; X64-FASTLZCNT:       # %bb.0:
; X64-FASTLZCNT-NEXT:    movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
; X64-FASTLZCNT-NEXT:    orq %rdi, %rax
; X64-FASTLZCNT-NEXT:    tzcntq %rax, %rax
; X64-FASTLZCNT-NEXT:    retq
;
; X86-FASTLZCNT-LABEL: cttz_i64_zero_test_knownneverzero:
; X86-FASTLZCNT:       # %bb.0:
; X86-FASTLZCNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-FASTLZCNT-NEXT:    testl %eax, %eax
; X86-FASTLZCNT-NEXT:    jne .LBB9_1
; X86-FASTLZCNT-NEXT:  # %bb.2:
; X86-FASTLZCNT-NEXT:    movl $-2147483648, %eax # imm = 0x80000000
; X86-FASTLZCNT-NEXT:    orl {{[0-9]+}}(%esp), %eax
; X86-FASTLZCNT-NEXT:    tzcntl %eax, %eax
; X86-FASTLZCNT-NEXT:    orl $32, %eax
; X86-FASTLZCNT-NEXT:    xorl %edx, %edx
; X86-FASTLZCNT-NEXT:    retl
; X86-FASTLZCNT-NEXT:  .LBB9_1:
; X86-FASTLZCNT-NEXT:    tzcntl %eax, %eax
; X86-FASTLZCNT-NEXT:    xorl %edx, %edx
; X86-FASTLZCNT-NEXT:    retl
  %o = or i64 %n, -9223372036854775808 ; 0x8000000000000000
  %tmp1 = call i64 @llvm.cttz.i64(i64 %o, i1 false)
  ret i64 %tmp1
}

define i32 @cttz_i32_osize(i32 %x) optsize {
; X86-LABEL: cttz_i32_osize:
; X86:       # %bb.0:
; X86-NOT:     rep
; X86-NEXT:    bsfl {{[0-9]+}}(%esp), %eax
; X86-NEXT:    retl
;
; X64-LABEL: cttz_i32_osize:
; X64:       # %bb.0:
; X64-NOT:     rep
; X64-NEXT:    bsfl %edi, %eax
; X64-NEXT:    retq
;
; X86-CLZ-LABEL: cttz_i32_osize:
; X86-CLZ:       # %bb.0:
; X86-CLZ-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-CLZ-NEXT:    retl
;
; X64-CLZ-LABEL: cttz_i32_osize:
; X64-CLZ:       # %bb.0:
; X64-CLZ-NEXT:    tzcntl %edi, %eax
; X64-CLZ-NEXT:    retq
;
; X64-FASTLZCNT-LABEL: cttz_i32_osize:
; X64-FASTLZCNT:       # %bb.0:
; X64-FASTLZCNT-NEXT:    tzcntl %edi, %eax
; X64-FASTLZCNT-NEXT:    retq
;
; X86-FASTLZCNT-LABEL: cttz_i32_osize:
; X86-FASTLZCNT:       # %bb.0:
; X86-FASTLZCNT-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-FASTLZCNT-NEXT:    retl
  %tmp = call i32 @llvm.cttz.i32( i32 %x, i1 true)
  ret i32 %tmp
}

define i32 @cttz_i32_msize(i32 %x) minsize {
; X86-LABEL: cttz_i32_msize:
; X86:       # %bb.0:
; X86-NOT:     rep
; X86-NEXT:    bsfl {{[0-9]+}}(%esp), %eax
; X86-NEXT:    retl
;
; X64-LABEL: cttz_i32_msize:
; X64:       # %bb.0:
; X64-NOT:     rep
; X64-NEXT:    bsfl %edi, %eax
; X64-NEXT:    retq
;
; X86-CLZ-LABEL: cttz_i32_msize:
; X86-CLZ:       # %bb.0:
; X86-CLZ-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-CLZ-NEXT:    retl
;
; X64-CLZ-LABEL: cttz_i32_msize:
; X64-CLZ:       # %bb.0:
; X64-CLZ-NEXT:    tzcntl %edi, %eax
; X64-CLZ-NEXT:    retq
;
; X64-FASTLZCNT-LABEL: cttz_i32_msize:
; X64-FASTLZCNT:       # %bb.0:
; X64-FASTLZCNT-NEXT:    tzcntl %edi, %eax
; X64-FASTLZCNT-NEXT:    retq
;
; X86-FASTLZCNT-LABEL: cttz_i32_msize:
; X86-FASTLZCNT:       # %bb.0:
; X86-FASTLZCNT-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-FASTLZCNT-NEXT:    retl
  %tmp = call i32 @llvm.cttz.i32( i32 %x, i1 true)
  ret i32 %tmp
}

define i64 @cttz_i32_sext(i32 %x) {
; X86-NOCMOV-LABEL: cttz_i32_sext:
; X86-NOCMOV:       # %bb.0:
; X86-NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NOCMOV-NEXT:    testl %eax, %eax
; X86-NOCMOV-NEXT:    je .LBB12_1
; X86-NOCMOV-NEXT:  # %bb.2: # %cond.false
; X86-NOCMOV-NEXT:    rep bsfl %eax, %eax
; X86-NOCMOV-NEXT:    xorl %edx, %edx
; X86-NOCMOV-NEXT:    retl
; X86-NOCMOV-NEXT:  .LBB12_1:
; X86-NOCMOV-NEXT:    movl $32, %eax
; X86-NOCMOV-NEXT:    xorl %edx, %edx
; X86-NOCMOV-NEXT:    retl
;
; X86-CMOV-LABEL: cttz_i32_sext:
; X86-CMOV:       # %bb.0:
; X86-CMOV-NEXT:    bsfl {{[0-9]+}}(%esp), %ecx
; X86-CMOV-NEXT:    movl $32, %eax
; X86-CMOV-NEXT:    cmovnel %ecx, %eax
; X86-CMOV-NEXT:    xorl %edx, %edx
; X86-CMOV-NEXT:    retl
;
; X64-LABEL: cttz_i32_sext:
; X64:       # %bb.0:
; X64-NEXT:    movl $32, %eax
; X64-NEXT:    rep bsfl %edi, %eax
; X64-NEXT:    retq
;
; X86-CLZ-LABEL: cttz_i32_sext:
; X86-CLZ:       # %bb.0:
; X86-CLZ-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-CLZ-NEXT:    xorl %edx, %edx
; X86-CLZ-NEXT:    retl
;
; X64-CLZ-LABEL: cttz_i32_sext:
; X64-CLZ:       # %bb.0:
; X64-CLZ-NEXT:    tzcntl %edi, %eax
; X64-CLZ-NEXT:    retq
;
; X64-FASTLZCNT-LABEL: cttz_i32_sext:
; X64-FASTLZCNT:       # %bb.0:
; X64-FASTLZCNT-NEXT:    tzcntl %edi, %eax
; X64-FASTLZCNT-NEXT:    retq
;
; X86-FASTLZCNT-LABEL: cttz_i32_sext:
; X86-FASTLZCNT:       # %bb.0:
; X86-FASTLZCNT-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-FASTLZCNT-NEXT:    xorl %edx, %edx
; X86-FASTLZCNT-NEXT:    retl
  %tmp = call i32 @llvm.cttz.i32( i32 %x, i1 false)
  %ext = sext i32 %tmp to i64
  ret i64 %ext
}

define i64 @cttz_i32_zext(i32 %x) {
; X86-NOCMOV-LABEL: cttz_i32_zext:
; X86-NOCMOV:       # %bb.0:
; X86-NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NOCMOV-NEXT:    testl %eax, %eax
; X86-NOCMOV-NEXT:    je .LBB13_1
; X86-NOCMOV-NEXT:  # %bb.2: # %cond.false
; X86-NOCMOV-NEXT:    rep bsfl %eax, %eax
; X86-NOCMOV-NEXT:    xorl %edx, %edx
; X86-NOCMOV-NEXT:    retl
; X86-NOCMOV-NEXT:  .LBB13_1:
; X86-NOCMOV-NEXT:    movl $32, %eax
; X86-NOCMOV-NEXT:    xorl %edx, %edx
; X86-NOCMOV-NEXT:    retl
;
; X86-CMOV-LABEL: cttz_i32_zext:
; X86-CMOV:       # %bb.0:
; X86-CMOV-NEXT:    bsfl {{[0-9]+}}(%esp), %ecx
; X86-CMOV-NEXT:    movl $32, %eax
; X86-CMOV-NEXT:    cmovnel %ecx, %eax
; X86-CMOV-NEXT:    xorl %edx, %edx
; X86-CMOV-NEXT:    retl
;
; X64-LABEL: cttz_i32_zext:
; X64:       # %bb.0:
; X64-NEXT:    movl $32, %eax
; X64-NEXT:    rep bsfl %edi, %eax
; X64-NEXT:    retq
;
; X86-CLZ-LABEL: cttz_i32_zext:
; X86-CLZ:       # %bb.0:
; X86-CLZ-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-CLZ-NEXT:    xorl %edx, %edx
; X86-CLZ-NEXT:    retl
;
; X64-CLZ-LABEL: cttz_i32_zext:
; X64-CLZ:       # %bb.0:
; X64-CLZ-NEXT:    tzcntl %edi, %eax
; X64-CLZ-NEXT:    retq
;
; X64-FASTLZCNT-LABEL: cttz_i32_zext:
; X64-FASTLZCNT:       # %bb.0:
; X64-FASTLZCNT-NEXT:    tzcntl %edi, %eax
; X64-FASTLZCNT-NEXT:    retq
;
; X86-FASTLZCNT-LABEL: cttz_i32_zext:
; X86-FASTLZCNT:       # %bb.0:
; X86-FASTLZCNT-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
; X86-FASTLZCNT-NEXT:    xorl %edx, %edx
; X86-FASTLZCNT-NEXT:    retl
  %tmp = call i32 @llvm.cttz.i32( i32 %x, i1 false)
  %ext = zext i32 %tmp to i64
  ret i64 %ext
}