i386.md (cmpdi): Fix operand predicates.

	* i386.md (cmpdi): Fix operand predicates.
	(cmpdi_ccno_1_rex64, cmpdi_minus_1_rex64, cmpdi_1_rex64,
	 cmpdi_1_insn_rex64): New patterns.
	(adddi3): Turn to expander.
	(adddi3_1, adddi3_carry_rex64, adddi3_cc_rex64): New patterns.
	(addsi3_carry_zext): New pattern.
	(adddi_?_rex64): New patterns and splitters.
	(addsi_?_zext): New patterns.
	(subsi3_carry_zext): New pattern.
	(subdi_?_rex64): New patterns and splitters.
	(iorsi_?_zext): New patterns.
	(iordi_?_rex64): New patterns and splitters.
	(iorsi_?_zext): New patterns.
	(iorsi_?_zext_imm): New patterns.
	(xorsi_?_zext): New patterns.
	(xordi_?_rex64): New patterns and splitters.
	(xorsi_?_zext): New patterns.
	(negdi*): New patterns.
	(one_cmpldi*): Likewise.
	(one_cmplsi*_zext, negsi*_zext): Likewise.
	(testqi_ext_3_rex64): New pattern.

From-SVN: r40819

2 files changed, 1370 insertions, 26 deletions

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 625ccfc..db5a254 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,27 @@
+Sat Mar 24 23:15:19 CET 2001  Jan Hubicka  <jh@suse.cz>
+
+	* i386.md (cmpdi): Fix operand predicates.
+	(cmpdi_ccno_1_rex64, cmpdi_minus_1_rex64, cmpdi_1_rex64,
+	 cmpdi_1_insn_rex64): New patterns.
+	(adddi3): Turn to expander.
+	(adddi3_1, adddi3_carry_rex64, adddi3_cc_rex64): New patterns.
+	(addsi3_carry_zext): New pattern.
+	(adddi_?_rex64): New patterns and splitters.
+	(addsi_?_zext): New patterns.
+	(subsi3_carry_zext): New pattern.
+	(subdi_?_rex64): New patterns and splitters.
+	(iorsi_?_zext): New patterns.
+	(iordi_?_rex64): New patterns and splitters.
+	(iorsi_?_zext): New patterns.
+	(iorsi_?_zext_imm): New patterns.
+	(xorsi_?_zext): New patterns.
+	(xordi_?_rex64): New patterns and splitters.
+	(xorsi_?_zext): New patterns.
+	(negdi*): New patterns.
+	(one_cmpldi*): Likewise.
+	(one_cmplsi*_zext, negsi*_zext): Likewise.
+	(testqi_ext_3_rex64): New pattern.
+
 Sat Mar 24 21:13:28 CET 2001  Jan Hubicka  <jh@suse.cz>
 
 	* i386-protos.h (ix86_split_long_move): Return void.
diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
index 24e21b1..0a490a6 100644
--- a/gcc/config/i386/i386.md
+++ b/gcc/config/i386/i386.md
@@ -1002,8 +1002,8 @@
 
 (define_expand "cmpdi"
   [(set (reg:CC 17)
-	(compare:CC (match_operand:DI 0 "general_operand" "")
-		    (match_operand:DI 1 "general_operand" "")))]
+	(compare:CC (match_operand:DI 0 "x86_64_general_operand" "")
+		    (match_operand:DI 1 "x86_64_general_operand" "")))]
   ""
   "
 {
@@ -1060,6 +1060,45 @@
   DONE;
 }")
 
+(define_insn "cmpdi_ccno_1_rex64"
+  [(set (reg 17)
+	(compare (match_operand:DI 0 "nonimmediate_operand" "r,?mr")
+		 (match_operand:DI 1 "const0_operand" "n,n")))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)"
+  "@
+   test{q}\\t{%0, %0|%0, %0}
+   cmp{q}\\t{%1, %0|%0, %1}"
+  [(set_attr "type" "test,icmp")
+   (set_attr "length_immediate" "0,1")
+   (set_attr "mode" "DI")])
+
+(define_insn "*cmpdi_minus_1_rex64"
+  [(set (reg 17)
+	(compare (minus:DI (match_operand:DI 0 "nonimmediate_operand" "rm,r")
+			   (match_operand:DI 1 "x86_64_general_operand" "re,mr"))
+		 (const_int 0)))]
+  "ix86_match_ccmode (insn, CCGOCmode)"
+  "cmp{q}\\t{%1, %0|%0, %1}"
+  [(set_attr "type" "icmp")
+   (set_attr "mode" "DI")])
+
+(define_expand "cmpdi_1_rex64"
+  [(set (reg:CC 17)
+	(compare:CC (match_operand:DI 0 "nonimmediate_operand" "")
+		    (match_operand:DI 1 "general_operand" "")))]
+  ""
+  "")
+
+(define_insn "cmpdi_1_insn_rex64"
+  [(set (reg 17)
+	(compare (match_operand:DI 0 "nonimmediate_operand" "mr,r")
+		 (match_operand:DI 1 "x86_64_general_operand" "re,mr")))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCmode)"
+  "cmp{q}\\t{%1, %0|%0, %1}"
+  [(set_attr "type" "icmp")
+   (set_attr "mode" "DI")])
+
+
 (define_insn "*cmpsi_ccno_1"
   [(set (reg 17)
 	(compare (match_operand:SI 0 "nonimmediate_operand" "r,?mr")
@@ -5100,12 +5139,20 @@
 ;	(plus:DI (match_operand:DI 1 "general_operand" "")
 ;		 (zero_extend:DI (match_operand:SI 2 "general_operand" ""))))]
 
-(define_insn "adddi3"
+(define_expand "adddi3"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "")
+	(plus:DI (match_operand:DI 1 "nonimmediate_operand" "")
+		 (match_operand:DI 2 "x86_64_general_operand" "")))
+   (clobber (reg:CC 17))]
+  ""
+  "ix86_expand_binary_operator (PLUS, DImode, operands); DONE;")
+
+(define_insn "*adddi3_1"
   [(set (match_operand:DI 0 "nonimmediate_operand" "=r,o")
 	(plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
 		 (match_operand:DI 2 "general_operand" "roiF,riF")))
    (clobber (reg:CC 17))]
-  ""
+  "!TARGET_64BIT"
   "#")
 
 (define_split
@@ -5125,6 +5172,29 @@
    split_di (operands+1, 1, operands+1, operands+4);
    split_di (operands+2, 1, operands+2, operands+5);")
 
+(define_insn "*adddi3_carry_rex64"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=rm,r")
+	  (plus:DI (plus:DI (ltu:DI (reg:CC 17) (const_int 0))
+			    (match_operand:DI 1 "nonimmediate_operand" "%0,0"))
+		   (match_operand:DI 2 "x86_64_general_operand" "re,rm")))
+   (clobber (reg:CC 17))]
+  "ix86_binary_operator_ok (PLUS, DImode, operands)"
+  "adc{q}\\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "pent_pair" "pu")
+   (set_attr "mode" "DI")
+   (set_attr "ppro_uops" "few")])
+
+(define_insn "*adddi3_cc_rex64"
+  [(set (reg:CC 17) (unspec:CC [(match_operand:DI 1 "nonimmediate_operand" "%0,0")
+				(match_operand:DI 2 "x86_64_general_operand" "re,rm")] 12))
+   (set (match_operand:DI 0 "nonimmediate_operand" "=rm,r")
+	(plus:DI (match_dup 1) (match_dup 2)))]
+  "TARGET_64BIT && ix86_binary_operator_ok (PLUS, DImode, operands)"
+  "add{q}\\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "DI")])
+
 (define_insn "*addsi3_carry"
   [(set (match_operand:SI 0 "nonimmediate_operand" "=rm,r")
 	  (plus:SI (plus:SI (ltu:SI (reg:CC 17) (const_int 0))
@@ -5138,6 +5208,20 @@
    (set_attr "mode" "SI")
    (set_attr "ppro_uops" "few")])
 
+(define_insn "*addsi3_carry_zext"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	  (zero_extend:DI 
+	    (plus:SI (plus:SI (ltu:SI (reg:CC 17) (const_int 0))
+			      (match_operand:SI 1 "nonimmediate_operand" "%0"))
+		     (match_operand:SI 2 "general_operand" "rim"))))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT && ix86_binary_operator_ok (PLUS, SImode, operands)"
+  "adc{l}\\t{%2, %k0|%k0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "pent_pair" "pu")
+   (set_attr "mode" "SI")
+   (set_attr "ppro_uops" "few")])
+
 (define_insn "*addsi3_cc"
   [(set (reg:CC 17) (unspec:CC [(match_operand:SI 1 "nonimmediate_operand" "%0,0")
 			        (match_operand:SI 2 "general_operand" "ri,rm")] 12))
@@ -5270,6 +5354,266 @@
   [(set_attr "type" "lea")
    (set_attr "mode" "SI")])
 
+(define_insn "*adddi_1_rex64"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=r,rm,r")
+	(plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0,r")
+		 (match_operand:DI 2 "x86_64_general_operand" "rme,re,re")))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT && ix86_binary_operator_ok (PLUS, DImode, operands)"
+  "*
+{
+  switch (get_attr_type (insn))
+    {
+    case TYPE_LEA:
+      operands[2] = SET_SRC (XVECEXP (PATTERN (insn), 0, 0));
+      return \"lea{q}\\t{%a2, %0|%0, %a2}\";
+
+    case TYPE_INCDEC:
+      if (! rtx_equal_p (operands[0], operands[1]))
+	abort ();
+      if (operands[2] == const1_rtx)
+        return \"inc{q}\\t%0\";
+      else if (operands[2] == constm1_rtx)
+        return \"dec{q}\\t%0\";
+      else
+	abort ();
+
+    default:
+      if (! rtx_equal_p (operands[0], operands[1]))
+	abort ();
+
+      /* Make things pretty and `subl $4,%eax' rather than `addl $-4, %eax'.
+	 Exceptions: -128 encodes smaller than 128, so swap sign and op.  */
+      if (GET_CODE (operands[2]) == CONST_INT
+          && (INTVAL (operands[2]) == 128
+	      || (INTVAL (operands[2]) < 0
+		  && INTVAL (operands[2]) != -128)))
+        {
+          operands[2] = GEN_INT (-INTVAL (operands[2]));
+          return \"sub{q}\\t{%2, %0|%0, %2}\";
+        }
+      return \"add{q}\\t{%2, %0|%0, %2}\";
+    }
+}"
+  [(set (attr "type")
+     (cond [(eq_attr "alternative" "2")
+	      (const_string "lea")
+	    ; Current assemblers are broken and do not allow @GOTOFF in
+	    ; ought but a memory context.
+	    (match_operand:DI 2 "pic_symbolic_operand" "")
+	      (const_string "lea")
+	    (match_operand:DI 2 "incdec_operand" "")
+	      (const_string "incdec")
+	   ]
+	   (const_string "alu")))
+   (set_attr "mode" "DI")])
+
+;; Convert lea to the lea pattern to avoid flags dependency.
+(define_split
+  [(set (match_operand:DI 0 "register_operand" "")
+	(plus:DI (match_operand:DI 1 "register_operand" "")
+		 (match_operand:DI 2 "x86_64_nonmemory_operand" "")))
+   (clobber (reg:CC 17))]
+  "reload_completed && TARGET_64BIT
+   && true_regnum (operands[0]) != true_regnum (operands[1])"
+  [(set (match_dup 0)
+	(plus:DI (match_dup 1)
+		 (match_dup 2)))]
+  "")
+
+(define_insn "*adddi_2_rex64"
+  [(set (reg 17)
+	(compare
+	  (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
+		   (match_operand:DI 2 "x86_64_general_operand" "rme,re"))
+	  (const_int 0)))			
+   (set (match_operand:DI 0 "nonimmediate_operand" "=r,rm")
+	(plus:DI (match_dup 1) (match_dup 2)))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCGOCmode)
+   && ix86_binary_operator_ok (PLUS, DImode, operands)
+   /* Current assemblers are broken and do not allow @GOTOFF in
+      ought but a memory context. */
+   && ! pic_symbolic_operand (operands[2], VOIDmode)"
+  "*
+{
+  switch (get_attr_type (insn))
+    {
+    case TYPE_INCDEC:
+      if (! rtx_equal_p (operands[0], operands[1]))
+	abort ();
+      if (operands[2] == const1_rtx)
+        return \"inc{q}\\t%0\";
+      else if (operands[2] == constm1_rtx)
+        return \"dec{q}\\t%0\";
+      else
+	abort ();
+
+    default:
+      if (! rtx_equal_p (operands[0], operands[1]))
+	abort ();
+      /* ???? We ought to handle there the 32bit case too
+	 - do we need new constrant?  */
+      /* Make things pretty and `subl $4,%eax' rather than `addl $-4, %eax'.
+	 Exceptions: -128 encodes smaller than 128, so swap sign and op.  */
+      if (GET_CODE (operands[2]) == CONST_INT
+          && (INTVAL (operands[2]) == 128
+	      || (INTVAL (operands[2]) < 0
+		  && INTVAL (operands[2]) != -128)))
+        {
+          operands[2] = GEN_INT (-INTVAL (operands[2]));
+          return \"sub{q}\\t{%2, %0|%0, %2}\";
+        }
+      return \"add{q}\\t{%2, %0|%0, %2}\";
+    }
+}"
+  [(set (attr "type")
+     (if_then_else (match_operand:DI 2 "incdec_operand" "")
+	(const_string "incdec")
+	(const_string "alu")))
+   (set_attr "mode" "DI")])
+
+(define_insn "*adddi_3"
+  [(set (reg 17)
+	(compare (neg:DI (match_operand:DI 2 "x86_64_general_operand" "rme"))
+		 (match_operand:DI 1 "x86_64_general_operand" "%0")))
+   (clobber (match_scratch:DI 0 "=r"))]
+  "ix86_match_ccmode (insn, CCZmode)
+   && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)
+   /* Current assemblers are broken and do not allow @GOTOFF in
+      ought but a memory context. */
+   && ! pic_symbolic_operand (operands[2], VOIDmode)"
+  "*
+{
+  switch (get_attr_type (insn))
+    {
+    case TYPE_INCDEC:
+      if (! rtx_equal_p (operands[0], operands[1]))
+	abort ();
+      if (operands[2] == const1_rtx)
+        return \"inc{q}\\t%0\";
+      else if (operands[2] == constm1_rtx)
+        return \"dec{q}\\t%0\";
+      else
+	abort ();
+
+    default:
+      if (! rtx_equal_p (operands[0], operands[1]))
+	abort ();
+      /* ???? We ought to handle there the 32bit case too
+	 - do we need new constrant?  */
+      /* Make things pretty and `subl $4,%eax' rather than `addl $-4, %eax'.
+	 Exceptions: -128 encodes smaller than 128, so swap sign and op.  */
+      if (GET_CODE (operands[2]) == CONST_INT
+          && (INTVAL (operands[2]) == 128
+	      || (INTVAL (operands[2]) < 0
+		  && INTVAL (operands[2]) != -128)))
+        {
+          operands[2] = GEN_INT (-INTVAL (operands[2]));
+          return \"sub{q}\\t{%2, %0|%0, %2}\";
+        }
+      return \"add{q}\\t{%2, %0|%0, %2}\";
+    }
+}"
+  [(set (attr "type")
+     (if_then_else (match_operand:DI 2 "incdec_operand" "")
+	(const_string "incdec")
+	(const_string "alu")))
+   (set_attr "mode" "DI")])
+
+; For comparisons against 1, -1 and 128, we may generate better code
+; by converting cmp to add, inc or dec as done by peephole2.  This pattern
+; is matched then.  We can't accept general immediate, because for
+; case of overflows,  the result is messed up.
+; This pattern also don't hold of 0x8000000000000000, since the value overflows
+; when negated.
+; Also carry flag is reversed compared to cmp, so this converison is valid
+; only for comparisons not depending on it.
+(define_insn "*adddi_4"
+  [(set (reg 17)
+	(compare (match_operand:DI 1 "nonimmediate_operand" "0")
+		 (match_operand:DI 2 "x86_64_immediate_operand" "e")))
+   (clobber (match_scratch:DI 0 "=rm"))]
+  "ix86_match_ccmode (insn, CCGCmode)"
+  "*
+{
+  switch (get_attr_type (insn))
+    {
+    case TYPE_INCDEC:
+      if (operands[2] == constm1_rtx)
+        return \"inc{q}\\t%0\";
+      else if (operands[2] == const1_rtx)
+        return \"dec{q}\\t%0\";
+      else
+	abort();
+
+    default:
+      if (! rtx_equal_p (operands[0], operands[1]))
+	abort ();
+      /* Make things pretty and `subl $4,%eax' rather than `addl $-4, %eax'.
+	 Exceptions: -128 encodes smaller than 128, so swap sign and op.  */
+      if ((INTVAL (operands[2]) == -128
+	   || (INTVAL (operands[2]) > 0
+	       && INTVAL (operands[2]) != 128)))
+	return \"sub{q}\\t{%2, %0|%0, %2}\";
+      operands[2] = GEN_INT (-INTVAL (operands[2]));
+      return \"add{q}\\t{%2, %0|%0, %2}\";
+    }
+}"
+  [(set (attr "type")
+     (if_then_else (match_operand:DI 2 "incdec_operand" "")
+	(const_string "incdec")
+	(const_string "alu")))
+   (set_attr "mode" "DI")])
+
+(define_insn "*adddi_5"
+  [(set (reg 17)
+	(compare
+	  (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0")
+		   (match_operand:DI 2 "x86_64_general_operand" "rme"))
+	  (const_int 0)))			
+   (clobber (match_scratch:DI 0 "=r"))]
+  "ix86_match_ccmode (insn, CCGOCmode)
+   && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)
+   /* Current assemblers are broken and do not allow @GOTOFF in
+      ought but a memory context. */
+   && ! pic_symbolic_operand (operands[2], VOIDmode)"
+  "*
+{
+  switch (get_attr_type (insn))
+    {
+    case TYPE_INCDEC:
+      if (! rtx_equal_p (operands[0], operands[1]))
+	abort ();
+      if (operands[2] == const1_rtx)
+        return \"inc{q}\\t%0\";
+      else if (operands[2] == constm1_rtx)
+        return \"dec{q}\\t%0\";
+      else
+	abort();
+
+    default:
+      if (! rtx_equal_p (operands[0], operands[1]))
+	abort ();
+      /* Make things pretty and `subl $4,%eax' rather than `addl $-4, %eax'.
+	 Exceptions: -128 encodes smaller than 128, so swap sign and op.  */
+      if (GET_CODE (operands[2]) == CONST_INT
+          && (INTVAL (operands[2]) == 128
+	      || (INTVAL (operands[2]) < 0
+		  && INTVAL (operands[2]) != -128)))
+        {
+          operands[2] = GEN_INT (-INTVAL (operands[2]));
+          return \"sub{q}\\t{%2, %0|%0, %2}\";
+        }
+      return \"add{q}\\t{%2, %0|%0, %2}\";
+    }
+}"
+  [(set (attr "type")
+     (if_then_else (match_operand:DI 2 "incdec_operand" "")
+	(const_string "incdec")
+	(const_string "alu")))
+   (set_attr "mode" "DI")])
+
+
 (define_insn "*addsi_1"
   [(set (match_operand:SI 0 "nonimmediate_operand" "=r,rm,r")
 	(plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,r")
@@ -5351,6 +5695,78 @@
   DONE;
 }")
 
+;; It may seem that nonimmediate operand is proper one for operand 1.
+;; The addsi_1 pattern allows nonimmediate operand at that place and
+;; we take care in ix86_binary_operator_ok to not allow two memory
+;; operands so proper swapping will be done in reload.  This allow
+;; patterns constructed from addsi_1 to match.
+(define_insn "addsi_1_zext"
+  [(set (match_operand:DI 0 "register_operand" "=r,r")
+	(zero_extend:DI
+	  (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,r")
+		   (match_operand:SI 2 "general_operand" "rmni,rni"))))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT && ix86_binary_operator_ok (PLUS, SImode, operands)"
+  "*
+{
+  switch (get_attr_type (insn))
+    {
+    case TYPE_LEA:
+      operands[2] = SET_SRC (XVECEXP (PATTERN (insn), 0, 0));
+      return \"lea{l}\\t{%a2, %k0|%k0, %a2}\";
+
+    case TYPE_INCDEC:
+      if (operands[2] == const1_rtx)
+        return \"inc{l}\\t%k0\";
+      else if (operands[2] == constm1_rtx)
+        return \"dec{l}\\t%k0\";
+      else
+	abort();
+
+    default:
+      /* Make things pretty and `subl $4,%eax' rather than `addl $-4, %eax'.
+	 Exceptions: -128 encodes smaller than 128, so swap sign and op.  */
+      if (GET_CODE (operands[2]) == CONST_INT
+          && (INTVAL (operands[2]) == 128
+	      || (INTVAL (operands[2]) < 0
+		  && INTVAL (operands[2]) != -128)))
+        {
+          operands[2] = GEN_INT (-INTVAL (operands[2]));
+          return \"sub{l}\\t{%2, %k0|%k0, %2}\";
+        }
+      return \"add{l}\\t{%2, %k0|%k0, %2}\";
+    }
+}"
+  [(set (attr "type")
+     (cond [(eq_attr "alternative" "1")
+	      (const_string "lea")
+	    ; Current assemblers are broken and do not allow @GOTOFF in
+	    ; ought but a memory context.
+	    (match_operand:SI 2 "pic_symbolic_operand" "")
+	      (const_string "lea")
+	    (match_operand:SI 2 "incdec_operand" "")
+	      (const_string "incdec")
+	   ]
+	   (const_string "alu")))
+   (set_attr "mode" "SI")])
+
+;; Convert lea to the lea pattern to avoid flags dependency.
+(define_split
+  [(set (match_operand:DI 0 "register_operand" "")
+	(zero_extend:DI
+	  (plus:SI (match_operand:SI 1 "register_operand" "")
+		   (match_operand:SI 2 "nonmemory_operand" ""))))
+   (clobber (reg:CC 17))]
+  "reload_completed
+   && true_regnum (operands[0]) != true_regnum (operands[1])"
+  [(set (match_dup 0)
+	(zero_extend:DI (subreg:SI (plus:DI (match_dup 1) (match_dup 2)) 0)))]
+  "
+{
+  operands[1] = gen_lowpart (Pmode, operands[1]);
+  operands[2] = gen_lowpart (Pmode, operands[2]);
+}")
+
 (define_insn "*addsi_2"
   [(set (reg 17)
 	(compare
@@ -5400,6 +5816,52 @@
 	(const_string "alu")))
    (set_attr "mode" "SI")])
 
+;; See comment for addsi_1_zext why we do use nonimmediate_operand
+(define_insn "*addsi_2_zext"
+  [(set (reg 17)
+	(compare
+	  (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
+		   (match_operand:SI 2 "general_operand" "rmni"))
+	  (const_int 0)))			
+   (set (match_operand:DI 0 "register_operand" "=r")
+	(zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCGOCmode)
+   && ix86_binary_operator_ok (PLUS, SImode, operands)
+   /* Current assemblers are broken and do not allow @GOTOFF in
+      ought but a memory context. */
+   && ! pic_symbolic_operand (operands[2], VOIDmode)"
+  "*
+{
+  switch (get_attr_type (insn))
+    {
+    case TYPE_INCDEC:
+      if (operands[2] == const1_rtx)
+        return \"inc{l}\\t%k0\";
+      else if (operands[2] == constm1_rtx)
+        return \"dec{l}\\t%k0\";
+      else
+	abort();
+
+    default:
+      /* Make things pretty and `subl $4,%eax' rather than `addl $-4, %eax'.
+	 Exceptions: -128 encodes smaller than 128, so swap sign and op.  */
+      if (GET_CODE (operands[2]) == CONST_INT
+          && (INTVAL (operands[2]) == 128
+	      || (INTVAL (operands[2]) < 0
+		  && INTVAL (operands[2]) != -128)))
+        {
+          operands[2] = GEN_INT (-INTVAL (operands[2]));
+          return \"sub{l}\\t{%2, %k0|%k0, %2}\";
+        }
+      return \"add{l}\\t{%2, %k0|%k0, %2}\";
+    }
+}"
+  [(set (attr "type")
+     (if_then_else (match_operand:SI 2 "incdec_operand" "")
+	(const_string "incdec")
+	(const_string "alu")))
+   (set_attr "mode" "SI")])
+
 (define_insn "*addsi_3"
   [(set (reg 17)
 	(compare (neg:SI (match_operand:SI 2 "general_operand" "rmni"))
@@ -5446,6 +5908,50 @@
 	(const_string "alu")))
    (set_attr "mode" "SI")])
 
+;; See comment for addsi_1_zext why we do use nonimmediate_operand
+(define_insn "*addsi_3_zext"
+  [(set (reg 17)
+	(compare (neg:SI (match_operand:SI 2 "general_operand" "rmni"))
+		 (match_operand:SI 1 "nonimmediate_operand" "%0")))
+   (set (match_operand:DI 0 "register_operand" "=r")
+	(zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCZmode)
+   && ix86_binary_operator_ok (PLUS, SImode, operands)
+   /* Current assemblers are broken and do not allow @GOTOFF in
+      ought but a memory context. */
+   && ! pic_symbolic_operand (operands[2], VOIDmode)"
+  "*
+{
+  switch (get_attr_type (insn))
+    {
+    case TYPE_INCDEC:
+      if (operands[2] == const1_rtx)
+        return \"inc{l}\\t%k0\";
+      else if (operands[2] == constm1_rtx)
+        return \"dec{l}\\t%k0\";
+      else
+	abort();
+
+    default:
+      /* Make things pretty and `subl $4,%eax' rather than `addl $-4, %eax'.
+	 Exceptions: -128 encodes smaller than 128, so swap sign and op.  */
+      if (GET_CODE (operands[2]) == CONST_INT
+          && (INTVAL (operands[2]) == 128
+	      || (INTVAL (operands[2]) < 0
+		  && INTVAL (operands[2]) != -128)))
+        {
+          operands[2] = GEN_INT (-INTVAL (operands[2]));
+          return \"sub{l}\\t{%2, %k0|%k0, %2}\";
+        }
+      return \"add{l}\\t{%2, %k0|%k0, %2}\";
+    }
+}"
+  [(set (attr "type")
+     (if_then_else (match_operand:SI 2 "incdec_operand" "")
+	(const_string "incdec")
+	(const_string "alu")))
+   (set_attr "mode" "SI")])
+
 ; For comparisons agains 1, -1 and 128, we may generate better code
 ; by converting cmp to add, inc or dec as done by peephole2.  This pattern
 ; is matched then.  We can't accept general immediate, because for
@@ -6188,15 +6694,22 @@
 
 ;; Subtract instructions
 
-;; %%% define_expand from the very first?
 ;; %%% splits for subsidi3
 
-(define_insn "subdi3"
+(define_expand "subdi3"
+  [(parallel [(set (match_operand:DI 0 "nonimmediate_operand" "")
+		   (minus:DI (match_operand:DI 1 "nonimmediate_operand" "")
+			     (match_operand:DI 2 "x86_64_general_operand" "")))
+	      (clobber (reg:CC 17))])]
+  ""
+  "ix86_expand_binary_operator (MINUS, DImode, operands); DONE;")
+
+(define_insn "*subdi3_1"
   [(set (match_operand:DI 0 "nonimmediate_operand" "=r,o")
 	(minus:DI (match_operand:DI 1 "nonimmediate_operand" "0,0")
 		  (match_operand:DI 2 "general_operand" "roiF,riF")))
    (clobber (reg:CC 17))]
-  ""
+  "!TARGET_64BIT"
   "#")
 
 (define_split
@@ -6216,6 +6729,56 @@
    split_di (operands+1, 1, operands+1, operands+4);
    split_di (operands+2, 1, operands+2, operands+5);")
 
+(define_insn "subdi3_carry_rex64"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=rm,r")
+	  (minus:DI (match_operand:DI 1 "nonimmediate_operand" "0,0")
+	    (plus:DI (ltu:DI (reg:CC 17) (const_int 0))
+	       (match_operand:DI 2 "x86_64_general_operand" "re,rm"))))
+   (clobber (reg:CC 17))]
+  "ix86_binary_operator_ok (MINUS, DImode, operands)"
+  "sbb{q}\\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "pent_pair" "pu")
+   (set_attr "ppro_uops" "few")
+   (set_attr "mode" "DI")])
+
+(define_insn "*subdi_1_rex64"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=rm,r")
+	(minus:DI (match_operand:DI 1 "nonimmediate_operand" "0,0")
+		  (match_operand:DI 2 "x86_64_general_operand" "re,rm")))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT && ix86_binary_operator_ok (MINUS, DImode, operands)"
+  "sub{q}\\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "DI")])
+
+(define_insn "*subdi_2_rex64"
+  [(set (reg 17)
+	(compare
+	  (minus:DI (match_operand:DI 1 "nonimmediate_operand" "0,0")
+		    (match_operand:DI 2 "x86_64_general_operand" "re,rm"))
+	  (const_int 0)))
+   (set (match_operand:DI 0 "nonimmediate_operand" "=rm,r")
+	(minus:DI (match_dup 1) (match_dup 2)))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCGOCmode)
+   && ix86_binary_operator_ok (MINUS, DImode, operands)"
+  "sub{q}\\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "DI")])
+
+(define_insn "*subdi_3_rex63"
+  [(set (reg 17)
+	(compare (match_operand:DI 1 "nonimmediate_operand" "0,0")
+		 (match_operand:DI 2 "x86_64_general_operand" "re,rm")))
+   (set (match_operand:DI 0 "nonimmediate_operand" "=rm,r")
+	(minus:DI (match_dup 1) (match_dup 2)))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCmode)
+   && ix86_binary_operator_ok (MINUS, SImode, operands)"
+  "sub{q}\\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "DI")])
+
+
 (define_insn "subsi3_carry"
   [(set (match_operand:SI 0 "nonimmediate_operand" "=rm,r")
 	  (minus:SI (match_operand:SI 1 "nonimmediate_operand" "0,0")
@@ -6229,6 +6792,20 @@
    (set_attr "ppro_uops" "few")
    (set_attr "mode" "SI")])
 
+(define_insn "subsi3_carry_zext"
+  [(set (match_operand:DI 0 "register_operand" "=rm,r")
+	  (zero_extend:DI
+	    (minus:SI (match_operand:SI 1 "register_operand" "0,0")
+	      (plus:SI (ltu:SI (reg:CC 17) (const_int 0))
+		 (match_operand:SI 2 "general_operand" "ri,rm")))))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT && ix86_binary_operator_ok (MINUS, SImode, operands)"
+  "sbb{l}\\t{%2, %k0|%k0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "pent_pair" "pu")
+   (set_attr "ppro_uops" "few")
+   (set_attr "mode" "SI")])
+
 (define_expand "subsi3"
   [(parallel [(set (match_operand:SI 0 "nonimmediate_operand" "")
 		   (minus:SI (match_operand:SI 1 "nonimmediate_operand" "")
@@ -6247,6 +6824,17 @@
   [(set_attr "type" "alu")
    (set_attr "mode" "SI")])
 
+(define_insn "*subsi_1_zext"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	(zero_extend:DI
+	  (minus:SI (match_operand:SI 1 "register_operand" "0")
+		    (match_operand:SI 2 "general_operand" "rim"))))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT && ix86_binary_operator_ok (MINUS, SImode, operands)"
+  "sub{l}\\t{%2, %k0|%k0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "SI")])
+
 (define_insn "*subsi_2"
   [(set (reg 17)
 	(compare
@@ -6261,6 +6849,22 @@
   [(set_attr "type" "alu")
    (set_attr "mode" "SI")])
 
+(define_insn "*subsi_2_zext"
+  [(set (reg 17)
+	(compare
+	  (minus:SI (match_operand:SI 1 "register_operand" "0")
+		    (match_operand:SI 2 "general_operand" "rim"))
+	  (const_int 0)))
+   (set (match_operand:DI 0 "register_operand" "=r")
+	(zero_extend:DI
+	  (minus:SI (match_dup 1)
+		    (match_dup 2))))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCGOCmode)
+   && ix86_binary_operator_ok (MINUS, SImode, operands)"
+  "sub{l}\\t{%2, %k0|%k0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "SI")])
+
 (define_insn "*subsi_3"
   [(set (reg 17)
 	(compare (match_operand:SI 1 "nonimmediate_operand" "0,0")
@@ -6273,6 +6877,20 @@
   [(set_attr "type" "alu")
    (set_attr "mode" "SI")])
 
+(define_insn "*subsi_3_zext"
+  [(set (reg 17)
+	(compare (match_operand:SI 1 "nonimmediate_operand" "0")
+		 (match_operand:SI 2 "general_operand" "rim")))
+   (set (match_operand:DI 0 "register_operand" "=r")
+	(zero_extend:DI
+	  (minus:SI (match_dup 1)
+		    (match_dup 2))))]
+  "ix86_match_ccmode (insn, CCmode)
+   && ix86_binary_operator_ok (MINUS, SImode, operands)"
+  "sub{q}\\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "DI")])
+
 (define_expand "subhi3"
   [(parallel [(set (match_operand:HI 0 "nonimmediate_operand" "")
 		   (minus:HI (match_operand:HI 1 "nonimmediate_operand" "")
@@ -6393,6 +7011,29 @@
 
 ;; Multiply instructions
 
+(define_expand "muldi3"
+  [(parallel [(set (match_operand:DI 0 "register_operand" "")
+		   (mult:DI (match_operand:DI 1 "register_operand" "")
+			    (match_operand:DI 2 "x86_64_general_operand" "")))
+	      (clobber (reg:CC 17))])]
+  "TARGET_64BIT"
+  "")
+
+(define_insn "*muldi3_1_rex64"
+  [(set (match_operand:DI 0 "register_operand" "=r,r,r")
+	(mult:DI (match_operand:DI 1 "nonimmediate_operand" "%rm,0,0")
+		 (match_operand:DI 2 "x86_64_general_operand" "K,e,mr")))
+   (clobber (reg:CC 17))]
+  "(GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)
+   && TARGET_64BIT"
+  "@
+   imul{q}\\t{%2, %1, %0|%0, %1, %2}
+   imul{q}\\t{%2, %1, %0|%0, %1, %2}
+   imul{q}\\t{%2, %0|%0, %2}"
+  [(set_attr "type" "imul")
+   (set_attr "prefix_0f" "0,0,1")
+   (set_attr "mode" "DI")])
+
 (define_expand "mulsi3"
   [(parallel [(set (match_operand:SI 0 "register_operand" "")
 		   (mult:SI (match_operand:SI 1 "register_operand" "")
@@ -6427,6 +7068,34 @@
    (set_attr "prefix_0f" "0,0,1")
    (set_attr "mode" "SI")])
 
+(define_insn "*mulsi3_1_zext"
+  [(set (match_operand:DI 0 "register_operand" "=r,r,r")
+	(zero_extend:DI
+	  (mult:SI (match_operand:SI 1 "nonimmediate_operand" "%rm,0,0")
+		   (match_operand:SI 2 "general_operand" "K,i,mr"))))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT
+   && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
+  ; For the {r,0,i} alternative (i.e., register <- register * immediate),
+  ; there are two ways of writing the exact same machine instruction
+  ; in assembly language.  One, for example, is:
+  ;
+  ;   imul $12, %eax
+  ;
+  ; while the other is:
+  ;
+  ;   imul $12, %eax, %eax
+  ;
+  ; The first is simply short-hand for the latter.  But, some assemblers,
+  ; like the SCO OSR5 COFF assembler, don't handle the first form.
+  "@
+   imul{l}\\t{%2, %1, %k0|%k0, %1, %2}
+   imul{l}\\t{%2, %1, %k0|%k0, %1, %2}
+   imul{l}\\t{%2, %k0|%k0, %2}"
+  [(set_attr "type" "imul")
+   (set_attr "prefix_0f" "0,0,1")
+   (set_attr "mode" "SI")])
+
 (define_expand "mulhi3"
   [(parallel [(set (match_operand:HI 0 "register_operand" "")
 		   (mult:HI (match_operand:HI 1 "register_operand" "")
@@ -6484,18 +7153,17 @@
    (set_attr "length_immediate" "0")
    (set_attr "mode" "QI")])
 
-(define_insn "umulsi3"
-  [(set (match_operand:SI 0 "register_operand" "=a")
-	(mult:SI (match_operand:SI 1 "register_operand" "%0")
-		 (match_operand:SI 2 "nonimmediate_operand" "rm")))
-   (clobber (match_operand:SI 3 "register_operand" "=d"))
+(define_insn "umulditi3"
+  [(set (match_operand:TI 0 "register_operand" "=A")
+	(mult:TI (zero_extend:TI (match_operand:DI 1 "register_operand" "%0"))
+		 (zero_extend:TI (match_operand:DI 2 "nonimmediate_operand" "rm"))))
    (clobber (reg:CC 17))]
-  ""
-  "mul{l}\\t%2"
+  "TARGET_64BIT"
+  "mul{q}\\t%2"
   [(set_attr "type" "imul")
    (set_attr "ppro_uops" "few")
    (set_attr "length_immediate" "0")
-   (set_attr "mode" "SI")])
+   (set_attr "mode" "DI")])
 
 ;; We can't use this pattern in 64bit mode, since it results in two separate 32bit registers
 (define_insn "umulsidi3"
@@ -6510,6 +7178,17 @@
    (set_attr "length_immediate" "0")
    (set_attr "mode" "SI")])
 
+(define_insn "mulditi3"
+  [(set (match_operand:TI 0 "register_operand" "=A")
+	(mult:TI (sign_extend:TI (match_operand:DI 1 "register_operand" "%0"))
+		 (sign_extend:TI (match_operand:DI 2 "nonimmediate_operand" "rm"))))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT"
+  "imul{q}\\t%2"
+  [(set_attr "type" "imul")
+   (set_attr "length_immediate" "0")
+   (set_attr "mode" "DI")])
+
 (define_insn "mulsidi3"
   [(set (match_operand:DI 0 "register_operand" "=A")
 	(mult:DI (sign_extend:DI (match_operand:SI 1 "register_operand" "%0"))
@@ -6521,6 +7200,24 @@
    (set_attr "length_immediate" "0")
    (set_attr "mode" "SI")])
 
+(define_insn "*umuldi3_highpart_rex64"
+  [(set (match_operand:DI 0 "register_operand" "=d")
+	(truncate:DI
+	  (lshiftrt:TI
+	    (mult:TI (zero_extend:TI
+		       (match_operand:DI 1 "register_operand" "%a"))
+		     (zero_extend:TI
+		       (match_operand:DI 2 "nonimmediate_operand" "rm")))
+	    (const_int 64))))
+   (clobber (match_scratch:DI 3 "=a"))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT"
+  "mul{q}\\t%2"
+  [(set_attr "type" "imul")
+   (set_attr "ppro_uops" "few")
+   (set_attr "length_immediate" "0")
+   (set_attr "mode" "DI")])
+
 (define_insn "umulsi3_highpart"
   [(set (match_operand:SI 0 "register_operand" "=d")
 	(truncate:SI
@@ -6539,6 +7236,23 @@
    (set_attr "length_immediate" "0")
    (set_attr "mode" "SI")])
 
+(define_insn "*smuldi3_highpart_rex64"
+  [(set (match_operand:DI 0 "register_operand" "=d")
+	(truncate:DI
+	  (lshiftrt:TI
+	    (mult:TI (sign_extend:TI
+		       (match_operand:DI 1 "register_operand" "%a"))
+		     (sign_extend:TI
+		       (match_operand:DI 2 "nonimmediate_operand" "rm")))
+	    (const_int 64))))
+   (clobber (match_scratch:DI 3 "=a"))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT"
+  "imul{q}\\t%2"
+  [(set_attr "type" "imul")
+   (set_attr "ppro_uops" "few")
+   (set_attr "mode" "DI")])
+
 (define_insn "smulsi3_highpart"
   [(set (match_operand:SI 0 "register_operand" "=d")
 	(truncate:SI
@@ -6556,6 +7270,23 @@
    (set_attr "ppro_uops" "few")
    (set_attr "mode" "SI")])
 
+(define_insn "*smulsi3_highpart_zext"
+  [(set (match_operand:DI 0 "register_operand" "=d")
+	(zero_extend:DI (truncate:SI
+	  (lshiftrt:DI
+	    (mult:DI (sign_extend:DI
+		       (match_operand:SI 1 "register_operand" "%a"))
+		     (sign_extend:DI
+		       (match_operand:SI 2 "nonimmediate_operand" "rm")))
+	    (const_int 32)))))
+   (clobber (match_scratch:SI 3 "=a"))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT"
+  "imul{l}\\t%2"
+  [(set_attr "type" "imul")
+   (set_attr "ppro_uops" "few")
+   (set_attr "mode" "SI")])
+
 ;; The patterns that match these are at the end of this file.
 
 (define_expand "mulxf3"
@@ -6641,6 +7372,93 @@
   "")
 
 ;; Remainder instructions.
+
+(define_expand "divmoddi4"
+  [(parallel [(set (match_operand:DI 0 "register_operand" "")
+		   (div:DI (match_operand:DI 1 "register_operand" "")
+			   (match_operand:DI 2 "nonimmediate_operand" "")))
+	      (set (match_operand:DI 3 "register_operand" "")
+		   (mod:DI (match_dup 1) (match_dup 2)))
+	      (clobber (reg:CC 17))])]
+  "TARGET_64BIT"
+  "")
+
+;; Allow to come the parameter in eax or edx to avoid extra moves.
+;; Penalize eax case sligthly because it results in worse scheduling
+;; of code.
+(define_insn "*divmoddi4_nocltd_rex64"
+  [(set (match_operand:DI 0 "register_operand" "=&a,?a")
+	(div:DI (match_operand:DI 2 "register_operand" "1,0")
+		(match_operand:DI 3 "nonimmediate_operand" "rm,rm")))
+   (set (match_operand:DI 1 "register_operand" "=&d,&d")
+	(mod:DI (match_dup 2) (match_dup 3)))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT && !optimize_size && !TARGET_USE_CLTD"
+  "#"
+  [(set_attr "type" "multi")])
+
+(define_insn "*divmoddi4_cltd_rex64"
+  [(set (match_operand:DI 0 "register_operand" "=a")
+	(div:DI (match_operand:DI 2 "register_operand" "a")
+		(match_operand:DI 3 "nonimmediate_operand" "rm")))
+   (set (match_operand:DI 1 "register_operand" "=&d")
+	(mod:DI (match_dup 2) (match_dup 3)))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT && (optimize_size || TARGET_USE_CLTD)"
+  "#"
+  [(set_attr "type" "multi")])
+
+(define_insn "*divmoddi_noext_rex64"
+  [(set (match_operand:DI 0 "register_operand" "=a")
+	(div:DI (match_operand:DI 1 "register_operand" "0")
+		(match_operand:DI 2 "nonimmediate_operand" "rm")))
+   (set (match_operand:DI 3 "register_operand" "=d")
+	(mod:DI (match_dup 1) (match_dup 2)))
+   (use (match_operand:DI 4 "register_operand" "3"))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT"
+  "idiv{q}\\t%2"
+  [(set_attr "type" "idiv")
+   (set_attr "mode" "DI")
+   (set_attr "ppro_uops" "few")])
+
+(define_split
+  [(set (match_operand:DI 0 "register_operand" "")
+	(div:DI (match_operand:DI 1 "register_operand" "")
+		(match_operand:DI 2 "nonimmediate_operand" "")))
+   (set (match_operand:DI 3 "register_operand" "")
+	(mod:DI (match_dup 1) (match_dup 2)))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT && reload_completed"
+  [(parallel [(set (match_dup 3)
+		   (ashiftrt:DI (match_dup 4) (const_int 63)))
+	      (clobber (reg:CC 17))])
+   (parallel [(set (match_dup 0)
+	           (div:DI (reg:DI 0) (match_dup 2)))
+	      (set (match_dup 3)
+		   (mod:DI (reg:DI 0) (match_dup 2)))
+	      (use (match_dup 3))
+	      (clobber (reg:CC 17))])]
+  "
+{
+  /* Avoid use of cltd in favour of a mov+shift.  */
+  if (!TARGET_USE_CLTD && !optimize_size)
+    {
+      if (true_regnum (operands[1]))
+        emit_move_insn (operands[0], operands[1]);
+      else
+	emit_move_insn (operands[3], operands[1]);
+      operands[4] = operands[3];
+    }
+  else
+    {
+      if (true_regnum (operands[1]))
+	abort();
+      operands[4] = operands[1];
+    }
+}")
+
+
 (define_expand "divmodsi4"
   [(parallel [(set (match_operand:SI 0 "register_operand" "")
 		   (div:SI (match_operand:SI 1 "register_operand" "")
@@ -6739,6 +7557,50 @@
    (set_attr "length_immediate" "0")
    (set_attr "mode" "SI")])
 
+(define_insn "udivmoddi4"
+  [(set (match_operand:DI 0 "register_operand" "=a")
+	(udiv:DI (match_operand:DI 1 "register_operand" "0")
+		 (match_operand:DI 2 "nonimmediate_operand" "rm")))
+   (set (match_operand:DI 3 "register_operand" "=&d")
+	(umod:DI (match_dup 1) (match_dup 2)))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT"
+  "xor{q}\\t%3, %3\;div{q}\\t%2"
+  [(set_attr "type" "multi")
+   (set_attr "length_immediate" "0")
+   (set_attr "mode" "DI")])
+
+(define_insn "*udivmoddi4_noext"
+  [(set (match_operand:DI 0 "register_operand" "=a")
+	(udiv:DI (match_operand:DI 1 "register_operand" "0")
+		 (match_operand:DI 2 "nonimmediate_operand" "rm")))
+   (set (match_operand:DI 3 "register_operand" "=d")
+	(umod:DI (match_dup 1) (match_dup 2)))
+   (use (match_dup 3))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT"
+  "div{q}\\t%2"
+  [(set_attr "type" "idiv")
+   (set_attr "ppro_uops" "few")
+   (set_attr "mode" "DI")])
+
+(define_split
+  [(set (match_operand:DI 0 "register_operand" "")
+	(udiv:DI (match_operand:DI 1 "register_operand" "")
+		 (match_operand:DI 2 "nonimmediate_operand" "")))
+   (set (match_operand:DI 3 "register_operand" "")
+	(umod:DI (match_dup 1) (match_dup 2)))
+   (clobber (reg:CC 17))]
+  "reload_completed && TARGET_64BIT"
+  [(set (match_dup 3) (const_int 0))
+   (parallel [(set (match_dup 0)
+		   (udiv:DI (match_dup 1) (match_dup 2)))
+	      (set (match_dup 3)
+		   (umod:DI (match_dup 1) (match_dup 2)))
+	      (use (match_dup 3))
+	      (clobber (reg:CC 17))])]
+  "")
+
 (define_insn "udivmodsi4"
   [(set (match_operand:SI 0 "register_operand" "=a")
 	(udiv:SI (match_operand:SI 1 "register_operand" "0")
@@ -6834,6 +7696,23 @@
 ;; On Pentium, "test imm, reg" is pairable only with eax, ax, and al.
 ;; Note that this excludes ah.
 
+(define_insn "*testdi_1_rex64"
+  [(set (reg 17)
+	(compare
+	  (and:DI (match_operand:DI 0 "nonimmediate_operand" "%*a,r,*a,r,rm")
+		  (match_operand:DI 1 "x86_64_szext_nonmemory_operand" "Z,Z,e,e,re"))
+	  (const_int 0)))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)"
+  "@
+   test{l}\\t{%k1, %k0|%k0, %k1} 
+   test{l}\\t{%k1, %k0|%k0, %k1} 
+   test{q}\\t{%1, %0|%0, %1} 
+   test{q}\\t{%1, %0|%0, %1} 
+   test{q}\\t{%1, %0|%0, %1}"
+  [(set_attr "type" "test")
+   (set_attr "modrm" "0,1,0,1,1")
+   (set_attr "mode" "SI,SI,DI,DI,DI")
+   (set_attr "pent_pair" "uv,np,uv,np,uv")])
 
 (define_insn "testsi_1"
   [(set (reg 17)
@@ -6990,16 +7869,32 @@
 		 (const_int 0)))]
   "ix86_match_ccmode (insn, CCNOmode)
    && (GET_MODE (operands[0]) == SImode
+       || (TARGET_64BIT && GET_MODE (operands[0]) == DImode)
+       || GET_MODE (operands[0]) == HImode
+       || GET_MODE (operands[0]) == QImode)"
+  "#")
+
+(define_insn "*testqi_ext_3_rex64"
+  [(set (reg 17)
+        (compare (zero_extract:DI
+		   (match_operand 0 "nonimmediate_operand" "rm")
+		   (match_operand:DI 1 "const_int_operand" "")
+		   (match_operand:DI 2 "const_int_operand" ""))
+		 (const_int 0)))]
+  "ix86_match_ccmode (insn, CCNOmode)
+   && TARGET_64BIT
+   && (GET_MODE (operands[0]) == SImode
+       || GET_MODE (operands[0]) == DImode
        || GET_MODE (operands[0]) == HImode
        || GET_MODE (operands[0]) == QImode)"
   "#")
 
 (define_split
   [(set (reg 17)
-        (compare (zero_extract:SI
+        (compare (zero_extract
 		   (match_operand 0 "nonimmediate_operand" "rm")
-		   (match_operand:SI 1 "const_int_operand" "")
-		   (match_operand:SI 2 "const_int_operand" ""))
+		   (match_operand 1 "const_int_operand" "")
+		   (match_operand 2 "const_int_operand" ""))
 		 (const_int 0)))]
   "ix86_match_ccmode (insn, CCNOmode)"
   [(set (reg:CCNO 17) (compare:CCNO (match_dup 3) (const_int 0)))]
@@ -7038,6 +7933,73 @@
 ;; and sometimes to QImode registers.  If this is considered useful,
 ;; it should be done with splitters.
 
+(define_expand "anddi3"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "")
+	(and:DI (match_operand:DI 1 "nonimmediate_operand" "")
+		(match_operand:DI 2 "x86_64_szext_general_operand" "")))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT"
+  "ix86_expand_binary_operator (AND, DImode, operands); DONE;")
+
+(define_insn "*anddi_1_rex64"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=r,rm,r,r")
+	(and:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0,0,qm")
+		(match_operand:DI 2 "x86_64_szext_general_operand" "Z,re,rm,L")))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT && ix86_binary_operator_ok (AND, DImode, operands)"
+  "*
+{
+  switch (get_attr_type (insn))
+    {
+    case TYPE_IMOVX:
+      {
+	enum machine_mode mode;
+
+	if (GET_CODE (operands[2]) != CONST_INT)
+	  abort ();
+        if (INTVAL (operands[2]) == 0xff)
+	  mode = QImode;
+	else if (INTVAL (operands[2]) == 0xffff)
+	  mode = HImode;
+	else
+	  abort ();
+	
+	operands[1] = gen_lowpart (mode, operands[1]);
+	if (mode == QImode)
+	  return \"movz{bq|x}\\t{%1,%0|%0, %1}\";
+	else
+	  return \"movz{wq|x}\\t{%1,%0|%0, %1}\";
+      }
+
+    default:
+      if (! rtx_equal_p (operands[0], operands[1]))
+	abort ();
+      if (get_attr_mode (insn) == MODE_SI)
+	return \"and{l}\\t{%k2, %k0|%k0, %k2}\";
+      else
+	return \"and{q}\\t{%2, %0|%0, %2}\";
+    }
+}"
+  [(set_attr "type" "alu,alu,alu,imovx")
+   (set_attr "length_immediate" "*,*,*,0")
+   (set_attr "mode" "SI,DI,DI,DI")])
+
+(define_insn "*anddi_2"
+  [(set (reg 17)
+	(compare (and:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0,0")
+			 (match_operand:DI 2 "x86_64_szext_general_operand" "Z,rem,re"))
+		 (const_int 0)))
+   (set (match_operand:DI 0 "nonimmediate_operand" "=r,r,rm")
+	(and:DI (match_dup 1) (match_dup 2)))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
+   && ix86_binary_operator_ok (AND, DImode, operands)"
+  "@
+   and{l}\\t{%k2, %k0|%k0, %k2} 
+   and{q}\\t{%2, %0|%0, %2} 
+   and{q}\\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "SI,DI,DI")])
+
 (define_expand "andsi3"
   [(set (match_operand:SI 0 "nonimmediate_operand" "")
 	(and:SI (match_operand:SI 1 "nonimmediate_operand" "")
@@ -7087,31 +8049,35 @@
    (set_attr "mode" "SI")])
 
 (define_split
-  [(set (match_operand:SI 0 "register_operand" "")
-	(and:SI (match_dup 0)
-		(const_int -65536)))
+  [(set (match_operand 0 "register_operand" "")
+	(and (match_dup 0)
+	     (const_int -65536)))
    (clobber (reg:CC 17))]
-  "optimize_size"
+  "optimize_size
+   && (GET_MODE (operands[0]) == SImode || GET_MODE (operands[0]) == HImode
+       || (TARGET_64BIT && GET_MODE (operands[0]) == DImode))"
   [(set (strict_low_part (match_dup 1)) (const_int 0))]
   "operands[1] = gen_lowpart (HImode, operands[0]);")
 
 (define_split
   [(set (match_operand 0 "q_regs_operand" "")
 	(and (match_dup 0)
-		(const_int -256)))
+	     (const_int -256)))
    (clobber (reg:CC 17))]
   "(optimize_size || !TARGET_PARTIAL_REG_STALL)
-   && (GET_MODE (operands[0]) == SImode || GET_MODE (operands[0]) == HImode)"
+   && (GET_MODE (operands[0]) == SImode || GET_MODE (operands[0]) == HImode
+       || (TARGET_64BIT && GET_MODE (operands[0]) == DImode))"
   [(set (strict_low_part (match_dup 1)) (const_int 0))]
   "operands[1] = gen_lowpart (QImode, operands[0]);")
 
 (define_split
-  [(set (match_operand 0 "q_regs_operand" "")
+  [(set (match_operand 0 "register_operand" "")
 	(and (match_dup 0)
 	     (const_int -65281)))
    (clobber (reg:CC 17))]
   "(optimize_size || !TARGET_PARTIAL_REG_STALL)
-   && (GET_MODE (operands[0]) == SImode || GET_MODE (operands[0]) == HImode)"
+   && (GET_MODE (operands[0]) == SImode || GET_MODE (operands[0]) == HImode
+       || (TARGET_64BIT && GET_MODE (operands[0]) == DImode))"
   [(parallel [(set (zero_extract:SI (match_dup 0)
 				    (const_int 8)
 				    (const_int 8))
@@ -7125,6 +8091,18 @@
 	      (clobber (reg:CC 17))])]
   "operands[0] = gen_lowpart (SImode, operands[0]);")
 
+;; See comment for addsi_1_zext why we do use nonimmediate_operand
+(define_insn "*andsi_1_zext"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	(zero_extend:DI
+	  (and:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
+		  (match_operand:SI 2 "general_operand" "rim"))))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT && ix86_binary_operator_ok (AND, SImode, operands)"
+  "and{l}\\t{%2, %k0|%k0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "SI")])
+
 (define_insn "*andsi_2"
   [(set (reg 17)
 	(compare (and:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
@@ -7138,6 +8116,20 @@
   [(set_attr "type" "alu")
    (set_attr "mode" "SI")])
 
+;; See comment for addsi_1_zext why we do use nonimmediate_operand
+(define_insn "*andsi_2_zext"
+  [(set (reg 17)
+	(compare (and:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
+			 (match_operand:SI 2 "general_operand" "rim"))
+		 (const_int 0)))
+   (set (match_operand:DI 0 "register_operand" "=r")
+	(zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
+   && ix86_binary_operator_ok (AND, SImode, operands)"
+  "and{l}\\t{%2, %k0|%k0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "SI")])
+
 (define_expand "andhi3"
   [(set (match_operand:HI 0 "nonimmediate_operand" "")
 	(and:HI (match_operand:HI 1 "nonimmediate_operand" "")
@@ -7367,6 +8359,53 @@
 ;; %%% This used to optimize known byte-wide and operations to memory.
 ;; If this is considered useful, it should be done with splitters.
 
+(define_expand "iordi3"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "")
+	(ior:DI (match_operand:DI 1 "nonimmediate_operand" "")
+		(match_operand:DI 2 "x86_64_general_operand" "")))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT"
+  "ix86_expand_binary_operator (IOR, DImode, operands); DONE;")
+
+(define_insn "*iordi_1_rex64"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=rm,r")
+	(ior:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
+		(match_operand:DI 2 "x86_64_general_operand" "re,rme")))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT
+   && ix86_binary_operator_ok (IOR, DImode, operands)"
+  "or{q}\\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "DI")])
+
+(define_insn "*iordi_2_rex64"
+  [(set (reg 17)
+	(compare (ior:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
+			 (match_operand:DI 2 "x86_64_general_operand" "rem,re"))
+		 (const_int 0)))
+   (set (match_operand:DI 0 "nonimmediate_operand" "=r,rm")
+	(ior:DI (match_dup 1) (match_dup 2)))]
+  "TARGET_64BIT
+   && ix86_match_ccmode (insn, CCNOmode)
+   && ix86_binary_operator_ok (IOR, DImode, operands)"
+  "or{q}\\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "DI")])
+
+(define_insn "*iordi_3_rex64"
+  [(set (reg 17)
+	(compare (ior:DI (match_operand:DI 1 "nonimmediate_operand" "%0")
+			 (match_operand:DI 2 "x86_64_general_operand" "rem"))
+		 (const_int 0)))
+   (clobber (match_scratch:DI 0 "=r"))]
+  "TARGET_64BIT
+   && ix86_match_ccmode (insn, CCNOmode)
+   && ix86_binary_operator_ok (IOR, DImode, operands)"
+  "or{q}\\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "DI")])
+
+
 (define_expand "iorsi3"
   [(set (match_operand:SI 0 "nonimmediate_operand" "")
 	(ior:SI (match_operand:SI 1 "nonimmediate_operand" "")
@@ -7385,6 +8424,28 @@
   [(set_attr "type" "alu")
    (set_attr "mode" "SI")])
 
+;; See comment for addsi_1_zext why we do use nonimmediate_operand
+(define_insn "*iorsi_1_zext"
+  [(set (match_operand:DI 0 "register_operand" "=rm")
+	(zero_extend:DI
+	  (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
+		  (match_operand:SI 2 "general_operand" "rim"))))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT && ix86_binary_operator_ok (IOR, SImode, operands)"
+  "or{l}\\t{%2, %k0|%k0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "SI")])
+
+(define_insn "*iorsi_1_zext_imm"
+  [(set (match_operand:DI 0 "register_operand" "=rm")
+	(ior:DI (zero_extend:DI (match_operand:SI 1 "register_operand" "%0"))
+		(match_operand:DI 2 "x86_64_zext_immediate_operand" "Z")))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT"
+  "or{l}\\t{%2, %k0|%k0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "SI")])
+
 (define_insn "*iorsi_2"
   [(set (reg 17)
 	(compare (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
@@ -7398,6 +8459,34 @@
   [(set_attr "type" "alu")
    (set_attr "mode" "SI")])
 
+;; See comment for addsi_1_zext why we do use nonimmediate_operand
+;; ??? Special case for immediate operand is missing - it is tricky.
+(define_insn "*iorsi_2_zext"
+  [(set (reg 17)
+	(compare (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
+			 (match_operand:SI 2 "general_operand" "rim"))
+		 (const_int 0)))
+   (set (match_operand:DI 0 "register_operand" "=r")
+	(zero_extend:DI (ior:SI (match_dup 1) (match_dup 2))))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
+   && ix86_binary_operator_ok (IOR, SImode, operands)"
+  "or{l}\\t{%2, %k0|%k0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "SI")])
+
+(define_insn "*iorsi_2_zext_imm"
+  [(set (reg 17)
+	(compare (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
+			 (match_operand 2 "x86_64_zext_immediate_operand" "Z"))
+		 (const_int 0)))
+   (set (match_operand:DI 0 "register_operand" "=r")
+	(ior:DI (zero_extend:DI (match_dup 1)) (match_dup 2)))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
+   && ix86_binary_operator_ok (IOR, SImode, operands)"
+  "or{l}\\t{%2, %k0|%k0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "SI")])
+
 (define_insn "*iorsi_3"
   [(set (reg 17)
 	(compare (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
@@ -7528,6 +8617,56 @@
 ;; %%% This used to optimize known byte-wide and operations to memory.
 ;; If this is considered useful, it should be done with splitters.
 
+(define_expand "xordi3"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "")
+	(xor:DI (match_operand:DI 1 "nonimmediate_operand" "")
+		(match_operand:DI 2 "x86_64_general_operand" "")))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT"
+  "ix86_expand_binary_operator (XOR, DImode, operands); DONE;")
+
+(define_insn "*xordi_1_rex64"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=rm,r")
+	(xor:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
+		(match_operand:DI 2 "x86_64_general_operand" "re,rm")))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT
+   && ix86_binary_operator_ok (XOR, DImode, operands)"
+  "@
+   xor{q}\\t{%2, %0|%0, %2} 
+   xor{q}\\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "DI,DI")])
+
+(define_insn "*xordi_2_rex64"
+  [(set (reg 17)
+	(compare (xor:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
+			 (match_operand:DI 2 "x86_64_general_operand" "rem,re"))
+		 (const_int 0)))
+   (set (match_operand:DI 0 "nonimmediate_operand" "=r,rm")
+	(xor:DI (match_dup 1) (match_dup 2)))]
+  "TARGET_64BIT
+   && ix86_match_ccmode (insn, CCNOmode)
+   && ix86_binary_operator_ok (XOR, DImode, operands)"
+  "@
+   xor{q}\\t{%2, %0|%0, %2} 
+   xor{q}\\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "DI,DI")])
+
+(define_insn "*xordi_3_rex64"
+  [(set (reg 17)
+	(compare (xor:DI (match_operand:DI 1 "nonimmediate_operand" "%0")
+			 (match_operand:DI 2 "x86_64_general_operand" "rem"))
+		 (const_int 0)))
+   (clobber (match_scratch:DI 0 "=r"))]
+  "TARGET_64BIT
+   && ix86_match_ccmode (insn, CCNOmode)
+   && ix86_binary_operator_ok (XOR, DImode, operands)"
+  "xor{q}\\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "DI")])
+
 (define_expand "xorsi3"
   [(set (match_operand:SI 0 "nonimmediate_operand" "")
 	(xor:SI (match_operand:SI 1 "nonimmediate_operand" "")
@@ -7546,6 +8685,29 @@
   [(set_attr "type" "alu")
    (set_attr "mode" "SI")])
 
+;; See comment for addsi_1_zext why we do use nonimmediate_operand
+;; Add speccase for immediates
+(define_insn "*xorsi_1_zext"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	(zero_extend:DI
+	  (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
+		  (match_operand:SI 2 "general_operand" "rim"))))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT && ix86_binary_operator_ok (XOR, SImode, operands)"
+  "xor{l}\\t{%2, %k0|%k0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "SI")])
+
+(define_insn "*xorsi_1_zext_imm"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	(xor:DI (zero_extend:DI (match_operand:SI 1 "register_operand" "%0"))
+		(match_operand:DI 2 "x86_64_zext_immediate_operand" "Z")))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT && ix86_binary_operator_ok (XOR, SImode, operands)"
+  "xor{l}\\t{%2, %k0|%k0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "SI")])
+
 (define_insn "*xorsi_2"
   [(set (reg 17)
 	(compare (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
@@ -7559,6 +8721,34 @@
   [(set_attr "type" "alu")
    (set_attr "mode" "SI")])
 
+;; See comment for addsi_1_zext why we do use nonimmediate_operand
+;; ??? Special case for immediate operand is missing - it is tricky.
+(define_insn "*xorsi_2_zext"
+  [(set (reg 17)
+	(compare (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
+			 (match_operand:SI 2 "general_operand" "rim"))
+		 (const_int 0)))
+   (set (match_operand:DI 0 "register_operand" "=r")
+	(zero_extend:DI (xor:SI (match_dup 1) (match_dup 2))))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
+   && ix86_binary_operator_ok (XOR, SImode, operands)"
+  "xor{l}\\t{%2, %k0|%k0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "SI")])
+
+(define_insn "*xorsi_2_zext_imm"
+  [(set (reg 17)
+	(compare (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
+			 (match_operand 2 "x86_64_zext_immediate_operand" "Z"))
+		 (const_int 0)))
+   (set (match_operand:DI 0 "register_operand" "=r")
+	(xor:DI (zero_extend:DI (match_dup 1)) (match_dup 2)))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
+   && ix86_binary_operator_ok (XOR, SImode, operands)"
+  "xor{l}\\t{%2, %k0|%k0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "SI")])
+
 (define_insn "*xorsi_3"
   [(set (reg 17)
 	(compare (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
@@ -7785,6 +8975,31 @@
   "split_di (operands+1, 1, operands+2, operands+3);
    split_di (operands+0, 1, operands+0, operands+1);")
 
+(define_insn "*negdi2_1_rex64"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=rm")
+	(neg:DI (match_operand:DI 1 "nonimmediate_operand" "0")))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT && ix86_unary_operator_ok (NEG, DImode, operands)"
+  "neg{q}\\t%0"
+  [(set_attr "type" "negnot")
+   (set_attr "mode" "DI")])
+
+;; The problem with neg is that it does not perform (compare x 0),
+;; it really performs (compare 0 x), which leaves us with the zero
+;; flag being the only useful item.
+
+(define_insn "*negdi2_cmpz_rex64"
+  [(set (reg:CCZ 17)
+	(compare:CCZ (neg:DI (match_operand:DI 1 "nonimmediate_operand" "0"))
+		     (const_int 0)))
+   (set (match_operand:DI 0 "nonimmediate_operand" "=rm")
+	(neg:DI (match_dup 1)))]
+  "TARGET_64BIT && ix86_unary_operator_ok (NEG, DImode, operands)"
+  "neg{q}\\t%0"
+  [(set_attr "type" "negnot")
+   (set_attr "mode" "DI")])
+
+
 (define_expand "negsi2"
   [(parallel [(set (match_operand:SI 0 "nonimmediate_operand" "")
 		   (neg:SI (match_operand:SI 1 "nonimmediate_operand" "")))
@@ -7801,6 +9016,18 @@
   [(set_attr "type" "negnot")
    (set_attr "mode" "SI")])
 
+;; Combine is quite creative about this pattern.
+(define_insn "*negsi2_1_zext"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	(lshiftrt:DI (neg:DI (ashift:DI (match_operand:DI 1 "register_operand" "0")
+					(const_int 32)))
+		     (const_int 32)))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT && ix86_unary_operator_ok (NEG, SImode, operands)"
+  "neg{l}\\t%k0"
+  [(set_attr "type" "negnot")
+   (set_attr "mode" "SI")])
+
 ;; The problem with neg is that it does not perform (compare x 0),
 ;; it really performs (compare 0 x), which leaves us with the zero
 ;; flag being the only useful item.
@@ -7816,6 +9043,23 @@
   [(set_attr "type" "negnot")
    (set_attr "mode" "SI")])
 
+(define_insn "*negsi2_cmpz_zext"
+  [(set (reg:CCZ 17)
+	(compare:CCZ (lshiftrt:DI
+		       (neg:DI (ashift:DI
+				 (match_operand:DI 1 "register_operand" "0")
+				 (const_int 32)))
+		       (const_int 32))
+		     (const_int 0)))
+   (set (match_operand:DI 0 "register_operand" "=r")
+	(lshiftrt:DI (neg:DI (ashift:DI (match_dup 1)
+					(const_int 32)))
+		     (const_int 32)))]
+  "TARGET_64BIT && ix86_unary_operator_ok (NEG, SImode, operands)"
+  "neg{l}\\t%k0"
+  [(set_attr "type" "negnot")
+   (set_attr "mode" "SI")])
+
 (define_expand "neghi2"
   [(parallel [(set (match_operand:HI 0 "nonimmediate_operand" "")
 		   (neg:HI (match_operand:HI 1 "nonimmediate_operand" "")))
@@ -8495,6 +9739,46 @@
 
 ;; One complement instructions
 
+(define_expand "one_cmpldi2"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "")
+	(not:DI (match_operand:DI 1 "nonimmediate_operand" "")))]
+  "TARGET_64BIT"
+  "ix86_expand_unary_operator (NOT, DImode, operands); DONE;")
+
+(define_insn "*one_cmpldi2_1_rex64"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=rm")
+	(not:DI (match_operand:DI 1 "nonimmediate_operand" "0")))]
+  "TARGET_64BIT && ix86_unary_operator_ok (NOT, DImode, operands)"
+  "not{q}\\t%0"
+  [(set_attr "type" "negnot")
+   (set_attr "mode" "DI")])
+
+(define_insn "*one_cmpldi2_2_rex64"
+  [(set (reg 17)
+	(compare (not:DI (match_operand:DI 1 "nonimmediate_operand" "0"))
+		 (const_int 0)))
+   (set (match_operand:DI 0 "nonimmediate_operand" "=rm")
+	(not:DI (match_dup 1)))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
+   && ix86_unary_operator_ok (NOT, DImode, operands)"
+  "#"
+  [(set_attr "type" "alu1")
+   (set_attr "mode" "DI")])
+
+(define_split
+  [(set (reg 17)
+	(compare (not:DI (match_operand:DI 1 "nonimmediate_operand" ""))
+		 (const_int 0)))
+   (set (match_operand:DI 0 "nonimmediate_operand" "")
+	(not:DI (match_dup 1)))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)"
+  [(parallel [(set (reg:CCNO 17)
+		   (compare:CCNO (xor:DI (match_dup 1) (const_int -1))
+				 (const_int 0)))
+	      (set (match_dup 0)
+		   (xor:DI (match_dup 1) (const_int -1)))])]
+  "")
+
 (define_expand "one_cmplsi2"
   [(set (match_operand:SI 0 "nonimmediate_operand" "")
 	(not:SI (match_operand:SI 1 "nonimmediate_operand" "")))]
@@ -8509,6 +9793,15 @@
   [(set_attr "type" "negnot")
    (set_attr "mode" "SI")])
 
+;; ??? Currently never generated - xor is used instead.
+(define_insn "*one_cmplsi2_1_zext"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	(zero_extend:DI (not:SI (match_operand:SI 1 "register_operand" "0"))))]
+  "TARGET_64BIT && ix86_unary_operator_ok (NOT, SImode, operands)"
+  "not{l}\\t%k0"
+  [(set_attr "type" "negnot")
+   (set_attr "mode" "SI")])
+
 (define_insn "*one_cmplsi2_2"
   [(set (reg 17)
 	(compare (not:SI (match_operand:SI 1 "nonimmediate_operand" "0"))
@@ -8535,6 +9828,33 @@
 		   (xor:SI (match_dup 1) (const_int -1)))])]
   "")
 
+;; ??? Currently never generated - xor is used instead.
+(define_insn "*one_cmplsi2_2_zext"
+  [(set (reg 17)
+	(compare (not:SI (match_operand:SI 1 "register_operand" "0"))
+		 (const_int 0)))
+   (set (match_operand:DI 0 "register_operand" "=r")
+	(zero_extend:DI (not:SI (match_dup 1))))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
+   && ix86_unary_operator_ok (NOT, SImode, operands)"
+  "#"
+  [(set_attr "type" "alu1")
+   (set_attr "mode" "SI")])
+
+(define_split
+  [(set (reg 17)
+	(compare (not:SI (match_operand:SI 1 "register_operand" ""))
+		 (const_int 0)))
+   (set (match_operand:DI 0 "register_operand" "")
+	(zero_extend:DI (not:SI (match_dup 1))))]
+  "ix86_match_ccmode (insn, CCNOmode)"
+  [(parallel [(set (reg:CCNO 17)
+		   (compare:CCNO (xor:SI (match_dup 1) (const_int -1))
+				 (const_int 0)))
+	      (set (match_dup 0)
+		   (zero_extend:DI (xor:SI (match_dup 1) (const_int -1))))])]
+  "")
+
 (define_expand "one_cmplhi2"
   [(set (match_operand:HI 0 "nonimmediate_operand" "")
 	(not:HI (match_operand:HI 1 "nonimmediate_operand" "")))]