i386.c (ix86_expand_branch, [...]): Handle TImode in 64-bit mode the same as DImode in 32-bit mode.

gcc/
2005-07-18  Jan Beulich  <jbeulich@novell.com>

	* config/i386/i386.c (ix86_expand_branch, ix86_expand_setcc,
	ix86_expand_carry_flag_compare, ix86_expand_int_movcc): Handle TImode
	in 64-bit mode the same as DImode in 32-bit mode.
	(ix86_expand_ashl_const, ix86_split_ashl, ix86_split_ashr,
	ix86_split_lshr): Likewise. Rename to no longer refer to a specific
	mode. Add new mode parameter.
	* config/i386/i386.h (CONST_OK_FOR_LETTER_P): Describe and handle 'O'.
	* config/i386/i386.md (cmpti, addti3, subti3, negti2, ashlti3, ashrti3,
	x86_64_shift_adj): New expanders.
	(*addti3_1, *subti3_1, *negti2_1, ashlti3_1, *ashlti3_2, ashrti3_1,
	*ashrti3_2, lshrti3_1, *lshrti3_2, x86_64_shld, x86_64_shrd): New
	insns.
	Respective new splitters. Use renamed shift splitter helpers in 32-bit
	DImode shift splitters.
	* config/i386/i386-protos.h (ix86_split_ashl, ix86_split_ashr,
	ix86_split_lshr): Renamed from ix86_split_[al]sh[rl]di. Added new
	mode parameter.

From-SVN: r102129

4 files changed, 490 insertions, 82 deletions

diff --git a/gcc/config/i386/i386-protos.h b/gcc/config/i386/i386-protos.h
index 957876c..dfbf11a 100644
--- a/gcc/config/i386/i386-protos.h
+++ b/gcc/config/i386/i386-protos.h
@@ -155,9 +155,9 @@ extern void ix86_expand_call (rtx, rtx, rtx, rtx, rtx, int);
 extern void x86_initialize_trampoline (rtx, rtx, rtx);
 extern rtx ix86_zero_extend_to_Pmode (rtx);
 extern void ix86_split_long_move (rtx[]);
-extern void ix86_split_ashldi (rtx *, rtx);
-extern void ix86_split_ashrdi (rtx *, rtx);
-extern void ix86_split_lshrdi (rtx *, rtx);
+extern void ix86_split_ashl (rtx *, rtx, enum machine_mode);
+extern void ix86_split_ashr (rtx *, rtx, enum machine_mode);
+extern void ix86_split_lshr (rtx *, rtx, enum machine_mode);
 extern rtx ix86_find_base_term (rtx);
 extern int ix86_check_movabs (rtx, int);
 
diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c
index 17d8ea0..8dd9698 100644
--- a/gcc/config/i386/i386.c
+++ b/gcc/config/i386/i386.c
@@ -7200,7 +7200,7 @@ split_di (rtx operands[], int num, rtx lo_half[], rtx hi_half[])
 	}
     }
 }
-/* Split one or more TImode RTL references into pairs of SImode
+/* Split one or more TImode RTL references into pairs of DImode
    references.  The RTL can be REG, offsettable MEM, integer constant, or
    CONST_DOUBLE.  "operands" is a pointer to an array of DImode RTL to
    split and "num" is its length.  lo_half and hi_half are output arrays
@@ -9344,10 +9344,12 @@ ix86_expand_branch (enum rtx_code code, rtx label)
     case DImode:
       if (TARGET_64BIT)
 	goto simple;
+    case TImode:
       /* Expand DImode branch into multiple compare+branch.  */
       {
 	rtx lo[2], hi[2], label2;
 	enum rtx_code code1, code2, code3;
+	enum machine_mode submode;
 
 	if (CONSTANT_P (ix86_compare_op0) && ! CONSTANT_P (ix86_compare_op1))
 	  {
@@ -9356,8 +9358,18 @@ ix86_expand_branch (enum rtx_code code, rtx label)
 	    ix86_compare_op1 = tmp;
 	    code = swap_condition (code);
 	  }
-	split_di (&ix86_compare_op0, 1, lo+0, hi+0);
-	split_di (&ix86_compare_op1, 1, lo+1, hi+1);
+	if (GET_MODE (ix86_compare_op0) == DImode)
+	  {
+	    split_di (&ix86_compare_op0, 1, lo+0, hi+0);
+	    split_di (&ix86_compare_op1, 1, lo+1, hi+1);
+	    submode = SImode;
+	  }
+	else
+	  {
+	    split_ti (&ix86_compare_op0, 1, lo+0, hi+0);
+	    split_ti (&ix86_compare_op1, 1, lo+1, hi+1);
+	    submode = DImode;
+	  }
 
 	/* When comparing for equality, we can use (hi0^hi1)|(lo0^lo1) to
 	   avoid two branches.  This costs one extra insn, so disable when
@@ -9371,15 +9383,15 @@ ix86_expand_branch (enum rtx_code code, rtx label)
 
 	    xor1 = hi[0];
 	    if (hi[1] != const0_rtx)
-	      xor1 = expand_binop (SImode, xor_optab, xor1, hi[1],
+	      xor1 = expand_binop (submode, xor_optab, xor1, hi[1],
 				   NULL_RTX, 0, OPTAB_WIDEN);
 
 	    xor0 = lo[0];
 	    if (lo[1] != const0_rtx)
-	      xor0 = expand_binop (SImode, xor_optab, xor0, lo[1],
+	      xor0 = expand_binop (submode, xor_optab, xor0, lo[1],
 				   NULL_RTX, 0, OPTAB_WIDEN);
 
-	    tmp = expand_binop (SImode, ior_optab, xor1, xor0,
+	    tmp = expand_binop (submode, ior_optab, xor1, xor0,
 				NULL_RTX, 0, OPTAB_WIDEN);
 
 	    ix86_compare_op0 = tmp;
@@ -9547,8 +9559,7 @@ ix86_expand_setcc (enum rtx_code code, rtx dest)
   rtx ret, tmp, tmpreg, equiv;
   rtx second_test, bypass_test;
 
-  if (GET_MODE (ix86_compare_op0) == DImode
-      && !TARGET_64BIT)
+  if (GET_MODE (ix86_compare_op0) == (TARGET_64BIT ? TImode : DImode))
     return 0; /* FAIL */
 
   gcc_assert (GET_MODE (dest) == QImode);
@@ -9603,7 +9614,7 @@ ix86_expand_carry_flag_compare (enum rtx_code code, rtx op0, rtx op1, rtx *pop)
 
   /* Do not handle DImode compares that go trought special path.  Also we can't
      deal with FP compares yet.  This is possible to add.  */
-  if ((mode == DImode && !TARGET_64BIT))
+  if (mode == (TARGET_64BIT ? TImode : DImode))
     return false;
   if (FLOAT_MODE_P (mode))
     {
@@ -9743,7 +9754,7 @@ ix86_expand_int_movcc (rtx operands[])
      HImode insns, we'd be swallowed in word prefix ops.  */
 
   if ((mode != HImode || TARGET_FAST_PREFIX)
-      && (mode != DImode || TARGET_64BIT)
+      && (mode != (TARGET_64BIT ? TImode : DImode))
       && GET_CODE (operands[2]) == CONST_INT
       && GET_CODE (operands[3]) == CONST_INT)
     {
@@ -11092,68 +11103,81 @@ ix86_split_long_move (rtx operands[])
   return;
 }
 
-/* Helper function of ix86_split_ashldi used to generate an SImode
+/* Helper function of ix86_split_ashl used to generate an SImode/DImode
    left shift by a constant, either using a single shift or
    a sequence of add instructions.  */
 
 static void
-ix86_expand_ashlsi3_const (rtx operand, int count)
+ix86_expand_ashl_const (rtx operand, int count, enum machine_mode mode)
 {
   if (count == 1)
-    emit_insn (gen_addsi3 (operand, operand, operand));
+    {
+      emit_insn ((mode == DImode
+		  ? gen_addsi3
+		  : gen_adddi3) (operand, operand, operand));
+    }
   else if (!optimize_size
 	   && count * ix86_cost->add <= ix86_cost->shift_const)
     {
       int i;
       for (i=0; i<count; i++)
-	emit_insn (gen_addsi3 (operand, operand, operand));
+	{
+	  emit_insn ((mode == DImode
+		      ? gen_addsi3
+		      : gen_adddi3) (operand, operand, operand));
+	}
     }
   else
-    emit_insn (gen_ashlsi3 (operand, operand, GEN_INT (count)));
+    emit_insn ((mode == DImode
+		? gen_ashlsi3
+		: gen_ashldi3) (operand, operand, GEN_INT (count)));
 }
 
 void
-ix86_split_ashldi (rtx *operands, rtx scratch)
+ix86_split_ashl (rtx *operands, rtx scratch, enum machine_mode mode)
 {
   rtx low[2], high[2];
   int count;
+  const int single_width = mode == DImode ? 32 : 64;
 
   if (GET_CODE (operands[2]) == CONST_INT)
     {
-      split_di (operands, 2, low, high);
-      count = INTVAL (operands[2]) & 63;
+      (mode == DImode ? split_di : split_ti) (operands, 2, low, high);
+      count = INTVAL (operands[2]) & (single_width * 2 - 1);
 
-      if (count >= 32)
+      if (count >= single_width)
 	{
 	  emit_move_insn (high[0], low[1]);
 	  emit_move_insn (low[0], const0_rtx);
 
-	  if (count > 32)
-	    ix86_expand_ashlsi3_const (high[0], count - 32);
+	  if (count > single_width)
+	    ix86_expand_ashl_const (high[0], count - single_width, mode);
 	}
       else
 	{
 	  if (!rtx_equal_p (operands[0], operands[1]))
 	    emit_move_insn (operands[0], operands[1]);
-	  emit_insn (gen_x86_shld_1 (high[0], low[0], GEN_INT (count)));
-	  ix86_expand_ashlsi3_const (low[0], count);
+	  emit_insn ((mode == DImode
+		     ? gen_x86_shld_1
+		     : gen_x86_64_shld) (high[0], low[0], GEN_INT (count)));
+	  ix86_expand_ashl_const (low[0], count, mode);
 	}
       return;
     }
 
-  split_di (operands, 1, low, high);
+  (mode == DImode ? split_di : split_ti) (operands, 1, low, high);
 
   if (operands[1] == const1_rtx)
     {
-      /* Assuming we've chosen a QImode capable registers, then 1LL << N
-	 can be done with two 32-bit shifts, no branches, no cmoves.  */
+      /* Assuming we've chosen a QImode capable registers, then 1 << N
+	 can be done with two 32/64-bit shifts, no branches, no cmoves.  */
       if (ANY_QI_REG_P (low[0]) && ANY_QI_REG_P (high[0]))
 	{
 	  rtx s, d, flags = gen_rtx_REG (CCZmode, FLAGS_REG);
 
 	  ix86_expand_clear (low[0]);
 	  ix86_expand_clear (high[0]);
-	  emit_insn (gen_testqi_ccz_1 (operands[2], GEN_INT (32)));
+	  emit_insn (gen_testqi_ccz_1 (operands[2], GEN_INT (single_width)));
 	  
 	  d = gen_lowpart (QImode, low[0]);
 	  d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
@@ -11167,7 +11191,7 @@ ix86_split_ashldi (rtx *operands, rtx scratch)
 	}
 
       /* Otherwise, we can get the same results by manually performing
-	 a bit extract operation on bit 5, and then performing the two
+	 a bit extract operation on bit 5/6, and then performing the two
 	 shifts.  The two methods of getting 0/1 into low/high are exactly
 	 the same size.  Avoiding the shift in the bit extract case helps
 	 pentium4 a bit; no one else seems to care much either way.  */
@@ -11176,29 +11200,39 @@ ix86_split_ashldi (rtx *operands, rtx scratch)
 	  rtx x;
 
 	  if (TARGET_PARTIAL_REG_STALL && !optimize_size)
-	    x = gen_rtx_ZERO_EXTEND (SImode, operands[2]);
+	    x = gen_rtx_ZERO_EXTEND (mode == DImode ? SImode : DImode, operands[2]);
 	  else
-	    x = gen_lowpart (SImode, operands[2]);
+	    x = gen_lowpart (mode == DImode ? SImode : DImode, operands[2]);
 	  emit_insn (gen_rtx_SET (VOIDmode, high[0], x));
 
-	  emit_insn (gen_lshrsi3 (high[0], high[0], GEN_INT (5)));
-	  emit_insn (gen_andsi3 (high[0], high[0], GEN_INT (1)));
+	  emit_insn ((mode == DImode
+		      ? gen_lshrsi3
+		      : gen_lshrdi3) (high[0], high[0], GEN_INT (mode == DImode ? 5 : 6)));
+	  emit_insn ((mode == DImode
+		      ? gen_andsi3
+		      : gen_anddi3) (high[0], high[0], GEN_INT (1)));
 	  emit_move_insn (low[0], high[0]);
-	  emit_insn (gen_xorsi3 (low[0], low[0], GEN_INT (1)));
+	  emit_insn ((mode == DImode
+		      ? gen_xorsi3
+		      : gen_xordi3) (low[0], low[0], GEN_INT (1)));
 	}
 
-      emit_insn (gen_ashlsi3 (low[0], low[0], operands[2]));
-      emit_insn (gen_ashlsi3 (high[0], high[0], operands[2]));
+      emit_insn ((mode == DImode
+		    ? gen_ashlsi3
+		    : gen_ashldi3) (low[0], low[0], operands[2]));
+      emit_insn ((mode == DImode
+		    ? gen_ashlsi3
+		    : gen_ashldi3) (high[0], high[0], operands[2]));
       return;
     }
 
   if (operands[1] == constm1_rtx)
     {
-      /* For -1LL << N, we can avoid the shld instruction, because we
-	 know that we're shifting 0...31 ones into a -1.  */
+      /* For -1 << N, we can avoid the shld instruction, because we
+	 know that we're shifting 0...31/63 ones into a -1.  */
       emit_move_insn (low[0], constm1_rtx);
       if (optimize_size)
-        emit_move_insn (high[0], low[0]);
+	emit_move_insn (high[0], low[0]);
       else
 	emit_move_insn (high[0], constm1_rtx);
     }
@@ -11207,53 +11241,71 @@ ix86_split_ashldi (rtx *operands, rtx scratch)
       if (!rtx_equal_p (operands[0], operands[1]))
 	emit_move_insn (operands[0], operands[1]);
 
-      split_di (operands, 1, low, high);
-      emit_insn (gen_x86_shld_1 (high[0], low[0], operands[2]));
+      (mode == DImode ? split_di : split_ti) (operands, 1, low, high);
+      emit_insn ((mode == DImode
+		  ? gen_x86_shld_1
+		  : gen_x86_64_shld) (high[0], low[0], operands[2]));
     }
 
-  emit_insn (gen_ashlsi3 (low[0], low[0], operands[2]));
+  emit_insn ((mode == DImode ? gen_ashlsi3 : gen_ashldi3) (low[0], low[0], operands[2]));
 
   if (TARGET_CMOVE && scratch)
     {
       ix86_expand_clear (scratch);
-      emit_insn (gen_x86_shift_adj_1 (high[0], low[0], operands[2], scratch));
+      emit_insn ((mode == DImode
+		  ? gen_x86_shift_adj_1
+		  : gen_x86_64_shift_adj) (high[0], low[0], operands[2], scratch));
     }
   else
     emit_insn (gen_x86_shift_adj_2 (high[0], low[0], operands[2]));
 }
 
 void
-ix86_split_ashrdi (rtx *operands, rtx scratch)
+ix86_split_ashr (rtx *operands, rtx scratch, enum machine_mode mode)
 {
   rtx low[2], high[2];
   int count;
+  const int single_width = mode == DImode ? 32 : 64;
 
   if (GET_CODE (operands[2]) == CONST_INT)
     {
-      split_di (operands, 2, low, high);
-      count = INTVAL (operands[2]) & 63;
+      (mode == DImode ? split_di : split_ti) (operands, 2, low, high);
+      count = INTVAL (operands[2]) & (single_width * 2 - 1);
 
-      if (count == 63)
+      if (count == single_width * 2 - 1)
 	{
 	  emit_move_insn (high[0], high[1]);
-	  emit_insn (gen_ashrsi3 (high[0], high[0], GEN_INT (31)));
+	  emit_insn ((mode == DImode
+		      ? gen_ashrsi3
+		      : gen_ashrdi3) (high[0], high[0],
+				      GEN_INT (single_width - 1)));
 	  emit_move_insn (low[0], high[0]);
 
 	}
-      else if (count >= 32)
+      else if (count >= single_width)
 	{
 	  emit_move_insn (low[0], high[1]);
 	  emit_move_insn (high[0], low[0]);
-	  emit_insn (gen_ashrsi3 (high[0], high[0], GEN_INT (31)));
-	  if (count > 32)
-	    emit_insn (gen_ashrsi3 (low[0], low[0], GEN_INT (count - 32)));
+	  emit_insn ((mode == DImode
+		      ? gen_ashrsi3
+		      : gen_ashrdi3) (high[0], high[0],
+				      GEN_INT (single_width - 1)));
+	  if (count > single_width)
+	    emit_insn ((mode == DImode
+			? gen_ashrsi3
+			: gen_ashrdi3) (low[0], low[0],
+					GEN_INT (count - single_width)));
 	}
       else
 	{
 	  if (!rtx_equal_p (operands[0], operands[1]))
 	    emit_move_insn (operands[0], operands[1]);
-	  emit_insn (gen_x86_shrd_1 (low[0], high[0], GEN_INT (count)));
-	  emit_insn (gen_ashrsi3 (high[0], high[0], GEN_INT (count)));
+	  emit_insn ((mode == DImode
+		      ? gen_x86_shrd_1
+		      : gen_x86_64_shrd) (low[0], high[0], GEN_INT (count)));
+	  emit_insn ((mode == DImode
+		      ? gen_ashrsi3
+		      : gen_ashrdi3) (high[0], high[0], GEN_INT (count)));
 	}
     }
   else
@@ -11261,17 +11313,26 @@ ix86_split_ashrdi (rtx *operands, rtx scratch)
       if (!rtx_equal_p (operands[0], operands[1]))
 	emit_move_insn (operands[0], operands[1]);
 
-      split_di (operands, 1, low, high);
+      (mode == DImode ? split_di : split_ti) (operands, 1, low, high);
 
-      emit_insn (gen_x86_shrd_1 (low[0], high[0], operands[2]));
-      emit_insn (gen_ashrsi3 (high[0], high[0], operands[2]));
+      emit_insn ((mode == DImode
+		  ? gen_x86_shrd_1
+		  : gen_x86_64_shrd) (low[0], high[0], operands[2]));
+      emit_insn ((mode == DImode
+		  ? gen_ashrsi3
+		  : gen_ashrdi3)  (high[0], high[0], operands[2]));
 
       if (TARGET_CMOVE && scratch)
 	{
 	  emit_move_insn (scratch, high[0]);
-	  emit_insn (gen_ashrsi3 (scratch, scratch, GEN_INT (31)));
-	  emit_insn (gen_x86_shift_adj_1 (low[0], high[0], operands[2],
-					  scratch));
+	  emit_insn ((mode == DImode
+		      ? gen_ashrsi3
+		      : gen_ashrdi3) (scratch, scratch,
+				      GEN_INT (single_width - 1)));
+	  emit_insn ((mode == DImode
+		      ? gen_x86_shift_adj_1
+		      : gen_x86_64_shift_adj) (low[0], high[0], operands[2],
+					 scratch));
 	}
       else
 	emit_insn (gen_x86_shift_adj_3 (low[0], high[0], operands[2]));
@@ -11279,30 +11340,38 @@ ix86_split_ashrdi (rtx *operands, rtx scratch)
 }
 
 void
-ix86_split_lshrdi (rtx *operands, rtx scratch)
+ix86_split_lshr (rtx *operands, rtx scratch, enum machine_mode mode)
 {
   rtx low[2], high[2];
   int count;
+  const int single_width = mode == DImode ? 32 : 64;
 
   if (GET_CODE (operands[2]) == CONST_INT)
     {
-      split_di (operands, 2, low, high);
-      count = INTVAL (operands[2]) & 63;
+      (mode == DImode ? split_di : split_ti) (operands, 2, low, high);
+      count = INTVAL (operands[2]) & (single_width * 2 - 1);
 
-      if (count >= 32)
+      if (count >= single_width)
 	{
 	  emit_move_insn (low[0], high[1]);
 	  ix86_expand_clear (high[0]);
 
-	  if (count > 32)
-	    emit_insn (gen_lshrsi3 (low[0], low[0], GEN_INT (count - 32)));
+	  if (count > single_width)
+	    emit_insn ((mode == DImode
+			? gen_lshrsi3
+			: gen_lshrdi3) (low[0], low[0],
+					GEN_INT (count - single_width)));
 	}
       else
 	{
 	  if (!rtx_equal_p (operands[0], operands[1]))
 	    emit_move_insn (operands[0], operands[1]);
-	  emit_insn (gen_x86_shrd_1 (low[0], high[0], GEN_INT (count)));
-	  emit_insn (gen_lshrsi3 (high[0], high[0], GEN_INT (count)));
+	  emit_insn ((mode == DImode
+		      ? gen_x86_shrd_1
+		      : gen_x86_64_shrd) (low[0], high[0], GEN_INT (count)));
+	  emit_insn ((mode == DImode
+		      ? gen_lshrsi3
+		      : gen_lshrdi3) (high[0], high[0], GEN_INT (count)));
 	}
     }
   else
@@ -11310,17 +11379,23 @@ ix86_split_lshrdi (rtx *operands, rtx scratch)
       if (!rtx_equal_p (operands[0], operands[1]))
 	emit_move_insn (operands[0], operands[1]);
 
-      split_di (operands, 1, low, high);
+      (mode == DImode ? split_di : split_ti) (operands, 1, low, high);
 
-      emit_insn (gen_x86_shrd_1 (low[0], high[0], operands[2]));
-      emit_insn (gen_lshrsi3 (high[0], high[0], operands[2]));
+      emit_insn ((mode == DImode
+		  ? gen_x86_shrd_1
+		  : gen_x86_64_shrd) (low[0], high[0], operands[2]));
+      emit_insn ((mode == DImode
+		  ? gen_lshrsi3
+		  : gen_lshrdi3) (high[0], high[0], operands[2]));
 
       /* Heh.  By reversing the arguments, we can reuse this pattern.  */
       if (TARGET_CMOVE && scratch)
 	{
 	  ix86_expand_clear (scratch);
-	  emit_insn (gen_x86_shift_adj_1 (low[0], high[0], operands[2],
-					  scratch));
+	  emit_insn ((mode == DImode
+		      ? gen_x86_shift_adj_1
+		      : gen_x86_64_shift_adj) (low[0], high[0], operands[2],
+					       scratch));
 	}
       else
 	emit_insn (gen_x86_shift_adj_2 (low[0], high[0], operands[2]));
diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
index 7a54cba..e140bd2 100644
--- a/gcc/config/i386/i386.h
+++ b/gcc/config/i386/i386.h
@@ -1218,7 +1218,7 @@ enum reg_class
    (C) == 'l' ? INDEX_REGS :					\
    NO_REGS)
 
-/* The letters I, J, K, L and M in a register constraint string
+/* The letters I, J, K, L, M, N, and O in a register constraint string
    can be used to stand for particular ranges of immediate operands.
    This macro defines what the ranges are.
    C is the letter, and VALUE is a constant value.
@@ -1230,6 +1230,7 @@ enum reg_class
    L is for andsi as zero-extending move.
    M is for shifts that can be executed by the "lea" opcode.
    N is for immediate operands for out/in instructions (0-255)
+   O is for TImode shifts.
    */
 
 #define CONST_OK_FOR_LETTER_P(VALUE, C)				\
@@ -1239,6 +1240,7 @@ enum reg_class
    : (C) == 'L' ? (VALUE) == 0xff || (VALUE) == 0xffff		\
    : (C) == 'M' ? (VALUE) >= 0 && (VALUE) <= 3			\
    : (C) == 'N' ? (VALUE) >= 0 && (VALUE) <= 255		\
+   : (C) == 'O' ? (VALUE) >= 0 && (VALUE) <= 127		\
    : 0)
 
 /* Similar, but for floating constants, and defining letters G and H.
diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
index e7f2e22..ba35331 100644
--- a/gcc/config/i386/i386.md
+++ b/gcc/config/i386/i386.md
@@ -478,6 +478,19 @@
 ;; actually generating RTL.  The bCOND or sCOND (emitted immediately
 ;; after the cmp) will actually emit the cmpM.
 
+(define_expand "cmpti"
+  [(set (reg:CC FLAGS_REG)
+	(compare:CC (match_operand:TI 0 "nonimmediate_operand" "")
+		    (match_operand:TI 1 "x86_64_general_operand" "")))]
+  "TARGET_64BIT"
+{
+  if (GET_CODE (operands[0]) == MEM && GET_CODE (operands[1]) == MEM)
+    operands[0] = force_reg (TImode, operands[0]);
+  ix86_compare_op0 = operands[0];
+  ix86_compare_op1 = operands[1];
+  DONE;
+})
+
 (define_expand "cmpdi"
   [(set (reg:CC FLAGS_REG)
 	(compare:CC (match_operand:DI 0 "nonimmediate_operand" "")
@@ -4694,6 +4707,42 @@
 
 ;; Add instructions
 
+;; %%% splits for addditi3
+
+(define_expand "addti3"
+  [(set (match_operand:TI 0 "nonimmediate_operand" "")
+	(plus:TI (match_operand:TI 1 "nonimmediate_operand" "")
+		 (match_operand:TI 2 "x86_64_general_operand" "")))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT"
+  "ix86_expand_binary_operator (PLUS, TImode, operands); DONE;")
+
+(define_insn "*addti3_1"
+  [(set (match_operand:TI 0 "nonimmediate_operand" "=r,o")
+	(plus:TI (match_operand:TI 1 "nonimmediate_operand" "%0,0")
+		 (match_operand:TI 2 "general_operand" "roiF,riF")))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT && ix86_binary_operator_ok (PLUS, TImode, operands)"
+  "#")
+
+(define_split
+  [(set (match_operand:TI 0 "nonimmediate_operand" "")
+	(plus:TI (match_operand:TI 1 "nonimmediate_operand" "")
+		 (match_operand:TI 2 "general_operand" "")))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT && reload_completed"
+  [(parallel [(set (reg:CC FLAGS_REG) (unspec:CC [(match_dup 1) (match_dup 2)]
+					  UNSPEC_ADD_CARRY))
+	      (set (match_dup 0) (plus:DI (match_dup 1) (match_dup 2)))])
+   (parallel [(set (match_dup 3)
+		   (plus:DI (plus:DI (ltu:DI (reg:CC FLAGS_REG) (const_int 0))
+				     (match_dup 4))
+			    (match_dup 5)))
+	      (clobber (reg:CC FLAGS_REG))])]
+  "split_ti (operands+0, 1, operands+0, operands+3);
+   split_ti (operands+1, 1, operands+1, operands+4);
+   split_ti (operands+2, 1, operands+2, operands+5);")
+
 ;; %%% splits for addsidi3
 ;  [(set (match_operand:DI 0 "nonimmediate_operand" "")
 ;	(plus:DI (match_operand:DI 1 "general_operand" "")
@@ -6392,6 +6441,41 @@
 
 ;; Subtract instructions
 
+;; %%% splits for subditi3
+
+(define_expand "subti3"
+  [(parallel [(set (match_operand:TI 0 "nonimmediate_operand" "")
+		   (minus:TI (match_operand:TI 1 "nonimmediate_operand" "")
+			     (match_operand:TI 2 "x86_64_general_operand" "")))
+	      (clobber (reg:CC FLAGS_REG))])]
+  "TARGET_64BIT"
+  "ix86_expand_binary_operator (MINUS, TImode, operands); DONE;")
+
+(define_insn "*subti3_1"
+  [(set (match_operand:TI 0 "nonimmediate_operand" "=r,o")
+	(minus:TI (match_operand:TI 1 "nonimmediate_operand" "0,0")
+		  (match_operand:TI 2 "general_operand" "roiF,riF")))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT && ix86_binary_operator_ok (MINUS, TImode, operands)"
+  "#")
+
+(define_split
+  [(set (match_operand:TI 0 "nonimmediate_operand" "")
+	(minus:TI (match_operand:TI 1 "nonimmediate_operand" "")
+		  (match_operand:TI 2 "general_operand" "")))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT && reload_completed"
+  [(parallel [(set (reg:CC FLAGS_REG) (compare:CC (match_dup 1) (match_dup 2)))
+	      (set (match_dup 0) (minus:DI (match_dup 1) (match_dup 2)))])
+   (parallel [(set (match_dup 3)
+		   (minus:DI (match_dup 4)
+			     (plus:DI (ltu:DI (reg:CC FLAGS_REG) (const_int 0))
+				      (match_dup 5))))
+	      (clobber (reg:CC FLAGS_REG))])]
+  "split_ti (operands+0, 1, operands+0, operands+3);
+   split_ti (operands+1, 1, operands+1, operands+4);
+   split_ti (operands+2, 1, operands+2, operands+5);")
+
 ;; %%% splits for subsidi3
 
 (define_expand "subdi3"
@@ -9198,6 +9282,43 @@
 
 ;; Negation instructions
 
+(define_expand "negti2"
+  [(parallel [(set (match_operand:TI 0 "nonimmediate_operand" "")
+		   (neg:TI (match_operand:TI 1 "nonimmediate_operand" "")))
+	      (clobber (reg:CC FLAGS_REG))])]
+  "TARGET_64BIT"
+  "ix86_expand_unary_operator (NEG, TImode, operands); DONE;")
+
+(define_insn "*negti2_1"
+  [(set (match_operand:TI 0 "nonimmediate_operand" "=ro")
+	(neg:TI (match_operand:TI 1 "general_operand" "0")))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT
+   && ix86_unary_operator_ok (NEG, TImode, operands)"
+  "#")
+
+(define_split
+  [(set (match_operand:TI 0 "nonimmediate_operand" "")
+	(neg:TI (match_operand:TI 1 "general_operand" "")))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT && reload_completed"
+  [(parallel
+    [(set (reg:CCZ FLAGS_REG)
+	  (compare:CCZ (neg:DI (match_dup 2)) (const_int 0)))
+     (set (match_dup 0) (neg:DI (match_dup 2)))])
+   (parallel
+    [(set (match_dup 1)
+	  (plus:DI (plus:DI (ltu:DI (reg:CC FLAGS_REG) (const_int 0))
+			    (match_dup 3))
+		   (const_int 0)))
+     (clobber (reg:CC FLAGS_REG))])
+   (parallel
+    [(set (match_dup 1)
+	  (neg:DI (match_dup 1)))
+     (clobber (reg:CC FLAGS_REG))])]
+  "split_ti (operands+1, 1, operands+2, operands+3);
+   split_ti (operands+0, 1, operands+0, operands+1);")
+
 (define_expand "negdi2"
   [(parallel [(set (match_operand:DI 0 "nonimmediate_operand" "")
 		   (neg:DI (match_operand:DI 1 "nonimmediate_operand" "")))
@@ -10079,6 +10200,92 @@
 ;; shift pair, instead using moves and sign extension for counts greater
 ;; than 31.
 
+(define_expand "ashlti3"
+  [(parallel [(set (match_operand:TI 0 "register_operand" "")
+		   (ashift:TI (match_operand:TI 1 "register_operand" "")
+			      (match_operand:QI 2 "nonmemory_operand" "")))
+	      (clobber (reg:CC FLAGS_REG))])]
+  "TARGET_64BIT"
+{
+  if (! immediate_operand (operands[2], QImode))
+    {
+      emit_insn (gen_ashlti3_1 (operands[0], operands[1], operands[2]));
+      DONE;
+    }
+  ix86_expand_binary_operator (ASHIFT, TImode, operands);
+  DONE;
+})
+
+(define_insn "ashlti3_1"
+  [(set (match_operand:TI 0 "register_operand" "=r")
+	(ashift:TI (match_operand:TI 1 "register_operand" "0")
+		   (match_operand:QI 2 "register_operand" "c")))
+   (clobber (match_scratch:DI 3 "=&r"))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT"
+  "#"
+  [(set_attr "type" "multi")])
+
+(define_insn "*ashlti3_2"
+  [(set (match_operand:TI 0 "register_operand" "=r")
+	(ashift:TI (match_operand:TI 1 "register_operand" "0")
+		   (match_operand:QI 2 "immediate_operand" "O")))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT"
+  "#"
+  [(set_attr "type" "multi")])
+
+(define_split
+  [(set (match_operand:TI 0 "register_operand" "")
+	(ashift:TI (match_operand:TI 1 "nonmemory_operand" "")
+		   (match_operand:QI 2 "register_operand" "")))
+   (clobber (match_scratch:DI 3 ""))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT && reload_completed"
+  [(const_int 0)]
+  "ix86_split_ashl (operands, operands[3], TImode); DONE;")
+
+(define_split
+  [(set (match_operand:TI 0 "register_operand" "")
+	(ashift:TI (match_operand:TI 1 "register_operand" "")
+		   (match_operand:QI 2 "immediate_operand" "")))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT && reload_completed"
+  [(const_int 0)]
+  "ix86_split_ashl (operands, NULL_RTX, TImode); DONE;")
+
+(define_insn "x86_64_shld"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "+r*m,r*m")
+        (ior:DI (ashift:DI (match_dup 0)
+		  (match_operand:QI 2 "nonmemory_operand" "J,c"))
+		(lshiftrt:DI (match_operand:DI 1 "register_operand" "r,r")
+		  (minus:QI (const_int 64) (match_dup 2)))))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT"
+  "@
+   shld{q}\t{%2, %1, %0|%0, %1, %2}
+   shld{q}\t{%s2%1, %0|%0, %1, %2}"
+  [(set_attr "type" "ishift")
+   (set_attr "prefix_0f" "1")
+   (set_attr "mode" "DI")
+   (set_attr "athlon_decode" "vector")])
+
+(define_expand "x86_64_shift_adj"
+  [(set (reg:CCZ FLAGS_REG)
+	(compare:CCZ (and:QI (match_operand:QI 2 "register_operand" "")
+			     (const_int 64))
+		     (const_int 0)))
+   (set (match_operand:DI 0 "register_operand" "")
+        (if_then_else:DI (ne (reg:CCZ FLAGS_REG) (const_int 0))
+			 (match_operand:DI 1 "register_operand" "")
+			 (match_dup 0)))
+   (set (match_dup 1)
+	(if_then_else:DI (ne (reg:CCZ FLAGS_REG) (const_int 0))
+			 (match_operand:DI 3 "register_operand" "r")
+			 (match_dup 1)))]
+  "TARGET_64BIT"
+  "")
+
 (define_expand "ashldi3"
   [(set (match_operand:DI 0 "shiftdi_operand" "")
 	(ashift:DI (match_operand:DI 1 "ashldi_input_operand" "")
@@ -10203,7 +10410,7 @@
    (match_dup 3)]
   "!TARGET_64BIT && TARGET_CMOVE"
   [(const_int 0)]
-  "ix86_split_ashldi (operands, operands[3]); DONE;")
+  "ix86_split_ashl (operands, operands[3], DImode); DONE;")
 
 (define_split
   [(set (match_operand:DI 0 "register_operand" "")
@@ -10212,7 +10419,7 @@
    (clobber (reg:CC FLAGS_REG))]
   "!TARGET_64BIT && (flag_peephole2 ? flow2_completed : reload_completed)"
   [(const_int 0)]
-  "ix86_split_ashldi (operands, NULL_RTX); DONE;")
+  "ix86_split_ashl (operands, NULL_RTX, DImode); DONE;")
 
 (define_insn "x86_shld_1"
   [(set (match_operand:SI 0 "nonimmediate_operand" "+r*m,r*m")
@@ -10778,6 +10985,76 @@
 
 ;; See comment above `ashldi3' about how this works.
 
+(define_expand "ashrti3"
+  [(parallel [(set (match_operand:TI 0 "register_operand" "")
+		   (ashiftrt:TI (match_operand:TI 1 "register_operand" "")
+				(match_operand:QI 2 "nonmemory_operand" "")))
+	      (clobber (reg:CC FLAGS_REG))])]
+  "TARGET_64BIT"
+{
+  if (! immediate_operand (operands[2], QImode))
+    {
+      emit_insn (gen_ashrti3_1 (operands[0], operands[1], operands[2]));
+      DONE;
+    }
+  ix86_expand_binary_operator (ASHIFTRT, TImode, operands);
+  DONE;
+})
+
+(define_insn "ashrti3_1"
+  [(set (match_operand:TI 0 "register_operand" "=r")
+	(ashiftrt:TI (match_operand:TI 1 "register_operand" "0")
+		     (match_operand:QI 2 "register_operand" "c")))
+   (clobber (match_scratch:DI 3 "=&r"))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT"
+  "#"
+  [(set_attr "type" "multi")])
+
+(define_insn "*ashrti3_2"
+  [(set (match_operand:TI 0 "register_operand" "=r")
+	(ashiftrt:TI (match_operand:TI 1 "register_operand" "0")
+		     (match_operand:QI 2 "immediate_operand" "O")))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT"
+  "#"
+  [(set_attr "type" "multi")])
+
+(define_split
+  [(set (match_operand:TI 0 "register_operand" "")
+	(ashiftrt:TI (match_operand:TI 1 "register_operand" "")
+		     (match_operand:QI 2 "register_operand" "")))
+   (clobber (match_scratch:DI 3 ""))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT && reload_completed"
+  [(const_int 0)]
+  "ix86_split_ashr (operands, operands[3], TImode); DONE;")
+
+(define_split
+  [(set (match_operand:TI 0 "register_operand" "")
+	(ashiftrt:TI (match_operand:TI 1 "register_operand" "")
+		     (match_operand:QI 2 "immediate_operand" "")))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT && reload_completed"
+  [(const_int 0)]
+  "ix86_split_ashr (operands, NULL_RTX, TImode); DONE;")
+
+(define_insn "x86_64_shrd"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "+r*m,r*m")
+        (ior:DI (ashiftrt:DI (match_dup 0)
+		  (match_operand:QI 2 "nonmemory_operand" "J,c"))
+		(ashift:DI (match_operand:DI 1 "register_operand" "r,r")
+		  (minus:QI (const_int 64) (match_dup 2)))))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT"
+  "@
+   shrd{q}\t{%2, %1, %0|%0, %1, %2}
+   shrd{q}\t{%s2%1, %0|%0, %1, %2}"
+  [(set_attr "type" "ishift")
+   (set_attr "prefix_0f" "1")
+   (set_attr "mode" "DI")
+   (set_attr "athlon_decode" "vector")])
+
 (define_expand "ashrdi3"
   [(set (match_operand:DI 0 "shiftdi_operand" "")
 	(ashiftrt:DI (match_operand:DI 1 "shiftdi_operand" "")
@@ -10887,7 +11164,7 @@
    (match_dup 3)]
   "!TARGET_64BIT && TARGET_CMOVE"
   [(const_int 0)]
-  "ix86_split_ashrdi (operands, operands[3]); DONE;")
+  "ix86_split_ashr (operands, operands[3], DImode); DONE;")
 
 (define_split
   [(set (match_operand:DI 0 "register_operand" "")
@@ -10896,7 +11173,7 @@
    (clobber (reg:CC FLAGS_REG))]
   "!TARGET_64BIT && (flag_peephole2 ? flow2_completed : reload_completed)"
   [(const_int 0)]
-  "ix86_split_ashrdi (operands, NULL_RTX); DONE;")
+  "ix86_split_ashr (operands, NULL_RTX, DImode); DONE;")
 
 (define_insn "x86_shrd_1"
   [(set (match_operand:SI 0 "nonimmediate_operand" "+r*m,r*m")
@@ -11275,6 +11552,60 @@
 
 ;; See comment above `ashldi3' about how this works.
 
+(define_expand "lshrti3"
+  [(parallel [(set (match_operand:TI 0 "register_operand" "")
+		   (lshiftrt:TI (match_operand:TI 1 "register_operand" "")
+			        (match_operand:QI 2 "nonmemory_operand" "")))
+	      (clobber (reg:CC FLAGS_REG))])]
+  "TARGET_64BIT"
+{
+  if (! immediate_operand (operands[2], QImode))
+    {
+      emit_insn (gen_lshrti3_1 (operands[0], operands[1], operands[2]));
+      DONE;
+    }
+  ix86_expand_binary_operator (LSHIFTRT, TImode, operands);
+  DONE;
+})
+
+(define_insn "lshrti3_1"
+  [(set (match_operand:TI 0 "register_operand" "=r")
+	(lshiftrt:TI (match_operand:TI 1 "register_operand" "0")
+		     (match_operand:QI 2 "register_operand" "c")))
+   (clobber (match_scratch:DI 3 "=&r"))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT"
+  "#"
+  [(set_attr "type" "multi")])
+
+(define_insn "*lshrti3_2"
+  [(set (match_operand:TI 0 "register_operand" "=r")
+	(lshiftrt:TI (match_operand:TI 1 "register_operand" "0")
+		     (match_operand:QI 2 "immediate_operand" "O")))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT"
+  "#"
+  [(set_attr "type" "multi")])
+
+(define_split 
+  [(set (match_operand:TI 0 "register_operand" "")
+	(lshiftrt:TI (match_operand:TI 1 "register_operand" "")
+		     (match_operand:QI 2 "register_operand" "")))
+   (clobber (match_scratch:DI 3 ""))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT && reload_completed"
+  [(const_int 0)]
+  "ix86_split_lshr (operands, operands[3], TImode); DONE;")
+
+(define_split 
+  [(set (match_operand:TI 0 "register_operand" "")
+	(lshiftrt:TI (match_operand:TI 1 "register_operand" "")
+		     (match_operand:QI 2 "immediate_operand" "")))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT && reload_completed"
+  [(const_int 0)]
+  "ix86_split_lshr (operands, NULL_RTX, TImode); DONE;")
+
 (define_expand "lshrdi3"
   [(set (match_operand:DI 0 "shiftdi_operand" "")
 	(lshiftrt:DI (match_operand:DI 1 "shiftdi_operand" "")
@@ -11367,7 +11698,7 @@
    (match_dup 3)]
   "!TARGET_64BIT && TARGET_CMOVE"
   [(const_int 0)]
-  "ix86_split_lshrdi (operands, operands[3]); DONE;")
+  "ix86_split_lshr (operands, operands[3], DImode); DONE;")
 
 (define_split 
   [(set (match_operand:DI 0 "register_operand" "")
@@ -11376,7 +11707,7 @@
    (clobber (reg:CC FLAGS_REG))]
   "!TARGET_64BIT && (flag_peephole2 ? flow2_completed : reload_completed)"
   [(const_int 0)]
-  "ix86_split_lshrdi (operands, NULL_RTX); DONE;")
+  "ix86_split_lshr (operands, NULL_RTX, DImode); DONE;")
 
 (define_expand "lshrsi3"
   [(set (match_operand:SI 0 "nonimmediate_operand" "")