Makefile.in (OBJS): Add doloop.o.

	* Makefile.in (OBJS): Add doloop.o.
	* doloop.c: New file.

	* final.c (insn_current_reference_address): Return 0 before final.
	* flags.h (flag_branch_on_count_reg): Fix typos in commentary.
	* jump.c (any_uncondjump_p): Likewise.
	* loop.c (indirect_jump_in_function): Make static.
	(strength_reduce): Call doloop_optimize.
	(insert_bct, instrument_loop_bct): Remove.
	* loop.h (doloop_optimize): Prototype.
	* recog.c (split_all_insns): Split all INSN_P.
	* toplev.c (flag_branch_on_count_reg): Default on.

	* config/c4x/c4x.c (c4x_optimization_options): Don't set
	flag_branch_on_count_reg.
	* config/i386/i386.c (override_options): Likewise.
	* config/rs6000/rs6000.c (optimization_options): Likewise.

	* config/i386/i386.md (decrement_and_branch_on_count): Remove.
	(doloop_end): New.
	(dbra_ge): Remove, as well as all it's splitters.

	* config/rs6000/rs6000.md (decrement_and_branch_on_count): Remove.
	(doloop_end): New.

	* config/ia64/ia64-protos.h (ar_lc_reg_operand): Declare.
	(ia64_register_move_cost): Declare.
	* config/ia64/ia64.c (ar_lc_reg_operand): New.
	(struct ia64_frame_info): Add ar_size.
	(ia64_compute_frame_size): Set it.
	(save_restore_insns): Save and restore ar.lc.
	(ia64_register_move_cost): New, moved from header file.  Handle
	application registers.
	(REG_AR_PFS, REG_AR_EC): Remove.  Replace with AR_*_REGNUM numbers.
	(emit_insn_group_barriers): Special case doloop_end_internal.
	(ia64_epilogue_uses): Mark ar.lc live at end.
	* config/ia64/ia64.h (AR_CCV_REGNUM, AR_LC_REGNUM): New registers.
	(AR_EC_REGNUM, AR_PFS_REGNUM): New registers.
	(FIRST_PSEUDO_REGISTER): Make room.
	(AR_M_REGNO_P, AR_I_REGNO_P, AR_REGNO_P): New.
	(FIXED_REGISTERS, CALL_USED_REGISTERS): Update.
	(REG_ALLOC_ORDER): Update.
	(HARD_REGNO_MODE_OK): Update.
	(REGISTER_NAMES): Update.
	(enum reg_class): Add AR_M_REGS and AR_I_REGS.
	(REG_CLASS_NAMES, REG_CLASS_CONTENTS): Update.
	(REGNO_REG_CLASS): Update.
	(LEGITIMATE_ADDRESS_DISP): Displacement range is 9 bits, not 10.
	(REGISTER_MOVE_COST): Move out of line.
	(PREDICATE_CODES): Update.
	* config/ia64/ia64.md (movdi patterns): Handle ar register classes.
	(addsi3_plus1_alt, adddi3_plus1_alt): New.
	(shladd_elim splitter): Allow constants in the predicate.
	(doloop_end, doloop_end_internal): New.

From-SVN: r35358

9 files changed, 293 insertions, 202 deletions

diff --git a/gcc/config/c4x/c4x.c b/gcc/config/c4x/c4x.c
index 9afe59a..261215f 100644
--- a/gcc/config/c4x/c4x.c
+++ b/gcc/config/c4x/c4x.c
@@ -295,10 +295,6 @@ c4x_optimization_options (level, size)
      instructions.  The benefit we gain we get by scheduling before
      register allocation is probably marginal anyhow.  */
   flag_schedule_insns = 0;
-
-  /* When optimizing, enable use of RPTB instruction.  */
-  if (level >= 1)
-    flag_branch_on_count_reg = 1;
 }
 
 
diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c
index 061d693..c5289ab 100644
--- a/gcc/config/i386/i386.c
+++ b/gcc/config/i386/i386.c
@@ -630,10 +630,6 @@ override_options ()
   if (flag_fast_math)
     target_flags &= ~MASK_IEEE_FP;
 
-  /* If we're planning on using `loop', use it.  */
-  if (TARGET_USE_LOOP && optimize)
-    flag_branch_on_count_reg = 1;
-
   /* It makes no sense to ask for just SSE builtins, so MMX is also turned
      on by -msse.  */
   if (TARGET_SSE)
diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
index eecb151..0c06124 100644
--- a/gcc/config/i386/i386.md
+++ b/gcc/config/i386/i386.md
@@ -8327,27 +8327,32 @@
 ;; This is all complicated by the fact that since this is a jump insn
 ;; we must handle our own reloads.
 
-(define_expand "decrement_and_branch_on_count"
-  [(parallel [(set (pc) (if_then_else
-			  (ne (match_operand:SI 0 "register_operand" "")
-			      (const_int 1))
-			  (label_ref (match_operand 1 "" ""))
-			  (pc)))
-	      (set (match_dup 0)
-		   (plus:SI (match_dup 0)
-			    (const_int -1)))
-	      (clobber (match_scratch:SI 2 ""))
-	      (clobber (reg:CC 17))])]
+(define_expand "doloop_end"
+  [(use (match_operand 0 "" ""))        ; loop pseudo
+   (use (match_operand 1 "" ""))        ; iterations; zero if unknown
+   (use (match_operand 2 "" ""))        ; max iterations
+   (use (match_operand 3 "" ""))        ; loop level 
+   (use (match_operand 4 "" ""))]       ; label
   "TARGET_USE_LOOP"
-  "")
+  "                                 
+{
+  /* Only use cloop on innermost loops.  */
+  if (INTVAL (operands[3]) > 1)
+    FAIL;
+  if (GET_MODE (operands[0]) != SImode)
+    FAIL;
+  emit_jump_insn (gen_doloop_end_internal (operands[4], operands[0],
+					   operands[0]));
+  DONE;
+}")
 
-(define_insn "*dbra_ne"
+(define_insn "doloop_end_internal"
   [(set (pc)
-	(if_then_else (ne (match_operand:SI 1 "register_operand" "c,*r,*r")
+	(if_then_else (ne (match_operand:SI 1 "register_operand" "c,?*r,?*r")
 			  (const_int 1))
 		      (label_ref (match_operand 0 "" ""))
 		      (pc)))
-   (set (match_operand:SI 2 "register_operand" "=1,*r,*m*r")
+   (set (match_operand:SI 2 "register_operand" "=1,1,*m*r")
 	(plus:SI (match_dup 1)
 		 (const_int -1)))
    (clobber (match_scratch:SI 3 "=X,X,r"))
@@ -8372,55 +8377,24 @@
 		      (const_string "ibr")
 		      (const_string "multi")))])
 
-(define_insn "*dbra_ge"
-  [(set (pc)
-	(if_then_else (ge (match_operand:SI 1 "register_operand" "c,*r,*r")
-			  (const_int 0))
-		      (label_ref (match_operand 0 "" ""))
-		      (pc)))
-   (set (match_operand:SI 2 "register_operand" "=1,*r,*m*r")
-	(plus:SI (match_dup 1)
-		 (const_int -1)))
-   (clobber (match_scratch:SI 3 "=X,X,r"))
-   (clobber (reg:CC 17))]
-  "TARGET_USE_LOOP && find_reg_note (insn, REG_NONNEG, 0)"
-  "*
-{
-  if (which_alternative != 0)
-    return \"#\";
-  if (get_attr_length (insn) == 2)
-    return \"loop\\t%l0\";
-  else
-    return \"dec{l}\\t%1\;jne\\t%l0\";
-}"
-  [(set (attr "type")
-	(if_then_else (and (eq_attr "alternative" "0")
-			   (and (ge (minus (match_dup 0) (pc))
-			            (const_int -128))
-			        (lt (minus (match_dup 0) (pc))
-			            (const_int 124))))
-		      (const_string "ibr")
-		      (const_string "multi")))
-   (set_attr "ppro_uops" "many")])
-
 (define_split
   [(set (pc)
 	(if_then_else (ne (match_operand:SI 1 "register_operand" "")
 			  (const_int 1))
 		      (match_operand 0 "" "")
 		      (pc)))
-   (set (match_operand:SI 2 "register_operand" "")
+   (set (match_dup 1)
 	(plus:SI (match_dup 1)
 		 (const_int -1)))
-   (clobber (match_scratch:SI 3 ""))
+   (clobber (match_scratch:SI 2 ""))
    (clobber (reg:CC 17))]
-  "TARGET_USE_LOOP && reload_completed
-   && ! (REGNO (operands[1]) == 2 && rtx_equal_p (operands[1], operands[2]))"
-  [(set (match_dup 2) (match_dup 1))
-   (parallel [(set (reg:CCZ 17)
-		   (compare:CCZ (plus:SI (match_dup 2) (const_int -1))
+  "TARGET_USE_LOOP
+   && reload_completed
+   && REGNO (operands[1]) != 2"
+  [(parallel [(set (reg:CCZ 17)
+		   (compare:CCZ (plus:SI (match_dup 1) (const_int -1))
 				 (const_int 0)))
-	      (set (match_dup 2) (plus:SI (match_dup 2) (const_int -1)))])
+	      (set (match_dup 1) (plus:SI (match_dup 1) (const_int -1)))])
    (set (pc) (if_then_else (ne (reg:CCZ 17) (const_int 0))
 			   (match_dup 0)
 			   (pc)))]
@@ -8432,12 +8406,15 @@
 			  (const_int 1))
 		      (match_operand 0 "" "")
 		      (pc)))
-   (set (match_operand:SI 2 "memory_operand" "")
+   (set (match_operand:SI 2 "nonimmediate_operand" "")
 	(plus:SI (match_dup 1)
 		 (const_int -1)))
    (clobber (match_scratch:SI 3 ""))
    (clobber (reg:CC 17))]
-  "TARGET_USE_LOOP && reload_completed"
+  "TARGET_USE_LOOP
+   && reload_completed
+   && (! REG_P (operands[2])
+       || ! rtx_equal_p (operands[1], operands[2]))"
   [(set (match_dup 3) (match_dup 1))
    (parallel [(set (reg:CCZ 17)
 		   (compare:CCZ (plus:SI (match_dup 3) (const_int -1))
@@ -8448,52 +8425,6 @@
 			   (match_dup 0)
 			   (pc)))]
   "")
-
-(define_split
-  [(set (pc)
-	(if_then_else (ge (match_operand:SI 1 "register_operand" "")
-			  (const_int 0))
-		      (match_operand 0 "" "")
-		      (pc)))
-   (set (match_operand:SI 2 "register_operand" "")
-	(plus:SI (match_dup 1)
-		 (const_int -1)))
-   (clobber (match_scratch:SI 3 ""))
-   (clobber (reg:CC 17))]
-  "TARGET_USE_LOOP && reload_completed
-   && ! (REGNO (operands[1]) == 2 && rtx_equal_p (operands[1], operands[2]))"
-  [(set (match_dup 2) (match_dup 1))
-   (parallel [(set (reg:CCNO 17)
-		   (compare:CCNO (plus:SI (match_dup 2) (const_int -1))
-				 (const_int 0)))
-	      (set (match_dup 2) (plus:SI (match_dup 2) (const_int -1)))])
-   (set (pc) (if_then_else (lt (reg:CCNO 17) (const_int 0))
-			   (match_dup 0)
-			   (pc)))]
-  "")
-  
-(define_split
-  [(set (pc)
-	(if_then_else (ge (match_operand:SI 1 "register_operand" "")
-			  (const_int 0))
-		      (match_operand 0 "" "")
-		      (pc)))
-   (set (match_operand:SI 2 "memory_operand" "")
-	(plus:SI (match_dup 1)
-		 (const_int -1)))
-   (clobber (match_scratch:SI 3 ""))
-   (clobber (reg:CC 17))]
-  "TARGET_USE_LOOP && reload_completed"
-  [(set (match_dup 3) (match_dup 1))
-   (parallel [(set (reg:CCNO 17)
-		   (compare:CCNO (plus:SI (match_dup 3) (const_int -1))
-				 (const_int 0)))
-	      (set (match_dup 3) (plus:SI (match_dup 3) (const_int -1)))])
-   (set (match_dup 2) (match_dup 3))
-   (set (pc) (if_then_else (lt (reg:CCNO 17) (const_int 0))
-			   (match_dup 0)
-			   (pc)))]
-  "")
 
 ;; Call instructions.
 
diff --git a/gcc/config/ia64/ia64-protos.h b/gcc/config/ia64/ia64-protos.h
index 00421f8..12ea516 100644
--- a/gcc/config/ia64/ia64-protos.h
+++ b/gcc/config/ia64/ia64-protos.h
@@ -59,6 +59,8 @@ extern void ia64_function_prologue PARAMS((FILE *, int));
 extern void ia64_funtion_epilogue PARAMS((FILE *, int));
 extern int ia64_direct_return PARAMS((void));
 extern int predicate_operator PARAMS((rtx, enum machine_mode));
+extern int ar_lc_reg_operand PARAMS((rtx, enum machine_mode));
+
 extern int ia64_move_ok PARAMS((rtx, rtx));
 
 extern void ia64_expand_load_address PARAMS((rtx, rtx));
@@ -102,6 +104,7 @@ extern int ia64_valid_type_attribute PARAMS((tree, tree, tree, tree));
 extern void ia64_encode_section_info PARAMS((tree));
 #endif /* TREE_CODE */
 
+extern int ia64_register_move_cost PARAMS((enum reg_class, enum reg_class));
 extern int ia64_epilogue_uses PARAMS((int));
 extern void ia64_file_start PARAMS((FILE *));
 extern void ia64_expand_prologue PARAMS((void));
diff --git a/gcc/config/ia64/ia64.c b/gcc/config/ia64/ia64.c
index 6b877b5..7153620 100644
--- a/gcc/config/ia64/ia64.c
+++ b/gcc/config/ia64/ia64.c
@@ -559,6 +559,19 @@ predicate_operator (op, mode)
   return ((GET_MODE (op) == mode || mode == VOIDmode)
 	  && (code == EQ || code == NE));
 }
+
+/* Return 1 if this is the ar.lc register.  */
+
+int
+ar_lc_reg_operand (op, mode)
+     register rtx op;
+     enum machine_mode mode;
+{
+  return (GET_MODE (op) == DImode
+	  && (mode == DImode || mode == VOIDmode)
+	  && GET_CODE (op) == REG
+	  && REGNO (op) == AR_LC_REGNUM);
+}
 
 /* Return 1 if the operands of a move are ok.  */
 
@@ -683,6 +696,7 @@ struct ia64_frame_info
   long fr_pad_size;		/* # bytes needed to align FP save area.  */
   long pr_size;			/* # bytes needed to store predicate regs.  */
   long br_size;			/* # bytes needed to store branch regs.  */
+  long ar_size;			/* # bytes needed to store AR regs.  */
   HARD_REG_SET mask;		/* mask of saved registers.  */
   int initialized;		/* != 0 is frame size already calculated.  */
 };
@@ -713,6 +727,7 @@ ia64_compute_frame_size (size)
   int fr_pad_size = 0;
   int pr_size = 0;
   int br_size = 0;
+  int ar_size = 0;
   int pretend_pad_size = 0;
   int tmp;
   int regno;
@@ -772,6 +787,13 @@ ia64_compute_frame_size (size)
   else
     fr_pad_size = 0;
 
+  /* AR.LC, for reasons unexplained, is call saved.  */
+  if (regs_ever_live[AR_LC_REGNUM])
+    {
+      SET_HARD_REG_BIT (mask, AR_LC_REGNUM);
+      ar_size = 8;
+    }
+
   /* If we have an odd number of words of pretend arguments written to the
      stack, then the FR save area will be unaligned.  We pad below this area
      to keep things 16 byte aligned.  This needs to be kept distinct, to
@@ -780,7 +802,7 @@ ia64_compute_frame_size (size)
   pretend_pad_size = current_function_pretend_args_size % 16;
 
   /* The 16 bytes is for the scratch area.  */
-  tmp = (size + gr_size + fr_pad_size + fr_size + pr_size + br_size
+  tmp = (size + gr_size + fr_pad_size + fr_size + pr_size + br_size + ar_size
 	 + current_function_outgoing_args_size + 16);
   tmp += (current_function_pretend_args_size
 	  ? current_function_pretend_args_size - 16
@@ -810,6 +832,7 @@ ia64_compute_frame_size (size)
   current_frame_info.fr_pad_size = fr_pad_size;
   current_frame_info.pr_size = pr_size;
   current_frame_info.br_size = br_size;
+  current_frame_info.ar_size = ar_size;
   COPY_HARD_REG_SET (current_frame_info.mask, mask);
   current_frame_info.initialized = reload_completed;
 
@@ -822,8 +845,11 @@ save_restore_insns (save_p)
 {
   rtx insn;
 
-  if (current_frame_info.gr_size + current_frame_info.fr_size
-      + current_frame_info.br_size + current_frame_info.pr_size)
+  if (current_frame_info.gr_size
+      + current_frame_info.fr_size
+      + current_frame_info.br_size
+      + current_frame_info.pr_size
+      + current_frame_info.ar_size)
     {
       rtx tmp_reg = gen_rtx_REG (DImode, GR_REG (2));
       rtx tmp_post_inc = gen_rtx_POST_INC (DImode, tmp_reg);
@@ -833,6 +859,7 @@ save_restore_insns (save_p)
 		       + current_frame_info.fr_pad_size
 		       + current_frame_info.br_size
 		       + current_frame_info.pr_size
+		       + current_frame_info.ar_size
 		       + current_frame_info.var_size
 		       + current_frame_info.pretend_size
 		       + current_frame_info.pretend_pad_size));
@@ -961,6 +988,29 @@ save_restore_insns (save_p)
 	    if (save_p)
 	      RTX_FRAME_RELATED_P (insn) = 1;
 	  }
+
+      if (TEST_HARD_REG_BIT (current_frame_info.mask, AR_LC_REGNUM))
+	{
+	  rtx src, dest;
+
+	  if (save_p)
+	    {
+	      src = gen_rtx_REG (DImode, AR_LC_REGNUM);
+	      dest = gen_rtx_MEM (DImode, tmp_post_inc);
+	    }
+	  else
+	    {
+	      src = gen_rtx_MEM (DImode, tmp_post_inc);
+	      dest = gen_rtx_REG (DImode, AR_LC_REGNUM);
+	    }
+
+	  insn = emit_insn (gen_movdi (tmp2_reg, src));
+	  if (save_p)
+	    RTX_FRAME_RELATED_P (insn) = 1;
+	  insn = emit_insn (gen_movdi (dest, tmp2_reg));
+	  if (save_p)
+	    RTX_FRAME_RELATED_P (insn) = 1;
+	}
     }
 }
 
@@ -2149,6 +2199,28 @@ ia64_print_operand (file, x, code)
   return;
 }
 
+/* Calulate the cost of moving data from a register in class FROM to
+   one in class TO.  */
+
+int
+ia64_register_move_cost (from, to)
+     enum reg_class from, to;
+{
+  int from_hard, to_hard;
+  int from_gr, to_gr;
+
+  from_hard = (from == BR_REGS || from == AR_M_REGS || from == AR_I_REGS);
+  to_hard = (to == BR_REGS || to == AR_M_REGS || to == AR_I_REGS);
+  from_gr = (from == GENERAL_REGS);
+  to_gr = (to == GENERAL_REGS);
+
+  if (from_hard && to_hard)
+    return 8;
+  else if ((from_hard && !to_gr) || (!from_gr && to_hard))
+    return 6;
+
+  return 2;
+}
 
 /* This function returns the register class required for a secondary
    register when copying between one of the registers in CLASS, and X,
@@ -2382,14 +2454,11 @@ ia64_override_options ()
    complex).  */
 #define REG_GP		(GR_REG (1))
 #define REG_RP		(BR_REG (0))
-#define REG_AR_PFS	(FIRST_PSEUDO_REGISTER)
 #define REG_AR_CFM	(FIRST_PSEUDO_REGISTER + 1)
-/* ??? This will eventually need to be a hard register.  */
-#define REG_AR_EC	(FIRST_PSEUDO_REGISTER + 2)
 /* This is used for volatile asms which may require a stop bit immediately
    before and after them.  */
-#define REG_VOLATILE	(FIRST_PSEUDO_REGISTER + 3)
-#define NUM_REGS	(FIRST_PSEUDO_REGISTER + 4)
+#define REG_VOLATILE	(FIRST_PSEUDO_REGISTER + 2)
+#define NUM_REGS	(FIRST_PSEUDO_REGISTER + 3)
 
 /* For each register, we keep track of how many times it has been
    written in the current instruction group.  If a register is written
@@ -2521,15 +2590,13 @@ rws_access_reg (regno, flags, pred)
 	  /* Branches have several RAW exceptions that allow to avoid
 	     barriers.  */
 
-	  if (REGNO_REG_CLASS (regno) == BR_REGS || regno == REG_AR_PFS)
+	  if (REGNO_REG_CLASS (regno) == BR_REGS || regno == AR_PFS_REGNUM)
 	    /* RAW dependencies on branch regs are permissible as long
 	       as the writer is a non-branch instruction.  Since we
 	       never generate code that uses a branch register written
 	       by a branch instruction, handling this case is
 	       easy.  */
-	    /* ??? This assumes that we don't emit br.cloop, br.cexit, br.ctop,
-	       br.wexit, br.wtop.  This is true currently.  */
-	      return 0;
+	    return 0;
 
 	  if (REGNO_REG_CLASS (regno) == PR_REGS
 	      && ! rws_sum[regno].written_by_fp)
@@ -2678,7 +2745,7 @@ rtx_needs_barrier (x, flags, pred)
       new_flags.is_write = 0;
       /* ??? Why is this here?  It seems unnecessary.  */
       need_barrier |= rws_access_reg (REG_GP, new_flags, pred);
-      need_barrier |= rws_access_reg (REG_AR_EC, new_flags, pred);
+      need_barrier |= rws_access_reg (AR_EC_REGNUM, new_flags, pred);
 
       /* Avoid multiple register writes, in case this is a pattern with
 	 multiple CALL rtx.  This avoids an abort in rws_access_reg.  */
@@ -2688,7 +2755,7 @@ rtx_needs_barrier (x, flags, pred)
 	{
 	  new_flags.is_write = 1;
 	  need_barrier |= rws_access_reg (REG_RP, new_flags, pred);
-	  need_barrier |= rws_access_reg (REG_AR_PFS, new_flags, pred);
+	  need_barrier |= rws_access_reg (AR_PFS_REGNUM, new_flags, pred);
 	  need_barrier |= rws_access_reg (REG_AR_CFM, new_flags, pred);
 	}
       break;
@@ -2877,7 +2944,7 @@ rtx_needs_barrier (x, flags, pred)
 	  /* Alloc must always be the first instruction.  Currently, we
 	     only emit it at the function start, so we don't need to worry
 	     about emitting a stop bit before it.  */
-	  need_barrier = rws_access_reg (REG_AR_PFS, flags, pred);
+	  need_barrier = rws_access_reg (AR_PFS_REGNUM, flags, pred);
 
 	  new_flags.is_write = 1;
 	  need_barrier |= rws_access_reg (REG_AR_CFM, new_flags, pred);
@@ -2892,7 +2959,7 @@ rtx_needs_barrier (x, flags, pred)
 
 	case 4: /* mov ar.pfs= */
 	  new_flags.is_write = 1;
-	  need_barrier = rws_access_reg (REG_AR_PFS, new_flags, pred);
+	  need_barrier = rws_access_reg (AR_PFS_REGNUM, new_flags, pred);
 	  break;
 
 	case 5: /* set_bsp  */
@@ -2920,10 +2987,10 @@ rtx_needs_barrier (x, flags, pred)
     case RETURN:
       new_flags.is_write = 0;
       need_barrier  = rws_access_reg (REG_RP, flags, pred);
-      need_barrier |= rws_access_reg (REG_AR_PFS, flags, pred);
+      need_barrier |= rws_access_reg (AR_PFS_REGNUM, flags, pred);
 
       new_flags.is_write = 1;
-      need_barrier |= rws_access_reg (REG_AR_EC, new_flags, pred);
+      need_barrier |= rws_access_reg (AR_EC_REGNUM, new_flags, pred);
       need_barrier |= rws_access_reg (REG_AR_CFM, new_flags, pred);
       break;
 
@@ -3042,6 +3109,12 @@ emit_insn_group_barriers (insns)
 	      if (INSN_CODE (insn) == CODE_FOR_epilogue_deallocate_stack)
 		pat = XVECEXP (pat, 0, 0);
 
+	      /* ??? Similarly, the pattern we use for br.cloop
+		 confuses the code above.  The second element of the
+		 vector is representative.  */
+	      else if (INSN_CODE (insn) == CODE_FOR_doloop_end_internal)
+		pat = XVECEXP (pat, 0, 1);
+
 	      memset (rws_insn, 0, sizeof (rws_insn));
 	      need_barrier |= rtx_needs_barrier (pat, flags, 0);
 
@@ -3164,6 +3237,9 @@ ia64_epilogue_uses (regno)
   if (regno == R_BR (0))
     return 1;
 
+  if (regno == AR_LC_REGNUM)
+    return 1;
+
   return 0;
 }
 
diff --git a/gcc/config/ia64/ia64.h b/gcc/config/ia64/ia64.h
index 6379ae7..ab298f4 100644
--- a/gcc/config/ia64/ia64.h
+++ b/gcc/config/ia64/ia64.h
@@ -535,12 +535,11 @@ while (0)
 /* Register Basics */
 
 /* Number of hardware registers known to the compiler.  
-   We have 128 general registers, 128 floating point registers, 64 predicate
-   registers, 8 branch registers, and one frame pointer register.  */
+   We have 128 general registers, 128 floating point registers,
+   64 predicate registers, 8 branch registers, one frame pointer,
+   and several "application" registers.  */
 
-/* ??? Should add ar.lc, ar.ec and probably also ar.pfs.  */
-
-#define FIRST_PSEUDO_REGISTER 330
+#define FIRST_PSEUDO_REGISTER 334
 
 /* Ranges for the various kinds of registers.  */
 #define ADDL_REGNO_P(REGNO) ((unsigned HOST_WIDE_INT) (REGNO) <= 3)
@@ -561,10 +560,23 @@ while (0)
 #define IN_REG(REGNO) ((REGNO) + 112)
 #define LOC_REG(REGNO) ((REGNO) + 32)
 
+#define AR_CCV_REGNUM	330
+#define AR_LC_REGNUM	331
+#define AR_EC_REGNUM	332
+#define AR_PFS_REGNUM	333
+
 #define IN_REGNO_P(REGNO) ((REGNO) >= IN_REG (0) && (REGNO) <= IN_REG (7))
 #define LOC_REGNO_P(REGNO) ((REGNO) >= LOC_REG (0) && (REGNO) <= LOC_REG (79))
 #define OUT_REGNO_P(REGNO) ((REGNO) >= OUT_REG (0) && (REGNO) <= OUT_REG (7))
 
+#define AR_M_REGNO_P(REGNO) ((REGNO) == AR_CCV_REGNUM)
+#define AR_I_REGNO_P(REGNO) ((REGNO) >= AR_LC_REGNUM \
+			     && (REGNO) < FIRST_PSEUDO_REGISTER)
+#define AR_REGNO_P(REGNO) ((REGNO) >= AR_CCV_REGNUM \
+			   && (REGNO) < FIRST_PSEUDO_REGISTER)
+
+
+
 /* ??? Don't really need two sets of macros.  I like this one better because
    it is less typing.  */
 #define R_GR(REGNO) GR_REG (REGNO)
@@ -619,14 +631,14 @@ while (0)
   0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,	\
   /* Branch registers.  */				\
   0, 0, 0, 0, 0, 0, 0, 0,				\
-  /*FP RA*/						\
-  1, 1,							\
+  /*FP RA CCV LC EC PFS */				\
+     1, 1,  1, 1, 1,  1					\
  }
 
-/* Like `FIXED_REGISTERS' but has 1 for each register that is clobbered (in
-   general) by function calls as well as for fixed registers.  This macro
-   therefore identifies the registers that are not available for general
-   allocation of values that must live across function calls.  */
+/* Like `FIXED_REGISTERS' but has 1 for each register that is clobbered
+   (in general) by function calls as well as for fixed registers.  This
+   macro therefore identifies the registers that are not available for
+   general allocation of values that must live across function calls.  */
 
 #define CALL_USED_REGISTERS \
 { /* General registers.  */				\
@@ -654,8 +666,8 @@ while (0)
   0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,	\
   /* Branch registers.  */				\
   1, 0, 0, 0, 0, 0, 1, 1,				\
-  /*FP RA*/						\
-  1, 1,							\
+  /*FP RA CCV LC EC PFS */				\
+     1, 1,  1, 1, 1,  1					\
 }
 
 /* Define this macro if the target machine has register windows.  This C
@@ -787,10 +799,10 @@ while (0)
   R_PR (0),								   \
   /* Special branch registers.  */					   \
   R_BR (0),								   \
-  /* Frame pointer.  Return address.  */				   \
+  /* Other fixed registers.  */						   \
   FRAME_POINTER_REGNUM, RETURN_ADDRESS_POINTER_REGNUM,			   \
+  AR_CCV_REGNUM, AR_LC_REGNUM, AR_EC_REGNUM, AR_PFS_REGNUM		   \
 }
-
 
 /* How Values Fit in Registers */
 
@@ -815,6 +827,7 @@ while (0)
   (FR_REGNO_P (REGNO) ? (MODE) != CCmode				\
    : PR_REGNO_P (REGNO) ? (MODE) == CCmode				\
    : GR_REGNO_P (REGNO) ? (MODE) != XFmode				\
+   : AR_REGNO_P (REGNO) ? (MODE) == DImode				\
    : 1)
 
 /* A C expression that is nonzero if it is desirable to choose register
@@ -877,6 +890,8 @@ enum reg_class
   GR_REGS,
   FR_REGS,
   GR_AND_FR_REGS,
+  AR_M_REGS,
+  AR_I_REGS,
   ALL_REGS,
   LIM_REG_CLASSES
 };
@@ -890,7 +905,8 @@ enum reg_class
    constants.  These names are used in writing some of the debugging dumps.  */
 #define REG_CLASS_NAMES \
 { "NO_REGS", "PR_REGS", "BR_REGS", "ADDL_REGS", "GR_REGS", \
-  "FR_REGS", "GR_AND_FR_REGS", "ALL_REGS" }
+  "FR_REGS", "GR_AND_FR_REGS", "AR_M_REGS", "AR_I_REGS", \
+  "ALL_REGS" }
 
 /* An initializer containing the contents of the register classes, as integers
    which are bit masks.  The Nth integer specifies the contents of class N.
@@ -901,35 +917,43 @@ enum reg_class
   /* NO_REGS.  */					\
   { 0x00000000, 0x00000000, 0x00000000, 0x00000000,	\
     0x00000000, 0x00000000, 0x00000000, 0x00000000,	\
-    0x00000000, 0x00000000, 0x000 },			\
+    0x00000000, 0x00000000, 0x0000 },			\
   /* PR_REGS.  */					\
   { 0x00000000, 0x00000000, 0x00000000, 0x00000000,	\
     0x00000000, 0x00000000, 0x00000000, 0x00000000,	\
-    0xFFFFFFFF, 0xFFFFFFFF, 0x000 },			\
+    0xFFFFFFFF, 0xFFFFFFFF, 0x0000 },			\
   /* BR_REGS.  */					\
   { 0x00000000, 0x00000000, 0x00000000, 0x00000000,	\
     0x00000000, 0x00000000, 0x00000000, 0x00000000,	\
-    0x00000000, 0x00000000, 0x0FF },			\
+    0x00000000, 0x00000000, 0x00FF },			\
   /* ADDL_REGS.  */					\
   { 0x0000000F, 0x00000000, 0x00000000, 0x00000000,	\
     0x00000000, 0x00000000, 0x00000000, 0x00000000,	\
-    0x00000000, 0x00000000, 0x000 },			\
+    0x00000000, 0x00000000, 0x0000 },			\
   /* GR_REGS.  */					\
   { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,	\
     0x00000000, 0x00000000, 0x00000000, 0x00000000,	\
-    0x00000000, 0x00000000, 0x300 },			\
+    0x00000000, 0x00000000, 0x0300 },			\
   /* FR_REGS.  */					\
   { 0x00000000, 0x00000000, 0x00000000, 0x00000000,	\
     0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,	\
-    0x00000000, 0x00000000, 0x000 },			\
+    0x00000000, 0x00000000, 0x0000 },			\
   /* GR_AND_FR_REGS.  */				\
   { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,	\
     0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,	\
-    0x00000000, 0x00000000, 0x300 },			\
+    0x00000000, 0x00000000, 0x0300 },			\
+  /* AR_M_REGS.  */					\
+  { 0x00000000, 0x00000000, 0x00000000, 0x00000000,	\
+    0x00000000, 0x00000000, 0x00000000, 0x00000000,	\
+    0x00000000, 0x00000000, 0x0400 },			\
+  /* AR_I_REGS.  */					\
+  { 0x00000000, 0x00000000, 0x00000000, 0x00000000,	\
+    0x00000000, 0x00000000, 0x00000000, 0x00000000,	\
+    0x00000000, 0x00000000, 0x3800 },			\
   /* ALL_REGS.  */					\
   { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,	\
     0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,	\
-    0xFFFFFFFF, 0xFFFFFFFF, 0x3FF },			\
+    0xFFFFFFFF, 0xFFFFFFFF, 0x3FFF },			\
 }
 
 /* A C expression whose value is a register class containing hard register
@@ -944,6 +968,8 @@ enum reg_class
  : FR_REGNO_P (REGNO) ? FR_REGS		\
  : PR_REGNO_P (REGNO) ? PR_REGS		\
  : BR_REGNO_P (REGNO) ? BR_REGS		\
+ : AR_M_REGNO_P (REGNO) ? AR_I_REGS	\
+ : AR_I_REGNO_P (REGNO) ? AR_M_REGS	\
  : NO_REGS)
 
 /* A macro whose definition is the name of the class to which a valid base
@@ -968,6 +994,8 @@ enum reg_class
  : (CHAR) == 'a' ? ADDL_REGS		\
  : (CHAR) == 'b' ? BR_REGS		\
  : (CHAR) == 'c' ? PR_REGS		\
+ : (CHAR) == 'd' ? AR_M_REGS		\
+ : (CHAR) == 'e' ? AR_I_REGS		\
  : NO_REGS)
 
 /* A C expression which is nonzero if register number NUM is suitable for use
@@ -1816,8 +1844,8 @@ do {									\
    && rtx_equal_p (R, XEXP (X, 0))					\
    && (GET_CODE (XEXP (X, 1)) == REG					\
        || (GET_CODE (XEXP (X, 1)) == CONST_INT				\
-	   && INTVAL (XEXP (X, 1)) >= -512				\
-	   && INTVAL (XEXP (X, 1)) < 512)))
+	   && INTVAL (XEXP (X, 1)) >= -256				\
+	   && INTVAL (XEXP (X, 1)) < 256)))
 
 #define GO_IF_LEGITIMATE_ADDRESS(MODE, X, LABEL) 			\
 do {									\
@@ -1926,10 +1954,7 @@ do {									\
    one in class TO.  */
 
 #define REGISTER_MOVE_COST(FROM, TO) \
-((FROM) == BR_REGS && (TO) == BR_REGS ? 8				\
- : (((FROM) == BR_REGS && (TO) != GENERAL_REGS)				\
-    || ((TO) == BR_REGS && (FROM) != GENERAL_REGS)) ? 6			\
- : 2)
+  ia64_register_move_cost((FROM), (TO))
 
 /* A C expression for the cost of moving data of mode M between a register and
    memory.  */
@@ -2363,7 +2388,7 @@ do {									\
   /* Branch registers.  */						\
   "b0", "b1", "b2", "b3", "b4", "b5", "b6", "b7",			\
   /* Frame pointer.  Return address.  */				\
-  "sfp", "retaddr"							\
+  "sfp", "retaddr", "ar.ccv", "ar.lc", "ar.ec", "ar.pfs"		\
 }
 
 /* If defined, a C initializer for an array of structures containing a name and
@@ -2749,7 +2774,8 @@ do {									\
 { "normal_comparison_operator", {EQ, NE, GT, LE, GTU, LEU}},		\
 { "adjusted_comparison_operator", {LT, GE, LTU, GEU}},			\
 { "call_multiple_values_operation", {PARALLEL}},			\
-{ "predicate_operator", {NE, EQ}},
+{ "predicate_operator", {NE, EQ}},					\
+{ "ar_lc_reg_operand", {REG}},
 
 /* An alias for a machine mode name.  This is the machine mode that elements of
    a jump-table should have.  */
diff --git a/gcc/config/ia64/ia64.md b/gcc/config/ia64/ia64.md
index 872ef6b..f430120 100644
--- a/gcc/config/ia64/ia64.md
+++ b/gcc/config/ia64/ia64.md
@@ -368,10 +368,12 @@
 (define_insn ""
   [(cond_exec
      (match_operator 2 "predicate_operator"
-       [(match_operand:CC 3 "register_operand" "c,c,c,c,c,c,c,c")
+       [(match_operand:CC 3 "register_operand" "c,c,c,c,c,c,c,c,c,c")
         (const_int 0)])
-     (set (match_operand:DI 0 "register_operand"  "=r,r,r, r,*f,*f, r,*b")
-	  (match_operand:DI 1 "nonmemory_operand" "rO,J,i,*f,rO,*f,*b,rO")))]
+     (set (match_operand:DI 0 "register_operand"
+			      "=r,r,r, r,*f,*f,   r,*b*e, r,*d")
+	  (match_operand:DI 1 "nonmemory_operand"
+			      "rO,J,i,*f,rO,*f,*b*e,  rO,*d,rO")))]
   "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
   "*
 {
@@ -383,6 +385,8 @@
     \"(%J2) setf.sig %0 = %r1\",
     \"(%J2) mov %0 = %1\",
     \"(%J2) mov %0 = %1\",
+    \"(%J2) mov %0 = %r1\",
+    \"(%J2) mov %0 = %1\",
     \"(%J2) mov %0 = %r1\"
   };
 
@@ -403,14 +407,14 @@
 
   return alt[which_alternative];
 }"
-  [(set_attr "type" "A,A,L,M,M,F,I,I")
+  [(set_attr "type" "A,A,L,M,M,F,I,I,M,M")
    (set_attr "predicable" "no")])
 
 (define_insn "*movdi_internal_astep"
   [(set (match_operand:DI 0 "destination_operand"
-			    "=r,r,r,r, m, r,*f,*f,*f, Q, r,*b")
+			    "=r,r,r,r, m, r,*f,*f,*f, Q,   r,*b*e, r,*d")
 	(match_operand:DI 1 "move_operand"
-			    "rO,J,i,m,rO,*f,rO,*f, Q,*f,*b,rO"))]
+			    "rO,J,i,m,rO,*f,rO,*f, Q,*f,*b*e,  rO,*d,rO"))]
   "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
   "*
 {
@@ -426,6 +430,8 @@
     \"ldf8 %0 = %1%P1\",
     \"stf8 %0 = %1%P0\",
     \"mov %0 = %1\",
+    \"mov %0 = %r1\",
+    \"mov %0 = %1\",
     \"mov %0 = %r1\"
   };
 
@@ -435,14 +441,14 @@
 
   return alt[which_alternative];
 }"
-  [(set_attr "type" "A,A,L,M,M,M,M,F,M,M,I,I")
+  [(set_attr "type" "A,A,L,M,M,M,M,F,M,M,I,I,M,M")
    (set_attr "predicable" "no")])
 
 (define_insn "*movdi_internal"
   [(set (match_operand:DI 0 "destination_operand"
-			    "=r,r,r,r, m, r,*f,*f,*f, Q, r,*b")
+			    "=r,r,r,r, m, r,*f,*f,*f, Q,   r,*b*e, r,*d")
 	(match_operand:DI 1 "move_operand"
-			    "rO,J,i,m,rO,*f,rO,*f, Q,*f,*b,rO"))]
+			    "rO,J,i,m,rO,*f,rO,*f, Q,*f,*b*e,  rO,*d,rO"))]
   "! TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
   "*
 {
@@ -458,6 +464,8 @@
     \"%,ldf8 %0 = %1%P1\",
     \"%,stf8 %0 = %1%P0\",
     \"%,mov %0 = %1\",
+    \"%,mov %0 = %r1\",
+    \"%,mov %0 = %1\",
     \"%,mov %0 = %r1\"
   };
 
@@ -467,7 +475,7 @@
 
   return alt[which_alternative];
 }"
-  [(set_attr "type" "A,A,L,M,M,M,M,F,M,M,I,I")])
+  [(set_attr "type" "A,A,L,M,M,M,M,F,M,M,I,I,M,M")])
 
 (define_split
   [(set (match_operand:DI 0 "register_operand" "")
@@ -1104,6 +1112,15 @@
   "add %0 = %1, %2, 1"
   [(set_attr "type" "A")])
 
+(define_insn "*addsi3_plus1_alt"
+  [(set (match_operand:SI 0 "register_operand" "=r")
+	(plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
+			  (const_int 2))
+		 (const_int 1)))]
+  ""
+  "add %0 = %1, %1, 1"
+  [(set_attr "type" "A")])
+
 (define_expand "subsi3"
   [(set (match_operand:SI 0 "register_operand" "")
 	(minus:SI (match_operand:SI 1 "reg_or_8bit_operand" "")
@@ -1299,6 +1316,18 @@
   "add %0 = %1, %2, 1"
   [(set_attr "type" "A")])
 
+;; This has some of the same problems as shladd.  We let the shladd
+;; eliminator hack handle it, which results in the 1 being forced into
+;; a register, but not more ugliness here.
+(define_insn "*adddi3_plus1_alt"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	(plus:DI (mult:DI (match_operand:DI 1 "register_operand" "r")
+			  (const_int 2))
+		 (const_int 1)))]
+  ""
+  "add %0 = %1, %1, 1"
+  [(set_attr "type" "A")])
+
 (define_insn "subdi3"
   [(set (match_operand:DI 0 "register_operand" "=r")
 	(minus:DI (match_operand:DI 1 "reg_or_8bit_operand" "rK")
@@ -1862,31 +1891,20 @@
 ;; doesn't succeed, then this remain a shladd pattern, and will be reloaded
 ;; incorrectly.
 
-(define_insn "*shladd_elim"
+(define_insn_and_split "*shladd_elim"
   [(set (match_operand:DI 0 "register_operand" "=&r")
 	(plus:DI (plus:DI (mult:DI (match_operand:DI 1 "register_operand" "r")
 				   (match_operand:DI 2 "shladd_operand" "n"))
-			  (match_operand:DI 3 "register_operand" "r"))
+			  (match_operand:DI 3 "nonmemory_operand" "r"))
 		 (match_operand:DI 4 "nonmemory_operand" "rI")))]
   "reload_in_progress"
-  "#"
-  [(set_attr "type" "unknown")])
-
-;; ??? Need to emit an instruction group barrier here because this gets split
-;; after md_reorg.
-
-(define_split
-  [(set (match_operand:DI 0 "register_operand" "")
-	(plus:DI (plus:DI (mult:DI (match_operand:DI 1 "register_operand" "")
-				   (match_operand:DI 2 "shladd_operand" ""))
-			  (match_operand:DI 3 "register_operand" ""))
-		 (match_operand:DI 4 "reg_or_14bit_operand" "")))]
+  "* abort ();"
   "reload_completed"
   [(set (match_dup 0) (plus:DI (mult:DI (match_dup 1) (match_dup 2))
 			       (match_dup 3)))
-   (unspec_volatile [(const_int 0)] 2)
    (set (match_dup 0) (plus:DI (match_dup 0) (match_dup 4)))]
-  "")
+  ""
+  [(set_attr "type" "unknown")])
 
 (define_insn "ashrdi3"
   [(set (match_operand:DI 0 "register_operand" "=r")
@@ -2465,6 +2483,42 @@
 
 ;; ::::::::::::::::::::
 ;; ::
+;; :: Counted loop operations
+;; ::
+;; ::::::::::::::::::::
+
+(define_expand "doloop_end"
+  [(use (match_operand 0 "" ""))	; loop pseudo
+   (use (match_operand 1 "" ""))	; iterations; zero if unknown
+   (use (match_operand 2 "" ""))	; max iterations
+   (use (match_operand 3 "" ""))	; loop level
+   (use (match_operand 4 "" ""))]	; label
+  ""
+  "
+{
+  /* Only use cloop on innermost loops.  */
+  if (INTVAL (operands[3]) > 1)
+    FAIL;
+  emit_jump_insn (gen_doloop_end_internal (gen_rtx_REG (DImode, AR_LC_REGNUM),
+					   operands[4]));
+  DONE;
+}")
+
+(define_insn "doloop_end_internal"
+  [(set (pc) (if_then_else (ne (match_operand:DI 0 "ar_lc_reg_operand" "")
+			       (const_int 0))
+		(label_ref (match_operand 1 "" ""))
+		(pc)))
+   (set (match_dup 0) (if_then_else:DI (ne (match_dup 0) (const_int 0))
+			 (match_dup 0)
+			 (plus:DI (match_dup 0) (const_int -1))))]
+  ""
+  "br.cloop.sptk.few %l1"
+  [(set_attr "type" "B")
+   (set_attr "predicable" "no")])
+
+;; ::::::::::::::::::::
+;; ::
 ;; :: Set flag operations
 ;; ::
 ;; ::::::::::::::::::::
@@ -2706,32 +2760,32 @@
 ;; Errata 72 workaround.
 (define_insn "*cmovdi_internal_astep"
   [(set (match_operand:DI 0 "nonimmediate_operand"
-			  "=r,*f,Q,*b,r,*f,Q,*b,r,*f,Q,*b")
+		"=r,*f,Q,*b*d*e,r,*f,Q,*b*d*e,r,*f,Q,*b*d*e")
 	(if_then_else:DI
 	  (match_operator:CC 4 "predicate_operator"
 	    [(match_operand:CC 1 "register_operand"
 			       "c,c,c,c,c,c,c,c,c,c,c,c")
 	     (const_int 0)])
 	  (match_operand:DI 2 "general_operand"
-		    "0,0,0,0,ri*f*b,rO,*f,r,ri*f*b,rO,*f,r")
+		"0,0,0,0,ri*f*b*d*e,rO,*f,r,ri*f*b*d*e,rO,*f,r")
 	  (match_operand:DI 3 "general_operand"
-		    "ri*f*b,rO,*f,r,0,0,0,0,ri*f*b,rO,*f,r")))]
+		"ri*f*b*d*e,rO,*f,r,0,0,0,0,ri*f*b*d*e,rO,*f,r")))]
   "TARGET_A_STEP"
   "* abort ();"
   [(set_attr "predicable" "no")])
 
 (define_insn "*cmovdi_internal"
   [(set (match_operand:DI 0 "nonimmediate_operand"
-			  "=r,m,*f,Q,*b,r,m,*f,Q,*b,r,m,*f,Q,*b")
+		"=r,m,*f,Q,*b*d*e,r,m,*f,Q,*b*d*e,r,m,*f,Q,*b*d*e")
 	(if_then_else:DI
 	  (match_operator:CC 4 "predicate_operator"
 	    [(match_operand:CC 1 "register_operand"
 			       "c,c,c,c,c,c,c,c,c,c,c,c,c,c,c")
 	     (const_int 0)])
 	  (match_operand:DI 2 "general_operand"
-		    "0,0,0,0,0,rim*f*b,rO,rOQ,*f,r,rim*f*b,rO,rOQ,*f,r")
+		"0,0,0,0,0,rim*f*b*d*e,rO,rOQ,*f,r,rim*f*b*d*e,rO,rOQ,*f,r")
 	  (match_operand:DI 3 "general_operand"
-		    "rim*f*b,rO,rOQ,*f,r,0,0,0,0,0,rim*f*b,rO,rOQ,*f,r")))]
+		"rim*f*b*d*e,rO,rOQ,*f,r,0,0,0,0,0,rim*f*b*d*e,rO,rOQ,*f,r")))]
   "! TARGET_A_STEP"
   "* abort ();"
   [(set_attr "predicable" "no")])
diff --git a/gcc/config/rs6000/rs6000.c b/gcc/config/rs6000/rs6000.c
index 16a55af..7644606 100644
--- a/gcc/config/rs6000/rs6000.c
+++ b/gcc/config/rs6000/rs6000.c
@@ -400,11 +400,6 @@ optimization_options (level, size)
      int level;
      int size ATTRIBUTE_UNUSED;
 {
-#ifdef HAVE_decrement_and_branch_on_count
-  /* When optimizing, enable use of BCT instruction.  */
-  if (level >= 1)
-      flag_branch_on_count_reg = 1;
-#endif
 }
 
 /* Do anything needed at the start of the asm file.  */
diff --git a/gcc/config/rs6000/rs6000.md b/gcc/config/rs6000/rs6000.md
index 52ef798..428fac8 100644
--- a/gcc/config/rs6000/rs6000.md
+++ b/gcc/config/rs6000/rs6000.md
@@ -13291,16 +13291,30 @@ operands[2] = GEN_INT (INTVAL (operands[1]) >> 32);
 ;; Define the subtract-one-and-jump insns, starting with the template
 ;; so loop.c knows what to generate.
 
-(define_expand "decrement_and_branch_on_count"
-  [(use (match_operand 0 "register_operand" ""))
-   (use (label_ref (match_operand 1 "" "")))]
+(define_expand "doloop_end"
+  [(use (match_operand 0 "" ""))	; loop pseudo
+   (use (match_operand 1 "" ""))	; iterations; zero if unknown
+   (use (match_operand 2 "" ""))	; max iterations
+   (use (match_operand 3 "" ""))	; loop level
+   (use (match_operand 4 "" ""))]	; label
   ""
   "
 {
+  /* Only use this on innermost loops.  */
+  if (INTVAL (operands[3]) > 1)
+    FAIL;
   if (TARGET_POWERPC64)
-    emit_jump_insn (gen_ctrdi (operands[0], operands[1]));
+    {
+      if (GET_MODE (operands[0]) != DImode)
+	FAIL;
+      emit_jump_insn (gen_ctrdi (operands[0], operands[4]));
+    }
   else
-    emit_jump_insn (gen_ctrsi (operands[0], operands[1]));
+    {
+      if (GET_MODE (operands[0]) != SImode)
+	FAIL;
+      emit_jump_insn (gen_ctrsi (operands[0], operands[4]));
+    }
   DONE;
 }")